diff options
Diffstat (limited to 'erts')
273 files changed, 41947 insertions, 19408 deletions
diff --git a/erts/Makefile.in b/erts/Makefile.in index fabf86db7c..2e63fc469e 100644 --- a/erts/Makefile.in +++ b/erts/Makefile.in @@ -1,19 +1,19 @@ # # %CopyrightBegin% -# -# Copyright Ericsson AB 2006-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 2006-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # include $(ERL_TOP)/make/target.mk @@ -87,15 +87,20 @@ endif # in the same directory... local_setup: @cd start_scripts && $(MAKE) + @echo `ls $(ERL_TOP)/bin/` @rm -f $(ERL_TOP)/bin/erl $(ERL_TOP)/bin/erlc $(ERL_TOP)/bin/cerl \ $(ERL_TOP)/bin/erl.exe $(ERL_TOP)/bin/erlc.exe \ $(ERL_TOP)/bin/escript $(ERL_TOP)/bin/escript.exe \ $(ERL_TOP)/bin/dialyzer $(ERL_TOP)/bin/dialyzer.exe \ $(ERL_TOP)/bin/typer $(ERL_TOP)/bin/typer.exe \ + $(ERL_TOP)/bin/run_test $(ERL_TOP)/bin/run_test.exe \ + $(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/ct_run.exe \ $(ERL_TOP)/bin/start*.boot $(ERL_TOP)/bin/start*.script @if [ "X$(TARGET)" = "Xwin32" ]; then \ cp $(ERL_TOP)/bin/$(TARGET)/dialyzer.exe $(ERL_TOP)/bin/dialyzer.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/typer.exe $(ERL_TOP)/bin/typer.exe; \ + cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/ct_run.exe; \ + cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/run_test.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erlc.exe $(ERL_TOP)/bin/erlc.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erl.exe $(ERL_TOP)/bin/erl.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/werl.exe $(ERL_TOP)/bin/werl.exe; \ @@ -115,6 +120,8 @@ local_setup: $(ERL_TOP)/erts/etc/unix/cerl.src > $(ERL_TOP)/bin/cerl; \ cp $(ERL_TOP)/bin/$(TARGET)/dialyzer $(ERL_TOP)/bin/dialyzer; \ cp $(ERL_TOP)/bin/$(TARGET)/typer $(ERL_TOP)/bin/typer; \ + cp $(ERL_TOP)/bin/$(TARGET)/ct_run $(ERL_TOP)/bin/ct_run; \ + ln -s $(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/run_test; \ cp $(ERL_TOP)/bin/$(TARGET)/erlc $(ERL_TOP)/bin/erlc; \ cp $(ERL_TOP)/bin/$(TARGET)/escript $(ERL_TOP)/bin/escript; \ chmod 755 $(ERL_TOP)/bin/erl $(ERL_TOP)/bin/erlc \ diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 3d935b7295..a1211bbf0c 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -386,14 +386,24 @@ AC_DEFUN(LM_SYS_IPV6, AC_CACHE_VAL(ac_cv_sys_ipv6_support, [ok_so_far=yes AC_TRY_COMPILE([#include <sys/types.h> -#include <netinet/in.h>], +#ifdef __WIN32__ +#include <winsock2.h> +#include <ws2tcpip.h> +#else +#include <netinet/in.h> +#endif], [struct in6_addr a6; struct sockaddr_in6 s6;], ok_so_far=yes, ok_so_far=no) if test $ok_so_far = yes; then ac_cv_sys_ipv6_support=yes else AC_TRY_COMPILE([#include <sys/types.h> -#include <netinet/in.h>], +#ifdef __WIN32__ +#include <winsock2.h> +#include <ws2tcpip.h> +#else +#include <netinet/in.h> +#endif], [struct in_addr6 a6; struct sockaddr_in6 s6;], ac_cv_sys_ipv6_support=in_addr6, ac_cv_sys_ipv6_support=no) fi @@ -512,6 +522,8 @@ dnl AC_DEFUN(LM_CHECK_THR_LIB, [ +NEED_NPTL_PTHREAD_H=no + dnl win32? AC_MSG_CHECKING([for native win32 threads]) if test "X$host_os" = "Xwin32"; then @@ -519,11 +531,13 @@ if test "X$host_os" = "Xwin32"; then THR_DEFS="-DWIN32_THREADS" THR_LIBS= THR_LIB_NAME=win32_threads + THR_LIB_TYPE=win32_threads else AC_MSG_RESULT(no) THR_DEFS= THR_LIBS= THR_LIB_NAME= + THR_LIB_TYPE=posix_unknown dnl Try to find POSIX threads @@ -584,8 +598,11 @@ dnl On ofs1 the '-pthread' switch should be used AC_MSG_WARN([result yes guessed because of cross compilation]) fi if test $nptl = yes; then + THR_LIB_TYPE=posix_nptl need_nptl_incldir=no - AC_CHECK_HEADER(nptl/pthread.h, need_nptl_incldir=yes) + AC_CHECK_HEADER(nptl/pthread.h, + [need_nptl_incldir=yes + NEED_NPTL_PTHREAD_H=yes]) if test $need_nptl_incldir = yes; then # Ahh... nptl_path="$C_INCLUDE_PATH:$CPATH" @@ -649,6 +666,19 @@ fi ]) +AC_DEFUN(ERL_INTERNAL_LIBS, +[ + +ERTS_INTERNAL_X_LIBS= + +AC_CHECK_LIB(kstat, kstat_open, +[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat]) +ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"]) + +AC_SUBST(ERTS_INTERNAL_X_LIBS) + +]) + dnl ---------------------------------------------------------------------- dnl dnl ERL_FIND_ETHR_LIB @@ -672,10 +702,14 @@ AC_DEFUN(ERL_FIND_ETHR_LIB, [ LM_CHECK_THR_LIB +ERL_INTERNAL_LIBS +ethr_have_native_atomics=no +ethr_have_native_spinlock=no ETHR_THR_LIB_BASE="$THR_LIB_NAME" +ETHR_THR_LIB_BASE_TYPE="$THR_LIB_TYPE" ETHR_DEFS="$THR_DEFS" -ETHR_X_LIBS="$THR_LIBS" +ETHR_X_LIBS="$THR_LIBS $ERTS_INTERNAL_X_LIBS" ETHR_LIBS= ETHR_LIB_NAME= @@ -687,6 +721,7 @@ ethr_lib_name=ethread case "$THR_LIB_NAME" in win32_threads) + ETHR_THR_LIB_BASE_DIR=win # * _WIN32_WINNT >= 0x0400 is needed for # TryEnterCriticalSection # * _WIN32_WINNT >= 0x0403 is needed for @@ -712,10 +747,128 @@ case "$THR_LIB_NAME" in if test $found_win32_winnt = no; then AC_MSG_ERROR([-D_WIN32_WINNT missing in CPPFLAGS]) fi + AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedCompareExchange64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedCompareExchange64(var, (__int64) 1, (__int64) 0); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()]) + + AC_CHECK_SIZEOF(void *) + case "$ac_cv_sizeof_void_p-$have_ilckd" in + 8-no) + ethr_have_native_atomics=no + ethr_have_native_spinlock=no;; + *) + ethr_have_native_atomics=yes + ethr_have_native_spinlock=yes;; + esac + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedDecrement64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedDecrement64(var); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedIncrement64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedIncrement64(var); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedExchangeAdd64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedExchangeAdd64(var, (__int64) 1); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedExchange64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedExchange64(var, (__int64) 1); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedAnd64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedAnd64(var, (__int64) 1); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedOr64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedOr64(var, (__int64) 1); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()]) + ;; pthread) + ETHR_THR_LIB_BASE_DIR=pthread AC_DEFINE(ETHR_PTHREADS, 1, [Define if you have pthreads]) case $host_os in openbsd*) @@ -771,9 +924,7 @@ case "$THR_LIB_NAME" in if test $usable_sigaltstack = no; then ETHR_DEFS="$ETHR_DEFS -DETHR_UNUSABLE_SIGALTSTACK" fi - - AC_DEFINE(ETHR_INIT_MUTEX_IN_CHILD_AT_FORK, 1, \ -[Define if mutexes should be reinitialized (instead of unlocked) in child at fork.]) ;; + ;; *) ;; esac @@ -799,6 +950,15 @@ case "$THR_LIB_NAME" in AC_DEFINE(ETHR_HAVE_MIT_PTHREAD_H, 1, \ [Define if the pthread.h header file is in pthread/mit directory.])) + if test $NEED_NPTL_PTHREAD_H = yes; then + AC_DEFINE(ETHR_NEED_NPTL_PTHREAD_H, 1, \ +[Define if you need the <nptl/pthread.h> header file.]) + fi + + AC_CHECK_HEADER(sched.h, \ + AC_DEFINE(ETHR_HAVE_SCHED_H, 1, \ +[Define if you have the <sched.h> header file.])) + AC_CHECK_HEADER(sys/time.h, \ AC_DEFINE(ETHR_HAVE_SYS_TIME_H, 1, \ [Define if you have the <sys/time.h> header file.])) @@ -814,39 +974,212 @@ case "$THR_LIB_NAME" in dnl Check for functions dnl - AC_CHECK_FUNC(pthread_atfork, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_ATFORK, 1, \ -[Define if you have the pthread_atfork function.])) - AC_CHECK_FUNC(pthread_mutexattr_settype, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE, 1, \ -[Define if you have the pthread_mutexattr_settype function.])) - AC_CHECK_FUNC(pthread_mutexattr_setkind_np, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP, 1, \ -[Define if you have the pthread_mutexattr_setkind_np function.])) AC_CHECK_FUNC(pthread_spin_lock, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \ -[Define if you have the pthread_spin_lock function.])) - test "$force_linux_pthread_rwlocks" = "yes" || { - force_linux_pthread_rwlocks=no - } - case "$force_linux_pthread_rwlocks-$host_os" in - no-linux*) # Writers may get starved - # TODO: write a test that tests the implementation - ;; - *) - AC_CHECK_FUNC(pthread_rwlock_init, \ - AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCK_INIT, 1, \ -[Define if you have a pthread_rwlock implementation that can be used.])) - ;; - esac + [ethr_have_native_spinlock=yes \ + AC_DEFINE(ETHR_HAVE_PTHREAD_SPIN_LOCK, 1, \ +[Define if you have the pthread_spin_lock function.])]) + + have_sched_yield=no + have_librt_sched_yield=no + AC_CHECK_FUNC(sched_yield, [have_sched_yield=yes]) + if test $have_sched_yield = no; then + AC_CHECK_LIB(rt, sched_yield, + [have_librt_sched_yield=yes + ETHR_X_LIBS="$ETHR_X_LIBS -lrt"]) + fi + if test $have_sched_yield = yes || test $have_librt_sched_yield = yes; then + AC_DEFINE(ETHR_HAVE_SCHED_YIELD, 1, [Define if you have the sched_yield() function.]) + AC_MSG_CHECKING([whether sched_yield() returns an int]) + sched_yield_ret_int=no + AC_TRY_COMPILE([ + #ifdef ETHR_HAVE_SCHED_H + #include <sched.h> + #endif + ], + [int sched_yield();], + [sched_yield_ret_int=yes]) + AC_MSG_RESULT([$sched_yield_ret_int]) + if test $sched_yield_ret_int = yes; then + AC_DEFINE(ETHR_SCHED_YIELD_RET_INT, 1, [Define if sched_yield() returns an int.]) + fi + fi + + have_pthread_yield=no + AC_CHECK_FUNC(pthread_yield, [have_pthread_yield=yes]) + if test $have_pthread_yield = yes; then + AC_DEFINE(ETHR_HAVE_PTHREAD_YIELD, 1, [Define if you have the pthread_yield() function.]) + AC_MSG_CHECKING([whether pthread_yield() returns an int]) + pthread_yield_ret_int=no + AC_TRY_COMPILE([ + #if defined(ETHR_NEED_NPTL_PTHREAD_H) + #include <nptl/pthread.h> + #elif defined(ETHR_HAVE_MIT_PTHREAD_H) + #include <pthread/mit/pthread.h> + #elif defined(ETHR_HAVE_PTHREAD_H) + #include <pthread.h> + #endif + ], + [int pthread_yield();], + [pthread_yield_ret_int=yes]) + AC_MSG_RESULT([$pthread_yield_ret_int]) + if test $pthread_yield_ret_int = yes; then + AC_DEFINE(ETHR_PTHREAD_YIELD_RET_INT, 1, [Define if pthread_yield() returns an int.]) + fi + fi + + have_pthread_rwlock_init=no + AC_CHECK_FUNC(pthread_rwlock_init, [have_pthread_rwlock_init=yes]) + if test $have_pthread_rwlock_init = yes; then + + ethr_have_pthread_rwlockattr_setkind_np=no + AC_CHECK_FUNC(pthread_rwlockattr_setkind_np, + [ethr_have_pthread_rwlockattr_setkind_np=yes]) + + if test $ethr_have_pthread_rwlockattr_setkind_np = yes; then + AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP, 1, \ +[Define if you have the pthread_rwlockattr_setkind_np() function.]) + + AC_MSG_CHECKING([for PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP]) + ethr_pthread_rwlock_writer_nonrecursive_initializer_np=no + AC_TRY_LINK([ + #if defined(ETHR_NEED_NPTL_PTHREAD_H) + #include <nptl/pthread.h> + #elif defined(ETHR_HAVE_MIT_PTHREAD_H) + #include <pthread/mit/pthread.h> + #elif defined(ETHR_HAVE_PTHREAD_H) + #include <pthread.h> + #endif + ], + [ + pthread_rwlockattr_t *attr; + return pthread_rwlockattr_setkind_np(attr, + PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + ], + [ethr_pthread_rwlock_writer_nonrecursive_initializer_np=yes]) + AC_MSG_RESULT([$ethr_pthread_rwlock_writer_nonrecursive_initializer_np]) + if test $ethr_pthread_rwlock_writer_nonrecursive_initializer_np = yes; then + AC_DEFINE(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP, 1, \ +[Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock attribute.]) + fi + fi + fi + + if test "$force_pthread_rwlocks" = "yes"; then + + AC_DEFINE(ETHR_FORCE_PTHREAD_RWLOCK, 1, \ +[Define if you want to force usage of pthread rwlocks]) + + if test $have_pthread_rwlock_init = yes; then + AC_MSG_WARN([Forced usage of pthread rwlocks. Note that this implementation may suffer from starvation issues.]) + else + AC_MSG_ERROR([User forced usage of pthread rwlock, but no such implementation was found]) + fi + fi + AC_CHECK_FUNC(pthread_attr_setguardsize, \ AC_DEFINE(ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE, 1, \ [Define if you have the pthread_attr_setguardsize function.])) + linux_futex=no + AC_MSG_CHECKING([for Linux futexes]) + AC_TRY_LINK([ + #include <sys/syscall.h> + #include <unistd.h> + #include <linux/futex.h> + #include <sys/time.h> + ], + [ + int i = 1; + syscall(__NR_futex, (void *) &i, FUTEX_WAKE, 1, + (void*)0,(void*)0, 0); + syscall(__NR_futex, (void *) &i, FUTEX_WAIT, 0, + (void*)0,(void*)0, 0); + return 0; + ], + linux_futex=yes) + AC_MSG_RESULT([$linux_futex]) + test $linux_futex = yes && AC_DEFINE(ETHR_HAVE_LINUX_FUTEX, 1, [Define if you have a linux futex implementation.]) + + AC_MSG_CHECKING([for GCC atomic operations]) + ethr_have_gcc_atomic_ops=no + AC_TRY_LINK([], + [ + long res; + volatile long val; + res = __sync_val_compare_and_swap(&val, (long) 1, (long) 0); + res = __sync_add_and_fetch(&val, (long) 1); + res = __sync_sub_and_fetch(&val, (long) 1); + res = __sync_fetch_and_and(&val, (long) 1); + res = __sync_fetch_and_or(&val, (long) 1); + ], + [ethr_have_native_atomics=yes + ethr_have_gcc_atomic_ops=yes]) + AC_MSG_RESULT([$ethr_have_gcc_atomic_ops]) + test $ethr_have_gcc_atomic_ops = yes && AC_DEFINE(ETHR_HAVE_GCC_ATOMIC_OPS, 1, [Define if you have gcc atomic operations]) + + case "$host_cpu" in + sun4u | sparc64 | sun4v) + ethr_have_native_atomics=yes;; + i86pc | i*86 | x86_64 | amd64) + ethr_have_native_atomics=yes;; + macppc | ppc | "Power Macintosh") + ethr_have_native_atomics=yes;; + tile) + ethr_have_native_atomics=yes;; + *) + ;; + esac + + AC_MSG_CHECKING([for a usable libatomic_ops implementation]) + case "x$with_libatomic_ops" in + xno | xyes | x) + libatomic_ops_include= + ;; + *) + if test -d "${with_libatomic_ops}/include"; then + libatomic_ops_include="-I$with_libatomic_ops/include" + CPPFLAGS="$CPPFLAGS $libatomic_ops_include" + else + AC_MSG_ERROR([libatomic_ops include directory $with_libatomic_ops/include not found]) + fi;; + esac + ethr_have_libatomic_ops=no + AC_TRY_LINK([#include "atomic_ops.h"], + [ + volatile AO_t x; + AO_t y; + int z; + + AO_nop_full(); + AO_store(&x, (AO_t) 0); + z = AO_load(&x); + z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); + ], + [ethr_have_native_atomics=yes + ethr_have_libatomic_ops=yes]) + AC_MSG_RESULT([$ethr_have_libatomic_ops]) + if test $ethr_have_libatomic_ops = yes; then + AC_CHECK_SIZEOF(AO_t, , + [ + #include <stdio.h> + #include "atomic_ops.h" + ]) + AC_DEFINE_UNQUOTED(ETHR_SIZEOF_AO_T, $ac_cv_sizeof_AO_t, [Define to the size of AO_t if libatomic_ops is used]) + + AC_DEFINE(ETHR_HAVE_LIBATOMIC_OPS, 1, [Define if you have libatomic_ops atomic operations]) + if test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then + AC_DEFINE(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS, 1, [Define if you prefer libatomic_ops native ethread implementations]) + fi + ETHR_DEFS="$ETHR_DEFS $libatomic_ops_include" + elif test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then + AC_MSG_ERROR([No usable libatomic_ops implementation found]) + fi + dnl Restore LIBS LIBS=$saved_libs dnl restore CPPFLAGS CPPFLAGS=$saved_cppflags + ;; *) ;; @@ -862,17 +1195,111 @@ fi if test "x$ETHR_THR_LIB_BASE" != "x"; then ETHR_DEFS="-DUSE_THREADS $ETHR_DEFS" - ETHR_LIBS="-l$ethr_lib_name $ETHR_X_LIBS" + ETHR_LIBS="-l$ethr_lib_name -lerts_internal_r $ETHR_X_LIBS" ETHR_LIB_NAME=$ethr_lib_name fi AC_CHECK_SIZEOF(void *) AC_DEFINE_UNQUOTED(ETHR_SIZEOF_PTR, $ac_cv_sizeof_void_p, [Define to the size of pointers]) -if test "X$disable_native_ethr_impls" = "Xyes"; then - AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) +AC_CHECK_SIZEOF(int) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF_INT, $ac_cv_sizeof_int, [Define to the size of int]) +AC_CHECK_SIZEOF(long) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG, $ac_cv_sizeof_long, [Define to the size of long]) +AC_CHECK_SIZEOF(long long) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG_LONG, $ac_cv_sizeof_long_long, [Define to the size of long long]) +AC_CHECK_SIZEOF(__int64) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT64, $ac_cv_sizeof___int64, [Define to the size of __int64]) + + +case X$erl_xcomp_bigendian in + X) ;; + Xyes|Xno) ac_cv_c_bigendian=$erl_xcomp_bigendian;; + *) AC_MSG_ERROR([Bad erl_xcomp_bigendian value: $erl_xcomp_bigendian]);; +esac + +AC_C_BIGENDIAN + +if test "$ac_cv_c_bigendian" = "yes"; then + AC_DEFINE(ETHR_BIGENDIAN, 1, [Define if bigendian]) fi +AC_ARG_ENABLE(native-ethr-impls, + AS_HELP_STRING([--disable-native-ethr-impls], + [disable native ethread implementations]), +[ case "$enableval" in + no) disable_native_ethr_impls=yes ;; + *) disable_native_ethr_impls=no ;; + esac ], disable_native_ethr_impls=no) + +test "X$disable_native_ethr_impls" = "Xyes" && + AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) + +AC_ARG_ENABLE(prefer-gcc-native-ethr-impls, + AS_HELP_STRING([--enable-prefer-gcc-native-ethr-impls], + [prefer gcc native ethread implementations]), +[ case "$enableval" in + yes) enable_prefer_gcc_native_ethr_impls=yes ;; + *) enable_prefer_gcc_native_ethr_impls=no ;; + esac ], enable_prefer_gcc_native_ethr_impls=no) + +test $enable_prefer_gcc_native_ethr_impls = yes && + AC_DEFINE(ETHR_PREFER_GCC_NATIVE_IMPLS, 1, [Define if you prefer gcc native ethread implementations]) + +AC_ARG_WITH(libatomic_ops, + AS_HELP_STRING([--with-libatomic_ops=PATH], + [specify and prefer usage of libatomic_ops in the ethread library])) + +AC_ARG_ENABLE(ethread-pre-pentium4-compatibility, + AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility], + [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]), +[ + case "$enable_ethread_pre_pentium4_compatibility" in + yes|no) ;; + *) enable_ethread_pre_pentium4_compatibility=check;; + esac +], +[enable_ethread_pre_pentium4_compatibility=check]) + +test "$cross_compiling" != "yes" || enable_ethread_pre_pentium4_compatibility=no + +case "$enable_ethread_pre_pentium4_compatibility-$host_cpu" in + check-i86pc | check-i*86) + AC_MSG_CHECKING([whether pre pentium 4 compatibility should forced]) + AC_RUN_IFELSE([ +#if defined(__GNUC__) +# if defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS) +# define CHECK_LIBATOMIC_OPS__ +# else +# define CHECK_GCC_ASM__ +# endif +#elif defined(ETHR_HAVE_LIBATOMIC_OPS) +# define CHECK_LIBATOMIC_OPS__ +#endif +#if defined(CHECK_LIBATOMIC_OPS__) +#include "atomic_ops.h" +#endif +int main(void) +{ +#if defined(CHECK_GCC_ASM__) + __asm__ __volatile__("mfence" : : : "memory"); +#elif defined(CHECK_LIBATOMIC_OPS__) + AO_nop_full(); +#endif + return 0; +} + ], + [enable_ethread_pre_pentium4_compatibility=no], + [enable_ethread_pre_pentium4_compatibility=yes], + [enable_ethread_pre_pentium4_compatibility=no]) + AC_MSG_RESULT([$enable_ethread_pre_pentium4_compatibility]);; + *) + ;; +esac + +test $enable_ethread_pre_pentium4_compatibility = yes && + AC_DEFINE(ETHR_PRE_PENTIUM4_COMPAT, 1, [Define if you want compatibilty with x86 processors before pentium4.]) + AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \ [Define if you have all ethread defines]) @@ -881,6 +1308,7 @@ AC_SUBST(ETHR_LIBS) AC_SUBST(ETHR_LIB_NAME) AC_SUBST(ETHR_DEFS) AC_SUBST(ETHR_THR_LIB_BASE) +AC_SUBST(ETHR_THR_LIB_BASE_DIR) ]) diff --git a/erts/autoconf/configure.vxworks b/erts/autoconf/configure.vxworks index 70d7bdbaf2..14fbf766dc 100755 --- a/erts/autoconf/configure.vxworks +++ b/erts/autoconf/configure.vxworks @@ -101,7 +101,6 @@ epmd_dir=${ERL_TOP}/erts/epmd/src os_mon_dir=${ERL_TOP}/lib/os_mon/c_src orber_dir=${ERL_TOP}/lib/orber/c_src ic_dir=${ERL_TOP}/lib/ic/c_src -asn1_dir=${ERL_TOP}/lib/asn1/c_src internal_tools_dir=${ERL_TOP} libdir=${ERL_TOP}/lib tsdir=$libdir/test_server/src @@ -122,7 +121,6 @@ CONFIG_FILES="${ERL_TOP}/erts/emulator/$host/Makefile $os_mon_dir/$host/Makefile $zlibdir/$host/Makefile $ic_dir/$host/Makefile - $asn1_dir/$host/Makefile $runtime_tools_dir/$host/Makefile $tools_dir/$host/Makefile $orber_dir/$host/Makefile" diff --git a/erts/autoconf/vxworks/sed.general b/erts/autoconf/vxworks/sed.general index f725a6f9ca..551458daf5 100644 --- a/erts/autoconf/vxworks/sed.general +++ b/erts/autoconf/vxworks/sed.general @@ -55,12 +55,16 @@ s|@ETHR_LIBS@|| s|@ETHR_LIB_NAME@|| s|@ETHR_DEFS@|| s|@ETHR_THR_LIB_BASE@|| +s|@ETHR_THR_LIB_BASE_DIR@|| s|@EMU_THR_DEFS@|| s|@EMU_THR_LIBS@|| s|@EMU_THR_LIB_NAME@|ethread| s|@ERTS_ENABLE_KERNEL_POLL@|no| +s|@ERTS_INTERNAL_X_LIBS@|| s|@cc_root@|/clearcase/otp/| # Define VxWorks even though cross-compiling. +s|@CROSS_COMPILING|yes| + s|@HCFLAGS@|-DVXWORKS| s|@HCLIBS@|| s|@ENABLE_ALLOC_TYPE_VARS@|| diff --git a/erts/autoconf/win32.config.cache b/erts/autoconf/win32.config.cache.static index 31dfe510cd..d25b1df9d9 100755 --- a/erts/autoconf/win32.config.cache +++ b/erts/autoconf/win32.config.cache.static @@ -61,7 +61,6 @@ ac_cv_func_fork=${ac_cv_func_fork=no} ac_cv_func_fork_works=${ac_cv_func_fork_works=no} ac_cv_func_fpsetmask=${ac_cv_func_fpsetmask=no} ac_cv_func_fstat=${ac_cv_func_fstat=yes} -ac_cv_func_getaddrinfo=${ac_cv_func_getaddrinfo=no} ac_cv_func_gethostbyaddr=${ac_cv_func_gethostbyaddr=no} ac_cv_func_gethostbyaddr_r=${ac_cv_func_gethostbyaddr_r=no} ac_cv_func_gethostbyname=${ac_cv_func_gethostbyname=no} @@ -71,7 +70,6 @@ ac_cv_func_gethostname=${ac_cv_func_gethostname=no} ac_cv_func_gethrtime=${ac_cv_func_gethrtime=no} ac_cv_func_getipnodebyaddr=${ac_cv_func_getipnodebyaddr=no} ac_cv_func_getipnodebyname=${ac_cv_func_getipnodebyname=no} -ac_cv_func_getnameinfo=${ac_cv_func_getnameinfo=no} ac_cv_func_getpagesize=${ac_cv_func_getpagesize=no} ac_cv_func_gettimeofday=${ac_cv_func_gettimeofday=no} ac_cv_func_gmtime_r=${ac_cv_func_gmtime_r=no} @@ -212,7 +210,6 @@ ac_cv_sizeof_void_p=${ac_cv_sizeof_void_p=4} ac_cv_struct_exception=${ac_cv_struct_exception=no} ac_cv_struct_sockaddr_sa_len=${ac_cv_struct_sockaddr_sa_len=no} ac_cv_struct_tm=${ac_cv_struct_tm=time.h} -ac_cv_sys_ipv6_support=${ac_cv_sys_ipv6_support=no} ac_cv_sys_multicast_support=${ac_cv_sys_multicast_support=no} ac_cv_type_char=${ac_cv_type_char=yes} ac_cv_type_int=${ac_cv_type_int=yes} diff --git a/erts/configure.in b/erts/configure.in index a14b10adbf..627f734409 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -61,6 +61,9 @@ if test x"${ERL_TOP}/erts" != x"$srcdir"; then fi erl_top=${ERL_TOP} +# Remove old configuration information +/bin/rm -f "$ERL_TOP/erts/CONF_INFO" + # echo XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX # echo X # echo "X srcdir = $srcdir" @@ -87,6 +90,13 @@ else host_os=$host fi +if test "$cross_compiling" = "yes"; then + CROSS_COMPILING=yes +else + CROSS_COMPILING=no +fi +AC_SUBST(CROSS_COMPILING) + ERL_XCOMP_SYSROOT_INIT AC_ISC_POSIX @@ -100,7 +110,8 @@ ENABLE_ALLOC_TYPE_VARS= AC_SUBST(ENABLE_ALLOC_TYPE_VARS) AC_ARG_ENABLE(bootstrap-only, -[ --enable-bootstrap-only enable bootstrap only configuration], +AS_HELP_STRING([--enable-bootstrap-only], + [enable bootstrap only configuration]), [ if test "X$enableval" = "Xyes"; then # Disable stuff not necessary in a bootstrap only system in order # to speed up things by reducing the amount of stuff needing to be @@ -116,46 +127,46 @@ AC_ARG_ENABLE(bootstrap-only, ]) AC_ARG_ENABLE(threads, -[ --enable-threads enable async thread support - --disable-threads disable async thread support], +AS_HELP_STRING([--enable-threads], [enable async thread support]) +AS_HELP_STRING([--disable-threads], [disable async thread support]), [ case "$enableval" in no) enable_threads=no ;; *) enable_threads=yes ;; esac ], enable_threads=unknown) AC_ARG_ENABLE(halfword-emulator, -[ --enable-halfword-emulator enable halfword emulator (only for 64bit builds) - --disable-halfword-emulator disable halfword emulator (only for 64bit builds)], +AS_HELP_STRING([--enable-halfword-emulator], + [enable halfword emulator (only for 64bit builds)]), [ case "$enableval" in no) enable_halfword_emualtor=no ;; *) enable_halfword_emulator=yes ;; esac ], enable_halfword_emulator=unknown) AC_ARG_ENABLE(smp-support, -[ --enable-smp-support enable smp support - --disable-smp-support disable smp support], +AS_HELP_STRING([--enable-smp-support], [enable smp support]) +AS_HELP_STRING([--disable-smp-support], [disable smp support]), [ case "$enableval" in no) enable_smp_support=no ;; *) enable_smp_support=yes ;; esac ], enable_smp_support=unknown) AC_ARG_WITH(termcap, -[ --with-termcap use termcap (default) - --without-termcap do not use any termcap libraries (ncurses,curses,termcap,termlib)], +AS_HELP_STRING([--with-termcap], [use termcap (default)]) +AS_HELP_STRING([--without-termcap], + [do not use any termcap libraries (ncurses,curses,termcap,termlib)]), [], [with_termcap=yes]) AC_ARG_ENABLE(hybrid-heap, -[ --enable-hybrid-heap enable hybrid heap - --disable-hybrid-heap disable hybrid heap], +AS_HELP_STRING([--enable-hybrid-heap], [enable hybrid heap]), [ case "$enableval" in no) enable_hybrid_heap=no ;; *) enable_hybrid_heap=yes ;; esac ], enable_hybrid_heap=unknown) AC_ARG_ENABLE(lock-checking, -[ --enable-lock-checking enable lock checking], +AS_HELP_STRING([--enable-lock-checking], [enable lock checking]), [ case "$enableval" in no) enable_lock_check=no ;; *) enable_lock_check=yes ;; @@ -164,16 +175,15 @@ AC_ARG_ENABLE(lock-checking, enable_lock_check=no) AC_ARG_ENABLE(lock-counter, -[ --enable-lock-counter enable lock counters - --disable-lock-counter disable lock counters], +AS_HELP_STRING([--enable-lock-counter], [enable lock counters]), [ case "$enableval" in no) enable_lock_count=no ;; *) enable_lock_count=yes ;; esac ], enable_lock_count=no) AC_ARG_ENABLE(kernel-poll, -[ --enable-kernel-poll enable kernel poll support - --disable-kernel-poll disable kernel poll support], +AS_HELP_STRING([--enable-kernel-poll], [enable kernel poll support]) +AS_HELP_STRING([--disable-kernel-poll], [disable kernel poll support]), [ case "$enableval" in no) enable_kernel_poll=no ;; *) enable_kernel_poll=yes ;; @@ -181,28 +191,27 @@ AC_ARG_ENABLE(kernel-poll, AC_ARG_ENABLE(sctp, -[ --enable-sctp enable sctp support - --disable-sctp disable sctp support], +AS_HELP_STRING([--enable-sctp], [enable sctp support]) +AS_HELP_STRING([--disable-sctp], [disable sctp support]), [ case "$enableval" in no) enable_sctp=no ;; *) enable_sctp=yes ;; esac ], enable_sctp=unknown) AC_ARG_ENABLE(hipe, -[ --enable-hipe enable hipe support - --disable-hipe disable hipe support]) +AS_HELP_STRING([--enable-hipe], [enable hipe support]) +AS_HELP_STRING([--disable-hipe], [disable hipe support])) AC_ARG_ENABLE(native-libs, -[ --enable-native-libs compile Erlang libraries to native code]) +AS_HELP_STRING([--enable-native-libs], + [compile Erlang libraries to native code])) AC_ARG_ENABLE(tsp, -[ --enable-tsp compile tsp app]) - -AC_ARG_ENABLE(elib-malloc, -[ --enable-elib-malloc use elib_malloc instead of normal malloc]) +AS_HELP_STRING([--enable-tsp], [compile tsp app])) AC_ARG_ENABLE(fp-exceptions, -[ --enable-fp-exceptions Use hardware floating point exceptions (default if hipe enabled)], +AS_HELP_STRING([--enable-fp-exceptions], + [use hardware floating point exceptions (default if hipe enabled)]), [ case "$enableval" in no) enable_fp_exceptions=no ;; *) enable_fp_exceptions=yes ;; @@ -210,7 +219,8 @@ AC_ARG_ENABLE(fp-exceptions, ],enable_fp_exceptions=auto) AC_ARG_ENABLE(darwin-universal, -[ --enable-darwin-universal build universal binaries on darwin i386], +AS_HELP_STRING([--enable-darwin-universal], + [build universal binaries on darwin i386]), [ case "$enableval" in no) enable_darwin_universal=no ;; *) enable_darwin_univeral=yes ;; @@ -219,7 +229,7 @@ AC_ARG_ENABLE(darwin-universal, AC_ARG_ENABLE(darwin-64bit, -[ --enable-darwin-64bit build 64bit binaries on darwin], +AS_HELP_STRING([--enable-darwin-64bit], [build 64bit binaries on darwin]), [ case "$enableval" in no) enable_darwin_64bit=no ;; *) enable_darwin_64bit=yes ;; @@ -227,7 +237,8 @@ AC_ARG_ENABLE(darwin-64bit, ],enable_darwin_64bit=no) AC_ARG_ENABLE(m64-build, -[ --enable-m64-build build 64bit binaries using the -m64 flag to (g)cc], +AS_HELP_STRING([--enable-m64-build], + [build 64bit binaries using the -m64 flag to (g)cc]), [ case "$enableval" in no) enable_m64_build=no ;; *) enable_m64_build=yes ;; @@ -235,7 +246,8 @@ AC_ARG_ENABLE(m64-build, ],enable_m64_build=no) AC_ARG_ENABLE(m32-build, -[ --enable-m32-build build 32bit binaries using the -m32 flag to (g)cc], +AS_HELP_STRING([--enable-m32-build], + [build 32bit binaries using the -m32 flag to (g)cc]), [ case "$enableval" in no) enable_m32_build=no ;; *) @@ -248,7 +260,7 @@ AC_ARG_ENABLE(m32-build, ],enable_m32_build=no) AC_ARG_ENABLE(fixalloc, -[ --disable-fixalloc disable the use of fix_alloc]) +AS_HELP_STRING([--disable-fixalloc], [disable the use of fix_alloc])) if test x${enable_fixalloc} = xno ; then AC_DEFINE(NO_FIX_ALLOC,[], [Define if you don't want the fix allocator in Erlang]) @@ -256,8 +268,9 @@ fi AC_SUBST(PERFCTR_PATH) AC_ARG_WITH(perfctr, -[ --with-perfctr=PATH specify location of perfctr include and lib - --without-perfctr don't use perfctr (default)]) +AS_HELP_STRING([--with-perfctr=PATH], + [specify location of perfctr include and lib]) +AS_HELP_STRING([--without-perfctr], [don't use perfctr (default)])) if test "x$with_perfctr" = "xno" -o "x$with_perfctr" = "x" ; then PERFCTR_PATH= @@ -271,26 +284,13 @@ else fi AC_ARG_ENABLE(clock-gettime, -[ --enable-clock-gettime Use clock-gettime for time correction], +AS_HELP_STRING([--enable-clock-gettime], + [use clock-gettime for time correction]), [ case "$enableval" in no) clock_gettime_correction=no ;; *) clock_gettime_correction=yes ;; esac ], clock_gettime_correction=unknown) -AC_ARG_ENABLE(native-ethr-impls, -[ --enable-native-ethr-impls enable native ethread implementations - --disable-native-ethr-impls disable native ethread implementations], -[ case "$enableval" in - no) disable_native_ethr_impls=yes ;; - *) disable_native_ethr_impls=no ;; - esac ], disable_native_ethr_impls=no) - -dnl Defined in libraries/megaco/configure.in but we need it here -dnl also in order to show it to the "top user" - -AC_ARG_ENABLE(megaco_flex_scanner_lineno, -[ --disable-megaco-flex-scanner-lineno disable megaco flex scanner lineno]) - dnl Magic test for clearcase. OTP_RELEASE= if test "${ERLANG_COMMERCIAL_BUILD}" != ""; then @@ -580,6 +580,11 @@ AC_SUBST(WFLAGS) AC_SUBST(CFLAG_RUNTIME_LIBRARY_PATH) AC_CHECK_SIZEOF(void *) # Needed for ARCH and smp checks below +if test "x$ac_cv_sizeof_void_p" = x8; then + AC_SUBST(EXTERNAL_WORD_SIZE, 64) +else + AC_SUBST(EXTERNAL_WORD_SIZE, 32) +fi dnl dnl Figure out operating system and cpu architecture @@ -1049,11 +1054,48 @@ if test $ERTS_BUILD_SMP_EMU = yes; then fi AC_DEFINE(ERTS_HAVE_SMP_EMU, 1, [Define if the smp emulator is built]) + + case "$ethr_have_native_atomics-$ethr_have_native_spinlock" in + yes-*) + ;; + + no-yes) + + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> $ERL_TOP/erts/CONF_INFO <<EOF + + No native atomic implementation available. + Fallbacks implemented using spinlocks will be + used. Note that the performance of the SMP + runtime system will suffer due to this. + +EOF + ;; + + no-no) + + test -f "$ERL_TOP/erts/CONF_INFO" || + echo "" > "$ERL_TOP/erts/CONF_INFO" + cat >> "$ERL_TOP/erts/CONF_INFO" <<EOF + + No native atomic implementation, nor no native + spinlock implementation available. Fallbacks + implemented using mutexes will be used. Note + that the performance of the SMP runtime system + will suffer much due to this. + +EOF + ;; + + esac + enable_threads=force fi AC_SUBST(ERTS_BUILD_SMP_EMU) +AC_CHECK_FUNCS([posix_fadvise]) # @@ -1148,17 +1190,15 @@ else enable_child_waiter_thread=yes ;; linux*) - AC_DEFINE(USE_RECURSIVE_MALLOC_MUTEX,[1], - [Define if malloc should use a recursive mutex]) AC_MSG_CHECKING([whether dlopen() needs to be called before first call to dlerror()]) - if test "x$ETHR_THR_LIB_BASE_NAME" != "xnptl"; then + if test "x$ETHR_THR_LIB_BASE_TYPE" != "xposix_nptl"; then AC_DEFINE(ERTS_NEED_DLOPEN_BEFORE_DLERROR,[1], [Define if dlopen() needs to be called before first call to dlerror()]) AC_MSG_RESULT(yes) else AC_MSG_RESULT(no) fi - if test "x$ETHR_THR_LIB_BASE_NAME" != "xnptl"; then + if test "x$ETHR_THR_LIB_BASE_TYPE" != "xposix_nptl"; then # Child waiter thread cannot be enabled disable_child_waiter_thread=yes enable_child_waiter_thread=no @@ -1220,13 +1260,7 @@ fi AC_SUBST(EMU_LOCK_CHECKING) -ERTS_INTERNAL_X_LIBS= - -AC_CHECK_LIB(kstat, kstat_open, -[AC_DEFINE(HAVE_KSTAT, 1, [Define if you have kstat]) -ERTS_INTERNAL_X_LIBS="$ERTS_INTERNAL_X_LIBS -lkstat"]) - -AC_SUBST(ERTS_INTERNAL_X_LIBS) +ERL_INTERNAL_LIBS dnl THR_LIBS and THR_DEFS are only used by odbc THR_LIBS=$ETHR_X_LIBS @@ -1271,8 +1305,7 @@ dnl zlib dnl ------------- AC_ARG_ENABLE(shared-zlib, -[ --enable-shared-zlib enable using shared zlib library - --disable-shared-zlib disable shared zlib, compile own zlib source (default)], +AS_HELP_STRING([--enable-shared-zlib], [enable using shared zlib library]), [ case "$enableval" in no) enable_shared_zlib=no ;; *) enable_shared_zlib=yes ;; @@ -1451,7 +1484,7 @@ AC_CHECK_HEADERS(fcntl.h limits.h unistd.h syslog.h dlfcn.h ieeefp.h \ sys/ioctl.h sys/time.h sys/uio.h \ sys/socket.h sys/sockio.h sys/socketio.h \ net/errno.h malloc.h mach-o/dyld.h arpa/nameser.h \ - pty.h util.h utmp.h langinfo.h poll.h) + pty.h util.h utmp.h langinfo.h poll.h sdkddkver.h) AC_CHECK_HEADER(sys/resource.h, [AC_DEFINE(HAVE_SYS_RESOURCE_H, 1, @@ -1651,18 +1684,62 @@ LIBS="$LIBS $EMU_THR_X_LIBS" dnl Check if we have these, in which case we'll try to build dnl inet_gethost with ipv6 support. -AC_CHECK_FUNC(getaddrinfo, have_getaddrinfo=yes, have_getaddrinfo=no) +AC_CHECK_HEADERS(windows.h) +AC_CHECK_HEADERS(winsock2.h) +AC_CHECK_HEADERS(ws2tcpip.h,[],[],[ +#ifdef HAVE_WINSOCK2_H +#include <winsock2.h> +#endif +#ifdef HAVE_WINDOWS_H +#include <windows.h> +#endif +]) +dnl AC_CHECK_FUNC(getaddrinfo, have_getaddrinfo=yes, have_getaddrinfo=no) +AC_MSG_CHECKING(for getaddrinfo) +AC_TRY_LINK([ +#include <stdlib.h> +#include <string.h> +#ifdef HAVE_WINSOCK2_H +#include <winsock2.h> +#endif +#ifdef HAVE_WINDOWS_H +#include <windows.h> +#endif +#ifdef HAVE_WS2TCPIP_H +#include <ws2tcpip.h> +#endif +#ifndef __WIN32__ +#include <sys/socket.h> +#include <netdb.h> +#endif +], +[ +getaddrinfo("","",NULL,NULL); +],have_getaddrinfo=yes, have_getaddrinfo=no) if test $have_getaddrinfo = yes; then + AC_MSG_RESULT([yes]) AC_MSG_CHECKING([whether getaddrinfo accepts enough flags]) - AC_TRY_RUN([ + AC_TRY_COMPILE([ #include <stdlib.h> #include <string.h> +#ifdef HAVE_WINSOCK2_H +#include <winsock2.h> +#endif +#ifdef HAVE_WINDOWS_H +#include <windows.h> +#endif +#ifdef HAVE_WS2TCPIP_H +#include <ws2tcpip.h> +#endif +#ifndef __WIN32__ #include <sys/socket.h> #include <netdb.h> -int main(int argc, char **argv) { +#endif +], +[ struct addrinfo hints, *ai; memset(&hints, 0, sizeof(hints)); - hints.ai_flags = (AI_CANONNAME|AI_V4MAPPED|AI_ADDRCONFIG); + hints.ai_flags = AI_CANONNAME; hints.ai_socktype = SOCK_STREAM; hints.ai_family = AF_INET6; if (getaddrinfo("::", NULL, &hints, &ai) == 0) { @@ -1671,32 +1748,57 @@ int main(int argc, char **argv) { } else { exit(1); } -} - ],, have_getaddrinfo=no, - [ - case X$erl_xcomp_getaddrinfo in - X) have_getaddrinfo=cross;; - Xyes|Xno) have_getaddrinfo=$erl_xcomp_getaddrinfo;; - *) AC_MSG_ERROR([Bad erl_xcomp_getaddrinfo value: $erl_xcomp_getaddrinfo]);; - esac - ]) +],, have_getaddrinfo=no) AC_MSG_RESULT($have_getaddrinfo) case $have_getaddrinfo in yes) AC_DEFINE(HAVE_GETADDRINFO, [1], [Define to 1 if you have a good `getaddrinfo' function.]);; - cross) - AC_MSG_WARN([result no guessed because of cross compilation]);; *) ;; esac +else + AC_MSG_RESULT([no]) +fi +AC_MSG_CHECKING(for getnameinfo) +AC_TRY_LINK([ +#include <stdlib.h> +#include <string.h> +#ifdef HAVE_WINSOCK2_H +#include <winsock2.h> +#endif +#ifdef HAVE_WINDOWS_H +#include <windows.h> +#endif +#ifdef HAVE_WS2TCPIP_H +#include <ws2tcpip.h> +#endif +#ifndef __WIN32__ +#include <sys/socket.h> +#include <netdb.h> +#endif +], +[ +getnameinfo(NULL,0,NULL,0,NULL,0,0); +],have_getnameinfo=yes, have_getnameinfo=no) +if test $have_getnameinfo = yes; then + AC_MSG_RESULT([yes]) + AC_DEFINE(HAVE_GETNAMEINFO, [1], + [Define to 1 if you have a good `getnameinfo' function.]) +else + AC_MSG_RESULT([no]) fi -AC_CHECK_FUNCS([getnameinfo getipnodebyname getipnodebyaddr gethostbyname2]) + + +AC_CHECK_FUNCS([getipnodebyname getipnodebyaddr gethostbyname2]) AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlopen \ pread pwrite writev memmove strerror strerror_r strncasecmp \ gethrtime localtime_r gmtime_r mmap mremap memcpy mallopt \ sbrk _sbrk __sbrk brk _brk __brk \ flockfile fstat strlcpy strlcat setsid posix2time setlocale nl_langinfo poll]) + +AC_CHECK_DECLS([posix2time],,,[#include <time.h>]) + if test "X$host" = "Xwin32"; then ac_cv_func_setvbuf_reversed=yes fi @@ -1757,6 +1859,15 @@ fi dnl Need by run_erl. AC_CHECK_FUNCS([openpty]) +dnl fdatasync syscall (Unix only) +AC_CHECK_FUNCS([fdatasync]) + +dnl Find which C libraries are required to use fdatasync +AC_SEARCH_LIBS(fdatasync, [rt]) + +AC_CHECK_HEADERS(net/if_dl.h ifaddrs.h netpacket/packet.h) +AC_CHECK_FUNCS([getifaddrs]) + dnl ---------------------------------------------------------------------- dnl Checks for features/quirks in the system that affects Erlang. dnl ---------------------------------------------------------------------- @@ -1818,6 +1929,27 @@ if test $processor_bind_functionality = yes; then AC_DEFINE(HAVE_PROCESSOR_BIND, 1, [Define if you have processor_bind functionality]) fi +AC_MSG_CHECKING([for cpuset_getaffinity/cpuset_setaffinity]) +AC_TRY_COMPILE([ +#include <sys/param.h> +#include <sys/cpuset.h> +], +[ + int res; + cpuset_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(1, &cpuset); + res = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cpuset_t), &cpuset); + res = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cpuset_t), &cpuset); + res = CPU_ISSET(1, &cpuset); + CPU_CLR(1, &cpuset); +], + cpuset_xetaffinity=yes, + cpuset_xetaffinity=no) +AC_MSG_RESULT([$cpuset_xetaffinity]) +if test $cpuset_xetaffinity = yes; then + AC_DEFINE(HAVE_CPUSET_xETAFFINITY, 1, [Define if you have cpuset_getaffinity/cpuset_setaffinity]) +fi AC_CACHE_CHECK([for 'end' symbol], erts_cv_have_end_symbol, @@ -2642,7 +2774,7 @@ static __inline__ int check_fpe(double f) * Implement SIGFPE handler based on CPU/OS combination */ -#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__i386__) || defined(__x86_64__))) || (defined(__OpenBSD__) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__)) +#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__i386__) || defined(__x86_64__))) || ((defined(__OpenBSD__) || defined(__NetBSD__)) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__)) #if defined(__linux__) && defined(__i386__) #if !defined(X86_FXSR_MAGIC) @@ -2756,6 +2888,11 @@ static void fpe_sig_action(int sig, siginfo_t *si, void *puc) struct fxsave64 *fxsave = uc->sc_fpstate; fxsave->fx_mxcsr = 0x1F80; fxsave->fx_fsw &= ~0xFF; +#elif defined(__NetBSD__) && defined(__x86_64__) + mcontext_t *mc = &uc->uc_mcontext; + struct fxsave64 *fxsave = (struct fxsave64 *)&mc->__fpregs; + fxsave->fx_mxcsr = 0x1F80; + fxsave->fx_fsw &= ~0xFF; #elif defined(__sun__) && defined(__x86_64__) mcontext_t *mc = &uc->uc_mcontext; struct fpchip_state *fpstate = &mc->fpregs.fp_reg_set.fpchip_state; @@ -2973,17 +3110,6 @@ if test "x$HIPE_ENABLED" = "xyes" ; then fi # -# Check if we should use elib_malloc. -# - -if test X${enable_elib_malloc} = Xyes; then - AC_DEFINE(ENABLE_ELIB_MALLOC,[],[Define to enable use of elib_malloc (a malloc() replacement)]) - AC_DEFINE(ELIB_HEAP_SBRK,[],[Elib sbrk]) - AC_DEFINE(ELIB_ALLOC_IS_CLIB,[],[Use elib malloc as clib]) - AC_DEFINE(ELIB_SORTED_BLOCKS,[],[Define to enable the use of sorted blocks when using elib_malloc]) -fi - -# # Check for working poll(). # AC_MSG_CHECKING([for working poll()]) @@ -3399,14 +3525,16 @@ AC_SUBST(SSL_LINK_WITH_KERBEROS) AC_SUBST(STATIC_KERBEROS_LIBS) AC_SUBST(SSL_LINK_WITH_ZLIB) AC_SUBST(STATIC_ZLIB_LIBS) -AC_SUBST(OPENSSL_CMD) std_ssl_locations="/usr/local /usr/sfw /opt/local /usr /usr/pkg /usr/local/openssl /usr/lib/openssl /usr/openssl /usr/local/ssl /usr/lib/ssl /usr/ssl" AC_ARG_WITH(ssl-zlib, -[ --with-ssl-zlib=PATH specify location of ZLib to be used by OpenSSL - --with-ssl-zlib link SSL with Zlib (default if found) - --without-ssl-zlib don't link SSL with ZLib]) +AS_HELP_STRING([--with-ssl-zlib=PATH], + [specify location of ZLib to be used by OpenSSL]) +AS_HELP_STRING([--with-ssl-zlib], + [link SSL with Zlib (default if found)]) +AS_HELP_STRING([--without-ssl-zlib], + [don't link SSL with ZLib])) if test "x$with_ssl_zlib" = "xno"; then @@ -3475,13 +3603,13 @@ fi AC_ARG_WITH(ssl, -[ --with-ssl=PATH specify location of OpenSSL include and lib - --with-ssl use SSL (default) - --without-ssl don't use SSL]) +AS_HELP_STRING([--with-ssl=PATH], [specify location of OpenSSL include and lib]) +AS_HELP_STRING([--with-ssl], [use SSL (default)]) +AS_HELP_STRING([--without-ssl], [don't use SSL])) AC_ARG_ENABLE(dynamic-ssl-lib, -[ --enable-dynamic-ssl-lib enable using dynamic openssl libraries - --disable-dynamic-ssl-lib disable using dynamic openssl libraries], +AS_HELP_STRING([--disable-dynamic-ssl-lib], + [disable using dynamic openssl libraries]), [ case "$enableval" in no) enable_dynamic_ssl=no ;; *) enable_dynamic_ssl=yes ;; @@ -3591,21 +3719,6 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in SSL_DYNAMIC_ONLY=yes fi SSL_BINDIR="$rdir/bin" -dnl Should one use EXEEXT or ac_exeext? - if test -f "$erl_xcomp_sysroot$SSL_BINDIR/openssl$EXEEXT"; then - if test "$cross_compiling" = "yes"; then - dnl Cannot test it; hope it is working... - OPENSSL_CMD="$SSL_BINDIR/openssl" - else - if "$SSL_BINDIR/openssl" version > /dev/null 2>&1; then - OPENSSL_CMD="$SSL_BINDIR/openssl" - else - is_real_ssl=no - fi - fi - else - is_real_ssl=no - fi if test "x$is_real_ssl" = "xyes" ; then SSL_INCLUDE="-I$dir/include" old_CPPFLAGS=$CPPFLAGS @@ -3669,7 +3782,6 @@ dnl Should one use EXEEXT or ac_exeext? SSL_RUNTIME_LIB="/usr/lib" SSL_LIB="$erl_xcomp_sysroot/usr/lib" SSL_BINDIR="/usr/sbin" - OPENSSL_CMD="$SSL_BINDIR/openssl" dnl OpenBSD requires us to link with -L and -l SSL_DYNAMIC_ONLY="yes" fi @@ -3755,7 +3867,6 @@ dnl so it is - be adoptable SSL_DYNAMIC_ONLY=yes fi SSL_INCLUDE="-I$with_ssl/include" - OPENSSL_CMD="$with_ssl/bin/openssl" SSL_APP=ssl CRYPTO_APP=crypto SSH_APP=ssh @@ -3961,9 +4072,9 @@ esac AC_ARG_WITH(javac, -[ --with-javac=JAVAC specify Java compiler to use - --with-javac use a Java compiler if found (default) - --without-javac don't use any Java compiler]) +AS_HELP_STRING([--with-javac=JAVAC], [specify Java compiler to use]) +AS_HELP_STRING([--with-javac], [use a Java compiler if found (default)]) +AS_HELP_STRING([--without-javac], [don't use any Java compiler])) dnl dnl Then there are a number of apps which needs a java compiler... @@ -4152,7 +4263,6 @@ dnl ../lib/ic/c_src/$host/Makefile:../lib/ic/c_src/Makefile.in ../lib/os_mon/c_src/$host/Makefile:../lib/os_mon/c_src/Makefile.in ../lib/ssl/c_src/$host/Makefile:../lib/ssl/c_src/Makefile.in - ../lib/ssl/examples/certs/$host/Makefile:../lib/ssl/examples/certs/Makefile.in ../lib/crypto/c_src/$host/Makefile:../lib/crypto/c_src/Makefile.in ../lib/orber/c_src/$host/Makefile:../lib/orber/c_src/Makefile.in ../lib/runtime_tools/c_src/$host/Makefile:../lib/runtime_tools/c_src/Makefile.in diff --git a/erts/doc/src/driver.xml b/erts/doc/src/driver.xml index 006a6160de..db455312ec 100644 --- a/erts/doc/src/driver.xml +++ b/erts/doc/src/driver.xml @@ -196,11 +196,14 @@ static ErlDrvData start(ErlDrvPort port, char *command) <p>We call disconnect to log out from the database. (This should have been done from Erlang, but just in case.)</p> <code type="none"><![CDATA[ - static int do_disconnect(our_data_t* data, ei_x_buff* x); +static int do_disconnect(our_data_t* data, ei_x_buff* x); static void stop(ErlDrvData drv_data) { - do_disconnect((our_data_t*)drv_data, NULL); + our_data_t* data = (our_data_t*)drv_data; + + do_disconnect(data, NULL); + driver_free(data); } ]]></code> <p>We use the binary format only to return data to the emulator; diff --git a/erts/doc/src/driver_entry.xml b/erts/doc/src/driver_entry.xml index e71b48bd92..dd949d4048 100644 --- a/erts/doc/src/driver_entry.xml +++ b/erts/doc/src/driver_entry.xml @@ -172,7 +172,7 @@ typedef struct erl_drv_entry { added to the driver list.) The driver should return 0, or if the driver can't initialize, -1.</p> </item> - <tag><marker id="start"/>int (*start)(ErlDrvPort port, char* command)</tag> + <tag><marker id="start"/>ErlDrvData (*start)(ErlDrvPort port, char* command)</tag> <item> <p>This is called when the driver is instantiated, when <c>open_port/2</c> is called. The driver should return a @@ -188,7 +188,9 @@ typedef struct erl_drv_entry { <p>This is called when the port is closed, with <c>port_close/1</c> or <c>Port ! {self(), close}</c>. Note that terminating the port owner process also closes the - port.</p> + port. If <c>drv_data</c> is a pointer to memory allocated in + <c>start</c>, then <c>stop</c> is the place to deallocate that + memory.</p> </item> <tag><marker id="output"/>void (*output)(ErlDrvData drv_data, char *buf, int len)</tag> <item> diff --git a/erts/doc/src/epmd.xml b/erts/doc/src/epmd.xml index 796ab3820b..474230cb38 100644 --- a/erts/doc/src/epmd.xml +++ b/erts/doc/src/epmd.xml @@ -4,7 +4,7 @@ <comref> <header> <copyright> - <year>1996</year><year>2009</year> + <year>1996</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -31,9 +31,23 @@ <rev>A</rev> <file>epmd.xml</file> </header> + <com>epmd</com> - <comsummary>Erlang Port Mapper Daemon </comsummary> + <comsummary> + <p>Erlang Port Mapper Daemon</p> + <taglist> + <tag><c><![CDATA[epmd [-d|-debug] [DbgExtra...] [-port No] [-daemon] [-relaxed_command_check]]]></c></tag> + <item> + <p>Starts the port mapper daemon</p> + </item> + <tag><c><![CDATA[epmd [-d|-debug] [-port No] [-names|-kill|-stop Name]]]></c></tag> + <item> + <p>Communicates with a running port mapper daemon</p> + </item> + </taglist> + </comsummary> <description> + <p>This daemon acts as a name server on all hosts involved in distributed Erlang computations. When an Erlang node starts, the node has a name and it obtains an address from the host @@ -46,48 +60,171 @@ The job of the <c><![CDATA[epmd]]></c> daemon is to keep track of which node name listens on which address. Hence, <c><![CDATA[epmd]]></c> map symbolic node names to machine addresses.</p> - <p>The daemon is started automatically by the Erlang start-up script.</p> - <p>The program <c><![CDATA[epmd]]></c> can also be used for a variety of other - purposes, for example checking the DNS (Domain Name System) - configuration of a host.</p> + + <p>The TCP/IP <c>epmd</c> daemon actually only keeps track of + the <c>Name</c> (first) part of an Erlang node name, the <c>Host</c> + part (whatever is after the <c><![CDATA[@]]></c> is implicit in the + node name where the <c>epmd</c> daemon was actually contacted, + as is the IP address where the Erlang node can be + reached. Consistent and correct TCP naming services are + therefore required for an Erlang network to function + correctly.</p> + + <taglist> + <tag>Starting the port mapper daemon</tag> + <item> + + <p>The daemon is started automatically by the <c>erl</c> + command if the node is to be distributed and there is no + running instance present. If automatically launched, + environment variables has to be used to alter the behavior of + the daemon. See the <seealso + marker="#environment_variables">Environment + variables</seealso> section below.</p> + + <p>If the -daemon argument is not given, the + <c><![CDATA[epmd]]></c> runs as a normal program with the + controlling terminal of the shell in which it is + started. Normally, it should run as a daemon.</p> + + <p>Regular start-up options are described in the + <seealso marker="#daemon_flags">Regular options</seealso> + section below.</p> + + <p>The <c>DbgExtra</c> options are described in the + <seealso marker="#debug_flags">DbgExtra options</seealso> + section below.</p> + + </item> + <tag>Communicating with a running port mapper daemon</tag> + <item> + + <p>Communicating with the running epmd daemon by means of the + <c>epmd</c> program is done primarily for debugging + purposes.</p> + + <p>The different queries are described in the <seealso + marker="#interactive_flags">Interactive options</seealso> + section below.</p> + + </item> + </taglist> </description> - <funcs> - <func> - <name>epmd [-daemon] </name> - <fsummary>Start a name server as a daemon</fsummary> - <desc> - <p>Starts a name server as a daemon. If it has no argument, the - <c><![CDATA[epmd]]></c> runs as a normal program with the controlling terminal - of the shell in which it is started. Normally, it should run as a - daemon.</p> - </desc> - </func> - <func> - <name>epmd -names</name> - <fsummary>Request the names of the registered Erlang nodes on this host</fsummary> - <desc> - <p>Requests the names of the local Erlang nodes <c><![CDATA[epmd]]></c> has - registered.</p> - </desc> - </func> - <func> - <name>epmd -kill</name> - <fsummary>Kill the <c><![CDATA[epmd]]></c>process</fsummary> - <desc> - <p>Kills the <c><![CDATA[epmd]]></c> process.</p> - </desc> - </func> - <func> - <name>epmd -help</name> - <fsummary>List options</fsummary> - <desc> - <p>Write short info about the usage including some debugging - options not listed here.</p> - </desc> - </func> - </funcs> + <section> + <marker id="daemon_flags"></marker> + <title>Regular options</title> + + <p>These options are available when starting the actual name server. The name server is normally started automatically by the <c>erl</c> command (if not already available), but it can also be started at i.e. system start-up.</p> + <taglist> + <tag><c><![CDATA[-port No]]></c></tag> + <item> + <p>Let this instance of epmd listen to another TCP port than + default 4369. This can also be set using the + <c><![CDATA[ERL_EPMD_PORT]]></c> environment variable, see the + section <seealso marker="#environment_variables">Environment + variables</seealso> below</p> + </item> + <tag><c><![CDATA[-d | -debug]]></c></tag> + <item> + + <p>Enable debug output. The more <c>-d</c> flags given, the more + debug output you will get (to a certain limit). This option is + most useful when the epmd daemon is not started as a daemon.</p> + </item> + <tag><c><![CDATA[-daemon]]></c></tag> + <item> + <p>Start epmd detached from the controlling terminal. Logging will end up in syslog when available and correctly configured. If the epmd daemon is started at boot, this option should definitely be used. It is also used when the <c>erl</c> command automatically starts <c>epmd</c>.</p> + </item> + <tag><c><![CDATA[-relaxed_command_check]]></c></tag> + <item> + <p>Start the epmd program with relaxed command checking (mostly for backward compatibility). This affects the following:</p> + <list type="bulleted"> + <item> + <p>With relaxed command checking, the <c>epmd</c> daemon can be killed from the localhost with i.e. <c>epmd -kill</c> even if there are active nodes registered. Normally only daemons with an empty node database can be killed with the <c>epmd -kill</c> command.</p> + </item> + <item> + <p>The <c>epmd -stop</c> command (and the corresponding messages to epmd, as can be given using <c>erl_interface/ei</c>) is normally always ignored, as it opens up for strange situation when two nodes of the same name can be alive at the same time. A node unregisters itself by just closing the connection to epmd, why the <c>stop</c> command was only intended for use in debugging situations.</p> + <p>With relaxed command checking enabled, you can forcibly unregister live nodes.</p> + </item> + </list> + <p>Relaxed command checking can also be enabled by setting the environment variable <c>ERL_EPMD_RELAXED_COMMAND_CHECK</c> prior to starting <c>epmd</c>.</p> + <p>Only use relaxed command checking on systems with very limited interactive usage.</p> + </item> + </taglist> + </section> <section> + <marker id="debug_flags"></marker> + <title>DbgExtra options</title> + <p>These options are purely for debugging and testing epmd clients, they should not be used in normal operation.</p> + + <taglist> + <tag><c><![CDATA[-packet_timeout Seconds]]></c></tag> + <item> + <p>Set the number of seconds a connection can be + inactive before epmd times out and closes the + connection (default 60).</p> + </item> + <tag><c><![CDATA[-delay_accept Seconds]]></c></tag> + <item> + <p>To simulate a busy server you can insert a delay between epmd + gets notified about that a new connection is requested and + when the connections gets accepted.</p> + </item> + <tag><c><![CDATA[-delay_write Seconds]]></c></tag> + <item> + <p>Also a simulation of a busy server. Inserts + a delay before a reply is sent.</p> + </item> + </taglist> + </section> + <section> + <marker id="interactive_flags"></marker> + <title>Interactive options</title> + <p>These options make <c>epmd</c> run as an interactive command displaying the results of sending queries ta an already running instance of <c>epmd</c>. The epmd contacted is always on the local node, but the <c>-port</c> option can be used to select between instances if several are running using different port on the host.</p> + <taglist> + <tag><c><![CDATA[-port No]]></c></tag> + <item> + <p>Contacts the <c>epmd</c> listening on the given TCP port number + (default 4369). This can also be set using the + <c><![CDATA[ERL_EPMD_PORT]]></c> environment variable, see the + section <seealso marker="#environment_variables">Environment + variables</seealso> below</p> + </item> + <tag><c><![CDATA[-names]]></c></tag> + <item> + <p>List names registered with the currently running epmd</p> + </item> + <tag><c><![CDATA[-kill]]></c></tag> + <item> + <p>Kill the currently running <c>epmd</c>.</p> + + <p>Killing the running <c>epmd</c> is only allowed if <c>epmd + -names</c> show an empty database or + <c>-relaxed_command_check</c> was given when the running + instance of <c>epmd</c> was started. Note that + <c>-relaxed_command_check</c> is given when starting the + daemon that is to accept killing when it has live nodes + registered. When running epmd interactively, + <c>-relaxed_command_check</c> has no effect. A daemon that is + started without relaxed command checking has to be killed + using i.e. signals or some other OS specific method if it has + active clients registered.</p> + </item> + <tag><c><![CDATA[-stop Name]]></c></tag> + <item> + <p>Forcibly unregister a live node from <c>epmd</c>'s database</p> + + <p>This command can only be used when contacting <c>epmd</c> + instances started with the <c>-relaxed_command_check</c> + flag. Note that relaxed command checking has to be enabled for + the <c>epmd</c> daemon contacted, When running epmd + interactively, + <c>-relaxed_command_check</c> has no effect.</p> + </item> + </taglist> + </section> + <section> <marker id="environment_variables"></marker> <title>Environment variables</title> <taglist> @@ -99,6 +236,15 @@ independent clusters of nodes, to co-exist on the same host. All nodes in a cluster must use the same epmd port number.</p> </item> + <tag><c><![CDATA[ERL_EPMD_RELAXED_COMMAND_CHECK]]></c></tag> + <item> + <p>If set prior to start, the <c>epmd</c> daemon will behave + as if the <c>-relaxed_command_check</c> option was given at + start-up. If consequently setting this option before starting + the Erlang virtual machine, the automatically started + <c>epmd</c> will accept the <c>-kill</c> and <c>-stop</c> + commands without restrictions.</p> + </item> </taglist> </section> @@ -116,5 +262,29 @@ silently be ignored. </p> </section> + <section> + <title>Access restrictions</title> + <p>The <c>epmd</c> daemon accepts messages from both localhost and + remote hosts. However, only the query commands are answered (and + acted upon) if the query comes from a remote host. It is always an + error to try to register a nodename if the client is not a process + located on the same host as the <c>epmd</c> instance is running on, + why such requests are considered hostile and the connection is + immediately closed.</p> + + <p>The queries accepted from remote nodes are:</p> + <list type="bulleted"> + <item> + <p>Port queries - i.e. on which port does the node with a given + name listen</p> + </item> + <item> + <p>Name listing - i.e. give a list of all names registered on + the host</p> + </item> + </list> + <p>To restrict access further, firewall software has to be used.</p> + </section> + </comref> diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index bb741c7836..77bd952d41 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -41,6 +41,26 @@ to scroll back to text which has scrolled off the screen. The <c><![CDATA[erl]]></c> program must be used, however, in pipelines or if you want to redirect standard input or output.</p> + <note><p>As of ERTS version 5.8 (OTP-R14A) the runtime system will by + default bind schedulers to logical processors using the + <c>default_bind</c> bind type if the amount of schedulers are + at least equal to the amount of logical processors configured, + binding of schedulers is supported, and a CPU topology is + available at startup. + </p><p> + If the Erlang runtime system is the only operating system + process that binds threads to logical processors, this + improves the performance of the runtime system. However, + if other operating system processes (as for example + another Erlang runtime system) also bind threads to + logical processors, there might be a performance penalty + instead. If this is the case you, are are advised to + unbind the schedulers using the + <seealso marker="#+sbt">+sbtu</seealso> command line argument, + or by invoking + <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, + unbound)</seealso>.</p> + </note> </description> <funcs> <func> @@ -521,6 +541,28 @@ <p>Calling <c>erlang:halt/1</c> with a string argument will still produce a crash dump.</p> </item> + <tag><c><![CDATA[+e Number]]></c></tag> + <item> + <p>Set max number of ETS tables.</p> + </item> + <tag><c><![CDATA[+ec]]></c></tag> + <item> + <p>Force the <c>compressed</c> option on all ETS tables. + Only intended for test and evaluation.</p> + </item> + <tag><c><![CDATA[+fnl]]></c></tag> + <item> + <p>The VM works with file names as if they are encoded using the ISO-latin-1 encoding, disallowing Unicode characters with codepoints beyond 255. This is default on operating systems that have transparent file naming, i.e. all Unixes except MacOSX.</p> + </item> + <tag><c><![CDATA[+fnu]]></c></tag> + <item> + <p>The VM works with file names as if they are encoded using UTF-8 (or some other system specific Unicode encoding). This is the default on operating systems that enforce Unicode encoding, i.e. Windows and MacOSX.</p> + <p>By enabling Unicode file name translation on systems where this is not default, you open up to the possibility that some file names can not be interpreted by the VM and therefore will be returned to the program as raw binaries. The option is therefore considered experimental.</p> + </item> + <tag><c><![CDATA[+fna]]></c></tag> + <item> + <p>Selection between <c>+fnl</c> and <c>+fnu</c> is done based on the current locale settings in the OS, meaning that if you have set your terminal for UTF-8 encoding, the filesystem is expected to use the same encoding for filenames (use with care).</p> + </item> <tag><c><![CDATA[+hms Size]]></c></tag> <item> <p>Sets the default heap size of processes to the size @@ -583,6 +625,24 @@ <item> <p>Force ets memory block to be moved on realloc.</p> </item> + <tag><marker id="+rg"><c><![CDATA[+rg ReaderGroupsLimit]]></c></marker></tag> + <item> + <p>Limits the amount of reader groups used by read/write locks + optimized for read operations in the Erlang runtime system. By + default the reader groups limit equals 8.</p> + <p>When the amount of schedulers is less than or equal to the reader + groups limit, each scheduler has its own reader group. When the + amount of schedulers is larger than the reader groups limit, + schedulers share reader groups. Shared reader groups degrades + read lock and read unlock performance while a large amount of + reader groups degrades write lock performance, so the limit is a + tradeoff between performance for read operations and performance + for write operations. Each reader group currently consumes 64 byte + in each read/write lock. Also note that a runtime system using + shared reader groups benefits from <seealso marker="#+sbt">binding + schedulers to logical processors</seealso>, since the reader groups + are distributed better between schedulers.</p> + </item> <tag><marker id="+S"><c><![CDATA[+S Schedulers:SchedulerOnline]]></c></marker></tag> <item> <p>Sets the amount of scheduler threads to create and scheduler @@ -647,8 +707,8 @@ <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, default_bind)</seealso>. </p></item> </taglist> - <p>Binding of schedulers are currently only supported on newer - Linux and Solaris systems.</p> + <p>Binding of schedulers is currently only supported on newer + Linux, Solaris, FreeBSD, and Windows systems.</p> <p>If no CPU topology is available when the <c>+sbt</c> flag is processed and <c>BindType</c> is any other type than <c>u</c>, the runtime system will fail to start. CPU @@ -657,6 +717,22 @@ that the <c>+sct</c> flag may have to be passed before the <c>+sbt</c> flag on the command line (in case no CPU topology has been automatically detected).</p> + <p>The runtime system will by default bind schedulers to logical + processors using the <c>default_bind</c> bind type if the amount + of schedulers are at least equal to the amount of logical + processors configured, binding of schedulers is supported, + and a CPU topology is available at startup. + </p> + <p><em>NOTE:</em> If the Erlang runtime system is the only operating + system process that binds threads to logical processors, this + improves the performance of the runtime system. However, if other + operating system processes (as for example another Erlang runtime + system) also bind threads to logical processors, there might be a + performance penalty instead. If this is the case you, are advised + to unbind the schedulers using the <c>+sbtu</c> command line + argument, or by invoking + <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, + unbound)</seealso>.</p> <p>For more information, see <seealso marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, SchedulerBindType)</seealso>. </p> @@ -777,14 +853,28 @@ <p>For more information, see <seealso marker="erlang#system_flag_cpu_topology">erlang:system_flag(cpu_topology, CpuTopology)</seealso>.</p> </item> + <tag><marker id="+swt"><c>+swt very_low|low|medium|high|very_high</c></marker></tag> + <item> + <p>Set scheduler wakeup threshold. Default is <c>medium</c>. + The threshold determines when to wake up sleeping schedulers + when more work than can be handled by currently awake schedulers + exist. A low threshold will cause earlier wakeups, and a high + threshold will cause later wakeups. Early wakeups will + distribute work over multiple schedulers faster, but work will + more easily bounce between schedulers. + </p> + <p><em>NOTE:</em> This flag may be removed or changed at any time + without prior notice. + </p> + </item> + <tag><marker id="sched_thread_stack_size"><c><![CDATA[+sss size]]></c></marker></tag> + <item> + <p>Suggested stack size, in kilowords, for scheduler threads. + Valid range is 4-8192 kilowords. The default stack size + is OS dependent.</p> + </item> </taglist> </item> - <tag><marker id="sched_thread_stack_size"><c><![CDATA[+sss size]]></c></marker></tag> - <item> - <p>Suggested stack size, in kilowords, for scheduler threads. - Valid range is 4-8192 kilowords. The default stack size - is OS dependent.</p> - </item> <tag><marker id="+t"><c><![CDATA[+t size]]></c></marker></tag> <item> <p>Set the maximum number of atoms the VM can handle. Default is 1048576.</p> @@ -838,6 +928,25 @@ <seealso marker="kernel:error_logger#warning_map/0">error_logger(3)</seealso> for further information.</p> </item> + <tag><c><![CDATA[+zFlag Value]]></c></tag> + <item> + <p>Miscellaneous flags.</p> + <taglist> + <tag><marker id="+zdbbl"><c>+zdbbl size</c></marker></tag> + <item> + <p>Set the distribution buffer busy limit + (<seealso marker="erlang#system_info_dist_buf_busy_limit">dist_buf_busy_limit</seealso>) + in kilobytes. Valid range is 1-2097151. Default is 1024.</p> + <p>A larger buffer limit will allow processes to buffer + more outgoing messages over the distribution. When the + buffer limit has been reached, sending processes will be + suspended until the buffer size has shrunk. The buffer + limit is per distribution channel. A higher limit will + give lower latency and higher throughput at the expense + of higher memory usage.</p> + </item> + </taglist> + </item> </taglist> </section> diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 5ec844e2ad..27887cbdf6 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -4,7 +4,7 @@ <cref> <header> <copyright> - <year>2001</year><year>2009</year> + <year>2001</year><year>2010</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -34,18 +34,29 @@ <lib>erl_nif</lib> <libsummary>API functions for an Erlang NIF library</libsummary> <description> - <warning><p>The NIF concept was introduced in R13B03 as an - EXPERIMENTAL feature. The interfaces may be changed in any way - in coming releases. The plan is however to lift the experimental label and - maintain interface backward compatibility from R14B.</p> - <p>Incompatible changes in <em>R13B04</em>:</p> + <note><p>The NIF concept is officially supported from R14B. NIF source code + written for earlier experimental versions might need adaption to run on R14B.</p> + <p>No incompatible changes between <em>R14B</em> and R14A.</p> + <p>Incompatible changes between <em>R14A</em> and R13B04:</p> + <list> + <item>Environment argument removed for <c>enif_alloc</c>, + <c>enif_realloc</c>, <c>enif_free</c>, <c>enif_alloc_binary</c>, + <c>enif_realloc_binary</c>, <c>enif_release_binary</c>, + <c>enif_alloc_resource</c>, <c>enif_release_resource</c>, + <c>enif_is_identical</c> and <c>enif_compare</c>.</item> + <item>Character encoding argument added to <c>enif_get_atom</c> + and <c>enif_make_existing_atom</c>.</item> + <item>Module argument added to <c>enif_open_resource_type</c> + while changing name spaces of resource types from global to module local.</item> + </list> + <p>Incompatible changes between <em>R13B04</em> and R13B03:</p> <list> <item>The function prototypes of the NIFs have changed to expect <c>argc</c> and <c>argv</c> arguments. The arity of a NIF is by that no longer limited to 3.</item> <item><c>enif_get_data</c> renamed as <c>enif_priv_data</c>.</item> <item><c>enif_make_string</c> got a third argument for character encoding.</item> </list> - </warning> + </note> <p>A NIF library contains native implementation of some functions of an Erlang module. The native implemented functions (NIFs) are @@ -109,9 +120,9 @@ ok the new directive <seealso marker="doc/reference_manual:code_loading#on_load">on_load</seealso> to automatically load the NIF library when the module is loaded.</p> - <note><p>A NIF must be exported or used locally by the module (or both). - An unused local stub function will be optimized away by the compiler - causing loading of the NIF library to fail.</p> + <note><p>A NIF does not have to be exported, it can be local to the module. + Note however that unused local stub functions will be optimized + away by the compiler causing loading of the NIF library to fail.</p> </note> <p>A loaded NIF library is tied to the Erlang module code version that loaded it. If the module is upgraded with a new version, the @@ -122,7 +133,7 @@ ok will be shared as well. To avoid unintentionally shared static data, each Erlang module code can keep its own private data. This private data can be set when the NIF library is loaded and - then retrieved by calling <seealso marker="#enif_priv_data">enif_priv_data()</seealso>.</p> + then retrieved by calling <seealso marker="#enif_priv_data">enif_priv_data</seealso>.</p> <p>There is no way to explicitly unload a NIF library. A library will be automatically unloaded when the module code that it belongs to is purged by the code server. A NIF library will also be unloaded if it is replaced @@ -137,14 +148,20 @@ ok <taglist> <tag>Read and write Erlang terms</tag> <item><p>Any Erlang terms can be passed to a NIF as function arguments and - be returned as function return values. The terms are of C-type <c>ERL_NIF_TERM</c> + be returned as function return values. The terms are of C-type + <seealso marker="#ERL_NIF_TERM">ERL_NIF_TERM</seealso> and can only be read or written using API functions. Most functions to read the content of a term are prefixed <c>enif_get_</c> and usually return true (or false) if the term was of the expected type (or not). The functions to write terms are all prefixed <c>enif_make_</c> and usually return the created <c>ERL_NIF_TERM</c>. There are also some functions to query terms, like <c>enif_is_atom</c>, <c>enif_is_identical</c> and - <c>enif_compare</c>.</p></item> + <c>enif_compare</c>.</p> + <p>All terms of type <c>ERL_NIF_TERM</c> belong to an environment of type + <seealso marker="#ErlNifEnv">ErlNifEnv</seealso>. The lifetime of a term is + controlled by the lifetime of its environment object. All API functions that read + or write terms has the environment, that the term belongs to, as the first + function argument.</p></item> <tag>Binaries</tag> <item><p>Terms of type binary are accessed with the help of the struct type <seealso marker="#ErlNifBinary">ErlNifBinary</seealso> @@ -172,28 +189,29 @@ ok <item><p>The use of resource objects is a way to return pointers to native data structures from a NIF in a safe way. A resource object is just a block of memory allocated with - <seealso marker="#enif_alloc_resource">enif_alloc_resource()</seealso>. + <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>. A handle ("safe pointer") to this memory block can then be returned to Erlang by the use of - <seealso marker="#enif_make_resource">enif_make_resource()</seealso>. + <seealso marker="#enif_make_resource">enif_make_resource</seealso>. The term returned by <c>enif_make_resource</c> is totally opaque in nature. It can be stored and passed between processses on the same node, but the only real end usage is to pass it back as argument to a NIF. - The NIF can then do <seealso marker="#enif_get_resource">enif_get_resource()</seealso> + The NIF can then do <seealso marker="#enif_get_resource">enif_get_resource</seealso> and get back a pointer to the memory block that is guaranteed to still be valid. A resource object will not be deallocated until the last handle term has been garbage collected by the VM and the resource has been - released with <seealso marker="#enif_release_resource">enif_release_resource()</seealso> + released with <seealso marker="#enif_release_resource">enif_release_resource</seealso> (not necessarily in that order).</p> <p>All resource objects are created as instances of some <em>resource type</em>. This makes resources from different modules to be distinguishable. A resource type is created by calling - <seealso marker="#enif_open_resource_type">enif_open_resource_type()</seealso> + <seealso marker="#enif_open_resource_type">enif_open_resource_type</seealso> when a library is loaded. Objects of that resource type can then later be allocated and <c>enif_get_resource</c> verifies that the resource is of the expected type. A resource type can have a user supplied destructor function that is automatically called when resources of that type are released (by either the garbage collector or <c>enif_release_resource</c>). Resource types - are uniquely identified by a supplied name string.</p> + are uniquely identified by a supplied name string and the name of the + implementing module.</p> <p>Resource types support upgrade in runtime by allowing a loaded NIF library to takeover an already existing resource type and thereby "inherit" all existing objects of that type. The destructor of the new @@ -208,7 +226,7 @@ ok <p/> <code type="none"> ERL_NIF_TERM term; - MyStruct* ptr = enif_alloc_resource(env, my_resource_type, sizeof(MyStruct)); + MyStruct* ptr = enif_alloc_resource(my_resource_type, sizeof(MyStruct)); /* initialize struct ... */ @@ -218,21 +236,31 @@ ok /* store 'ptr' in static variable, private data or other resource object */ } else { - enif_release_resource(env, obj); + enif_release_resource(obj); /* resource now only owned by "Erlang" */ } return term; } </code> - + <p>Another usage of resource objects is to create binary terms with + user defined memory management. + <seealso marker="#enif_make_resource_binary">enif_make_resource_binary</seealso> + will create a binary term that is connected to a resource object. The + destructor of the resource will be called when the binary is garbage + collected, at which time the binary data can be released. An example of + this can be a binary term consisting of data from a <c>mmap</c>'ed file. + The destructor can then do <c>munmap</c> to release the memory + region.</p> </item> <tag>Threads and concurrency</tag> <item><p>A NIF is thread-safe without any explicit synchronization as long as it acts as a pure function and only reads the supplied arguments. As soon as you write towards a shared state either through static variables or <seealso marker="#enif_priv_data">enif_priv_data</seealso> - you need to supply your own explicit synchronization. Resource objects - will also require synchronization if you treat them as mutable.</p> + you need to supply your own explicit synchronization. This includes terms + in process independent environments that are shared between threads. + Resource objects will also require synchronization if you treat them as + mutable.</p> <p>The library initialization callbacks <c>load</c>, <c>reload</c> and <c>upgrade</c> are all thread-safe even for shared state data.</p> <p>Avoid doing lengthy work in NIF calls as that may degrade the @@ -265,7 +293,7 @@ ok and there is no previously loaded library for this module.</p> <p><c>*priv_data</c> can be set to point to some private data that the library needs in order to keep a state between NIF - calls. <c>enif_priv_data()</c> will return this pointer. + calls. <c>enif_priv_data</c> will return this pointer. <c>*priv_data</c> will be initialized to NULL when <c>load</c> is called.</p> <p><c>load_info</c> is the second argument to <seealso @@ -317,19 +345,37 @@ ok <item> <p>Variables of type <c>ERL_NIF_TERM</c> can refer to any Erlang term. This is an opaque type and values of it can only by used either as - arguments to API functions or as return values from NIFs. A variable of - type <c>ERL_NIF_TERM</c> is only valid until the NIF call, where it was - obtained, returns.</p> + arguments to API functions or as return values from NIFs. All + <c>ERL_NIF_TERM</c>'s belong to an environment + (<seealso marker="#ErlNifEnv">ErlNifEnv</seealso>). A term can not be + destructed individually, it is valid until its environment is destructed.</p> </item> <tag><marker id="ErlNifEnv"/>ErlNifEnv</tag> <item> - <p><c>ErlNifEnv</c> contains information about the context in - which a NIF call is made. This pointer should not be - dereferenced in any way, but only passed on to API - functions. An <c>ErlNifEnv</c> pointer is only valid until - the function, where it was supplied as argument, - returns. It is thus useless and dangerous to store <c>ErlNifEnv</c> - pointers in between NIF calls.</p> + <p><c>ErlNifEnv</c> represents an environment that can host Erlang terms. + All terms in an environment are valid as long as the environment is valid. + <c>ErlNifEnv</c> is an opaque type and pointers to it can only be passed + on to API functions. There are two types of environments; process + bound and process independent.</p> + <p>A <em>process bound environment</em> is passed as the first argument to all NIFs. + All function arguments passed to a NIF will belong to that environment. + The return value from a NIF must also be a term belonging to the same + environment. + In addition a process bound environment contains transient information + about the calling Erlang process. The environment is only valid in the + thread where it was supplied as argument until the NIF returns. It is + thus useless and dangerous to store pointers to process bound + environments between NIF calls. </p> + <p>A <em>process independent environment</em> is created by calling + <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>. It can be + used to store terms beteen NIF calls and to send terms with + <seealso marker="#enif_send">enif_send</seealso>. A process + independent environment with all its terms is valid until you explicitly + invalidates it with <seealso marker="#enif_free_env">enif_free_env</seealso> + or <c>enif_send</c>.</p> + <p>All elements of a list/tuple must belong to the same environment as the + list/tuple itself. Terms can be copied between environments with + <seealso marker="#enif_make_copy">enif_make_copy</seealso>.</p> </item> <tag><marker id="ErlNifFunc"/>ErlNifFunc</tag> <item> @@ -363,7 +409,18 @@ typedef struct { <p><c>ErlNifBinary</c> contains transient information about an inspected binary term. <c>data</c> is a pointer to a buffer of <c>size</c> bytes with the raw content of the binary.</p> + <p>Note that <c>ErlNifBinary</c> is a semi-opaque type and you are + only allowed to read fields <c>size</c> and <c>data</c>.</p> </item> + <tag><marker id="ErlNifPid"/>ErlNifPid</tag> + <item> + <p><c>ErlNifPid</c> is a process identifier (pid). In contrast to + pid terms (instances of <c>ERL_NIF_TERM</c>), <c>ErlNifPid</c>'s are self + contained and not bound to any + <seealso marker="#ErlNifEnv">environment</seealso>. <c>ErlNifPid</c> + is an opaque type.</p> + </item> + <tag><marker id="ErlNifResourceType"/>ErlNifResourceType</tag> <item> <p>Each instance of <c>ErlNifResourceType</c> represent a class of @@ -388,9 +445,9 @@ typedef enum { ERL_NIF_LATIN1 }ErlNifCharEncoding; </code> - <p>The character encoding used in strings. The only supported - encoding is currently <c>ERL_NIF_LATIN1</c> for iso-latin-1 - (8-bit ascii).</p> + <p>The character encoding used in strings and atoms. The only + supported encoding is currently <c>ERL_NIF_LATIN1</c> for + iso-latin-1 (8-bit ascii).</p> </item> <tag><marker id="ErlNifSysInfo"/>ErlNifSysInfo</tag> <item> @@ -398,6 +455,10 @@ typedef enum { to return information about the runtime system. Contains currently the exact same content as <seealso marker="erl_driver#ErlDrvSysInfo">ErlDrvSysInfo</seealso>.</p> </item> + <tag><marker id="ErlNifSInt64"/>ErlNifSInt64</tag> + <item><p>A native signed 64-bit integer type.</p></item> + <tag><marker id="ErlNifUInt64"/>ErlNifUInt64</tag> + <item><p>A native unsigned 64-bit integer type.</p></item> </taglist> </section> @@ -407,24 +468,40 @@ typedef enum { <fsummary>Allocate dynamic memory.</fsummary> <desc><p>Allocate memory of <c>size</c> bytes. Return NULL if allocation failed.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin)</nametext></name> + <func><name><ret>int</ret><nametext>enif_alloc_binary(size_t size, ErlNifBinary* bin)</nametext></name> <fsummary>Create a new binary.</fsummary> <desc><p>Allocate a new binary of size <c>size</c> bytes. Initialize the structure pointed to by <c>bin</c> to refer to the allocated binary. The binary must either be released by - <seealso marker="#enif_release_binary">enif_release_binary()</seealso> + <seealso marker="#enif_release_binary">enif_release_binary</seealso> or ownership transferred to an Erlang term with - <seealso marker="#enif_make_binary">enif_make_binary()</seealso>. + <seealso marker="#enif_make_binary">enif_make_binary</seealso>. An allocated (and owned) <c>ErlNifBinary</c> can be kept between NIF calls.</p> - <p>Return false if allocation failed.</p> + <p>Return true on success or false if allocation failed.</p> </desc> </func> - <func><name><ret>void*</ret><nametext>enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned size)</nametext></name> + <func><name><ret>ErlNifEnv*</ret><nametext>enif_alloc_env()</nametext></name> + <fsummary>Create a new environment</fsummary> + <desc><p>Allocate a new process independent environment. The environment can + be used to hold terms that is not bound to any process. Such terms can + later be copied to a process environment with + <seealso marker="#enif_make_copy">enif_make_copy</seealso> + or be sent to a process as a message with <seealso marker="#enif_send">enif_send</seealso>.</p> + <p>Return pointer to the new environment.</p> + </desc> + </func> + <func><name><ret>void*</ret><nametext>enif_alloc_resource(ErlNifResourceType* type, unsigned size)</nametext></name> <fsummary>Allocate a memory managed resource object</fsummary> <desc><p>Allocate a memory managed resource object of type <c>type</c> and size <c>size</c> bytes.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_compare(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name> + <func><name><ret>void</ret><nametext>enif_clear_env(ErlNifEnv* env)</nametext></name> + <fsummary>Clear an environment for reuse.</fsummary> + <desc><p>Free all terms in an environment and clear it for reuse. The environment must + have been allocated with <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>. + </p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_compare(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name> <fsummary>Compare two terms</fsummary> <desc><p>Return an integer less than, equal to, or greater than zero if <c>lhs</c> is found, respectively, to be less than, @@ -434,77 +511,104 @@ typedef enum { </func> <func><name><ret>void</ret><nametext>enif_cond_broadcast(ErlNifCond *cnd)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_broadcast">erl_drv_cond_broadcast()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_broadcast">erl_drv_cond_broadcast</seealso>. </p></desc> </func> <func><name><ret>ErlNifCond*</ret><nametext>enif_cond_create(char *name)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_create">erl_drv_cond_create()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_create">erl_drv_cond_create</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_cond_destroy(ErlNifCond *cnd)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_destroy">erl_drv_cond_destroy()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_destroy">erl_drv_cond_destroy</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_cond_signal(ErlNifCond *cnd)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_signal">erl_drv_cond_signal()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_signal">erl_drv_cond_signal</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_cond_wait(ErlNifCond *cnd, ErlNifMutex *mtx)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_wait">erl_drv_cond_wait()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_cond_wait">erl_drv_cond_wait</seealso>. </p></desc> </func> <func><name><ret>int</ret><nametext>enif_equal_tids(ErlNifTid tid1, ErlNifTid tid2)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_equal_tids">erl_drv_equal_tids()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_equal_tids">erl_drv_equal_tids</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_free(ErlNifEnv* env, void* ptr)</nametext></name> <fsummary>Free dynamic memory</fsummary> <desc><p>Free memory allocated by <c>enif_alloc</c>.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_get_atom(ErlNifEnv* env, - ERL_NIF_TERM term, char* buf, unsigned size) - </nametext></name> + <func><name><ret>void</ret><nametext>enif_free_env(ErlNifEnv* env)</nametext></name> + <fsummary>Free an environment allocated with enif_alloc_env</fsummary> + <desc><p>Free an environment allocated with <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>. + All terms created in the environment will be freed as well.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_get_atom(ErlNifEnv* env, ERL_NIF_TERM term, char* buf, unsigned size, ErlNifCharEncoding encode)</nametext></name> <fsummary>Get the text representation of an atom term</fsummary> <desc><p>Write a null-terminated string, in the buffer pointed to by <c>buf</c> of size <c>size</c>, consisting of the string - representation of the atom <c>term</c>. Return the number of bytes - written (including terminating null character) or 0 if + representation of the atom <c>term</c> with encoding + <seealso marker="#ErlNifCharEncoding">encode</seealso>. Return + the number of bytes written (including terminating null character) or 0 if <c>term</c> is not an atom with maximum length of <c>size-1</c>.</p></desc> </func> + <func><name><ret>int</ret><nametext>enif_get_atom_length(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len, ErlNifCharEncoding encode)</nametext></name> + <fsummary>Get the length of atom <c>term</c>.</fsummary> + <desc><p>Set <c>*len</c> to the length (number of bytes excluding + terminating null character) of the atom <c>term</c> with encoding + <c>encode</c>. Return true on success or false if <c>term</c> is not an + atom.</p></desc> + </func> <func><name><ret>int</ret><nametext>enif_get_double(ErlNifEnv* env, ERL_NIF_TERM term, double* dp)</nametext></name> <fsummary>Read a floating-point number term.</fsummary> <desc><p>Set <c>*dp</c> to the floating point value of - <c>term</c> or return false if <c>term</c> is not a float.</p></desc> + <c>term</c>. Return true on success or false if <c>term</c> is not a float.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_int(ErlNifEnv* env, ERL_NIF_TERM term, int* ip)</nametext></name> - <fsummary>Read an integer term.</fsummary> + <fsummary>Read an integer term</fsummary> <desc><p>Set <c>*ip</c> to the integer value of - <c>term</c> or return false if <c>term</c> is not an integer or is - outside the bounds of type <c>int</c></p></desc> + <c>term</c>. Return true on success or false if <c>term</c> is not an + integer or is outside the bounds of type <c>int</c>.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_get_int64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifSInt64* ip)</nametext></name> + <fsummary>Read a 64-bit integer term</fsummary> + <desc><p>Set <c>*ip</c> to the integer value of + <c>term</c>. Return true on success or false if <c>term</c> is not an + integer or is outside the bounds of a signed 64-bit integer.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid)</nametext></name> + <fsummary>Read an local pid term</fsummary> + <desc><p>If <c>term</c> is the pid of a node local process, initialize the + pid variable <c>*pid</c> from it and return true. Otherwise return false. + No check if the process is alive is done.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_list_cell(ErlNifEnv* env, ERL_NIF_TERM list, ERL_NIF_TERM* head, ERL_NIF_TERM* tail)</nametext></name> <fsummary>Get head and tail from a list</fsummary> <desc><p>Set <c>*head</c> and <c>*tail</c> from - <c>list</c> or return false if <c>list</c> is not a non-empty - list.</p></desc> + <c>list</c> and return true, or return false if <c>list</c> is not a + non-empty list.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_get_list_length(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len)</nametext></name> + <fsummary>Get the length of list <c>term</c>.</fsummary> + <desc><p>Set <c>*len</c> to the length of list <c>term</c> and return true, + or return false if <c>term</c> is not a list.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_long(ErlNifEnv* env, ERL_NIF_TERM term, long int* ip)</nametext></name> <fsummary>Read an long integer term.</fsummary> - <desc><p>Set <c>*ip</c> to the long integer value of - <c>term</c> or return false if <c>term</c> is not an integer or is + <desc><p>Set <c>*ip</c> to the long integer value of <c>term</c> and + return true, or return false if <c>term</c> is not an integer or is outside the bounds of type <c>long int</c>.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)</nametext></name> <fsummary>Get the pointer to a resource object</fsummary> - <desc><p>Set <c>*objp</c> to point to the resource object referred to by <c>term</c>. - The pointer is valid until the calling NIF returns and should not be released.</p> - <p>Return false if <c>term</c> is not a handle to a resource object + <desc><p>Set <c>*objp</c> to point to the resource object referred to by <c>term</c>.</p> + <p>Return true on success or false if <c>term</c> is not a handle to a resource object of type <c>type</c>.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_string(ErlNifEnv* env, @@ -529,27 +633,32 @@ typedef enum { <c>*arity</c> to the number of elements. Note that the array is read-only and <c>(*array)[N-1]</c> will be the Nth element of the tuple. <c>*array</c> is undefined if the arity of the tuple - is zero.</p><p>Return false if <c>term</c> is not a + is zero.</p><p>Return true on success or false if <c>term</c> is not a tuple.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_get_uint(ErlNifEnv* env, ERL_NIF_TERM term, unsigned int* ip)</nametext></name> <fsummary>Read an unsigned integer term.</fsummary> - <desc><p>Set <c>*ip</c> to the unsigned integer value of - <c>term</c> or return false if <c>term</c> is not an unsigned integer or is - outside the bounds of type <c>unsigned int</c></p></desc> + <desc><p>Set <c>*ip</c> to the unsigned integer value of <c>term</c> and + return true, or return false if <c>term</c> is not an unsigned integer or + is outside the bounds of type <c>unsigned int</c>.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_get_uint64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifUInt64* ip)</nametext></name> + <fsummary>Read an unsigned 64-bit integer term.</fsummary> + <desc><p>Set <c>*ip</c> to the unsigned integer value of <c>term</c> and + return true, or return false if <c>term</c> is not an unsigned integer or + is outside the bounds of an unsigned 64-bit integer.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_get_ulong(ErlNifEnv* env, ERL_NIF_TERM term, unsigned long* ip)</nametext></name> <fsummary>Read an unsigned integer term.</fsummary> - <desc><p>Set <c>*ip</c> to the unsigned long integer value of - <c>term</c> or return false if <c>term</c> is not an unsigned integer or is - outside the bounds of type <c>unsigned long</c></p></desc> + <desc><p>Set <c>*ip</c> to the unsigned long integer value of <c>term</c> + and return true, or return false if <c>term</c> is not an unsigned integer or is + outside the bounds of type <c>unsigned long</c>.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_inspect_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, ErlNifBinary* bin)</nametext></name> <fsummary>Inspect the content of a binary</fsummary> <desc><p>Initialize the structure pointed to by <c>bin</c> with information about the binary term - <c>bin_term</c>. Return false if <c>bin_term</c> is not a binary.</p></desc> + <c>bin_term</c>. Return true on success or false if <c>bin_term</c> is not a binary.</p></desc> </func> <func><name><ret>int</ret><nametext>enif_inspect_iolist_as_binary(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifBinary* bin) @@ -558,7 +667,7 @@ typedef enum { <desc><p>Initialize the structure pointed to by <c>bin</c> with one continuous buffer with the same byte content as <c>iolist</c>. As with inspect_binary, the data pointed to by <c>bin</c> is transient and does - not need to be released. Return false if <c>iolist</c> is not an + not need to be released. Return true on success or false if <c>iolist</c> is not an iolist.</p> </desc> </func> @@ -578,7 +687,7 @@ typedef enum { <fsummary>Determine if a term is a fun</fsummary> <desc><p>Return true if <c>term</c> is a fun.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_is_identical(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name> + <func><name><ret>int</ret><nametext>enif_is_identical(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)</nametext></name> <fsummary>Erlang operator =:=</fsummary> <desc><p>Return true if the two terms are identical. Corresponds to the Erlang operators <c>=:=</c> and @@ -592,15 +701,35 @@ typedef enum { <fsummary>Determine if a term is a port</fsummary> <desc><p>Return true if <c>term</c> is a port.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_is_ref(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name> <fsummary>Determine if a term is a reference</fsummary> <desc><p>Return true if <c>term</c> is a reference.</p></desc> </func> + <func><name><ret>int</ret><nametext>enif_is_tuple(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name> + <fsummary>Determine if a term is a tuple</fsummary> + <desc><p>Return true if <c>term</c> is a tuple.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term)</nametext></name> + <fsummary>Determine if a term is a list</fsummary> + <desc><p>Return true if <c>term</c> is a list.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_keep_resource(void* obj)</nametext></name> + <fsummary>Add a reference to a resource object</fsummary> + <desc><p>Add a reference to resource object <c>obj</c> obtained from + <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>. + Each call to <c>enif_keep_resource</c> for an object must be balanced by + a call to <seealso marker="#enif_release_resource">enif_release_resource</seealso> + before the object will be destructed.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_atom(ErlNifEnv* env, const char* name)</nametext></name> <fsummary>Create an atom term</fsummary> - <desc><p>Create an atom term from the C-string <c>name</c>. Unlike other terms, atom - terms may be saved and used between NIF calls.</p></desc> + <desc><p>Create an atom term from the null-terminated C-string <c>name</c> + with iso-latin-1 encoding.</p></desc> + </func> + <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len)</nametext></name> + <fsummary>Create an atom term</fsummary> + <desc><p>Create an atom term from the string <c>name</c> with length <c>len</c>. + Null-characters are treated as any other characters.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_badarg(ErlNifEnv* env)</nametext></name> <fsummary>Make a badarg exception.</fsummary> @@ -613,21 +742,40 @@ typedef enum { <c>bin</c> should be considered read-only for the rest of the NIF call and then as released.</p></desc> </func> + <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)</nametext></name> + <fsummary>Make a copy of a term.</fsummary> + <desc><p>Make a copy of term <c>src_term</c>. The copy will be created in + environment <c>dst_env</c>. The source term may be located in any + environment.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_double(ErlNifEnv* env, double d)</nametext></name> <fsummary>Create a floating-point term</fsummary> <desc><p>Create a floating-point term from a <c>double</c>.</p></desc> </func> - <func><name><ret>int</ret><nametext>enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom)</nametext></name> + <func><name><ret>int</ret><nametext>enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding encode)</nametext></name> <fsummary>Create an existing atom term</fsummary> <desc><p>Try to create the term of an already existing atom from - the C-string <c>name</c>. If the atom already exist store the - term in <c>*atom</c> and return true, otherwise return - false.</p></desc> + the null-terminated C-string <c>name</c> with encoding + <seealso marker="#ErlNifCharEncoding">encode</seealso>. If the atom + already exists store the term in <c>*atom</c> and return true, otherwise + return false.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding encoding)</nametext></name> + <fsummary>Create an existing atom term</fsummary> + <desc><p>Try to create the term of an already existing atom from the + string <c>name</c> with length <c>len</c> and encoding + <seealso marker="#ErlNifCharEncoding">encode</seealso>. Null-characters + are treated as any other characters. If the atom already exists store the term + in <c>*atom</c> and return true, otherwise return false.</p></desc> </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_int(ErlNifEnv* env, int i)</nametext></name> <fsummary>Create an integer term</fsummary> <desc><p>Create an integer term.</p></desc> </func> + <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_int64(ErlNifEnv* env, ErlNifSInt64 i)</nametext></name> + <fsummary>Create an integer term</fsummary> + <desc><p>Create an integer term from a signed 64-bit integer.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_list(ErlNifEnv* env, unsigned cnt, ...)</nametext></name> <fsummary>Create a list term.</fsummary> <desc><p>Create an ordinary list term of length <c>cnt</c>. Expects @@ -662,7 +810,7 @@ typedef enum { <fsummary>Create an integer term from a long int</fsummary> <desc><p>Create an integer term from a <c>long int</c>.</p></desc> </func> - <func><name><ret>unsigned char*</ret><nametext>enif_make_new_binary(ErlNifEnv* env, unsigned size, ERL_NIF_TERM* termp)</nametext></name> + <func><name><ret>unsigned char*</ret><nametext>enif_make_new_binary(ErlNifEnv* env, size_t size, ERL_NIF_TERM* termp)</nametext></name> <fsummary>Allocate and create a new binary term</fsummary> <desc><p>Allocate a binary of size <c>size</c> bytes and create an owning term. The binary data is mutable until the calling NIF returns. This is a @@ -672,6 +820,10 @@ typedef enum { reallocated.</p><p>Return a pointer to the raw binary data and set <c>*termp</c> to the binary term.</p></desc> </func> + <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_pid(ErlNifEnv* env, const ErlNifPid* pid)</nametext></name> + <fsummary>Make a pid term</fsummary> + <desc><p>Make a pid term from <c>*pid</c>.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_ref(ErlNifEnv* env)</nametext></name> <fsummary>Create a reference.</fsummary> <desc><p>Create a reference like <seealso marker="erlang#make_ref-0">erlang:make_ref/0</seealso>.</p></desc> @@ -687,13 +839,42 @@ typedef enum { same node. Other operations such as matching or <c>term_to_binary</c> will have unpredictable (but harmless) results.</p></desc> </func> + <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_resource_binary(ErlNifEnv* env, void* obj, const void* data, size_t size)</nametext></name> + <fsummary>Create a custom binary term</fsummary> + <desc><p>Create a binary term that is memory managed by a resource object + <c>obj</c> obtained by <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>. + The returned binary term will consist of <c>size</c> bytes pointed to + by <c>data</c>. This raw binary data must be kept readable and unchanged + until the destructor of the resource is called. The binary data may be + stored external to the resource object in which case it is the responsibility + of the destructor to release the data.</p> + <p>Several binary terms may be managed by the same resource object. The + destructor will not be called until the last binary is garbage collected. + This can be useful as a way to return different parts of a larger binary + buffer.</p> + <p>As with <seealso marker="#enif_make_resource">enif_make_resource</seealso>, + no ownership transfer is done. The resource still needs to be released with + <seealso marker="#enif_release_resource">enif_release_resource</seealso>.</p> + </desc> + </func> + <func><name><ret>ErlNifPid*</ret><nametext>enif_self(ErlNifEnv* caller_env, ErlNifPid* pid)</nametext></name> + <fsummary>Get the pid of the calling process.</fsummary> + <desc><p>Initialize the pid variable <c>*pid</c> to represent the + calling process. Return <c>pid</c>.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_string(ErlNifEnv* env, const char* string, ErlNifCharEncoding encoding)</nametext></name> <fsummary>Create a string.</fsummary> <desc><p>Create a list containing the characters of the null-terminated string <c>string</c> with encoding <seealso marker="#ErlNifCharEncoding">encoding</seealso>.</p></desc> </func> + <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_string_len(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding encoding)</nametext></name> + <fsummary>Create a string.</fsummary> + <desc><p>Create a list containing the characters of the string <c>string</c> with + length <c>len</c> and encoding <seealso marker="#ErlNifCharEncoding">encoding</seealso>. + Null-characters are treated as any other characters.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_sub_binary(ErlNifEnv* - env, ERL_NIF_TERM bin_term, unsigned pos, unsigned size)</nametext></name> + env, ERL_NIF_TERM bin_term, size_t pos, size_t size)</nametext></name> <fsummary>Make a subbinary term.</fsummary> <desc><p>Make a subbinary of binary <c>bin_term</c>, starting at zero-based position <c>pos</c> with a length of <c>size</c> bytes. @@ -731,36 +912,41 @@ typedef enum { <fsummary>Create an unsigned integer term</fsummary> <desc><p>Create an integer term from an <c>unsigned int</c>.</p></desc> </func> + <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i)</nametext></name> + <fsummary>Create an unsigned integer term</fsummary> + <desc><p>Create an integer term from an unsigned 64-bit integer.</p></desc> + </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_ulong(ErlNifEnv* env, unsigned long i)</nametext></name> <fsummary>Create an integer term from an unsigned long int</fsummary> <desc><p>Create an integer term from an <c>unsigned long int</c>.</p></desc> </func> <func><name><ret>ErlNifMutex*</ret><nametext>enif_mutex_create(char *name)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_create">erl_drv_mutex_create()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_create">erl_drv_mutex_create</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_mutex_destroy(ErlNifMutex *mtx)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_destroy">erl_drv_mutex_destroy()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_destroy">erl_drv_mutex_destroy</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_mutex_lock(ErlNifMutex *mtx)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_lock">erl_drv_mutex_lock()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_lock">erl_drv_mutex_lock</seealso>. </p></desc> </func> <func><name><ret>int</ret><nametext>enif_mutex_trylock(ErlNifMutex *mtx)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_trylock">erl_drv_mutex_trylock()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_trylock">erl_drv_mutex_trylock</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_mutex_unlock(ErlNifMutex *mtx)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_unlock">erl_drv_mutex_unlock()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_mutex_unlock">erl_drv_mutex_unlock</seealso>. </p></desc> </func> - <func><name><ret>ErlNifResourceType*</ret><nametext>enif_open_resource_type(ErlNifEnv* env, const char* name, + <func><name><ret>ErlNifResourceType*</ret><nametext>enif_open_resource_type(ErlNifEnv* env, + const char* module_str, const char* name, ErlNifResourceDtor* dtor, ErlNifResourceFlags flags, ErlNifResourceFlags* tried)</nametext></name> <fsummary>Create or takeover a resource type</fsummary> <desc><p>Create or takeover a resource type identified by the string @@ -774,10 +960,10 @@ typedef enum { The supplied destructor <c>dtor</c> will be called both for existing instances as well as new instances not yet created by the calling NIF library.</item> </taglist> - <p>The two flag values can be combined with bitwise-or. To avoid unintentional - name clashes a good practice is to include the module name as part of the - type <c>name</c>. The <c>dtor</c> may be <c>NULL</c> in case no destructor - is needed.</p> + <p>The two flag values can be combined with bitwise-or. The name of the + resource type is local to the calling module. Argument <c>module_str</c> + is not (yet) used and must be NULL. The <c>dtor</c> may be <c>NULL</c> + in case no destructor is needed.</p> <p>On success, return a pointer to the resource type and <c>*tried</c> will be set to either <c>ERL_NIF_RT_CREATE</c> or <c>ERL_NIF_RT_TAKEOVER</c> to indicate what was actually done. @@ -794,129 +980,157 @@ typedef enum { <c>reload</c> or <c>upgrade</c>.</p> <p>Was previously named <c>enif_get_data</c>.</p></desc> </func> - <func><name><ret>void</ret><nametext>enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size)</nametext></name> + <func><name><ret>void</ret><nametext>enif_realloc_binary(ErlNifBinary* bin, size_t size)</nametext></name> <fsummary>Change the size of a binary.</fsummary> <desc><p>Change the size of a binary <c>bin</c>. The source binary may be read-only, in which case it will be left untouched and a mutable copy is allocated and assigned to <c>*bin</c>.</p></desc> </func> - <func><name><ret>void</ret><nametext>enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin)</nametext></name> + <func><name><ret>void</ret><nametext>enif_release_binary(ErlNifBinary* bin)</nametext></name> <fsummary>Release a binary.</fsummary> - <desc><p>Release a binary obtained - from <c>enif_alloc_binary</c>.</p></desc> + <desc><p>Release a binary obtained from <c>enif_alloc_binary</c>.</p></desc> </func> - <func><name><ret>void</ret><nametext>enif_release_resource(ErlNifEnv* env, void* obj)</nametext></name> + <func><name><ret>void</ret><nametext>enif_release_resource(void* obj)</nametext></name> <fsummary>Release a resource object.</fsummary> - <desc><p>Release a resource object obtained from <c>enif_alloc_resource</c>. - The object may still be alive if it is referred to by Erlang terms. Each call to - <c>enif_release_resource</c> must correspond to a previous call to <c>enif_alloc_resource</c>. - References made by <c>enif_make_resource</c> can only be released by the garbage collector.</p></desc> + <desc><p>Remove a reference to resource object <c>obj</c>obtained from + <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>. + The resource object will be destructed when the last reference is removed. + Each call to <c>enif_release_resource</c> must correspond to a previous + call to <c>enif_alloc_resource</c> or + <seealso marker="#enif_keep_resource">enif_keep_resource</seealso>. + References made by <seealso marker="#enif_make_resource">enif_make_resource</seealso> + can only be removed by the garbage collector.</p></desc> </func> <func><name><ret>ErlNifRWLock*</ret><nametext>enif_rwlock_create(char *name)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_create">erl_drv_rwlock_create()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_create">erl_drv_rwlock_create</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_rwlock_destroy(ErlNifRWLock *rwlck)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_destroy">erl_drv_rwlock_destroy()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_destroy">erl_drv_rwlock_destroy</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_rwlock_rlock(ErlNifRWLock *rwlck)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rlock">erl_drv_rwlock_rlock()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rlock">erl_drv_rwlock_rlock</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_rwlock_runlock(ErlNifRWLock *rwlck)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_runlock">erl_drv_rwlock_runlock()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_runlock">erl_drv_rwlock_runlock</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_rwlock_rwlock(ErlNifRWLock *rwlck)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwlock">erl_drv_rwlock_rwlock()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwlock">erl_drv_rwlock_rwlock</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_rwlock_rwunlock(ErlNifRWLock *rwlck)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwunlock">erl_drv_rwlock_rwunlock()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_rwunlock">erl_drv_rwlock_rwunlock</seealso>. </p></desc> </func> <func><name><ret>int</ret><nametext>enif_rwlock_tryrlock(ErlNifRWLock *rwlck)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrlock">erl_drv_rwlock_tryrlock()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrlock">erl_drv_rwlock_tryrlock</seealso>. </p></desc> </func> <func><name><ret>int</ret><nametext>enif_rwlock_tryrwlock(ErlNifRWLock *rwlck)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrwlock">erl_drv_rwlock_tryrwlock()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrwlock">erl_drv_rwlock_tryrwlock</seealso>. </p></desc> </func> - <func><name><ret>unsigned</ret><nametext>enif_sizeof_resource(ErlNifEnv* env, void* obj)</nametext></name> + <func><name><ret>unsigned</ret><nametext>enif_send(ErlNifEnv* env, ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)</nametext></name> + <fsummary>Send a message to a process.</fsummary> + <desc><p>Send a message to a process.</p> + <taglist> + <tag><c>env</c></tag> + <item>The environment of the calling process. Must be NULL if and + only if calling from a created thread.</item> + <tag><c>*to_pid</c></tag> + <item>The pid of the receiving process. The pid should refer to a process on the local node.</item> + <tag><c>msg_env</c></tag> + <item>The environment of the message term. Must be a process + independent environment allocated with + <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>.</item> + <tag><c>msg</c></tag> + <item>The message term to send.</item> + </taglist> + <p>Return true on success, or false if <c>*to_pid</c> does not refer to an alive local process.</p> + <p>The message environment <c>msg_env</c> with all its terms (including + <c>msg</c>) will be invalidated by a successful call to <c>enif_send</c>. The environment + should either be freed with <seealso marker="#enif_free_env">enif_free_env</seealso> + of cleared for reuse with <seealso marker="#enif_clear_env">enif_clear_env</seealso>.</p> + <p>This function is only thread-safe when the emulator with SMP support is used. + It can only be used in a non-SMP emulator from a NIF-calling thread.</p> + </desc> + </func> + <func><name><ret>unsigned</ret><nametext>enif_sizeof_resource(void* obj)</nametext></name> <fsummary>Get the byte size of a resource object</fsummary> <desc><p>Get the byte size of a resource object <c>obj</c> obtained by - <c>enif_alloc_resource</c>.</p></desc> + <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>.</p></desc> </func> <func> <name><ret>void</ret><nametext>enif_system_info(ErlNifSysInfo *sys_info_ptr, size_t size)</nametext></name> <fsummary>Get information about the Erlang runtime system</fsummary> - <desc><p>Same as <seealso marker="erl_driver#driver_system_info">driver_system_info()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#driver_system_info">driver_system_info</seealso>. </p></desc> </func> <func><name><ret>int</ret><nametext>enif_thread_create(char *name,ErlNifTid *tid,void * (*func)(void *),void *args,ErlNifThreadOpts *opts)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_create">erl_drv_thread_create()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_create">erl_drv_thread_create</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_thread_exit(void *resp)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_exit">erl_drv_thread_exit()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_exit">erl_drv_thread_exit</seealso>. </p></desc> </func> <func><name><ret>int</ret><nametext>enif_thread_join(ErlNifTid, void **respp)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_join">erl_drv_thread_join ()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_join">erl_drv_thread_join </seealso>. </p></desc> </func> <func><name><ret>ErlNifThreadOpts*</ret><nametext>enif_thread_opts_create(char *name)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_create">erl_drv_thread_opts_create()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_create">erl_drv_thread_opts_create</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_thread_opts_destroy(ErlNifThreadOpts *opts)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_destroy">erl_drv_thread_opts_destroy()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_opts_destroy">erl_drv_thread_opts_destroy</seealso>. </p></desc> </func> <func><name><ret>ErlNifTid</ret><nametext>enif_thread_self(void)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_self">erl_drv_thread_self()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_thread_self">erl_drv_thread_self</seealso>. </p></desc> </func> <func><name><ret>int</ret><nametext>enif_tsd_key_create(char *name, ErlNifTSDKey *key)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_create">erl_drv_tsd_key_create()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_create">erl_drv_tsd_key_create</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_tsd_key_destroy(ErlNifTSDKey key)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_destroy">erl_drv_tsd_key_destroy()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_key_destroy">erl_drv_tsd_key_destroy</seealso>. </p></desc> </func> <func><name><ret>void*</ret><nametext>enif_tsd_get(ErlNifTSDKey key)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get</seealso>. </p></desc> </func> <func><name><ret>void</ret><nametext>enif_tsd_set(ErlNifTSDKey key, void *data)</nametext></name> <fsummary></fsummary> - <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_set">erl_drv_tsd_set()</seealso>. + <desc><p>Same as <seealso marker="erl_driver#erl_drv_tsd_set">erl_drv_tsd_set</seealso>. </p></desc> </func> </funcs> <section> <title>SEE ALSO</title> - <p><seealso marker="erlang#load_nif-2">load_nif(3)</seealso></p> + <p><seealso marker="erlang#load_nif-2">erlang:load_nif/2</seealso></p> </section> </cref> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index cd9bb85f5c..78d58a1e56 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -80,7 +80,7 @@ iolist() = [char() | binary() | iolist()] </desc> </func> <func> - <name>adler32(Data) -> int()</name> + <name>erlang:adler32(Data) -> int()</name> <fsummary>Compute adler32 checksum</fsummary> <type> <v>Data = iodata()</v> @@ -90,7 +90,7 @@ iolist() = [char() | binary() | iolist()] </desc> </func> <func> - <name>adler32(OldAdler, Data) -> int()</name> + <name>erlang:adler32(OldAdler, Data) -> int()</name> <fsummary>Compute adler32 checksum</fsummary> <type> <v>OldAdler = int()</v> @@ -102,17 +102,17 @@ iolist() = [char() | binary() | iolist()] <c>Data</c>.</p> <p>The following code:</p> <code> - X = adler32(Data1), - Y = adler32(X,Data2). + X = erlang:adler32(Data1), + Y = erlang:adler32(X,Data2). </code> <p>- would assign the same value to <c>Y</c> as this would:</p> <code> - Y = adler32([Data1,Data2]). + Y = erlang:adler32([Data1,Data2]). </code> </desc> </func> <func> - <name>adler32_combine(FirstAdler, SecondAdler, SecondSize) -> int()</name> + <name>erlang:adler32_combine(FirstAdler, SecondAdler, SecondSize) -> int()</name> <fsummary>Combine two adler32 checksums</fsummary> <type> <v>FirstAdler = SecondAdler = int()</v> @@ -124,14 +124,14 @@ iolist() = [char() | binary() | iolist()] the second checksum to be known.</p> <p>The following code:</p> <code> - Y = adler32(Data1), - Z = adler32(Y,Data2). + Y = erlang:adler32(Data1), + Z = erlang:adler32(Y,Data2). </code> <p>- would assign the same value to <c>Z</c> as this would:</p> <code> - X = adler32(Data1), - Y = adler32(Data2), - Z = adler32_combine(X,Y,iolist_size(Data2)). + X = erlang:adler32(Data1), + Y = erlang:adler32(Data2), + Z = erlang:adler32_combine(X,Y,iolist_size(Data2)). </code> </desc> </func> @@ -147,7 +147,7 @@ iolist() = [char() | binary() | iolist()] <c>Tuple1</c>, and contains the elements in <c>Tuple1</c> followed by <c>Term</c> as the last element. Semantically equivalent to - <c>list_to_tuple(tuple_to_list(Tuple ++ [Term])</c>, but much + <c>list_to_tuple(tuple_to_list(Tuple) ++ [Term])</c>, but much faster.</p> <pre> > <input>erlang:append_element({one, two}, three).</input> @@ -253,6 +253,54 @@ iolist() = [char() | binary() | iolist()] </desc> </func> <func> + <name>binary_part(Subject, PosLen) -> binary()</name> + <fsummary>Extracts a part of a binary</fsummary> + <type> + <v>Subject = binary()</v> + <v>PosLen = {Start,Length}</v> + <v>Start = int()</v> + <v>Length = int()</v> + </type> + <desc> + <p>Extracts the part of the binary described by <c>PosLen</c>.</p> + + <p>Negative length can be used to extract bytes at the end of a binary:</p> + +<code> +1> Bin = <<1,2,3,4,5,6,7,8,9,10>>. +2> binary_part(Bin,{byte_size(Bin), -5)). +<<6,7,8,9,10>> +</code> + + <p>If <c>PosLen</c> in any way references outside the binary, a <c>badarg</c> exception is raised.</p> + + <p><c>Start</c> is zero-based, i.e:</p> +<code> +1> Bin = <<1,2,3>> +2> binary_part(Bin,{0,2}). +<<1,2>> +</code> + + <p>See the STDLIB module <c>binary</c> for details about the <c>PosLen</c> semantics.</p> + + <p>Allowed in guard tests.</p> + </desc> + </func> + <func> + <name>binary_part(Subject, Start, Length) -> binary()</name> + <fsummary>Extracts a part of a binary</fsummary> + <type> + <v>Subject = binary()</v> + <v>Start = int()</v> + <v>Length = int()</v> + </type> + <desc> + <p>The same as <c>binary_part(Subject, {Pos, Len})</c>.</p> + + <p>Allowed in guard tests.</p> + </desc> + </func> + <func> <name>binary_to_atom(Binary, Encoding) -> atom()</name> <fsummary>Convert from text representation to an atom</fsummary> <type> @@ -318,6 +366,11 @@ iolist() = [char() | binary() | iolist()] corresponding to the bytes from position <c>Start</c> to position <c>Stop</c> in <c>Binary</c>. Positions in the binary are numbered starting from 1.</p> + + <note><p>This function's indexing style of using one-based indices for + binaries is deprecated. New code should use the functions in + the STDLIB module <c>binary</c> instead. They consequently + use the same (zero-based) style of indexing.</p></note> </desc> </func> <func> @@ -354,7 +407,7 @@ iolist() = [char() | binary() | iolist()] </desc> </func> <func> - <name>erlang:binary_to_term(Binary, Opts) -> term()</name> + <name>binary_to_term(Binary, Opts) -> term()</name> <fsummary>Decode an Erlang external term format binary</fsummary> <type> <v>Opts = [safe]</v> @@ -446,7 +499,7 @@ iolist() = [char() | binary() | iolist()] <name>erlang:cancel_timer(TimerRef) -> Time | false</name> <fsummary>Cancel a timer</fsummary> <type> - <v>TimerRef = ref()</v> + <v>TimerRef = reference()</v> <v>Time = int()</v> </type> <desc> @@ -500,7 +553,7 @@ false</pre> </desc> </func> <func> - <name>crc32(Data) -> int()</name> + <name>erlang:crc32(Data) -> int()</name> <fsummary>Compute crc32 (IEEE 802.3) checksum</fsummary> <type> <v>Data = iodata()</v> @@ -510,7 +563,7 @@ false</pre> </desc> </func> <func> - <name>crc32(OldCrc, Data) -> int()</name> + <name>erlang:crc32(OldCrc, Data) -> int()</name> <fsummary>Compute crc32 (IEEE 802.3) checksum</fsummary> <type> <v>OldCrc = int()</v> @@ -522,17 +575,17 @@ false</pre> <c>Data</c>.</p> <p>The following code:</p> <code> - X = crc32(Data1), - Y = crc32(X,Data2). + X = erlang:crc32(Data1), + Y = erlang:crc32(X,Data2). </code> <p>- would assign the same value to <c>Y</c> as this would:</p> <code> - Y = crc32([Data1,Data2]). + Y = erlang:crc32([Data1,Data2]). </code> </desc> </func> <func> - <name>crc32_combine(FirstCrc, SecondCrc, SecondSize) -> int()</name> + <name>erlang:crc32_combine(FirstCrc, SecondCrc, SecondSize) -> int()</name> <fsummary>Combine two crc32 (IEEE 802.3) checksums</fsummary> <type> <v>FirstCrc = SecondCrc = int()</v> @@ -544,14 +597,14 @@ false</pre> the second checksum to be known.</p> <p>The following code:</p> <code> - Y = crc32(Data1), - Z = crc32(Y,Data2). + Y = erlang:crc32(Data1), + Z = erlang:crc32(Y,Data2). </code> <p>- would assign the same value to <c>Z</c> as this would:</p> <code> - X = crc32(Data1), - Y = crc32(Data2), - Z = crc32_combine(X,Y,iolist_size(Data2)). + X = erlang:crc32(Data1), + Y = erlang:crc32(Data2), + Z = erlang:crc32_combine(X,Y,iolist_size(Data2)). </code> </desc> </func> @@ -571,7 +624,7 @@ false</pre> </desc> </func> <func> - <name>decode_packet(Type,Bin,Options) -> {ok,Packet,Rest} | {more,Length} | {error,Reason}</name> + <name>erlang:decode_packet(Type,Bin,Options) -> {ok,Packet,Rest} | {more,Length} | {error,Reason}</name> <fsummary>Extracts a protocol packet from a binary</fsummary> <type> <v>Bin = binary()</v> @@ -707,18 +760,18 @@ false</pre> </desc> </func> <func> - <name>erlang:demonitor(MonitorRef) -> true</name> + <name>demonitor(MonitorRef) -> true</name> <fsummary>Stop monitoring</fsummary> <type> - <v>MonitorRef = ref()</v> + <v>MonitorRef = reference()</v> </type> <desc> <p>If <c>MonitorRef</c> is a reference which the calling process obtained by calling - <seealso marker="#monitor/2">erlang:monitor/2</seealso>, + <seealso marker="#monitor/2">monitor/2</seealso>, this monitoring is turned off. If the monitoring is already turned off, nothing happens.</p> - <p>Once <c>erlang:demonitor(MonitorRef)</c> has returned it is + <p>Once <c>demonitor(MonitorRef)</c> has returned it is guaranteed that no <c>{'DOWN', MonitorRef, _, _, _}</c> message due to the monitor will be placed in the callers message queue in the future. A <c>{'DOWN', MonitorRef, _, _, _}</c> message @@ -726,10 +779,10 @@ false</pre> the call, though. Therefore, in most cases, it is advisable to remove such a <c>'DOWN'</c> message from the message queue after monitoring has been stopped. - <seealso marker="#demonitor/2">erlang:demonitor(MonitorRef, [flush])</seealso> can be used instead of - <c>erlang:demonitor(MonitorRef)</c> if this cleanup is wanted.</p> + <seealso marker="#demonitor/2">demonitor(MonitorRef, [flush])</seealso> can be used instead of + <c>demonitor(MonitorRef)</c> if this cleanup is wanted.</p> <note> - <p>Prior to OTP release R11B (erts version 5.5) <c>erlang:demonitor/1</c> + <p>Prior to OTP release R11B (erts version 5.5) <c>demonitor/1</c> behaved completely asynchronous, i.e., the monitor was active until the "demonitor signal" reached the monitored entity. This had one undesirable effect, though. You could never know when @@ -747,10 +800,10 @@ false</pre> </desc> </func> <func> - <name>erlang:demonitor(MonitorRef, OptionList) -> true|false</name> + <name>demonitor(MonitorRef, OptionList) -> true|false</name> <fsummary>Stop monitoring</fsummary> <type> - <v>MonitorRef = ref()</v> + <v>MonitorRef = reference()</v> <v>OptionList = [Option]</v> <v>Option = flush</v> <v>Option = info</v> @@ -759,8 +812,8 @@ false</pre> <p>The returned value is <c>true</c> unless <c>info</c> is part of <c>OptionList</c>. </p> - <p><c>erlang:demonitor(MonitorRef, [])</c> is equivalent to - <seealso marker="#demonitor/1">erlang:demonitor(MonitorRef)</seealso>.</p> + <p><c>demonitor(MonitorRef, [])</c> is equivalent to + <seealso marker="#demonitor/1">demonitor(MonitorRef)</seealso>.</p> <p>Currently the following <c>Option</c>s are valid:</p> <taglist> <tag><c>flush</c></tag> @@ -768,11 +821,11 @@ false</pre> <p>Remove (one) <c>{_, MonitorRef, _, _, _}</c> message, if there is one, from the callers message queue after monitoring has been stopped.</p> - <p>Calling <c>erlang:demonitor(MonitorRef, [flush])</c> + <p>Calling <c>demonitor(MonitorRef, [flush])</c> is equivalent to the following, but more efficient:</p> <code type="none"> - erlang:demonitor(MonitorRef), + demonitor(MonitorRef), receive {_, MonitorRef, _, _, _} -> true @@ -810,7 +863,7 @@ false</pre> </note> <p>Failure: <c>badarg</c> if <c>OptionList</c> is not a list, or if <c>Option</c> is not a valid option, or the same failure as for - <seealso marker="#demonitor/1">erlang:demonitor/1</seealso></p> + <seealso marker="#demonitor/1">demonitor/1</seealso></p> </desc> </func> <func> @@ -891,7 +944,7 @@ b</pre> </desc> </func> <func> - <name>erlang:error(Reason)</name> + <name>error(Reason)</name> <fsummary>Stop execution with a given reason</fsummary> <type> <v>Reason = term()</v> @@ -904,7 +957,7 @@ b</pre> function first). Since evaluating this function causes the process to terminate, it has no return value.</p> <pre> -> <input>catch erlang:error(foobar).</input> +> <input>catch error(foobar).</input> {'EXIT',{foobar,[{erl_eval,do_apply,5}, {erl_eval,expr,5}, {shell,exprs,6}, @@ -913,7 +966,7 @@ b</pre> </desc> </func> <func> - <name>erlang:error(Reason, Args)</name> + <name>error(Reason, Args)</name> <fsummary>Stop execution with a given reason</fsummary> <type> <v>Reason = term()</v> @@ -1430,7 +1483,7 @@ os_prompt%</pre> </desc> </func> <func> - <name>erlang:integer_to_list(Integer, Base) -> string()</name> + <name>integer_to_list(Integer, Base) -> string()</name> <fsummary>Text representation of an integer</fsummary> <type> <v>Integer = int()</v> @@ -1440,7 +1493,7 @@ os_prompt%</pre> <p>Returns a string which corresponds to the text representation of <c>Integer</c> in base <c>Base</c>.</p> <pre> -> <input>erlang:integer_to_list(1023, 16).</input> +> <input>integer_to_list(1023, 16).</input> "3FF"</pre> </desc> </func> @@ -1879,7 +1932,7 @@ os_prompt%</pre> </desc> </func> <func> - <name>erlang:list_to_integer(String, Base) -> int()</name> + <name>list_to_integer(String, Base) -> int()</name> <fsummary>Convert from text representation to an integer</fsummary> <type> <v>String = string()</v> @@ -1889,7 +1942,7 @@ os_prompt%</pre> <p>Returns an integer whose text representation in base <c>Base</c> is <c>String</c>.</p> <pre> -> <input>erlang:list_to_integer("3FF", 16).</input> +> <input>list_to_integer("3FF", 16).</input> 1023</pre> <p>Failure: <c>badarg</c> if <c>String</c> contains a bad representation of an integer.</p> @@ -1981,16 +2034,18 @@ os_prompt%</pre> <v>Text = string()</v> </type> <desc> - <warning> - <p>This BIF is still an experimental feature. The interface - may be changed in any way in future releases.</p><p>In - R13B03 the return value on failure was + <note> + <p>In releases older than OTP R14B, NIFs were an + experimental feature. Versions of OTP older than R14B might + have different and possibly incompatible NIF semantics and + interfaces. For example, in R13B03 the return value on + failure was <c>{error,Reason,Text}</c>.</p> - </warning> + </note> <p>Loads and links a dynamic library containing native implemented functions (NIFs) for a module. <c>Path</c> is a file path to the sharable object/dynamic library file minus - the OS-dependant file extension (.so for Unix and .ddl for + the OS-dependent file extension (.so for Unix and .dll for Windows). See <seealso marker="erl_nif">erl_nif</seealso> on how to implement a NIF library.</p> <p><c>LoadInfo</c> can be any term. It will be passed on to @@ -2107,7 +2162,7 @@ os_prompt%</pre> </desc> </func> <func> - <name>make_ref() -> ref()</name> + <name>make_ref() -> reference()</name> <fsummary>Return an almost unique reference</fsummary> <desc> <p>Returns an almost unique reference.</p> @@ -2156,7 +2211,7 @@ os_prompt%</pre> </desc> </func> <func> - <name>erlang:max(Term1, Term2) -> Maximum</name> + <name>max(Term1, Term2) -> Maximum</name> <fsummary>Return the largest of two term</fsummary> <type> <v>Term1 = Term2 = Maximum = term()</v> @@ -2405,7 +2460,7 @@ os_prompt%</pre> </desc> </func> <func> - <name>erlang:min(Term1, Term2) -> Minimum</name> + <name>min(Term1, Term2) -> Minimum</name> <fsummary>Return the smallest of two term</fsummary> <type> <v>Term1 = Term2 = Minimum = term()</v> @@ -2433,7 +2488,7 @@ os_prompt%</pre> </desc> </func> <func> - <name>erlang:monitor(Type, Item) -> MonitorRef</name> + <name>monitor(Type, Item) -> MonitorRef</name> <fsummary>Start monitoring</fsummary> <type> <v>Type = process</v> @@ -2469,7 +2524,7 @@ os_prompt%</pre> <note> <p>When a process is monitored by registered name, the process that has the registered name at the time when - <c>erlang:monitor/2</c> is called will be monitored. + <c>monitor/2</c> is called will be monitored. The monitor will not be effected, if the registered name is unregistered.</p> </note> @@ -2503,20 +2558,20 @@ os_prompt%</pre> </item> </taglist> <note> - <p>If/when <c>erlang:monitor/2</c> is extended (e.g. to + <p>If/when <c>monitor/2</c> is extended (e.g. to handle other item types than <c>process</c>), other possible values for <c>Object</c>, and <c>Info</c> in the <c>'DOWN'</c> message will be introduced.</p> </note> <p>The monitoring is turned off either when the <c>'DOWN'</c> message is sent, or when - <seealso marker="#demonitor/1">erlang:demonitor/1</seealso> + <seealso marker="#demonitor/1">demonitor/1</seealso> is called.</p> <p>If an attempt is made to monitor a process on an older node (where remote process monitoring is not implemented or one where remote process monitoring by registered name is not implemented), the call fails with <c>badarg</c>.</p> - <p>Making several calls to <c>erlang:monitor/2</c> for the same + <p>Making several calls to <c>monitor/2</c> for the same <c>Item</c> is not an error; it results in as many, completely independent, monitorings.</p> <note> @@ -2591,6 +2646,37 @@ os_prompt%</pre> </desc> </func> <func> + <name>erlang:nif_error(Reason)</name> + <fsummary>Stop execution with a given reason</fsummary> + <type> + <v>Reason = term()</v> + </type> + <desc> + <p>Works exactly like + <seealso marker="#error/1">erlang:error/1</seealso>, + but Dialyzer thinks that this BIF will return an arbitrary term. + When used in a stub function for a NIF to generate an + exception when the NIF library is not loaded, Dialyzer + will not generate false warnings.</p> + </desc> + </func> + <func> + <name>erlang:nif_error(Reason, Args)</name> + <fsummary>Stop execution with a given reason</fsummary> + <type> + <v>Reason = term()</v> + <v>Args = [term()]</v> + </type> + <desc> + <p>Works exactly like + <seealso marker="#error/2">erlang:error/2</seealso>, + but Dialyzer thinks that this BIF will return an arbitrary term. + When used in a stub function for a NIF to generate an + exception when the NIF library is not loaded, Dialyzer + will not generate false warnings.</p> + </desc> + </func> + <func> <name>node() -> Node</name> <fsummary>Name of the local node</fsummary> <type> @@ -2606,7 +2692,7 @@ os_prompt%</pre> <name>node(Arg) -> Node</name> <fsummary>At which node is a pid, port or reference located</fsummary> <type> - <v>Arg = pid() | port() | ref()</v> + <v>Arg = pid() | port() | reference()</v> <v>Node = node()</v> </type> <desc> @@ -2683,8 +2769,10 @@ os_prompt%</pre> Otherwise, some other point in time is chosen. It is also guaranteed that subsequent calls to this BIF returns continuously increasing values. Hence, the return value from - <c>now()</c> can be used to generate unique time-stamps. It - can only be used to check the local time of day if + <c>now()</c> can be used to generate unique time-stamps, + and if it is called in a tight loop on a fast machine + the time of the node can become skewed.</p> + <p>It can only be used to check the local time of day if the time-zone info of the underlying operating system is properly configured.</p> </desc> @@ -2693,14 +2781,17 @@ os_prompt%</pre> <name>open_port(PortName, PortSettings) -> port()</name> <fsummary>Open a port</fsummary> <type> - <v>PortName = {spawn, Command} | {spawn_driver, Command} | {spawn_executable, Command} | {fd, In, Out}</v> + <v>PortName = {spawn, Command} | {spawn_driver, Command} | {spawn_executable, FileName} | {fd, In, Out}</v> <v> Command = string()</v> + <v> FileName = [ FileNameChar ] | binary()</v> + <v> FileNameChar = int() (1..255 or any Unicode codepoint, see description)</v> <v> In = Out = int()</v> <v>PortSettings = [Opt]</v> - <v> Opt = {packet, N} | stream | {line, L} | {cd, Dir} | {env, Env} | {args, [ string() ]} | {arg0, string()} | exit_status | use_stdio | nouse_stdio | stderr_to_stdout | in | out | binary | eof</v> + <v> Opt = {packet, N} | stream | {line, L} | {cd, Dir} | {env, Env} | {args, [ ArgString ]} | {arg0, ArgString} | exit_status | use_stdio | nouse_stdio | stderr_to_stdout | in | out | binary | eof</v> <v> N = 1 | 2 | 4</v> <v> L = int()</v> <v> Dir = string()</v> + <v> ArgString = [ FileNameChar ] | binary()</v> <v> Env = [{Name, Val}]</v> <v> Name = string()</v> <v> Val = string() | false</v> @@ -2763,7 +2854,26 @@ os_prompt%</pre> executed, the appropriate command interpreter will implicitly be invoked, but there will still be no command argument expansion or implicit PATH search.</p> - + + <p>The name of the executable as well as the arguments + given in <c>args</c> and <c>arg0</c> is subject to + Unicode file name translation if the system is running + in Unicode file name mode. To avoid + translation or force i.e. UTF-8, supply the executable + and/or arguments as a binary in the correct + encoding. See the <seealso + marker="kernel:file">file</seealso> module, the + <seealso marker="kernel:file#native_name_encoding/0"> + file:native_name_encoding/0</seealso> function and the + <seealso marker="stdlib:unicode_usage">stdlib users guide + </seealso> for details.</p> + + <note>The characters in the name (if given as a list) + can only be > 255 if the Erlang VM is started in + Unicode file name translation mode, otherwise the name + of the executable is limited to the ISO-latin-1 + character set.</note> + <p>If the <c>Command</c> cannot be run, an error exception, with the posix error code as the reason, is raised. The error reason may differ between operating @@ -2866,6 +2976,21 @@ os_prompt%</pre> should not be given in this list. The proper executable name will automatically be used as argv[0] where applicable.</p> + <p>When the Erlang VM is running in Unicode file name + mode, the arguments can contain any Unicode characters and + will be translated into whatever is appropriate on the + underlying OS, which means UTF-8 for all platforms except + Windows, which has other (more transparent) ways of + dealing with Unicode arguments to programs. To avoid + Unicode translation of arguments, they can be supplied as + binaries in whatever encoding is deemed appropriate.</p> + + <note>The characters in the arguments (if given as a + list of characters) can only be > 255 if the Erlang + VM is started in Unicode file name mode, + otherwise the arguments are limited to the + ISO-latin-1 character set.</note> + <p>If one, for any reason, wants to explicitly set the program name in the argument vector, the <c>arg0</c> option can be used.</p> @@ -2881,6 +3006,9 @@ os_prompt%</pre> responds to this is highly system dependent and no specific effect is guaranteed.</p> + <p>The unicode file name translation rules of the + <c>args</c> option apply to this option as well.</p> + </item> <tag><c>exit_status</c></tag> @@ -3147,7 +3275,7 @@ os_prompt%</pre> </desc> </func> <func> - <name>erlang:port_command(Port, Data, OptionList) -> true|false</name> + <name>port_command(Port, Data, OptionList) -> true|false</name> <fsummary>Send data to a port</fsummary> <type> <v>Port = port() | atom()</v> @@ -3183,10 +3311,6 @@ os_prompt%</pre> <note> <p>More options may be added in the future.</p> </note> - <note> - <p><c>erlang:port_command/3</c> is currently not auto imported, but - it is planned to be auto imported in OTP R14.</p> - </note> <p>Failures:</p> <taglist> <tag><c>badarg</c></tag> @@ -3815,11 +3939,11 @@ os_prompt%</pre> <tag><c>{monitored_by, Pids}</c></tag> <item> <p>A list of pids that are monitoring the process (with - <c>erlang:monitor/2</c>).</p> + <c>monitor/2</c>).</p> </item> <tag><c>{monitors, Monitors}</c></tag> <item> - <p>A list of monitors (started by <c>erlang:monitor/2</c>) + <p>A list of monitors (started by <c>monitor/2</c>) that are active for the process. For a local process monitor or a remote process monitor by pid, the list item is <c>{process, Pid}</c>, and for a remote process @@ -4008,7 +4132,7 @@ os_prompt%</pre> terminate, it has no return value - unless the arguments are invalid, in which case the function <em>returns the error reason</em>, that is <c>badarg</c>. If you want to be really sure not to return you can call - <c>erlang:error(erlang:raise(Class, Reason, Stacktrace))</c> + <c>error(erlang:raise(Class, Reason, Stacktrace))</c> and hope to distinguish exceptions later.</p> </desc> </func> @@ -4016,7 +4140,7 @@ os_prompt%</pre> <name>erlang:read_timer(TimerRef) -> int() | false</name> <fsummary>Number of milliseconds remaining for a timer</fsummary> <type> - <v>TimerRef = ref()</v> + <v>TimerRef = reference()</v> </type> <desc> <p><c>TimerRef</c> is a timer reference returned by @@ -4039,7 +4163,7 @@ os_prompt%</pre> <name>erlang:ref_to_list(Ref) -> string()</name> <fsummary>Text representation of a reference</fsummary> <type> - <v>Ref = ref()</v> + <v>Ref = reference()</v> </type> <desc> <p>Returns a string which corresponds to the text @@ -4218,7 +4342,7 @@ true</pre> <v>Dest = pid() | RegName </v> <v> LocalPid = pid() (of a process, alive or dead, on the local node)</v> <v>Msg = term()</v> - <v>TimerRef = ref()</v> + <v>TimerRef = reference()</v> </type> <desc> <p>Starts a timer which will send the message <c>Msg</c> @@ -4431,7 +4555,7 @@ true</pre> </desc> </func> <func> - <name>spawn(Node, Module, Function, ArgumentList) -> pid()</name> + <name>spawn(Node, Module, Function, Args) -> pid()</name> <fsummary>Create a new process with a function as entry point on a given node</fsummary> <type> <v>Node = node()</v> @@ -4609,7 +4733,7 @@ true</pre> <tag><c>monitor</c></tag> <item> <p>Monitor the new process (just like - <seealso marker="#monitor/2">erlang:monitor/2</seealso> does).</p> + <seealso marker="#monitor/2">monitor/2</seealso> does).</p> </item> <tag><c>{priority, Level}</c></tag> <item> @@ -4747,7 +4871,7 @@ true</pre> <v> LocalPid = pid() (of a process, alive or dead, on the local node)</v> <v> RegName = atom()</v> <v>Msg = term()</v> - <v>TimerRef = ref()</v> + <v>TimerRef = reference()</v> </type> <desc> <p>Starts a timer which will send the message @@ -5093,9 +5217,9 @@ true</pre> schedulers actually have bound as requested, call <seealso marker="#system_info_scheduler_bindings">erlang:system_info(scheduler_bindings)</seealso>. </p> - <p>Schedulers can currently only be bound on newer Linux - and Solaris systems, but more systems will be supported - in the future. + <p>Schedulers can currently only be bound on newer Linux, + Solaris, FreeBSD, and Windows systems, but more systems will be + supported in the future. </p> <p>In order for the runtime system to be able to bind schedulers, the CPU topology needs to be known. If the runtime system fails @@ -5103,10 +5227,21 @@ true</pre> For more information on how to define the CPU topology, see <seealso marker="#system_flag_cpu_topology">erlang:system_flag(cpu_topology, CpuTopology)</seealso>. </p> - <p><em>NOTE:</em> If other programs on the system have bound - to processors, e.g. another Erlang runtime system, you - may lose performance when binding schedulers. Therefore, - schedulers are by default not bound.</p> + <p>The runtime system will by default bind schedulers to logical + processors using the <c>default_bind</c> bind type if the amount + of schedulers are at least equal to the amount of logical + processors configured, binding of schedulers is supported, + and a CPU topology is available at startup. + </p> + <p><em>NOTE:</em> If the Erlang runtime system is the only + operating system process that binds threads to logical processors, + this improves the performance of the runtime system. However, + if other operating system processes (as for example another Erlang + runtime system) also bind threads to logical processors, there + might be a performance penalty instead. If this is the case you, + are are advised to unbind the schedulers using the + <seealso marker="erl#+sbt">+sbtu</seealso> command line argument, + or <c>erlang:system_flag(scheduler_bind_type, unbound)</c>.</p> <p>Schedulers can be bound in different ways. The <c>How</c> argument determines how schedulers are bound. <c>How</c> can currently be one of:</p> @@ -5271,7 +5406,7 @@ true</pre> <p>Returns <c>{Allocator, Version, Features, Settings}.</c></p> <p>Types:</p> <list type="bulleted"> - <item><c>Allocator = undefined | elib_malloc | glibc</c></item> + <item><c>Allocator = undefined | glibc</c></item> <item><c>Version = [int()]</c></item> <item><c>Features = [atom()]</c></item> <item><c>Settings = [{Subsystem, [{Parameter, Value}]}]</c></item> @@ -5286,7 +5421,7 @@ true</pre> implementation used. If <c>Allocator</c> equals <c>undefined</c>, the <c>malloc()</c> implementation used could not be identified. Currently - <c>elib_malloc</c> and <c>glibc</c> can be identified.</p> + <c>glibc</c> can be identified.</p> </item> <item> <p><c>Version</c> is a list of integers (but not a @@ -5363,6 +5498,16 @@ true</pre> <seealso marker="#system_info_allocator_tuple">erlang:system_info({allocator, Alloc})</seealso>. </p> </item> + <tag><c>build_type</c></tag> + <item> + <p>Returns an atom describing the build type of the runtime + system. This is normally the atom <c>opt</c> for optimized. + Other possible return values are <c>debug</c>, <c>purify</c>, + <c>quantify</c>, <c>purecov</c>, <c>gcov</c>, <c>valgrind</c>, + <c>gprof</c>, and <c>lcnt</c>. Possible return values + may be added and/or removed at any time without prior notice. + </p> + </item> <tag><c>c_compiler_used</c></tag> <item> <p>Returns a two-tuple describing the C compiler used when @@ -5440,7 +5585,7 @@ true</pre> <c>CpuTopology</c> type to change. </p> </item> - <tag><c>{cpu_topology, defined}</c></tag> + <tag><marker id="system_info_cpu_topology_defined"><c>{cpu_topology, defined}</c></marker></tag> <item> <p>Returns the user defined <c>CpuTopology</c>. For more information see the documentation of @@ -5450,12 +5595,14 @@ true</pre> argument. </p> </item> - <tag><c>{cpu_topology, detected}</c></tag> + <tag><marker id="system_info_cpu_topology_detected"><c>{cpu_topology, detected}</c></marker></tag> <item> <p>Returns the automatically detected <c>CpuTopology</c>. The emulator currently only detects the CPU topology on some newer - linux and solaris systems. For more information see the - documentation of the + Linux, Solaris, FreeBSD, and Windows systems. On Windows system with + more than 32 logical processors the CPU topology is not detected. + </p> + <p>For more information see the documentation of the <seealso marker="#system_info_cpu_topology">cpu_topology</seealso> argument. </p> @@ -5513,52 +5660,16 @@ true</pre> </item> <tag><c>elib_malloc</c></tag> <item> - <p>If the emulator uses the <c>elib_malloc</c> memory - allocator, a list of two-element tuples containing status - information is returned; otherwise, <c>false</c> is - returned. The list currently contains the following - two-element tuples (all sizes are presented in bytes):</p> - <taglist> - <tag><c>{heap_size, Size}</c></tag> - <item> - <p>Where <c>Size</c> is the current heap size.</p> - </item> - <tag><c>{max_alloced_size, Size}</c></tag> - <item> - <p>Where <c>Size</c> is the maximum amount of memory - allocated on the heap since the emulator started.</p> - </item> - <tag><c>{alloced_size, Size}</c></tag> - <item> - <p>Where <c>Size</c> is the current amount of memory - allocated on the heap.</p> - </item> - <tag><c>{free_size, Size}</c></tag> - <item> - <p>Where <c>Size</c> is the current amount of free - memory on the heap.</p> - </item> - <tag><c>{no_alloced_blocks, No}</c></tag> - <item> - <p>Where <c>No</c> is the current number of allocated - blocks on the heap.</p> - </item> - <tag><c>{no_free_blocks, No}</c></tag> - <item> - <p>Where <c>No</c> is the current number of free blocks - on the heap.</p> - </item> - <tag><c>{smallest_alloced_block, Size}</c></tag> - <item> - <p>Where <c>Size</c> is the size of the smallest - allocated block on the heap.</p> - </item> - <tag><c>{largest_free_block, Size}</c></tag> - <item> - <p>Where <c>Size</c> is the size of the largest free - block on the heap.</p> - </item> - </taglist> + <p>This option will be removed in a future release. + The return value will always be <c>false</c> since + the elib_malloc allocator has been removed.</p> + </item> + <tag><marker id="system_info_dist_buf_busy_limit"><c>dist_buf_busy_limit</c></marker></tag> + <item> + <p>Returns the value of the distribution buffer busy limit + in bytes. This limit can be set on startup by passing the + <seealso marker="erl#+zdbbl">+zdbbl</seealso> command line + flag to <c>erl</c>.</p> </item> <tag><c>fullsweep_after</c></tag> <item> @@ -5634,11 +5745,34 @@ true</pre> information see the <seealso marker="erts:crash_dump">"How to interpret the Erlang crash dumps"</seealso> chapter in the ERTS User's Guide.</p> </item> - <tag><c>logical_processors</c></tag> + <tag><marker id="logical_processors"><c>logical_processors</c></marker></tag> + <item> + <p>Returns the detected number of logical processors configured + on the system. The return value is either an integer, or + the atom <c>unknown</c> if the emulator wasn't able to + detect logical processors configured. + </p> + </item> + <tag><marker id="logical_processors_available"><c>logical_processors_available</c></marker></tag> + <item> + <p>Returns the detected number of logical processors available to + the Erlang runtime system. The return value is either an + integer, or the atom <c>unknown</c> if the emulator wasn't + able to detect logical processors available. The number + of logical processors available is less than or equal to + the number of <seealso marker="#logical_processors_online">logical + processors online</seealso>. + </p> + </item> + <tag><marker id="logical_processors_online"><c>logical_processors_online</c></marker></tag> <item> - <p>Returns the number of logical processors detected on the - system as an integer or the atom <c>unknown</c> if the - emulator wasn't able to detect any. + <p>Returns the detected number of logical processors online on + the system. The return value is either an integer, + or the atom <c>unknown</c> if the emulator wasn't able to + detect logical processors online. The number of logical + processors online is less than or equal to the number of + <seealso marker="#logical_processors">logical processors + configured</seealso>. </p> </item> <tag><c>machine</c></tag> @@ -5843,6 +5977,26 @@ true</pre> <c>get_tcw</c> in "Match Specifications in Erlang", <seealso marker="erts:match_spec#get_tcw">ERTS User's Guide</seealso>.</p> </item> + <tag><marker id="update_cpu_info"><c>update_cpu_info</c></marker></tag> + <item> + <p>The runtime system rereads the CPU information available and + updates its internally stored information about the + <seealso marker="#system_info_cpu_topology_detected">detected CPU + topology</seealso> and the amount of logical processors + <seealso marker="#logical_processors">configured</seealso>, + <seealso marker="#logical_processors_online">online</seealso>, and + <seealso marker="#logical_processors_available">available</seealso>. + If the CPU information has changed since the last time it was read, + the atom <c>changed</c> is returned; otherwise, the atom + <c>unchanged</c> is returned. If the CPU information has changed + you probably want to + <seealso marker="#system_flag_schedulers_online">adjust the amount + of schedulers online</seealso>. You typically want to have as + many schedulers online as + <seealso marker="#logical_processors_available">logical processors + available</seealso>. + </p> + </item> <tag><marker id="system_info_version"><c>version</c></marker></tag> <item> <p>Returns a string containing the version number of the @@ -6659,6 +6813,17 @@ true</pre> See also <seealso marker="#trace_pattern/3">erlang:trace_pattern/3</seealso>.</p> </item> + <tag><c>call_time</c></tag> + <item> + <p>Return the call time values for this function or + <c>true</c> for the pseudo function <c>on_load</c> if call + time tracing is active. Returns <c>false</c> otherwise. + The call time values returned, <c>[{Pid, Count, S, Us}]</c>, + is a list of each process that has executed the function and its specific counters. + See also + <seealso marker="#trace_pattern/3">erlang:trace_pattern/3</seealso>.</p> + </item> + <tag><c>all</c></tag> <item> <p>Return a list containing the <c>{Item, Value}</c> tuples @@ -6761,13 +6926,13 @@ true</pre> </item> <tag><c>restart</c></tag> <item> - <p>For the <c>FlagList</c> option <c>call_count</c>: + <p>For the <c>FlagList</c> option <c>call_count</c> and <c>call_time</c>: restart the existing counters. The behaviour is undefined for other <c>FlagList</c> options.</p> </item> <tag><c>pause</c></tag> <item> - <p>For the <c>FlagList</c> option <c>call_count</c>: pause + <p>For the <c>FlagList</c> option <c>call_count</c> and <c>call_time</c>: pause the existing counters. The behaviour is undefined for other <c>FlagList</c> options.</p> </item> @@ -6822,6 +6987,23 @@ true</pre> <p>The counter value can be read with <seealso marker="#trace_info/2">erlang:trace_info/2</seealso>.</p> </item> + <tag><c>call_time</c></tag> + <item> + <p>Starts (<c>MatchSpec == true</c>) or stops + (<c>MatchSpec == false</c>) call time tracing for all + types of function calls. For every function a counter is + incremented when the function is called. Time spent in the function + is accumulated in two other counters, seconds and micro-seconds. + The counters are stored for each call traced process.</p> + <p>If call time tracing is started while already running, + the count and time is restarted from zero. Running counters can be + paused with <c>MatchSpec == pause</c>. Paused and running + counters can be restarted from zero with + <c>MatchSpec == restart</c>.</p> + <p>The counter value can be read with + <seealso marker="#trace_info/2">erlang:trace_info/2</seealso>.</p> + </item> + </taglist> <p>The <c>global</c> and <c>local</c> options are mutually exclusive and <c>global</c> is the default (if no options are @@ -6829,7 +7011,7 @@ true</pre> perform a kind of local tracing, and can also not be combined with <c>global</c>. A function can be either globally or locally traced. If global tracing is specified for a - specified set of functions; local, meta and call count + specified set of functions; local, meta, call time and call count tracing for the matching set of local functions will be disabled, and vice versa.</p> <p>When disabling trace, the option must match the type of trace diff --git a/erts/doc/src/match_spec.xml b/erts/doc/src/match_spec.xml index b9f955e4db..f0390c9db8 100644 --- a/erts/doc/src/match_spec.xml +++ b/erts/doc/src/match_spec.xml @@ -60,7 +60,7 @@ <section> <title>Grammar</title> - <p>A match_spec can be described in this <em>informal</em> grammar:</p> + <p>A match_spec used in tracing can be described in this <em>informal</em> grammar:</p> <list type="bulleted"> <item>MatchExpression ::= [ MatchFunction, ... ] </item> @@ -117,6 +117,52 @@ <c><![CDATA[display]]></c> | <c><![CDATA[caller]]></c> | <c><![CDATA[set_tcw]]></c> | <c><![CDATA[silent]]></c></item> </list> + + <p>A match_spec used in ets can be described in this <em>informal</em> grammar:</p> + <list type="bulleted"> + <item>MatchExpression ::= [ MatchFunction, ... ] + </item> + <item>MatchFunction ::= { MatchHead, MatchConditions, MatchBody } + </item> + <item>MatchHead ::= MatchVariable | <c><![CDATA['_']]></c> | { MatchHeadPart, ... } + </item> + <item>MatchHeadPart ::= term() | MatchVariable | <c><![CDATA['_']]></c></item> + <item>MatchVariable ::= '$<number>' + </item> + <item>MatchConditions ::= [ MatchCondition, ...] | <c><![CDATA[[]]]></c></item> + <item>MatchCondition ::= { GuardFunction } | + { GuardFunction, ConditionExpression, ... } + </item> + <item>BoolFunction ::= <c><![CDATA[is_atom]]></c> | <c><![CDATA[is_constant]]></c> | + <c><![CDATA[is_float]]></c> | <c><![CDATA[is_integer]]></c> | <c><![CDATA[is_list]]></c> | + <c><![CDATA[is_number]]></c> | <c><![CDATA[is_pid]]></c> | <c><![CDATA[is_port]]></c> | + <c><![CDATA[is_reference]]></c> | <c><![CDATA[is_tuple]]></c> | <c><![CDATA[is_binary]]></c> | + <c><![CDATA[is_function]]></c> | <c><![CDATA[is_record]]></c> | <c><![CDATA[is_seq_trace]]></c> | + <c><![CDATA['and']]></c> | <c><![CDATA['or']]></c> | <c><![CDATA['not']]></c> | <c><![CDATA['xor']]></c> | + <c><![CDATA[andalso]]></c> | <c><![CDATA[orelse]]></c></item> + <item>ConditionExpression ::= ExprMatchVariable | { GuardFunction } | + { GuardFunction, ConditionExpression, ... } | TermConstruct + </item> + <item>ExprMatchVariable ::= MatchVariable (bound in the MatchHead) | + <c><![CDATA['$_']]></c> | <c><![CDATA['$$']]></c></item> + <item>TermConstruct = {{}} | {{ ConditionExpression, ... }} | + <c><![CDATA[[]]]></c> | [ConditionExpression, ...] | NonCompositeTerm | Constant + </item> + <item>NonCompositeTerm ::= term() (not list or tuple) + </item> + <item>Constant ::= {<c><![CDATA[const]]></c>, term()} + </item> + <item>GuardFunction ::= BoolFunction | <c><![CDATA[abs]]></c> | + <c><![CDATA[element]]></c> | <c><![CDATA[hd]]></c> | <c><![CDATA[length]]></c> | <c><![CDATA[node]]></c> | + <c><![CDATA[round]]></c> | <c><![CDATA[size]]></c> | <c><![CDATA[tl]]></c> | <c><![CDATA[trunc]]></c> | + <c><![CDATA['+']]></c> | <c><![CDATA['-']]></c> | <c><![CDATA['*']]></c> | <c><![CDATA['div']]></c> | + <c><![CDATA['rem']]></c> | <c><![CDATA['band']]></c> | <c><![CDATA['bor']]></c> | <c><![CDATA['bxor']]></c> | + <c><![CDATA['bnot']]></c> | <c><![CDATA['bsl']]></c> | <c><![CDATA['bsr']]></c> | <c><![CDATA['>']]></c> | + <c><![CDATA['>=']]></c> | <c><![CDATA['<']]></c> | <c><![CDATA['=<']]></c> | <c><![CDATA['=:=']]></c> | + <c><![CDATA['==']]></c> | <c><![CDATA['=/=']]></c> | <c><![CDATA['/=']]></c> | <c><![CDATA[self]]></c> | + <c><![CDATA[get_tcw]]></c></item> + <item>MatchBody ::= [ ConditionExpression, ... ]</item> + </list> </section> <section> @@ -453,8 +499,8 @@ <section> <title>Differences between match specifications in ETS and tracing</title> <p>ETS match specifications are there to produce a return - value. Usually the expression contains one single - <c><![CDATA[ActionTerm]]></c> which defines the return value without having + value. Usually the <c><![CDATA[MatchBody]]></c> contains one single + <c><![CDATA[ConditionExpression]]></c> which defines the return value without having any side effects. Calls with side effects are not allowed in the ETS context.</p> <p>When tracing there is no return value to produce, the @@ -530,7 +576,7 @@ the atom 'strider' and the tuple arity is 3 and return the whole object.</p> <code type="none"><![CDATA[ -[{{strider,'_'.'_'}, +[{{strider,'_','_'}, [], ['$_']}] ]]></code> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index a4867adf22..77181d3407 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -30,6 +30,1320 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 5.8.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix format_man_pages so it handles all man sections + and remove warnings/errors in various man pages. </p> + <p> + Own Id: OTP-8600</p> + </item> + <item> + <p> + The <c>configure</c> command line argument <seealso + marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--enable-ethread-pre-pentium4-compatibility</seealso> + had no effect. This option is now also automatically + enabled if required on the build machine.</p> + <p> + Own Id: OTP-8847</p> + </item> + <item> + <p> + Windows 2003 and Windows XP pre SP3 would sometimes not + start the Erlang R14B VM at all due to a bug in the cpu + topology detection. The bug affects Windows only, no + other platform is even remotely affected. The bug is now + corrected.</p> + <p> + Own Id: OTP-8876</p> + </item> + <item> + <p> + The HiPE run-time in the 64-bit emulator could do a + 64-bit write to a 32-bit struct field. It happened to be + harmless on Intel/AMD processors. Corrected. (Thanks to + Mikael Pettersson.)</p> + <p> + Own Id: OTP-8877</p> + </item> + <item> + <p> + A bug in <seealso + marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get()</seealso> + and <seealso + marker="erl_nif#enif_tsd_get">enif_tsd_get()</seealso> + could cause an emulator crash. These functions are + currently not used in OTP. That is, the crash only occur + on systems with user implemented NIF libraries, or + drivers that use one of these functions.</p> + <p> + Own Id: OTP-8889</p> + </item> + <item> + <p> + Calling <c>erlang:system_info({cpu_topology, + CpuTopologyType})</c> with another <c>CpuTopologyType</c> + element than one of the documented atoms <c>defined</c>, + <c>detected</c>, or <c>used</c> caused an emulator crash. + (Thanks to Paul Guyot)</p> + <p> + Own Id: OTP-8914</p> + </item> + <item> + <p> + The ERTS internal rwlock implementation could get into an + inconsistent state. This bug was very seldom triggered, + but could be during heavy contention. The bug was + introduced in R14B (erts-5.8.1).</p> + <p> + The bug was most likely to be triggered when using the + <c>read_concurrency</c> option on an ETS table that was + frequently accessed from multiple processes doing lots of + writes and reads. That is, in a situation where you + typically don't want to use the <c>read_concurrency</c> + option in the first place.</p> + <p> + Own Id: OTP-8925 Aux Id: OTP-8544 </p> + </item> + <item> + <p> + Tracing to port could cause an emulator crash when + unloading the trace driver.</p> + <p> + Own Id: OTP-8932</p> + </item> + <item> + <p> + Removed use of CancelIoEx on Windows that had been shown + to cause problems with some drivers.</p> + <p> + Own Id: OTP-8937</p> + </item> + <item> + <p> + The fallback implementation used when no native atomic + implementation was found did not compile. (Thanks to + Patrick Baggett, and Tuncer Ayaz)</p> + <p> + Own Id: OTP-8944</p> + </item> + <item> + <p> + Some integer values used during load balancing could + under rare circumstances wrap causing a load unbalance + between schedulers.</p> + <p> + Own Id: OTP-8950</p> + </item> + <item> + <p> + The windows VM now correctly handles appending to large + files (> 4GB).</p> + <p> + Own Id: OTP-8958</p> + </item> + <item> + <p> + Name resolving of IPv6 addresses has been implemented for + Windows versions that support it. The use of ancient + resolver flags (AI_V4MAPPED | AI_ADDRCONFIG) to the + getaddrinfo() function has been removed since e.g FreeBSD + regard mapped IPv4 addresses to be a security problem and + the semantics of the address configured flag is + uncertain.</p> + <p> + Own Id: OTP-8969</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The help texts produced by the <c>configure</c> scripts + in the top directory and in the erts directory have been + aligned and cleaned up.</p> + <p> + Own Id: OTP-8859</p> + </item> + <item> + <p> + When the runtime system had fewer schedulers than logical + processors, the system could get an unnecessarily large + amount reader groups.</p> + <p> + Own Id: OTP-8861</p> + </item> + <item> + <p> + <c>run_rel</c> has been updated to support Solaris's + /dev/ptmx device and to load the necessary STREAMS + modules so that <c>to_erl</c> can provide terminal echo + of keyboard input. (Thanks to Ryan Tilder.)</p> + <p> + Own Id: OTP-8878</p> + </item> + <item> + <p> + The Erlang VM now supports Unicode filenames. The feature + is turned on by default on systems where Unicode + filenames are mandatory (Windows and MacOSX), but can be + enabled on other systems with the '+fnu' emulator option. + Enabling the Unicode filename feature on systems where it + is not default is however considered experimental and not + to be used for production. Together with the Unicode file + name support, the concept of "raw filenames" is + introduced, which means filenames provided without + implicit unicode encoding translation. Raw filenames are + provided as binaries, not lists. For further information, + see stdlib users guide and the chapter about using + Unicode in Erlang. Also see the file module manual page.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8887</p> + </item> + <item> + <p>Buffer overflows have been prevented in <c>erlc</c>, + <c>dialyzer</c>, <c>typer</c>, <c>run_test</c>, + <c>heart</c>, <c>escript</c>, and <c>erlexec</c>.</p> + (Thanks to Michael Santos.) + <p> + Own Id: OTP-8892</p> + </item> + <item> + <p> + The runtime system is now less eager to suspend processes + sending messages over the distribution. The default value + of the distribution buffer busy limit has also been + increased from 128 KB to 1 MB. This in order to improve + throughput.</p> + <p> + Own Id: OTP-8901</p> + </item> + <item> + <p> + The distribution buffer busy limit can now be configured + at system startup. For more information see the + documentation of the <c>erl</c> <seealso + marker="erl#+zdbbl">+zdbbl</seealso> command line flag. + (Thanks to Scott Lystig Fritchie)</p> + <p> + Own Id: OTP-8912</p> + </item> + <item> + <p> + The inet driver internal buffer stack implementation has + been rewritten in order to reduce lock contention.</p> + <p> + Own Id: OTP-8916</p> + </item> + <item> + <p> + New ETS option <c>compressed</c>, to enable a more + compact storage format at the expence of heavier table + operations. For test and evaluation, <c>erl +ec</c> can + be used to force compression on all ETS tables.</p> + <p> + Own Id: OTP-8922 Aux Id: seq11658 </p> + </item> + <item> + <p> + There is now a new function inet:getifaddrs/0 modeled + after C library function getifaddrs() on BSD and LInux + that reports existing interfaces and their addresses on + the host. This replaces the undocumented and unsupported + inet:getiflist/0 and inet:ifget/2.</p> + <p> + Own Id: OTP-8926</p> + </item> + <item> + <p> + Support for detection of CPU topology and binding of + schedulers on FreeBSD 8 have been added. (Thanks to Paul + Guyot)</p> + <p> + Own Id: OTP-8939</p> + </item> + <item> + <p> + Several bugs related to hibernate/3 and HiPE have been + corrected. (Thanks to Paul Guyot.)</p> + <p> + Own Id: OTP-8952</p> + </item> + <item> + <p> + Support for soft and hard links on Windows versions and + filesystems that support them is added.</p> + <p> + Own Id: OTP-8955</p> + </item> + <item> + <p> + The win32 virtual machine is now linked large address + aware. his allows the Erlang VM to use up to 3 gigs of + address space on Windows instead of the default of 2 + gigs.</p> + <p> + Own Id: OTP-8956</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 5.8.1.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix that the documentation top index generator can + handle an Ericsson internal application group. </p> + <p> + Own Id: OTP-8875</p> + </item> + <item> + <p>In embedded mode, on_load handlers that called + <c>code:priv_dir/1</c> or other functions in <c>code</c> + would hang the system. Since the <c>crypto</c> + application now contains an on_loader handler that calls + <c>code:priv_dir/1</c>, including the <c>crypto</c> + application in the boot file would prevent the system + from starting.</p> + <p>Also extended the <c>-init_debug</c> option to print + information about on_load handlers being run to + facilitate debugging.</p> + <p> + Own Id: OTP-8902 Aux Id: seq11703 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 5.8.1.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Windows 2003 and Windows XP pre SP3 would sometimes not + start the Erlang R14B VM at all due to a bug in the cpu + topology detection. The bug affects Windows only, no + other platform is even remotely affected. The bug is now + corrected.</p> + <p> + Own Id: OTP-8876</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 5.8.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Very small floating point numbers generated errors when + converting from list to float in some versions of the VM, + this is now corrected so that i.e. + list_to_float("1.0e-324"). returns 0.0 in all versions of + Erlang.</p> + <p> + Own Id: OTP-7178</p> + </item> + <item> + <p> + Windows Vista and Windows 7 file system virtualization, + which makes "old style" windows programs execute in a + file system sandbox, was previously unintentionally + turned on for the Erlang VM. This is now corrected so + that i.e. writes to C:\Program Files\. without + administrator privileges will fail.</p> + <p> + Own Id: OTP-7405</p> + </item> + <item> + <p> + Fix faulty 64-bit integer term output from drivers. Large + 64-bits integers did not generate correct bignums and + could even cause emulator crash. Only affects drivers + using ERL_DRV_INT64 or ERL_DRV_UINT64, introduced in + R13B03.</p> + <p> + Own Id: OTP-8716</p> + </item> + <item> + <p> + Fixed: inet:setopts(S, [{linger,{true,2}}]) returned + {error,einval} for SCTP sockets. The inet_drv had a bug + when checking the option size.</p> + <p> + Own Id: OTP-8726 Aux Id: seq11617 </p> + </item> + <item> + <p>Fix libm linking with --as-needed flag + <p> + When building with "--as-needed" linker flags on Linux + the build will fail. This has now been fixed.</p> + <p> + (Thanks to Christian Faulhammer)</p></p> + <p> + Own Id: OTP-8728</p> + </item> + <item> + <p> + gen_udp:connect/3 was broken for SCTP enabled builds. It + did not detect remote end errors as it should.</p> + <p> + Own Id: OTP-8729</p> + </item> + <item> + <p>Reduce the risk of integer wrapping in bin vheap size + counting.</p> <p>The vheap size series will now use the + golden ratio instead of doubling and fibonacci + sequences.</p> + <p> + Own Id: OTP-8730</p> + </item> + <item> + <p> + ETS ordered_set containing <c>[]</c> as key could cause + strange thing to happen, like an infinite hanging + <c>ets:select</c>.</p> + <p> + Own Id: OTP-8732</p> + </item> + <item> + <p>reference() has been substituted for ref() in the + documentation.</p> + <p> + Own Id: OTP-8733</p> + </item> + <item> + <p> + When a native compiled module called a not loaded + non-native compiled module that had an on_load function, + the export entries were trashed after code loading so on + the next call from the native compiled module to the + non-native compiled the emulator crashed. This bug has + now been fixed.</p> + <p> + Own Id: OTP-8736</p> + </item> + <item> + <p> + HiPE-enabled Erlang VMs running on BSD systems sometimes + generated messages like "Yikes! erts_alloc() returned + misaligned address 0x8016a512c". Fixed. (Thanks to Mikael + Pettersson.)</p> + <p> + Own Id: OTP-8769</p> + </item> + <item> + <p> + A race condition in <c>erts_poll()</c> could cause delay + of poll for I/O.</p> + <p> + Own Id: OTP-8773</p> + </item> + <item> + <p> + Removed some potential vulnerabilities from the Erlang + Port Mapper Daemon (epmd) and straightened up access + control. Also removed hazardous interfaces allowing + anyone on a machine to forcefully unregister other nodes. + This means that the ei_unregister/erl_unregister + interfaces in erl_interface is rendered not only error + prone and mystifying as before, but totally ineffective. + The old behaviour of unchecked node unregistering can be + restored if needed, see epmd documentation for details.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8780</p> + </item> + <item> + <p> Building in a source tree without prebuilt platform + independent build results failed on the SSL examples + when: </p> <list><item> cross building. This has been + solved by not building the SSL examples during a cross + build. </item><item> building on Windows. </item></list> + <p> + Own Id: OTP-8791</p> + </item> + <item> + <p> + inet:getsockopt for SCTP sctp_default_send_param had a + bug to not initialize required feilds causing random + answers. It is now corrected.</p> + <p> + Own Id: OTP-8795 Aux Id: seq11655 </p> + </item> + <item> + <p> + The hipe_bifs:get_hrvtime/0 BIF now always returns a real + value even if the "perfctr" Linux kernel extension is not + available. It used to return a dummy value. (Thanks to + Mikael Pettersson.)</p> + <p> + Own Id: OTP-8798</p> + </item> + <item> + <p> + Calling a native-code compiled module with an + <c>on_load</c> function could cause a crash. (Thanks to + Mikael Pettersson.)</p> + <p> + Own Id: OTP-8799</p> + </item> + <item> + <p>The emulator could crash while writing a crash dump if + native-compiled modules had been loaded. (Thanks to Paul + Guyot.)</p> + <p> + Own Id: OTP-8801</p> + </item> + <item> + <p> + The garbage collector could crash if invoked from + native-compiled code after a call to a BIF. (Thanks to + Paul Guyot.)</p> + <p> + Own Id: OTP-8821</p> + </item> + <item> + <p> + A rare memory leak in binary:matches is removed</p> + <p> + Own Id: OTP-8823</p> + </item> + <item> + <p>For a socket in the HTTP packet mode, the return value + from <c>gen_tcp:recv/2,3</c> if there is an error in the + header will be <c>{ok,{http_error,String}}</c> instead of + <c>{error,{http_error,String}}</c> to be consistent with + <c>ssl:recv/2,3</c>.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8831</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + ets:select_reverse/{1,2,3} are now documented.</p> + <p> + Own Id: OTP-7863</p> + </item> + <item> + <p> + External format of integers changed to make full use of + all 32 bits of INTEGER_EXT. This is a compatible change + as old code can read full 32-bit integers but only + produce 28-bit integers as INTEGER_EXT.</p> + <p> + Own Id: OTP-8540 Aux Id: seq11534 </p> + </item> + <item> + <p> + Large parts of the <c>ethread</c> library have been + rewritten. The <c>ethread</c> library is an Erlang + runtime system internal, portable thread library used by + the runtime system itself.</p> + <p> + Most notable improvement is a reader optimized rwlock + implementation which dramatically improve the performance + of read-lock/read-unlock operations on multi processor + systems by avoiding ping-ponging of the rwlock cache + lines. The reader optimized rwlock implementation is used + by miscellaneous rwlocks in the runtime system that are + known to be read-locked frequently, and can be enabled on + ETS tables by passing the <seealso + marker="stdlib:ets#new_2_read_concurrency">{read_concurrency, + true}</seealso> option upon table creation. See the + documentation of <seealso + marker="stdlib:ets#new/2">ets:new/2</seealso> for more + information. The reader optimized rwlock implementation + can be fine tuned when starting the runtime system. For + more information, see the documentation of the <seealso + marker="erts:erl#+rg">+rg</seealso> command line argument + of <c>erl</c>.</p> + <p> + There is also a new implementation of rwlocks that is not + optimized for readers. Both implementations interleaves + readers and writers during contention as opposed to, + e.g., the NPTL (Linux) pthread rwlock implementation + which use either a reader or writer preferred strategy. + The reader/writer preferred strategy is problematic since + it starves threads doing the non-preferred operation.</p> + <p> + The new rwlock implementations in general performs better + in ERTS than common pthread implementations. However, in + some extremely heavily contended cases this is not the + case. Such heavy contention can more or less only appear + on ETS tables. This when multiple processes do very large + amounts of write locked operations simultaneously on the + same table. Such use of ETS is bad regardless of rwlock + implementation, will never scale, and is something we + strongly advise against.</p> + <p> + The new rwlock implementations depend on atomic + operations. If no native atomic implementation is found, + a fallback solution will be used. Using the fallback + implies a performance degradation. That is, it is more + important now than before to build OTP with a native + atomic implementation.</p> + <p> + The <c>ethread</c> library contains native atomic + implementations for, x86 (32 and 64 bit), powerpc (32 + bit), sparc V9 (32 and 64 bit), and tilera (32 bit). On + other hardware gcc's builtin support for atomic memory + access will be used if such exists. If no such support is + found, <c>configure</c> will warn about no atomic + implementation available.</p> + <p> + The <c>ethread</c> library can now also use the + <c>libatomic_ops</c> library for atomic memory accesses. + This makes it possible for the Erlang runtime system to + utilize optimized native atomic operations on more + platforms than before. If <c>configure</c> warns about no + atomic implementation available, try using the + <c>libatomic_ops</c> library. Use the <seealso + marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--with-libatomic_ops=PATH</seealso> + <c>configure</c> command line argument when specifying + where the <c>libatomic_ops</c> installation is located. + The <c>libatomic_ops</c> library can be downloaded from: + <url + href="http://www.hpl.hp.com/research/linux/atomic_ops/">http://www.hpl.hp.com/research/linux/atomic_ops/</url></p> + <p> + The changed API of the <c>ethread</c> library has also + caused modifications in the Erlang runtime system. + Preparations for the to come "delayed deallocation" + feature has also been done since it depends on the + <c>ethread</c> library.</p> + <p> + <em>Note</em>: When building for x86, the <c>ethread</c> + library will now use instructions that first appeared on + the pentium 4 processor. If you want the runtime system + to be compatible with older processors (back to 486) you + need to pass the <seealso + marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--enable-ethread-pre-pentium4-compatibility</seealso> + <c>configure</c> command line argument when configuring + the system.</p> + <p> + Own Id: OTP-8544</p> + </item> + <item> + <p> + erlang:localtime_to_universaltime({{2008, 8, 1}, {0, 0, + 0}},true) when TZ=UTC now behaves consistently on all + Unix platforms.</p> + <p> + The problem fixed was originally reported by Paul Guyot + on erlang-bugs mailing list:</p> + <p> + http://www.erlang.org/pipermail/erlang-bugs/2008-November/001077.html</p> + <p> + Own Id: OTP-8580</p> + </item> + <item> + <p> + Optimization reducing memory consumption by two words per + ETS object.</p> + <p> + Own Id: OTP-8737</p> + </item> + <item> + <p> + Fixes for unsupported halfword-emulator</p> + <p> + Own Id: OTP-8745</p> + </item> + <item> + <p> + NIF 64-bit integer support; <c>enif_get_int64</c>, + <c>enif_get_uint64</c>, <c>enif_make_int64</c>, + <c>enif_make_uint64</c>.</p> + <p> + Own Id: OTP-8746</p> + </item> + <item> + <p> + Alignment of trailing data in messages has been adjusted. + This in order to be able to pass data of any type as + trailing data in the future.</p> + <p> + Own Id: OTP-8754</p> + </item> + <item> + <p> + The obsolete/driver.h header file has been removed. It + has been obsolete and deprecated since R8B. Drivers that + still include obsolete/driver.h must be updated to + include erl_driver.h.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8758</p> + </item> + <item> + <p> + Added erlang:system_info(build_type) which makes it + easier to chose drivers, NIF libraries, etc based on + build type of the runtime system.</p> + <p> + The NIF library for crypto can now be built for valgrind + and/or debug as separate NIF libraries that will be + automatically loaded if the runtime system has been built + with a matching build type.</p> + <p> + Own Id: OTP-8760</p> + </item> + <item> + <p> + Further lessened the memory requirements of ETS objects.</p> + <p> + Own Id: OTP-8762</p> + </item> + <item> + <p>The broken elib_malloc alternate memory allocator has + been removed. <c>erlang:system_info(elib_malloc)</c> will + always return <c>false</c>, and in R15, + <c>erlang:system_info(elib_malloc)</c> will fail with a + <c>badarg</c> exception.</p> + <p> + Own Id: OTP-8764</p> + </item> + <item> + <p> + Calling <c>erlang:system_info/1</c> with the new argument + <c>update_cpu_info</c> will make the runtime system + reread and update the internally stored CPU information. + For more information see the documentation of <seealso + marker="erlang#update_cpu_info">erlang:system_info(update_cpu_info)</seealso>.</p> + <p> + The CPU topology is now automatically detected on Windows + systems with less than 33 logical processors. The runtime + system will now, also on Windows, by default bind + schedulers to logical processors using the + <c>default_bind</c> bind type if the amount of schedulers + is at least equal to the amount of logical processors + configured, binding of schedulers is supported, and a CPU + topology is available at startup.</p> + <p> + Own Id: OTP-8765</p> + </item> + <item> + <p> + The SMP ERTS internal child waiter thread used on Linux + system with NPTL was unintentionally disabled during + cross compilation rewrites (OTP-8323 in R13B03). It has + now been re-enabled. Enabling it again gives a slight + performance improvement.</p> + <p> + Own Id: OTP-8774</p> + </item> + <item> + <p> + <c>epmd</c> used to generate a message to the syslog when + it started up, which could be annoying. This has been + changed to only generate the message if the debug swith + is given. (Thanks to Michael Santos.)</p> + <p> + Own Id: OTP-8775</p> + </item> + <item> + <p> + The scheduler wakeup threshold is now possible to adjust + at system boot. For more information see the <seealso + marker="erl#+swt">+swt</seealso> command line argument of + <c>erl</c>.</p> + <p> + Own Id: OTP-8811</p> + </item> + <item> + <p> + The undocumented function inet:ifget/2 has been improved + to return interface hardware address (MAC) on platforms + supporting getaddrinfo() (such as BSD unixes). Note it + still does not work on all platforms for example not + Windows nor Solaris, so the function is still + undocumented.</p> + <p> + Buffer overflow and field init bugs for inet:ifget/2 and + inet:getservbyname/2 has also been fixed.</p> + <p> + Thanks to Michael Santos.</p> + <p> + Own Id: OTP-8816</p> + </item> + <item> + <p> + Optimizations for MIPS when using gcc atomics. (Thanks to + Steve Vinoski)</p> + <p> + Own Id: OTP-8834</p> + </item> + <item> + <p> + Lock optimization in timer functionality.</p> + <p> + Own Id: OTP-8835</p> + </item> + </list> + </section> + + + <section><title>Known Bugs and Problems</title> + <list> + <item> + <p>Fix epmd and build environment to build on VxWorks</p> + <p> + Own Id: OTP-8838</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 5.8</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Driver threads, such as async threads, using <seealso + marker="erl_driver#ErlDrvPDL">port data locks</seealso> + peeked at the port status field without proper locking + when looking up the driver queue.</p> + <p> + Own Id: OTP-8475</p> + </item> + <item> + <p> + The use of <c>mmap()</c> was unnecessarily disabled when + cross compiling.</p> + <p> + The <c>configure</c> arguments <c>--with-ssl</c>, and + <c>--with-odbc</c> refused to accept libraries outside of + <c>$erl_xcomp_sysroot</c> when cross compiling for no + good reason.</p> + <p> + The <c>configure</c> argument <c>--with-odbc</c> didn't + handle the value <c>yes</c> correct.</p> + <p> + The <c>configure</c> arguments <c>--with-odbc</c>, and + <c>--without-odbc</c> have also been added to the + configure help.</p> + <p> + (Thanks to Steve Vinoski for reporting these issues)</p> + <p> + Own Id: OTP-8484</p> + </item> + <item> + <p> + A call to the BIF <c>unregister(RegName)</c> when a port + had the name <c>RegName</c> registered in the runtime + system without SMP support caused a runtime system crash. + (Thanks to Per Hedeland for the bugfix and test case.)</p> + <p> + Own Id: OTP-8487</p> + </item> + <item> + <p> + The runtime system crashed if fewer logical processors + were found than reported by <c>sysconf( + SC_NPROCESSORS_CONF)</c>.</p> + <p> + Own Id: OTP-8549</p> + </item> + <item> + <p> + Fix memory management bug causing crash of non-SMP + emulator with async threads enabled. The bug did first + appear in R13B03.</p> + <p> + Own Id: OTP-8591 Aux Id: seq11554 </p> + </item> + <item> + <p> + Port locks could be prematurely destroyed.</p> + <p> + Own Id: OTP-8612</p> + </item> + <item> + <p>The <c>empd</c> program could loop and consume 100% + CPU time if an unexpected error ocurred in + <c>listen()</c> or <c>accept()</c>. Now <c>epmd</c> will + terminate if a non-recoverable error occurs. (Thanks to + Michael Santos.)</p> + <p> + Own Id: OTP-8618</p> + </item> + <item> + <p> + When kernel poll has been enabled, a livelock could in + rare circumstances occur. Problem reported by Chetan + Ahuja, fix by Mikael Pettersson.</p> + <p> + Own Id: OTP-8632</p> + </item> + <item> + <p> + Windows: Closing port of program that stalled without + reading all data could deadlock scheduler thread.</p> + <p> + Own Id: OTP-8641</p> + </item> + <item> + <p> + On some combination of Montavista Linux on Cavium Octeon + processors, some socket-related system calls returned + other numbers than -1 for errors. This caused a core dump + in inet_drv.c. Now the code works around this problem.</p> + <p> + Own Id: OTP-8654</p> + </item> + <item> + <p> + Missing memory barriers in <c>erts_poll()</c> could cause + the runtime system to hang indefinitely.</p> + <p> + Own Id: OTP-8658</p> + </item> + <item> + <p> + <c>ethr_rwmutex_tryrlock()</c> acquired and refused to + acquire a lock with inverted logic. The lock was however + never acquired in a thread unsafe manner. (Thanks to JR + Zhang for noting this issue)</p> + <p> + Own Id: OTP-8662</p> + </item> + <item> + <p> + Extreme combinations of register/unregister in a highly + parallell SMP application could crash the VM. The error + is corrected.</p> + <p> + Own Id: OTP-8663</p> + </item> + <item> + <p> + On Windows, files are now opened with FILE_SHARE_DELETE + to get closer to Unix semantics.</p> + <p> + Own Id: OTP-8667</p> + </item> + <item> + <p> + <c>erlang:system_info(multi_scheduling)</c> sometimes + erroneously returned <c>enabled</c> when it should have + returned <c>blocked</c>.</p> + <p> + Own Id: OTP-8675</p> + </item> + <item> + <p> + Fix bug causing <c>erlang:decode_packet</c> and + <c>enif_make_string</c> to generate faulty strings with + negative character values for ascii values larger than + 127. (Thanks to Paul Guyot)</p> + <p> + Own Id: OTP-8685</p> + </item> + <item> + <p> + <c>open_port/2</c> with the <c>spawn</c> and + <c>spawn_executable</c> options can include an + <c>{env,Env}</c> option. In some cases unsetting + variables would not work on Unix (typically if more + variables were unset than were actually present in the + environment).</p> + <p> + Own Id: OTP-8701</p> + </item> + <item> + <p> + A user defined CPU topology set via a call to <seealso + marker="erlang#system_flag_cpu_topology">erlang:system_flag(cpu_topology, + CPUTopology)</seealso> was not properly verified, and + could in worst case cause an emulator crash. The emulator + crash could only occur when a user defined CPU topology + already existed and was redefined.</p> + <p> + Own Id: OTP-8710</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The grammar for match specifications in ERTS users guide + only described the tracing dialect of match + specifications. An informal grammar for the ETS dialect + is added.</p> + <p> + Own Id: OTP-8086 Aux Id: seq11333 </p> + </item> + <item> + <p> + The module binary from EEP31 (and EEP9) is implemented.</p> + <p> + Own Id: OTP-8217</p> + </item> + <item> + <p> + New NIF API function <c>enif_make_new_binary</c></p> + <p> + Own Id: OTP-8474</p> + </item> + <item> + <p> + The guard BIF <c>is_boolean/1</c> (introduced in R10B) + has now been included in the lists of BIFs allowed in + guards in the Reference Manual.</p> + <p> + Own Id: OTP-8477</p> + </item> + <item> + <p> + Added function <c>zip:foldl/3</c> to iterate over zip + archives.</p> + <p> + Added functions to create and extract escripts. See + <c>escript:create/2</c> and <c>escript:extract/2</c>.</p> + <p> + The undocumented function <c>escript:foldl/3</c> has been + removed. The same functionality can be achieved with the + more flexible functions <c>escript:extract/2</c> and + <c>zip:foldl/3</c>.</p> + <p> + Record fields has been annotated with type info. Source + files as been adapted to fit within 80 chars and trailing + whitespace has been removed.</p> + <p> + Own Id: OTP-8521</p> + </item> + <item> + <p>A regular expression with many levels of parenthesis + could cause a buffer overflow. That has been corrected. + (Thanks to Michael Santos.)</p> + <p> + Own Id: OTP-8539</p> + </item> + <item> + <p> + <c>erlang:decode_packet(httph_bin,..)</c> could return + corrupt header strings or even crash the VM. This has + been fixed. It only happened on 32-bit VM if the header + name was unknown and between 16 and 20 characters long. + Sockets with simular <c>packet</c> option did not suffer + from this bug.</p> + <p> + Own Id: OTP-8548</p> + </item> + <item> + <p>New NIF features:</p> <list><item> Send messages from + a NIF, or from thread created by NIF, to any local + process (<c>enif_send</c>) </item><item> Store terms + between NIF calls (<c>enif_alloc_env</c>, + <c>enif_make_copy</c>) </item><item> Create binary terms + with user defined memory management + (<c>enif_make_resource_binary</c>) </item></list> <p>And + some incompatible changes made to the API. For more + information see the warning text in <seealso + marker="erl_nif">erl_nif(3)</seealso>.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8555</p> + </item> + <item> + <p>If the '<c>fop</c>' program (needed for building PDF + files) cannot not be found, it is now possible to build + the HTML and man pages anyway (there will also be dummy + PDF files with no real content created). (Thanks to + Tuncer Ayaz.)</p> + <p> + Own Id: OTP-8559</p> + </item> + <item> + <p>When defining macros the closing right parenthesis + before the dot is now mandatory.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8562</p> + </item> + <item> + <p>Local and imported functions now override the + auto-imported BIFs when the names clash. The pre R14 + behaviour was that auto-imported BIFs would override + local functions. To avoid that old programs change + behaviour, the following will generate an error:</p> + <list><item><p>Doing a call without explicit module name + to a local function having a name clashing with the name + of an auto-imported BIF that was present (and + auto-imported) before OTP R14A</p></item> + <item><p>Explicitly importing a function having a name + clashing with the name of an autoimported BIF that was + present (and autoimported) before OTP R14A</p></item> + <item><p>Using any form of the old compiler directive + <c>nowarn_bif_clash</c></p></item> </list> <p>If the BIF + was added or auto-imported in OTP R14A or later, + overriding it with an import or a local function will + only result in a warning,</p> <p>To resolve clashes, you + can either use the explicit module name <c>erlang</c> to + call the BIF, or you can remove the auto-import of that + specific BIF by using the new compiler directive + <c>-compile({no_auto_import,[F/A]}).</c>, which makes all + calls to the local or imported function without explicit + module name pass without warnings or errors.</p> <p>The + change makes it possible to add auto-imported BIFs + without breaking or silently changing old code in the + future. However some current code ingeniously utilizing + the old behaviour or the <c>nowarn_bif_clash</c> compiler + directive, might need changing to be accepted by the + compiler.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8579</p> + </item> + <item> + <p> + A bug in re that could cause certain regular expression + matches never to terminate is corrected. (Thanks to + Michael Santos and Gordon Guthrie.)</p> + <p> + Own Id: OTP-8589</p> + </item> + <item> + <p>The erlang:open_port spawn and spawn_executable + directives can include an <c>{env,Env}</c> directive to + set up environment variables for the spawned process. A + bug prevented applications from using <c>{env,Env}</c> to + set an environment variable whose value ended with a + '<c>=</c>' (equal sign) character; the trailing equal + sign was mistaken as an indication that an environment + variable was to be cleared from the environment of the + spawned process. (Thanks to Steve Vinoski.)</p> + <p> + Own Id: OTP-8614</p> + </item> + <item> + <p><c>receive</c> statements that can only read out a + newly created reference are now specially optimized so + that it will execute in constant time regardless of the + number of messages in the receive queue for the process. + That optimization will benefit calls to + <c>gen_server:call()</c>. (See <c>gen:do_call/4</c> for + an example of a receive statement that will be + optimized.)</p> + <p> + Own Id: OTP-8623</p> + </item> + <item> + <p> + The functions file:advise/4 and file:datasync/1 have been + added. (Thanks to Filipe David Manana.)</p> + <p> + Own Id: OTP-8637</p> + </item> + <item> + <p> + New NIF API functions: <c>enif_make_atom_len</c>, + <c>enif_make_existing_atom_len</c>, + <c>enif_make_string_len</c>, <c>enif_get_atom_length</c>, + <c>enif_get_list_length</c>, <c>enif_is_list</c>, + <c>enif_is_tuple</c> (by Tuncer Ayaz)</p> + <p> + Own Id: OTP-8640</p> + </item> + <item> + <p> + Support for using gcc's built-in functions for atomic + memory access has been added. This functionallity will be + used if available and no other native atomic + implementation in ERTS is available.</p> + <p> + Own Id: OTP-8659</p> + </item> + <item> + <p> + The number of spinlocks used when implementing atomic + fall-backs when no native atomic implementation is + available has been increased from 16 to 1024.</p> + <p> + Own Id: OTP-8660</p> + </item> + <item> + <p> + Writer preferred pthread read/write locks has been + enabled on Linux.</p> + <p> + Own Id: OTP-8661</p> + </item> + <item> + <p> + The runtime system will by default bind schedulers to + logical processors using the <c>default_bind</c> bind + type if the amount of schedulers are at least equal to + the amount of logical processors configured, binding of + schedulers is supported, and a CPU topology is available + at startup. </p> + <p> + <em>NOTE:</em> If the Erlang runtime system is the only + operating system process that binds threads to logical + processors, this improves the performance of the runtime + system. However, if other operating system processes (as + for example another Erlang runtime system) also bind + threads to logical processors, there might be a + performance penalty instead. If this is the case you, are + are advised to unbind the schedulers using the <seealso + marker="erl#+sbt">+sbtu</seealso> command line argument, + or by invoking <seealso + marker="erlang#system_flag_scheduler_bind_type">erlang:system_flag(scheduler_bind_type, + unbound)</seealso>.</p> + <p> + Own Id: OTP-8666</p> + </item> + <item> + <p> + The recently added BIFs erlang:min/2, erlang:max/2 and + erlang:port_command/3 are now auto-imported (as they were + originally intended to be). Due to the recent compiler + change (OTP-8579), the only impact on old code defining + it's own min/2, max/2 or port_command/3 functions will be + a warning, the local functions will still be used. The + warning can be removed by using + -compile({no_auto_import,[min/2,max/2,port_command/3]}). + in the source file.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8669 Aux Id: OTP-8579 </p> + </item> + <item> + <p> + There is a new option 'exclusive' to file:open/2 that + uses the OS O_EXCL flag where supported to open the file + in exclusive mode.</p> + <p> + Own Id: OTP-8670</p> + </item> + <item> + <p> + Now, binary_to_term/2 is auto-imported. This will cause a + compile warning if and only if a module has got a local + function with that name.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8671</p> + </item> + <item> + <p> + Alignment of scheduler data and run queues were adjusted.</p> + <p> + Own Id: OTP-8673</p> + </item> + <item> + <p>Call time breakpoint tracing</p> <list><item>Introduce + a <c>call_time</c> option to + <c>erlang:trace_pattern/3</c>.This option enables call + time breakpoint tracing on code that is executed by + processes with call tracing enabled. Call time tracing + stores the number of calls and the time spent of each + function with this trace pattern enabled. The information + can be retrieved with <c>erlang:trace_info/2</c></item> + <item>Add a scheduler array for BpData. To solve the + issue of multiple schedulers constantly updating the head + pointer to the bp data wheel, each scheduler now has its + own entrypoint to the wheel. This head pointer can be + updated without a locking being taken.</item></list> + <p>Teach call count tracing to use atomics</p> + <list><item>Call count previously used a global lock for + accessing and writing its counter in the breakpoint. This + is now changed to atomics instead.</item> <item>The + change will let call count tracing and cprof to scale + better when increasing the number of + schedulers.</item></list> + <p> + Own Id: OTP-8677</p> + </item> + <item> + <p><c>eprof</c> has been reimplemented with support in + the Erlang virtual machine and is now both faster (i.e. + slows down the code being measured less) and scales much + better. In measurements we saw speed-ups compared to the + old eprof ranging from 6 times (for sequential code that + only uses one scheduler/core) up to 84 times (for + parallel code that uses 8 cores).</p> + <p>Note: The API for the <c>eprof</c> has been cleaned up + and extended. See the documentation.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8706</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 5.7.5.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Driver threads, such as async threads, using <seealso + marker="erl_driver#ErlDrvPDL">port data locks</seealso> + peeked at the port status field without proper locking + when looking up the driver queue.</p> + <p> + Own Id: OTP-8475</p> + </item> + <item> + <p> + A call to the BIF <c>unregister(RegName)</c> when a port + had the name <c>RegName</c> registered in the runtime + system without SMP support caused a runtime system crash. + (Thanks to Per Hedeland for the bugfix and test case.)</p> + <p> + Own Id: OTP-8487</p> + </item> + <item> + <p> + Fix memory management bug causing crash of non-SMP + emulator with async threads enabled. The bug did first + appear in R13B03.</p> + <p> + Own Id: OTP-8591 Aux Id: seq11554 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 5.7.5</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 9f10a0ffaa..f04df354a8 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -74,7 +74,7 @@ else ifeq ($(TYPE),gcov) PURIFY = TYPEMARKER = .gcov -TYPE_FLAGS = @DEBUG_CFLAGS@ -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE= +TYPE_FLAGS = @DEBUG_CFLAGS@ -DERTS_GCOV -DNO_JUMP_TABLE -fprofile-arcs -ftest-coverage -O0 -DERTS_CAN_INLINE=0 -DERTS_INLINE= ifneq ($(findstring solaris,$(TARGET)),solaris) LIBS += -lgcov endif @@ -199,6 +199,14 @@ MKDIR = @MKDIR@ USING_MINGW=@MIXED_CYGWIN_MINGW@ +ifeq ($(TARGET),win32) +LIB_PREFIX= +LIB_SUFFIX=.lib +else +LIB_PREFIX=lib +LIB_SUFFIX=.a +endif + OMIT_OMIT_FP=no ifeq (@EMU_LOCK_CHECKING@,yes) @@ -279,16 +287,11 @@ endif ifeq ($(TARGET),win32) LIBS += -L$(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE) -lepcre -DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/epcre.lib else -LIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a -DEPLIBS += \ - $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/libepcre.a \ - $(ERL_TOP)/erts/lib/internal/$(TARGET)/liberts_internal.a -# rem liberts_internal.a +LIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) endif -ELIB_FLAGS = -DENABLE_ELIB_MALLOC -DELIB_ALLOC_IS_CLIB -DELIB_HEAP_SBRK +DEPLIBS += $(ERL_TOP)/erts/emulator/pcre/obj/$(TARGET)/$(TYPE)/$(LIB_PREFIX)epcre$(LIB_SUFFIX) PERFCTR_PATH=@PERFCTR_PATH@ USE_PERFCTR=@USE_PERFCTR@ @@ -305,7 +308,19 @@ LIBSCTP = @LIBSCTP@ ORG_THR_LIBS=@EMU_THR_LIBS@ THR_LIB_NAME=@EMU_THR_LIB_NAME@ -THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER),$(ORG_THR_LIBS)) +ifneq ($(strip $(THR_LIB_NAME)),) +DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal_r$(TYPEMARKER)$(LIB_SUFFIX) \ + $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)ethread$(TYPEMARKER)$(LIB_SUFFIX) +else +DEPLIBS += $(ERL_TOP)/erts/lib/internal/$(TARGET)/$(LIB_PREFIX)erts_internal$(TYPEMARKER)$(LIB_SUFFIX) +endif + +THR_LIBS=$(subst -l$(THR_LIB_NAME),-l$(THR_LIB_NAME)$(TYPEMARKER), \ + $(subst -lerts_internal_r,-lerts_internal_r$(TYPEMARKER),$(ORG_THR_LIBS))) + +LIBS += $(THR_LIBS) + +ifneq ($(findstring erts_internal_r, $(THR_LIBS)),erts_internal_r) ifeq ($(findstring vxworks,$(TARGET)),vxworks) ERTS_INTERNAL_LIB=erts_internal @@ -317,7 +332,9 @@ ERTS_INTERNAL_LIB=erts_internal endif endif -LIBS += $(THR_LIBS) -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) +LIBS += -l$(ERTS_INTERNAL_LIB)$(TYPEMARKER) + +endif # erts_internal_r LIBS += @LIBRT@ @@ -341,7 +358,6 @@ ERLANG_OSTYPE = @ERLANG_OSTYPE@ ENABLE_ALLOC_TYPE_VARS += @ERLANG_OSTYPE@ -EMULATOR_EXECUTABLE_SAE = beam_evm$(TF_MARKER) EMULATOR_EXECUTABLE_ELIB = beam.elib$(TF_MARKER) ifeq ($(TARGET), win32) EMULATOR_EXECUTABLE = beam$(TF_MARKER).dll @@ -406,7 +422,7 @@ endif @set -e ; cd zlib && $(MAKE) clean @set -e ; cd pcre && $(MAKE) clean -.PHONY: all sae zlib pcre clean +.PHONY: all zlib pcre clean docs: @@ -442,8 +458,6 @@ release_spec: all ifeq ($(ERLANG_OSTYPE), unix) $(INSTALL_PROGRAM) $(BINDIR)/$(CS_EXECUTABLE) $(RELSYSDIR)/bin endif - $(INSTALL_DIR) $(RELEASE_PATH)/usr/include/obsolete - $(INSTALL_DATA) obsolete/driver.h $(RELEASE_PATH)/usr/include/obsolete endif endif @@ -491,8 +505,10 @@ ifdef HIPE_ENABLED OPCODE_TABLES += hipe/hipe_ops.tab endif -$(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/beam_opcodes.c: $(OPCODE_TABLES) - LANG=C $(PERL) utils/beam_makeops -outdir $(TTF_DIR) \ +$(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/beam_opcodes.c: $(OPCODE_TABLES) utils/beam_makeops + LANG=C $(PERL) utils/beam_makeops \ + -wordsize @EXTERNAL_WORD_SIZE@ \ + -outdir $(TTF_DIR) \ -emulator $(OPCODE_TABLES) # bif and atom table @@ -517,8 +533,9 @@ TABLES= $(TARGET)/erl_bif_table.c $(TARGET)/erl_bif_table.h \ $(TARGET)/erl_atom_table.c $(TARGET)/erl_atom_table.h \ $(TARGET)/erl_pbifs.c -$(TABLES): $(ATOMS) $(BIFS) - LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET) $^ +$(TABLES): $(ATOMS) $(BIFS) utils/make_tables + LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\ + $(ATOMS) $(BIFS) $(TTF_DIR)/erl_alloc_types.h: beam/erl_alloc.types utils/make_alloc_types LANG=C $(PERL) utils/make_alloc_types -src $< -dst $@ $(ENABLE_ALLOC_TYPE_VARS) @@ -637,13 +654,6 @@ $(BINDIR)/$(CS_EXECUTABLE): $(CS_SRC) $(CS_PURIFY) $(CC) $(CS_LDFLAGS) -o $(BINDIR)/$(CS_EXECUTABLE) \ $(CS_CFLAGS) $(COMMON_INCLUDES) $(CS_SRC) $(CS_LIBS) -$(OBJDIR)/%.elib.o: beam/%.c - $(CC) $(ELIB_FLAGS) $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ - -# Disable vfork() for sae (then we don't need the child_setup program) -$(OBJDIR)/sys_sae.o: sys/$(ERLANG_OSTYPE)/sys.c - $(CC) -DDISABLE_VFORK=1 $(CFLAGS) $(INCLUDES) -c $< -o $@ - $(OBJDIR)/%.kp.o: sys/common/%.c $(CC) -DERTS_KERNEL_POLL_VERSION $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS)) $(INCLUDES) -c $< -o $@ @@ -652,9 +662,6 @@ $(OBJDIR)/%.nkp.o: sys/common/%.c ifeq ($(GCC),yes) -$(OBJDIR)/erl_obsolete.o: beam/erl_obsolete.c - $(CC) $(subst -Wstrict-prototypes, , $(subst -O2, $(GEN_OPT_FLGS), $(CFLAGS))) $(INCLUDES) -c $< -o $@ - $(OBJDIR)/erl_goodfit_alloc.o: beam/erl_goodfit_alloc.c $(CC) $(subst -O2, $(GEN_OPT_FLGS) $(UNROLL_FLG), $(CFLAGS)) $(INCLUDES) -c $< -o $@ endif @@ -689,8 +696,6 @@ endif INIT_OBJS = $(OBJDIR)/erl_main.o $(PRELOAD) -INIT_OBJS_SAE = $(OBJDIR)/erl9_start.o - EMU_OBJS = \ $(OBJDIR)/beam_emu.o $(OBJDIR)/beam_opcodes.o \ $(OBJDIR)/beam_load.o $(OBJDIR)/beam_bif_load.o \ @@ -731,11 +736,12 @@ RUN_OBJS = \ $(OBJDIR)/erl_fun.o $(OBJDIR)/erl_bif_port.o \ $(OBJDIR)/erl_term.o $(OBJDIR)/erl_node_tables.o \ $(OBJDIR)/erl_monitors.o $(OBJDIR)/erl_process_dump.o \ - $(OBJDIR)/erl_obsolete.o $(OBJDIR)/erl_bif_timer.o \ + $(OBJDIR)/erl_bif_timer.o $(OBJDIR)/erl_cpu_topology.o \ $(OBJDIR)/erl_drv_thread.o $(OBJDIR)/erl_bif_chksum.o \ $(OBJDIR)/erl_bif_re.o $(OBJDIR)/erl_unicode.o \ $(OBJDIR)/packet_parser.o $(OBJDIR)/safe_hash.o \ - $(OBJDIR)/erl_zlib.o $(OBJDIR)/erl_nif.o + $(OBJDIR)/erl_zlib.o $(OBJDIR)/erl_nif.o \ + $(OBJDIR)/erl_bif_binary.o ifeq ($(TARGET),win32) DRV_OBJS = \ @@ -754,15 +760,13 @@ OS_OBJS = \ $(OBJDIR)/sys_time.o \ $(OBJDIR)/sys_interrupt.o \ $(OBJDIR)/sys_env.o \ - $(OBJDIR)/dosmap.o \ - $(OBJDIR)/elib_malloc.o + $(OBJDIR)/dosmap.o else OS_OBJS = \ $(OBJDIR)/sys.o \ $(OBJDIR)/driver_tab.o \ $(OBJDIR)/unix_efile.o \ $(OBJDIR)/gzio.o \ - $(OBJDIR)/elib_malloc.o \ $(OBJDIR)/elib_memmove.o ifeq ($(findstring vxworks,$(TARGET)),vxworks) @@ -794,7 +798,8 @@ endif OS_OBJS += $(OBJDIR)/erl_mseg.o \ $(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \ - $(OBJDIR)/erl_mtrace_sys_wrap.o + $(OBJDIR)/erl_mtrace_sys_wrap.o \ + $(OBJDIR)/erl_sys_common_misc.o HIPE_x86_OS_OBJS=$(HIPE_x86_$(OPSYS)_OBJS) HIPE_x86_OBJS=$(OBJDIR)/hipe_x86.o $(OBJDIR)/hipe_x86_glue.o $(OBJDIR)/hipe_x86_bifs.o $(OBJDIR)/hipe_x86_signal.o $(OBJDIR)/hipe_x86_stack.o $(HIPE_x86_OS_OBJS) @@ -825,17 +830,6 @@ endif BASE_OBJS = $(RUN_OBJS) $(EMU_OBJS) $(OS_OBJS) $(EXTRA_BASE_OBJS) OBJS = $(BASE_OBJS) $(DRV_OBJS) -OBJS_SAE = $(subst sys.o,sys_sae.o,$(OBJS)) - -ELIB_C_FILES = beam/elib_malloc.c \ - beam/elib_memmove.c \ - beam/erl_bif_info.c \ - beam/utils.c \ - beam/erl_alloc.c - -MOD_OBJS_ELIB = $(patsubst %.c,$(OBJDIR)/%.o,$(notdir $(ELIB_C_FILES))) -OBJS_ELIB = $(patsubst %.o,%.elib.o,$(MOD_OBJS_ELIB)) \ - $(filter-out $(MOD_OBJS_ELIB),$(OBJS)) ######################################## # HiPE section @@ -928,14 +922,6 @@ $(BINDIR)/$(EMULATOR_EXECUTABLE): $(INIT_OBJS) $(OBJS) $(DEPLIBS) $(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE) \ $(HIPEBEAMLDFLAGS) $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS) $(LIBS) -$(BINDIR)/$(EMULATOR_EXECUTABLE_ELIB): $(INIT_OBJS) $(OBJS_ELIB) $(DEPLIBS) - $(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE_ELIB) \ - $(LDFLAGS) $(DEXPORT) $(INIT_OBJS) $(OBJS_ELIB) $(LIBS) - -$(BINDIR)/$(EMULATOR_EXECUTABLE_SAE): $(INIT_OBJS_SAE) $(OBJS_SAE) $(DEPLIBS) - $(PURIFY) $(LD) -o $(BINDIR)/$(EMULATOR_EXECUTABLE_SAE) \ - $(LDFLAGS) $(DEXPORT) $(INIT_OBJS_SAE) $(OBJS_SAE) $(LIBS) - endif # @@ -1030,7 +1016,7 @@ depend: $(DEP_CC) $(DEP_FLAGS) $(TARGET_SRC) \ | $(SED_DEPEND) >> $(TARGET)/depend.mk ifneq ($(TARGET),win32) - $(DEP_CC) $(DEP_FLAGS) $(ELIB_FLAGS) $(ELIB_C_FILES) \ + $(DEP_CC) $(DEP_FLAGS) $(ELIB_C_FILES) \ | $(SED_ELIB_DEPEND) >> $(TARGET)/depend.mk endif ifdef HIPE_ENABLED diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index e2a79d6e4f..b97705ed96 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -41,8 +41,7 @@ static erts_smp_rwmtx_t atom_table_lock; #define atom_read_unlock() erts_smp_rwmtx_runlock(&atom_table_lock) #define atom_write_lock() erts_smp_rwmtx_rwlock(&atom_table_lock) #define atom_write_unlock() erts_smp_rwmtx_rwunlock(&atom_table_lock) -#define atom_init_lock() erts_smp_rwmtx_init(&atom_table_lock, \ - "atom_tab") + #if 0 #define ERTS_ATOM_PUT_OPS_STAT #endif @@ -304,12 +303,17 @@ init_atom_table(void) HashFunctions f; int i; Atom a; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; #ifdef ERTS_ATOM_PUT_OPS_STAT erts_smp_atomic_init(&atom_put_ops, 0); #endif - atom_init_lock(); + erts_smp_rwmtx_init_opt(&atom_table_lock, &rwmtx_opt, "atom_tab"); + f.hash = (H_FUN) atom_hash; f.cmp = (HCMP_FUN) atom_cmp; f.alloc = (HALLOC_FUN) atom_alloc; diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 9ce21089ba..327620772f 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -65,6 +65,7 @@ atom EXIT='EXIT' atom aborted atom abs_path atom absoluteURI +atom ac atom active atom all atom all_but_first @@ -100,8 +101,15 @@ atom band atom big atom bif_return_trap atom binary +atom binary_bin_to_list_trap +atom binary_copy_trap +atom binary_longest_prefix_trap +atom binary_longest_suffix_trap +atom binary_match_trap +atom binary_matches_trap atom block atom blocked +atom bm atom bnot atom bor atom bxor @@ -111,10 +119,12 @@ atom bsl atom bsr atom bsr_anycrlf atom bsr_unicode +atom build_type atom busy_dist_port atom busy_port atom call atom call_count +atom call_time atom caller atom capture atom case_clause @@ -369,6 +379,7 @@ atom old_heap_size atom on_load atom open atom open_error +atom opt atom or atom ordered_set atom orelse @@ -423,6 +434,7 @@ atom raw atom re atom re_pattern atom re_run_trap +atom read_concurrency atom ready_input atom ready_output atom ready_async @@ -454,6 +466,7 @@ atom scheduler atom scheduler_id atom schedulers_online atom scheme +atom scope atom sensitive atom sequential_tracer atom sequential_trace_token diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 8462f1c7fd..6ae9736141 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -337,7 +337,6 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) ep->code[0] == BIF_ARG_1 && ep->code[4] != 0) { ep->address = (void *) ep->code[4]; - ep->code[3] = 0; ep->code[4] = 0; } } @@ -368,7 +367,6 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) BIF_RET(am_true); } - static void set_default_trace_pattern(Eterm module) { @@ -403,7 +401,7 @@ check_process_code(Process* rp, Module* modp) BeamInstr* end; Eterm* sp; #ifndef HYBRID /* FIND ME! */ - ErlFunThing* funp; + struct erl_off_heap_header* oh; int done_gc = 0; #endif @@ -471,27 +469,30 @@ check_process_code(Process* rp, Module* modp) #ifndef HYBRID /* FIND ME! */ rescan: - for (funp = MSO(rp).funs; funp; funp = funp->next) { - BeamInstr* fun_code; - - fun_code = funp->fe->address; - - if (INSIDE((BeamInstr *) funp->fe->address)) { - if (done_gc) { - return am_true; - } else { - /* - * Try to get rid of this fun by garbage collecting. - * Clear both fvalue and ftrace to make sure they - * don't hold any funs. - */ - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - done_gc = 1; - FLAGS(rp) |= F_NEED_FULLSWEEP; - (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); - goto rescan; + for (oh = MSO(rp).first; oh; oh = oh->next) { + if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { + ErlFunThing* funp = (ErlFunThing*) oh; + BeamInstr* fun_code; + + fun_code = funp->fe->address; + + if (INSIDE((BeamInstr *) funp->fe->address)) { + if (done_gc) { + return am_true; + } else { + /* + * Try to get rid of this fun by garbage collecting. + * Clear both fvalue and ftrace to make sure they + * don't hold any funs. + */ + rp->freason = EXC_NULL; + rp->fvalue = NIL; + rp->ftrace = NIL; + done_gc = 1; + FLAGS(rp) |= F_NEED_FULLSWEEP; + (void) erts_garbage_collect(rp, 0, rp->arg_reg, rp->arity); + goto rescan; + } } } } diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 6278ff6bad..31910888d1 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -30,6 +30,7 @@ #include "error.h" #include "erl_binary.h" #include "beam_bp.h" +#include "erl_term.h" /* ************************************************************************* ** Macros @@ -100,6 +101,11 @@ do { \ (b)->prev = (a); \ } while (0) + +#define BREAK_IS_BIF (1) +#define BREAK_IS_ERL (0) + + /* ************************************************************************* ** Local prototypes */ @@ -114,7 +120,7 @@ static int set_break(Eterm mfa[3], int specified, static int set_module_break(Module *modp, Eterm mfa[3], int specified, Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid); -static int set_function_break(Module *modp, BeamInstr *pc, +static int set_function_break(Module *modp, BeamInstr *pc, int bif, Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid); @@ -122,11 +128,29 @@ static int clear_break(Eterm mfa[3], int specified, BeamInstr break_op); static int clear_module_break(Module *modp, Eterm mfa[3], int specified, BeamInstr break_op); -static int clear_function_break(Module *modp, BeamInstr *pc, +static int clear_function_break(Module *modp, BeamInstr *pc, int bif, BeamInstr break_op); static BpData *is_break(BeamInstr *pc, BeamInstr break_op); - +static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op); + +/* bp_hash */ +#define BP_TIME_ADD(pi0, pi1) \ + do { \ + Uint r; \ + (pi0)->count += (pi1)->count; \ + (pi0)->s_time += (pi1)->s_time; \ + (pi0)->us_time += (pi1)->us_time; \ + r = (pi0)->us_time / 1000000; \ + (pi0)->s_time += r; \ + (pi0)->us_time = (pi0)->us_time % 1000000; \ + } while(0) + +static void bp_hash_init(bp_time_hash_t *hash, Uint n); +static void bp_hash_rehash(bp_time_hash_t *hash, Uint n); +static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem); +static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t *sitem); +static void bp_hash_delete(bp_time_hash_t *hash); /* ************************************************************************* @@ -156,25 +180,20 @@ erts_set_mtrace_break(Eterm mfa[3], int specified, Binary *match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); } +/* set breakpoint data for on exported bif entry */ + void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, Eterm tracer_pid) { - BpDataTrace *bdt; ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + set_function_break(NULL, pc, BREAK_IS_BIF, match_spec, (BeamInstr) BeamOp(op_i_mtrace_breakpoint), 0, tracer_pid); +} - bdt = (BpDataTrace *) pc[-4]; - if (bdt) { - MatchSetUnref(bdt->match_spec); - MatchSetRef(match_spec); - bdt->match_spec = match_spec; - bdt->tracer_pid = tracer_pid; - } else { - bdt = Alloc(sizeof(BpDataTrace)); - BpInit((BpData *) bdt, 0); - MatchSetRef(match_spec); - bdt->match_spec = match_spec; - bdt->tracer_pid = tracer_pid; - pc[-4] = (BeamInstr) bdt; - } +void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) { + set_function_break(NULL, pc, BREAK_IS_BIF, NULL, (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); +} + +void erts_clear_time_trace_bif(BeamInstr *pc) { + clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_time_breakpoint)); } int @@ -191,7 +210,12 @@ erts_set_count_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { (BeamInstr) BeamOp(op_i_count_breakpoint), count_op, NIL); } - +int +erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op count_op) { + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + return set_break(mfa, specified, NULL, + (BeamInstr) BeamOp(op_i_time_breakpoint), count_op, NIL); +} int erts_clear_trace_break(Eterm mfa[3], int specified) { @@ -209,17 +233,7 @@ erts_clear_mtrace_break(Eterm mfa[3], int specified) { void erts_clear_mtrace_bif(BeamInstr *pc) { - BpDataTrace *bdt; - ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); - - bdt = (BpDataTrace *) pc[-4]; - if (bdt) { - if (bdt->match_spec) { - MatchSetUnref(bdt->match_spec); - } - Free(bdt); - } - pc[-4] = (BeamInstr) NULL; + clear_function_break(NULL, pc, BREAK_IS_BIF, (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); } int @@ -237,6 +251,13 @@ erts_clear_count_break(Eterm mfa[3], int specified) { } int +erts_clear_time_break(Eterm mfa[3], int specified) { + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + return clear_break(mfa, specified, + (BeamInstr) BeamOp(op_i_time_breakpoint)); +} + +int erts_clear_break(Eterm mfa[3], int specified) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); return clear_break(mfa, specified, 0); @@ -253,7 +274,7 @@ int erts_clear_function_break(Module *modp, BeamInstr *pc) { ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); ASSERT(modp); - return clear_function_break(modp, pc, 0); + return clear_function_break(modp, pc, BREAK_IS_ERL, 0); } @@ -265,9 +286,12 @@ BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, Uint32 *ret_flags, Eterm *tracer_pid) { Eterm tpid1, tpid2; - BpDataTrace *bdt = (BpDataTrace *) pc[-4]; + BpData **bds = (BpData **) (pc)[-4]; + BpDataTrace *bdt = NULL; + ASSERT(bds); ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + bdt = (BpDataTrace *) bds[bp_sched2ix_proc(p)]; ASSERT(bdt); bdt = (BpDataTrace *) bdt->next; ASSERT(bdt); @@ -286,7 +310,7 @@ erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, bdt->tracer_pid = tpid2; ErtsSmpBPUnlock(bdt); } - pc[-4] = (BeamInstr) bdt; + bds[bp_sched2ix_proc(p)] = (BpData *) bdt; return bdt->orig_instr; } @@ -298,12 +322,15 @@ erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, Uint32 erts_bif_mtrace(Process *p, BeamInstr *pc, Eterm *args, int local, Eterm *tracer_pid) { - BpDataTrace *bdt = (BpDataTrace *) pc[-4]; + BpData **bds = (BpData **) (pc)[-4]; + BpDataTrace *bdt = NULL; + ASSERT(tracer_pid); - if (bdt) { + if (bds) { Eterm tpid1, tpid2; Uint32 flags; + bdt = (BpDataTrace *)bds[bp_sched2ix_proc(p)]; ErtsSmpBPLock(bdt); tpid1 = tpid2 = bdt->tracer_pid; @@ -364,24 +391,6 @@ erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_r } int -erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret) { - BpDataTrace *bdt = (BpDataTrace *) pc[-4]; - - if (bdt) { - if (match_spec_ret) { - *match_spec_ret = bdt->match_spec; - } - if (tracer_pid_ret) { - ErtsSmpBPLock(bdt); - *tracer_pid_ret = bdt->tracer_pid; - ErtsSmpBPUnlock(bdt); - } - return !0; - } - return 0; -} - -int erts_is_native_break(BeamInstr *pc) { #ifdef HIPE ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); @@ -399,15 +408,69 @@ erts_is_count_break(BeamInstr *pc, Sint *count_ret) { if (bdc) { if (count_ret) { - ErtsSmpBPLock(bdc); - *count_ret = bdc->count; - ErtsSmpBPUnlock(bdc); + *count_ret = (Sint) erts_smp_atomic_read(&bdc->acount); } return !0; } return 0; } +int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { + Uint i, ix; + bp_time_hash_t hash; + Uint size; + Eterm *hp, t; + bp_data_time_item_t *item = NULL; + BpDataTime *bdt = (BpDataTime *) is_break(pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + + if (bdt) { + if (retval) { + /* collect all hashes to one hash */ + bp_hash_init(&hash, 64); + /* foreach threadspecific hash */ + for (i = 0; i < bdt->n; i++) { + bp_data_time_item_t *sitem; + + /* foreach hash bucket not NIL*/ + for(ix = 0; ix < bdt->hash[i].n; ix++) { + item = &(bdt->hash[i].item[ix]); + if (item->pid != NIL) { + sitem = bp_hash_get(&hash, item); + if (sitem) { + BP_TIME_ADD(sitem, item); + } else { + bp_hash_put(&hash, item); + } + } + } + } + /* *retval should be NIL or term from previous bif in export entry */ + + if (hash.used > 0) { + size = (5 + 2)*hash.used; + hp = HAlloc(p, size); + + for(ix = 0; ix < hash.n; ix++) { + item = &(hash.item[ix]); + if (item->pid != NIL) { + t = TUPLE4(hp, item->pid, + make_small(item->count), + make_small(item->s_time), + make_small(item->us_time)); + hp += 5; + *retval = CONS(hp, t, *retval); hp += 2; + } + } + } + bp_hash_delete(&hash); + } + return !0; + } + + return 0; +} + + BeamInstr * erts_find_local_func(Eterm mfa[3]) { Module *modp; @@ -432,6 +495,351 @@ erts_find_local_func(Eterm mfa[3]) { return NULL; } +/* bp_hash */ +ERTS_INLINE Uint bp_sched2ix() { +#ifdef ERTS_SMP + ErtsSchedulerData *esdp; + esdp = erts_get_scheduler_data(); + return esdp->no - 1; +#else + return 0; +#endif +} +static void bp_hash_init(bp_time_hash_t *hash, Uint n) { + Uint size = sizeof(bp_data_time_item_t)*n; + Uint i; + + hash->n = n; + hash->used = 0; + + hash->item = (bp_data_time_item_t *)Alloc(size); + sys_memzero(hash->item, size); + + for(i = 0; i < n; ++i) { + hash->item[i].pid = NIL; + } +} + +static void bp_hash_rehash(bp_time_hash_t *hash, Uint n) { + bp_data_time_item_t *item = NULL; + Uint size = sizeof(bp_data_time_item_t)*n; + Uint ix; + Uint hval; + + item = (bp_data_time_item_t *)Alloc(size); + sys_memzero(item, size); + + for( ix = 0; ix < n; ++ix) { + item[ix].pid = NIL; + } + + /* rehash, old hash -> new hash */ + + for( ix = 0; ix < hash->n; ix++) { + if (hash->item[ix].pid != NIL) { + + hval = ((hash->item[ix].pid) >> 4) % n; /* new n */ + + while (item[hval].pid != NIL) { + hval = (hval + 1) % n; + } + item[hval].pid = hash->item[ix].pid; + item[hval].count = hash->item[ix].count; + item[hval].s_time = hash->item[ix].s_time; + item[hval].us_time = hash->item[ix].us_time; + } + } + + Free(hash->item); + hash->n = n; + hash->item = item; +} +static ERTS_INLINE bp_data_time_item_t * bp_hash_get(bp_time_hash_t *hash, bp_data_time_item_t *sitem) { + Eterm pid = sitem->pid; + Uint hval = (pid >> 4) % hash->n; + bp_data_time_item_t *item = NULL; + + item = hash->item; + + while (item[hval].pid != pid) { + if (item[hval].pid == NIL) return NULL; + hval = (hval + 1) % hash->n; + } + + return &(item[hval]); +} + +static ERTS_INLINE bp_data_time_item_t * bp_hash_put(bp_time_hash_t *hash, bp_data_time_item_t* sitem) { + Uint hval; + float r = 0.0; + bp_data_time_item_t *item; + + /* make sure that the hash is not saturated */ + /* if saturated, rehash it */ + + r = hash->used / (float) hash->n; + + if (r > 0.7f) { + bp_hash_rehash(hash, hash->n * 2); + } + /* Do hval after rehash */ + hval = (sitem->pid >> 4) % hash->n; + + /* find free slot */ + item = hash->item; + + while (item[hval].pid != NIL) { + hval = (hval + 1) % hash->n; + } + item = &(hash->item[hval]); + + item->pid = sitem->pid; + item->s_time = sitem->s_time; + item->us_time = sitem->us_time; + item->count = sitem->count; + hash->used++; + + return item; +} + +static void bp_hash_delete(bp_time_hash_t *hash) { + hash->n = 0; + hash->used = 0; + Free(hash->item); + hash->item = NULL; +} + +static void bp_time_diff(bp_data_time_item_t *item, /* out */ + process_breakpoint_time_t *pbt, /* in */ + Uint ms, Uint s, Uint us) { + int dms,ds,dus; + + dms = ms - pbt->ms; + ds = s - pbt->s; + dus = us - pbt->us; + + /* get_sys_now may return zero difftime, + * this is ok. + */ + + ASSERT(dms >= 0 || ds >= 0 || dus >= 0); + + if (dus < 0) { + dus += 1000000; + ds -= 1; + } + if (ds < 0) { + ds += 1000000; + } + + item->s_time = ds; + item->us_time = dus; +} + +void erts_schedule_time_break(Process *p, Uint schedule) { + Uint ms, s, us; + process_breakpoint_time_t *pbt = NULL; + bp_data_time_item_t sitem, *item = NULL; + bp_time_hash_t *h = NULL; + BpDataTime *pbdt = NULL; + + ASSERT(p); + + pbt = ERTS_PROC_GET_CALL_TIME(p); + + if (pbt) { + + switch(schedule) { + case ERTS_BP_CALL_TIME_SCHEDULE_EXITING : + break; + case ERTS_BP_CALL_TIME_SCHEDULE_OUT : + /* When a process is scheduled _out_, + * timestamp it and add its delta to + * the previous breakpoint. + */ + + pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + if (pbdt) { + get_sys_now(&ms,&s,&us); + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->id; + sitem.count = 0; + + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + break; + case ERTS_BP_CALL_TIME_SCHEDULE_IN : + /* When a process is scheduled _in_, + * timestamp it and remove the previous + * timestamp in the psd. + */ + get_sys_now(&ms,&s,&us); + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + break; + default : + ASSERT(0); + /* will never happen */ + break; + } + } /* pbt */ +} + +/* call_time breakpoint + * Accumulated times are added to the previous bp, + * not the current one. The current one is saved + * for future reference. + * The previous breakpoint is stored in the process it self, the psd. + * We do not need to store in a stack frame. + * There is no need for locking, each thread has its own + * area in each bp to save data. + * Since we need to diffrentiate between processes for each bp, + * every bp has a hash (per thread) to process-bp statistics. + * - egil + */ + +void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type) { + Uint ms,s,us; + process_breakpoint_time_t *pbt = NULL; + bp_data_time_item_t sitem, *item = NULL; + bp_time_hash_t *h = NULL; + BpDataTime *pbdt = NULL; + + ASSERT(p); + ASSERT(p->status == P_RUNNING); + + /* get previous timestamp and breakpoint + * from the process psd */ + + pbt = ERTS_PROC_GET_CALL_TIME(p); + get_sys_now(&ms,&s,&us); + + switch(type) { + /* get pbt + * timestamp = t0 + * lookup bdt from code + * set ts0 to pbt + * add call count here? + */ + case ERTS_BP_CALL_TIME_CALL: + case ERTS_BP_CALL_TIME_TAIL_CALL: + + if (pbt) { + ASSERT(pbt->pc); + /* add time to previous code */ + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->id; + sitem.count = 0; + + /* previous breakpoint */ + pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + + /* if null then the breakpoint was removed */ + if (pbdt) { + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + + } else { + /* first call of process to instrumented function */ + pbt = Alloc(sizeof(process_breakpoint_time_t)); + (void *) ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCK_MAIN, pbt); + } + /* add count to this code */ + sitem.pid = p->id; + sitem.count = 1; + sitem.s_time = 0; + sitem.us_time = 0; + + /* this breakpoint */ + ASSERT(bdt); + h = &(bdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + + pbt->pc = pc; + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + break; + + case ERTS_BP_CALL_TIME_RETURN: + /* get pbt + * lookup bdt from code + * timestamp = t1 + * get ts0 from pbt + * get item from bdt->hash[bp_hash(p->id)] + * ack diff (t1, t0) to item + */ + + if(pbt) { + /* might have been removed due to + * trace_pattern(false) + */ + ASSERT(pbt->pc); + + bp_time_diff(&sitem, pbt, ms, s, us); + sitem.pid = p->id; + sitem.count = 0; + + /* previous breakpoint */ + pbdt = (BpDataTime *) get_break(p, pbt->pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); + + /* beware, the trace_pattern might have been removed */ + if (pbdt) { + h = &(pbdt->hash[bp_sched2ix_proc(p)]); + + ASSERT(h); + ASSERT(h->item); + + item = bp_hash_get(h, &sitem); + if (!item) { + item = bp_hash_put(h, &sitem); + } else { + BP_TIME_ADD(item, &sitem); + } + } + + pbt->pc = pc; + pbt->ms = ms; + pbt->s = s; + pbt->us = us; + } + break; + default : + ASSERT(0); + /* will never happen */ + break; + } +} /* ************************************************************************* @@ -489,26 +897,35 @@ static int set_module_break(Module *modp, Eterm mfa[3], int specified, ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if ((specified < 2 || mfa[1] == ((Eterm) code_ptr[3])) && (specified < 3 || ((int) mfa[2]) == ((int) code_ptr[4]))) { - BeamInstr *pc = code_ptr+5; + BeamInstr *pc = code_ptr+5; num_processed += - set_function_break(modp, pc, match_spec, + set_function_break(modp, pc, BREAK_IS_ERL, match_spec, break_op, count_op, tracer_pid); } } return num_processed; } -static int set_function_break(Module *modp, BeamInstr *pc, +static int set_function_break(Module *modp, BeamInstr *pc, int bif, Binary *match_spec, BeamInstr break_op, enum erts_break_op count_op, Eterm tracer_pid) { - BpData *bd, **r; + + BeamInstr **code_base = NULL; + BpData *bd, **r, ***rs; size_t size; - BeamInstr **code_base = (BeamInstr **)modp->code; + Uint ix = 0; - ASSERT(code_base); - ASSERT(code_base <= (BeamInstr **)pc); - ASSERT((BeamInstr **)pc < code_base + (modp->code_length/sizeof(BeamInstr *))); + if (bif == BREAK_IS_ERL) { + code_base = (BeamInstr **)modp->code; + ASSERT(code_base); + ASSERT(code_base <= (BeamInstr **)pc); + ASSERT((BeamInstr **)pc < code_base + (modp->code_length/sizeof(BeamInstr *))); + } else { + ASSERT(*pc == (BeamInstr) em_apply_bif); + ASSERT(modp == NULL); + } + /* * Currently no trace support for native code. */ @@ -519,6 +936,7 @@ static int set_function_break(Module *modp, BeamInstr *pc, if ( (bd = is_break(pc, break_op))) { if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { + BpDataTrace *bdt = (BpDataTrace *) bd; Binary *old_match_spec; @@ -531,20 +949,44 @@ static int set_function_break(Module *modp, BeamInstr *pc, ErtsSmpBPUnlock(bdt); MatchSetUnref(old_match_spec); } else { + BpDataCount *bdc = (BpDataCount *) bd; + erts_aint_t count = 0; + erts_aint_t res = 0; + ASSERT(! match_spec); ASSERT(is_nil(tracer_pid)); - if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { - BpDataCount *bdc = (BpDataCount *) bd; - ErtsSmpBPLock(bdc); + if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { if (count_op == erts_break_stop) { - if (bdc->count >= 0) { - bdc->count = -bdc->count-1; /* Stop call counter */ + count = erts_smp_atomic_read(&bdc->acount); + if (count >= 0) { + while(1) { + res = erts_smp_atomic_cmpxchg(&bdc->acount, -count - 1, count); + if ((res == count) || count < 0) break; + count = res; + } } } else { - bdc->count = 0; /* Reset call counter */ + /* Reset call counter */ + erts_smp_atomic_set(&bdc->acount, 0); } - ErtsSmpBPUnlock(bdc); + + } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + BpDataTime *bdt = (BpDataTime *) bd; + Uint i = 0; + + ERTS_SMP_LC_ASSERT(erts_smp_is_system_blocked(0)); + + if (count_op == erts_break_stop) { + bdt->pause = 1; + } else { + bdt->pause = 0; + for (i = 0; i < bdt->n; i++) { + bp_hash_delete(&(bdt->hash[i])); + bp_hash_init(&(bdt->hash[i]), 32); + } + } + } else { ASSERT (! count_op); } @@ -558,43 +1000,63 @@ static int set_function_break(Module *modp, BeamInstr *pc, ASSERT(! match_spec); ASSERT(is_nil(tracer_pid)); if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { - if (count_op == erts_break_reset - || count_op == erts_break_stop) { + if (count_op == erts_break_reset || count_op == erts_break_stop) { /* Do not insert a new breakpoint */ return 1; } size = sizeof(BpDataCount); + } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + if (count_op == erts_break_reset || count_op == erts_break_stop) { + /* Do not insert a new breakpoint */ + return 1; + } + size = sizeof(BpDataTime); } else { ASSERT(! count_op); ASSERT(break_op == (BeamInstr) BeamOp(op_i_debug_breakpoint)); size = sizeof(BpDataDebug); } } - r = (BpData **) (pc-4); + rs = (BpData ***) (pc-4); + if (! *rs) { + size_t ssize = sizeof(BeamInstr) * erts_no_schedulers; + *rs = (BpData **) Alloc(ssize); + sys_memzero(*rs, ssize); + } + + r = &((*rs)[0]); + if (! *r) { ASSERT(*pc != (BeamInstr) BeamOp(op_i_trace_breakpoint)); ASSERT(*pc != (BeamInstr) BeamOp(op_i_mtrace_breakpoint)); ASSERT(*pc != (BeamInstr) BeamOp(op_i_debug_breakpoint)); ASSERT(*pc != (BeamInstr) BeamOp(op_i_count_breakpoint)); + ASSERT(*pc != (BeamInstr) BeamOp(op_i_time_breakpoint)); /* First breakpoint; create singleton ring */ bd = Alloc(size); BpInit(bd, *pc); - *pc = break_op; *r = bd; + if (bif == BREAK_IS_ERL) { + *pc = break_op; + } } else { ASSERT(*pc == (BeamInstr) BeamOp(op_i_trace_breakpoint) || *pc == (BeamInstr) BeamOp(op_i_mtrace_breakpoint) || *pc == (BeamInstr) BeamOp(op_i_debug_breakpoint) || - *pc == (BeamInstr) BeamOp(op_i_count_breakpoint)); + *pc == (BeamInstr) BeamOp(op_i_time_breakpoint) || + *pc == (BeamInstr) BeamOp(op_i_count_breakpoint) || + *pc == (BeamInstr) em_apply_bif); if (*pc == (BeamInstr) BeamOp(op_i_debug_breakpoint)) { /* Debug bp must be last, so if it is also first; * it must be singleton. */ - ASSERT(BpSingleton(*r)); + ASSERT(BpSingleton(*r)); /* Insert new bp first in the ring, i.e second to last. */ bd = Alloc(size); BpInitAndSpliceNext(bd, *pc, *r); - *pc = break_op; - } else if ((*r)->prev->orig_instr + if (bif == BREAK_IS_ERL) { + *pc = break_op; + } + } else if ((*r)->prev->orig_instr == (BeamInstr) BeamOp(op_i_debug_breakpoint)) { /* Debug bp last in the ring; insert new second to last. */ bd = Alloc(size); @@ -608,6 +1070,11 @@ static int set_function_break(Module *modp, BeamInstr *pc, *r = bd; } } + for (ix = 1; ix < erts_no_schedulers; ++ix) { + (*rs)[ix] = (*rs)[0]; + } + + bd->this_instr = break_op; /* Init the bp type specific data */ if (break_op == (BeamInstr) BeamOp(op_i_trace_breakpoint) || break_op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { @@ -617,12 +1084,25 @@ static int set_function_break(Module *modp, BeamInstr *pc, MatchSetRef(match_spec); bdt->match_spec = match_spec; bdt->tracer_pid = tracer_pid; + } else if (break_op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + BpDataTime *bdt = (BpDataTime *) bd; + Uint i = 0; + + bdt->pause = 0; + bdt->n = erts_no_schedulers; + bdt->hash = Alloc(sizeof(bp_time_hash_t)*(bdt->n)); + + for (i = 0; i < bdt->n; i++) { + bp_hash_init(&(bdt->hash[i]), 32); + } } else if (break_op == (BeamInstr) BeamOp(op_i_count_breakpoint)) { BpDataCount *bdc = (BpDataCount *) bd; + erts_smp_atomic_init(&bdc->acount, 0); + } - bdc->count = 0; + if (bif == BREAK_IS_ERL) { + ++(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); } - ++(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); return 1; } @@ -672,33 +1152,51 @@ static int clear_module_break(Module *m, Eterm mfa[3], int specified, BeamInstr *pc = code_ptr + 5; num_processed += - clear_function_break(m, pc, break_op); + clear_function_break(m, pc, BREAK_IS_ERL, break_op); } } return num_processed; } -static int clear_function_break(Module *m, BeamInstr *pc, BeamInstr break_op) { +static int clear_function_break(Module *m, BeamInstr *pc, int bif, BeamInstr break_op) { BpData *bd; - BeamInstr **code_base = (BeamInstr **)m->code; - - ASSERT(code_base); - ASSERT(code_base <= (BeamInstr **)pc); - ASSERT((BeamInstr **)pc < code_base + (m->code_length/sizeof(BeamInstr *))); + Uint ix = 0; + BeamInstr **code_base = NULL; + + if (bif == BREAK_IS_ERL) { + code_base = (BeamInstr **)m->code; + ASSERT(code_base); + ASSERT(code_base <= (BeamInstr **)pc); + ASSERT((BeamInstr **)pc < code_base + (m->code_length/sizeof(BeamInstr *))); + } else { + ASSERT(*pc == (BeamInstr) em_apply_bif); + ASSERT(m == NULL); + } + /* * Currently no trace support for native code. */ if (erts_is_native_break(pc)) { return 0; } + while ( (bd = is_break(pc, break_op))) { /* Remove all breakpoints of this type. * There should be only one of each type, * but break_op may be 0 which matches any type. */ BeamInstr op; - BpData **r = (BpData **) (pc-4); + BpData ***rs = (BpData ***) (pc - 4); + BpData **r = NULL; + +#ifdef DEBUG + for (ix = 1; ix < erts_no_schedulers; ++ix) { + ASSERT((*rs)[ix] == (*rs)[0]); + } +#endif + r = &((*rs)[0]); + ASSERT(*r); /* Find opcode for this breakpoint */ if (break_op) { @@ -714,8 +1212,11 @@ static int clear_function_break(Module *m, BeamInstr *pc, BeamInstr break_op) { if (BpSingleton(bd)) { ASSERT(*r == bd); /* Only one breakpoint to remove */ - *r = NULL; - *pc = bd->orig_instr; + if (bif == BREAK_IS_ERL) { + *pc = bd->orig_instr; + } + Free(*rs); + *rs = NULL; } else { BpData *bd_prev = bd->prev; @@ -727,7 +1228,9 @@ static int clear_function_break(Module *m, BeamInstr *pc, BeamInstr break_op) { bd_prev->orig_instr = bd->orig_instr; } else if (bd_prev == *r) { /* We removed the first breakpoint in the ring */ - *pc = bd->orig_instr; + if (bif == BREAK_IS_ERL) { + *pc = bd->orig_instr; + } } else { bd_prev->orig_instr = bd->orig_instr; } @@ -736,13 +1239,53 @@ static int clear_function_break(Module *m, BeamInstr *pc, BeamInstr break_op) { op == (BeamInstr) BeamOp(op_i_mtrace_breakpoint)) { BpDataTrace *bdt = (BpDataTrace *) bd; - MatchSetUnref(bdt->match_spec); } + if (op == (BeamInstr) BeamOp(op_i_time_breakpoint)) { + BpDataTime *bdt = (BpDataTime *) bd; + Uint i = 0; + Uint j = 0; + Process *h_p = NULL; + bp_data_time_item_t *item = NULL; + process_breakpoint_time_t *pbt = NULL; + + /* remove all psd associated with the hash + * and then delete the hash. + * ... sigh ... + */ + + for( i = 0; i < bdt->n; ++i) { + if (bdt->hash[i].used) { + for (j = 0; j < bdt->hash[i].n; ++j) { + item = &(bdt->hash[i].item[j]); + if (item->pid != NIL) { + h_p = process_tab[internal_pid_index(item->pid)]; + if (h_p) { + pbt = ERTS_PROC_SET_CALL_TIME(h_p, ERTS_PROC_LOCK_MAIN, NULL); + if (pbt) { + Free(pbt); + } + } + } + } + } + bp_hash_delete(&(bdt->hash[i])); + } + Free(bdt->hash); + bdt->hash = NULL; + bdt->n = 0; + } Free(bd); - ASSERT(((BeamInstr) code_base[MI_NUM_BREAKPOINTS]) > 0); - --(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); - } + if (bif == BREAK_IS_ERL) { + ASSERT(((BeamInstr) code_base[MI_NUM_BREAKPOINTS]) > 0); + --(*(BeamInstr*)&code_base[MI_NUM_BREAKPOINTS]); + } + if (*rs) { + for (ix = 1; ix < erts_no_schedulers; ++ix) { + (*rs)[ix] = (*rs)[0]; + } + } + } /* while bd != NULL */ return 1; } @@ -755,32 +1298,63 @@ static int clear_function_break(Module *m, BeamInstr *pc, BeamInstr break_op) { ** returned. The program counter must point to the first executable ** (breakpoint) instruction of the function. */ -static BpData *is_break(BeamInstr *pc, BeamInstr break_op) { + +BpData *erts_get_time_break(Process *p, BeamInstr *pc) { + return get_break(p, pc, (BeamInstr) BeamOp(op_i_time_breakpoint)); +} + +static BpData *get_break(Process *p, BeamInstr *pc, BeamInstr break_op) { ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); if (! erts_is_native_break(pc)) { - BpData *bd = (BpData *) pc[-4]; - - if (break_op == 0) { - return bd; - } - if (*pc == break_op) { - ASSERT(bd); - return bd->next; - } - if (! bd){ + BpData **rs = (BpData **) pc[-4]; + BpData *bd = NULL, *ebd = NULL; + + if (! rs) { return NULL; } + + bd = ebd = rs[bp_sched2ix_proc(p)]; + ASSERT(bd); + if (bd->this_instr == break_op) { + return bd; + } + bd = bd->next; - while (bd != (BpData *) pc[-4]) { + while (bd != ebd) { ASSERT(bd); - if (bd->orig_instr == break_op) { - bd = bd->next; + if (bd->this_instr == break_op) { ASSERT(bd); return bd; - } else { - bd = bd->next; } + bd = bd->next; } } return NULL; } + +static BpData *is_break(BeamInstr *pc, BeamInstr break_op) { + BpData **rs = (BpData **) pc[-4]; + BpData *bd = NULL, *ebd = NULL; + ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + + if (! rs) { + return NULL; + } + + bd = ebd = rs[bp_sched2ix()]; + ASSERT(bd); + if ( (break_op == 0) || (bd->this_instr == break_op)) { + return bd; + } + + bd = bd->next; + while (bd != ebd) { + ASSERT(bd); + if (bd->this_instr == break_op) { + ASSERT(bd); + return bd; + } + bd = bd->next; + } + return NULL; +} diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 786cbbe9d9..bd8a7249a7 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -27,28 +27,46 @@ -/* -** Common struct to all bp_data_* -** -** Two gotchas: -** -** 1) The type of bp_data structure in the ring is deduced from the -** orig_instr field of the structure _before_ in the ring, except for -** the first structure in the ring that has its instruction in -** pc[0] of the code to execute. -** -** 2) pc[-4] points to the _last_ structure in the ring before the -** breakpoints are being executed. -** -** So, as an example, when a breakpointed function starts to execute, -** the first instruction that is a breakpoint instruction at pc[0] finds -** its data at ((BpData *) pc[-4])->next and has to cast that pointer -** to the correct bp_data type. +/* A couple of gotchas: + * + * The breakpoint structure from BeamInstr, + * In beam_emu where the instruction counter pointer, I (or pc), + * points to the *current* instruction. At that time, if the instruction + * is a breakpoint instruction the pc looks like the following, + * + * I[-5] | op_i_func_info_IaaI | scheduler specific entries + * I[-4] | BpData** bpa | --> | BpData * bdas1 | ... | BpData * bdasN | + * I[-3] | Tagged Module | | | + * I[-2] | Tagged Function | V V + * I[-1] | Arity | BpData -> BpData -> BpData -> BpData + * I[0] | The bp instruction | ^ * the bp wheel * | + * |------------------------------ + * + * Common struct to all bp_data_* + * + * 1) The type of bp_data structure in the ring is deduced from the + * orig_instr field of the structure _before_ in the ring, except for + * the first structure in the ring that has its instruction in + * pc[0] of the code to execute. + * This is valid as long as you don't search for the function while it is + * being executed by something else. Or is in the middle of its rotation for + * any other reason. + * A key, the bp beam instruction, is included for this reason. + * + * 2) pc[-4][sched_id - 1] points to the _last_ structure in the ring before the + * breakpoints are being executed. + * + * So, as an example, when a breakpointed function starts to execute, + * the first instruction that is a breakpoint instruction at pc[0] finds + * its data at ((BpData **) pc[-4][sched_id - 1])->next and has to cast that pointer + * to the correct bp_data type. */ + typedef struct bp_data { struct bp_data *next; /* Doubly linked ring pointers */ struct bp_data *prev; /* -"- */ BeamInstr orig_instr; /* The original instruction to execute */ + BeamInstr this_instr; /* key */ } BpData; /* ** All the following bp_data_.. structs must begin the same way @@ -57,26 +75,67 @@ typedef struct bp_data { typedef struct bp_data_trace { struct bp_data *next; struct bp_data *prev; - BeamInstr orig_instr; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ Binary *match_spec; - Eterm tracer_pid; + Eterm tracer_pid; } BpDataTrace; typedef struct bp_data_debug { struct bp_data *next; struct bp_data *prev; - BeamInstr orig_instr; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ } BpDataDebug; -typedef struct bp_data_count { /* Call count */ +typedef struct bp_data_count { /* Call count */ struct bp_data *next; struct bp_data *prev; BeamInstr orig_instr; - Sint count; + BeamInstr this_instr; /* key */ + erts_smp_atomic_t acount; } BpDataCount; +typedef struct { + Eterm pid; + Sint count; + Uint s_time; + Uint us_time; +} bp_data_time_item_t; + +typedef struct { + Uint n; + Uint used; + bp_data_time_item_t *item; +} bp_time_hash_t; + +typedef struct bp_data_time { /* Call time */ + struct bp_data *next; + struct bp_data *prev; + BeamInstr orig_instr; + BeamInstr this_instr; /* key */ + Uint pause; + Uint n; + bp_time_hash_t *hash; +} BpDataTime; + +typedef struct { + Uint ms; + Uint s; + Uint us; + BeamInstr *pc; +} process_breakpoint_time_t; /* used within psd */ + extern erts_smp_spinlock_t erts_bp_lock; +#define ERTS_BP_CALL_TIME_SCHEDULE_IN (0) +#define ERTS_BP_CALL_TIME_SCHEDULE_OUT (1) +#define ERTS_BP_CALL_TIME_SCHEDULE_EXITING (2) + +#define ERTS_BP_CALL_TIME_CALL (0) +#define ERTS_BP_CALL_TIME_RETURN (1) +#define ERTS_BP_CALL_TIME_TAIL_CALL (2) + #ifdef ERTS_SMP #define ErtsSmpBPLock(BDC) erts_smp_spin_lock(&erts_bp_lock) #define ErtsSmpBPUnlock(BDC) erts_smp_spin_unlock(&erts_bp_lock) @@ -85,31 +144,46 @@ extern erts_smp_spinlock_t erts_bp_lock; #define ErtsSmpBPUnlock(BDC) #endif -#define ErtsCountBreak(pc,instr_result) \ -do { \ - BpDataCount *bdc = (BpDataCount *) (pc)[-4]; \ - \ +ERTS_INLINE Uint bp_sched2ix(void); + +#ifdef ERTS_SMP +#define bp_sched2ix_proc(p) ((p)->scheduler_data->no - 1) +#else +#define bp_sched2ix_proc(p) (0) +#endif + +#define ErtsCountBreak(p, pc,instr_result) \ +do { \ + BpData **bds = (BpData **) (pc)[-4]; \ + BpDataCount *bdc = NULL; \ + Uint ix = bp_sched2ix_proc( (p) ); \ + erts_aint_t count = 0; \ + \ ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \ - ASSERT(bdc); \ - bdc = (BpDataCount *) bdc->next; \ - ASSERT(bdc); \ - (pc)[-4] = (BeamInstr) bdc; \ - ErtsSmpBPLock(bdc); \ - if (bdc->count >= 0) bdc->count++; \ - ErtsSmpBPUnlock(bdc); \ - *(instr_result) = bdc->orig_instr; \ + ASSERT(bds); \ + bdc = (BpDataCount *) bds[ix]; \ + bdc = (BpDataCount *) bdc->next; \ + ASSERT(bdc); \ + bds[ix] = (BpData *) bdc; \ + count = erts_smp_atomic_read(&bdc->acount); \ + if (count >= 0) erts_smp_atomic_inc(&bdc->acount); \ + *(instr_result) = bdc->orig_instr; \ } while (0) -#define ErtsBreakSkip(pc,instr_result) \ -do { \ - BpData *bd = (BpData *) (pc)[-4]; \ - \ +#define ErtsBreakSkip(p, pc,instr_result) \ +do { \ + BpData **bds = (BpData **) (pc)[-4]; \ + BpData *bd = NULL; \ + Uint ix = bp_sched2ix_proc( (p) ); \ + \ ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \ - ASSERT(bd); \ - bd = bd->next; \ - ASSERT(bd); \ - (pc)[-4] = (BeamInstr) bd; \ - *(instr_result) = bd->orig_instr; \ + ASSERT(bds); \ + bd = bds[ix]; \ + ASSERT(bd); \ + bd = bd->next; \ + ASSERT(bd); \ + bds[ix] = bd; \ + *(instr_result) = bd->orig_instr; \ } while (0) enum erts_break_op{ @@ -159,6 +233,17 @@ int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, Eterm *tracer_pid_ret); int erts_is_native_break(BeamInstr *pc); int erts_is_count_break(BeamInstr *pc, Sint *count_ret); +int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time); + +void erts_trace_time_break(Process *p, BeamInstr *pc, BpDataTime *bdt, Uint type); +void erts_schedule_time_break(Process *p, Uint out); +int erts_set_time_break(Eterm mfa[3], int specified, enum erts_break_op); +int erts_clear_time_break(Eterm mfa[3], int specified); + +int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time); +void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op); +void erts_clear_time_trace_bif(BeamInstr *pc); +BpData *erts_get_time_break(Process *p, BeamInstr *pc); BeamInstr *erts_find_local_func(Eterm mfa[3]); diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 23b267d5cd..2855241b91 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -48,7 +48,6 @@ void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); -static void print_big(int to, void *to_arg, Eterm* addr); static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr); Eterm erts_debug_same_2(Process* p, Eterm term1, Eterm term2) @@ -125,7 +124,7 @@ erts_debug_breakpoint_2(Process* p, Eterm MFA, Eterm bool) BIF_ERROR(p, BADARG); } -#if 0 /* XXX:PaN - not used */ +#if 0 /* Kept for conveninence when hard debugging. */ void debug_dump_code(BeamInstr *I, int num) { BeamInstr *code_ptr = I; @@ -157,6 +156,25 @@ void debug_dump_code(BeamInstr *I, int num) } #endif +BIF_RETTYPE +erts_debug_instructions_0(BIF_ALIST_0) +{ + int i = 0; + Uint needed = num_instructions * 2; + Eterm* hp; + Eterm res = NIL; + + for (i = 0; i < num_instructions; i++) { + needed += 2*strlen(opc[i].name); + } + hp = HAlloc(BIF_P, needed); + for (i = num_instructions-1; i >= 0; i--) { + Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, strlen(opc[i].name)); + res = erts_bld_cons(&hp, 0, s, res); + } + return res; +} + Eterm erts_debug_disassemble_1(Process* p, Eterm addr) { @@ -312,6 +330,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) BeamInstr packed = 0; /* Accumulator for packed operations. */ BeamInstr args[8]; /* Arguments for this instruction. */ BeamInstr* ap; /* Pointer to arguments. */ + BeamInstr* unpacked; /* Unpacked arguments */ start_prog = opc[op].pack; @@ -360,6 +379,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) *ap++ = packed & BEAM_LOOSE_MASK; packed >>= BEAM_LOOSE_SHIFT; break; +#ifdef ARCH_64 + case 'w': /* Shift 32 steps */ + *ap++ = packed & BEAM_WIDE_MASK; + packed >>= BEAM_WIDE_SHIFT; + break; +#endif case 'p': *sp++ = *--ap; break; @@ -386,7 +411,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'x': /* x(N) */ if (reg_index(ap[0]) == 0) { - erts_print(to, to_arg, "X[0]"); + erts_print(to, to_arg, "x[0]"); } else { erts_print(to, to_arg, "x(%d)", reg_index(ap[0])); } @@ -506,6 +531,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) ap++; break; case 'P': /* Byte offset into tuple (see beam_load.c) */ + case 'Q': /* Like 'P', but packable */ erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm)) - 1); ap++; break; @@ -526,9 +552,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) * Print more information about certain instructions. */ + unpacked = ap; ap = addr + size; switch (op) { - case op_i_select_val_sfI: + case op_i_select_val_rfI: + case op_i_select_val_xfI: + case op_i_select_val_yfI: { int n = ap[-1]; @@ -540,7 +569,24 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_jump_on_val_sfII: + case op_i_select_tuple_arity_rfI: + case op_i_select_tuple_arity_xfI: + case op_i_select_tuple_arity_yfI: + { + int n = ap[-1]; + + while (n > 0) { + Uint arity = arityval(ap[0]); + erts_print(to, to_arg, " {%d} f(" HEXF ")", arity, ap[1]); + ap += 2; + size += 2; + n--; + } + } + break; + case op_i_jump_on_val_rfII: + case op_i_jump_on_val_xfII: + case op_i_jump_on_val_yfII: { int n; for (n = ap[-2]; n > 0; n--) { @@ -550,39 +596,46 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_select_big_sf: - while (ap[0]) { - Eterm *bigp = (Eterm *) ap; - int arity = thing_arityval(*bigp); - print_big(to, to_arg, bigp); - size += TermWords(arity+1); - ap += TermWords(arity+1); - erts_print(to, to_arg, " f(" HEXF ") ", ap[0]); - ap++; - size++; + case op_i_jump_on_val_zero_rfI: + case op_i_jump_on_val_zero_xfI: + case op_i_jump_on_val_zero_yfI: + { + int n; + for (n = ap[-1]; n > 0; n--) { + erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); + ap++; + size++; + } + } + break; + case op_i_put_tuple_rI: + case op_i_put_tuple_xI: + case op_i_put_tuple_yI: + { + int n = unpacked[-1]; + + while (n > 0) { + if (!is_header(ap[0])) { + erts_print(to, to_arg, " %T", (Eterm) ap[0]); + } else { + switch ((ap[0] >> 2) & 0x03) { + case R_REG_DEF: + erts_print(to, to_arg, " x(0)"); + break; + case X_REG_DEF: + erts_print(to, to_arg, " x(%d)", ap[0] >> 4); + break; + case Y_REG_DEF: + erts_print(to, to_arg, " y(%d)", ap[0] >> 4); + break; + } + } + ap++, size++, n--; + } } - ap++; - size++; break; } erts_print(to, to_arg, "\n"); return size; } - -static void -print_big(int to, void *to_arg, Eterm* addr) -{ - int i; - int k; - - i = BIG_SIZE(addr); - if (BIG_SIGN(addr)) - erts_print(to, to_arg, "-#integer(%d) = {", i); - else - erts_print(to, to_arg, "#integer(%d) = {", i); - erts_print(to, to_arg, "0x%x", BIG_DIGIT(addr, 0)); - for (k = 1; k < i; k++) - erts_print(to, to_arg, ",0x%x", BIG_DIGIT(addr, k)); - erts_print(to, to_arg, "}"); -} diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 5b2f032afc..16741aa2d7 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -228,9 +228,10 @@ BeamInstr* em_call_traced_function; ** for the refering variable (one of these), and rouge references ** will most likely cause chaos. */ -BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ -BeamInstr beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */ +BeamInstr beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +BeamInstr beam_return_trace[1]; /* OpCode(i_return_trace) */ +BeamInstr beam_exception_trace[1]; /* UGLY also OpCode(i_return_trace) */ +BeamInstr beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ /* * All Beam instructions in numerical order. @@ -343,6 +344,8 @@ extern int count_instructions; #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N))) #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N))) #define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N))) +#define Qb(N) (N) +#define Ib(N) (N) #define x(N) reg[N] #define y(N) E[N] #define r(N) x##N @@ -471,6 +474,13 @@ extern int count_instructions; HEAP_SPACE_VERIFIED(need); \ } while (0) +#define TestHeapPutList(Need, Reg) \ + do { \ + TestHeap((Need), 1); \ + PutList(Reg, r(0), r(0), StoreSimpleDest); \ + CHECK_TERM(r(0)); \ + } while (0) + #ifdef HYBRID #ifdef INCREMENTAL #define TestGlobalHeap(Nh, Live, hp) \ @@ -515,6 +525,11 @@ extern int count_instructions; SWAPIN; \ } while (0) +#define PutTuple(Dst, Arity) \ + do { \ + Dst = make_tuple(HTOP); \ + pt_arity = (Arity); \ + } while (0) /* * Check that we haven't used the reductions and jump to function pointed to by @@ -673,6 +688,11 @@ extern int count_instructions; SET_I((BeamInstr *) CallDest); \ Dispatch(); +#define MoveJump(Src) \ + r(0) = (Src); \ + SET_I((BeamInstr *) Arg(0)); \ + Goto(*I); + #define GetList(Src, H, T) do { \ Eterm* tmp_ptr = list_val(Src); \ H = CAR(tmp_ptr); \ @@ -722,16 +742,8 @@ extern int count_instructions; (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \ } while (0) -#define PutTuple(Arity, Src, Dest) \ - ASSERT(is_arity_value(Arity)); \ - Dest = make_tuple(HTOP); \ - HTOP[0] = (Arity); \ - HTOP[1] = (Src); \ - HTOP += 2 - -#define Put(Word) *HTOP++ = (Word) - #define EqualImmed(X, Y, Action) if (X != Y) { Action; } +#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; } @@ -983,8 +995,41 @@ extern int count_instructions; #define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; } #define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; } -static BifFunction translate_gc_bif(void* gcf); -static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf); +/* + * process_main() is already huge, so we want to avoid inlining + * into it. Especially functions that are seldom used. + */ +#ifdef __GNUC__ +# define NOINLINE __attribute__((__noinline__)) +#else +# define NOINLINE +#endif + +/* + * The following functions are called directly by process_main(). + * Don't inline them. + */ +static BifFunction translate_gc_bif(void* gcf) NOINLINE; +static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, + Eterm* reg, BifFunction bf) NOINLINE; +static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, + Eterm* reg, Eterm func) NOINLINE; +static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; +static BeamInstr* apply(Process* p, Eterm module, Eterm function, + Eterm args, Eterm* reg) NOINLINE; +static int hibernate(Process* c_p, Eterm module, Eterm function, + Eterm args, Eterm* reg) NOINLINE; +static BeamInstr* call_fun(Process* p, int arity, + Eterm* reg, Eterm args) NOINLINE; +static BeamInstr* apply_fun(Process* p, Eterm fun, + Eterm args, Eterm* reg) NOINLINE; +static Eterm new_fun(Process* p, Eterm* reg, + ErlFunEntry* fe, int num_free) NOINLINE; + + +/* + * Functions not directly called by process_main(). OK to inline. + */ static BeamInstr* next_catch(Process* c_p, Eterm *reg); static void terminate_proc(Process* c_p, Eterm Value); static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc); @@ -992,16 +1037,6 @@ static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, Eterm args); static struct StackTrace * get_trace_from_exc(Eterm exc); static Eterm make_arglist(Process* c_p, Eterm* reg, int a); -static Eterm call_error_handler(Process* p, BeamInstr* ip, Eterm* reg); -static Eterm call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg); -static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity); -static BeamInstr* apply(Process* p, Eterm module, Eterm function, - Eterm args, Eterm* reg); -static int hibernate(Process* c_p, Eterm module, Eterm function, - Eterm args, Eterm* reg); -static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args); -static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg); -static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free); #if defined(VXWORKS) static int init_done; @@ -1145,6 +1180,8 @@ void process_main(void) Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */ + Eterm pt_arity; /* Used by do_put_tuple */ + ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */ @@ -1245,6 +1282,52 @@ void process_main(void) #define STORE_ARITH_RESULT(res) StoreBifResult(2, (res)); #define ARITH_FUNC(name) erts_gc_##name + { + Eterm increment_reg_val; + Eterm increment_val; + Uint live; + Eterm result; + + OpCase(i_increment_yIId): + increment_reg_val = yb(Arg(0)); + goto do_increment; + + OpCase(i_increment_xIId): + increment_reg_val = xb(Arg(0)); + goto do_increment; + + OpCase(i_increment_rIId): + increment_reg_val = r(0); + I--; + + do_increment: + increment_val = Arg(1); + if (is_small(increment_reg_val)) { + Sint i = signed_val(increment_reg_val) + increment_val; + ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); + if (MY_IS_SSMALL(i)) { + result = make_small(i); + store_result: + StoreBifResult(3, result); + } + } + + live = Arg(2); + SWAPOUT; + reg[0] = r(0); + reg[live] = increment_reg_val; + reg[live+1] = make_small(increment_val); + result = erts_gc_mixed_plus(c_p, reg, live); + r(0) = reg[0]; + SWAPIN; + ERTS_HOLE_CHECK(c_p); + if (is_value(result)) { + goto store_result; + } + ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); + goto find_func_info; + } + OpCase(i_plus_jId): { Eterm result; @@ -1308,6 +1391,52 @@ void process_main(void) } Next(1); + { + Eterm is_eq_exact_lit_val; + + OpCase(i_is_eq_exact_literal_xfc): + is_eq_exact_lit_val = xb(Arg(0)); + I++; + goto do_is_eq_exact_literal; + + OpCase(i_is_eq_exact_literal_yfc): + is_eq_exact_lit_val = yb(Arg(0)); + I++; + goto do_is_eq_exact_literal; + + OpCase(i_is_eq_exact_literal_rfc): + is_eq_exact_lit_val = r(0); + + do_is_eq_exact_literal: + if (!eq(Arg(1), is_eq_exact_lit_val)) { + ClauseFail(); + } + Next(2); + } + + { + Eterm is_ne_exact_lit_val; + + OpCase(i_is_ne_exact_literal_xfc): + is_ne_exact_lit_val = xb(Arg(0)); + I++; + goto do_is_ne_exact_literal; + + OpCase(i_is_ne_exact_literal_yfc): + is_ne_exact_lit_val = yb(Arg(0)); + I++; + goto do_is_ne_exact_literal; + + OpCase(i_is_ne_exact_literal_rfc): + is_ne_exact_lit_val = r(0); + + do_is_ne_exact_literal: + if (eq(Arg(1), is_ne_exact_lit_val)) { + ClauseFail(); + } + Next(2); + } + OpCase(i_move_call_only_fcr): { r(0) = Arg(1); } @@ -1391,6 +1520,17 @@ void process_main(void) NextPF(1, next); } + OpCase(move_x1_c): { + x(1) = Arg(0); + Next(1); + } + + OpCase(move_x2_c): { + x(2) = Arg(0); + Next(1); + } + + OpCase(return): { SET_I(c_p->cp); /* @@ -1404,16 +1544,6 @@ void process_main(void) Goto(*I); } - OpCase(test_heap_1_put_list_Iy): { - BeamInstr *next; - - PreFetch(2, next); - TestHeap(Arg(0), 1); - PutList(yb(Arg(1)), r(0), r(0), StoreSimpleDest); - CHECK_TERM(r(0)); - NextPF(2, next); - } - /* * Send is almost a standard call-BIF with two arguments, except for: * 1) It cannot be traced. @@ -1446,24 +1576,36 @@ void process_main(void) goto find_func_info; } - OpCase(i_element_jssd): { - Eterm index; - Eterm tuple; - - /* - * Inlined version of element/2 for speed. - */ - GetArg2(1, index, tuple); - if (is_small(index) && is_tuple(tuple)) { - Eterm* tp = tuple_val(tuple); - - if ((signed_val(index) >= 1) && - (signed_val(index) <= arityval(*tp))) { - Eterm result = tp[signed_val(index)]; - StoreBifResult(3, result); - } - } - } + { + Eterm element_index; + Eterm element_tuple; + + OpCase(i_element_xjsd): + element_tuple = xb(Arg(0)); + I++; + goto do_element; + + OpCase(i_element_yjsd): + element_tuple = yb(Arg(0)); + I++; + goto do_element; + + OpCase(i_element_rjsd): + element_tuple = r(0); + /* Fall through */ + + do_element: + GetArg1(1, element_index); + if (is_small(element_index) && is_tuple(element_tuple)) { + Eterm* tp = tuple_val(element_tuple); + + if ((signed_val(element_index) >= 1) && + (signed_val(element_index) <= arityval(*tp))) { + Eterm result = tp[signed_val(element_index)]; + StoreBifResult(2, result); + } + } + } /* Fall through */ OpCase(badarg_j): @@ -1471,24 +1613,32 @@ void process_main(void) c_p->freason = BADARG; goto lb_Cl_error; - OpCase(i_fast_element_jIsd): { - Eterm tuple; - - /* - * Inlined version of element/2 for even more speed. - * The first argument is an untagged integer >= 1. - * The second argument is guaranteed to be a register operand. - */ - GetArg1(2, tuple); - if (is_tuple(tuple)) { - Eterm* tp = tuple_val(tuple); - tmp_arg2 = Arg(1); - if (tmp_arg2 <= arityval(*tp)) { - Eterm result = tp[tmp_arg2]; - StoreBifResult(3, result); - } - } + { + Eterm fast_element_tuple; + + OpCase(i_fast_element_rjId): + fast_element_tuple = r(0); + + do_fast_element: + if (is_tuple(fast_element_tuple)) { + Eterm* tp = tuple_val(fast_element_tuple); + Eterm pos = Arg(1); /* Untagged integer >= 1 */ + if (pos <= arityval(*tp)) { + Eterm result = tp[pos]; + StoreBifResult(2, result); + } + } goto badarg; + + OpCase(i_fast_element_xjId): + fast_element_tuple = xb(Arg(0)); + I++; + goto do_fast_element; + + OpCase(i_fast_element_yjId): + fast_element_tuple = yb(Arg(0)); + I++; + goto do_fast_element; } OpCase(catch_yf): @@ -1539,6 +1689,10 @@ void process_main(void) /* * Skeleton for receive statement: * + * recv_mark L1 Optional + * call make_ref/monitor Optional + * ... + * recv_set L1 Optional * L1: <-------------------+ * <-----------+ | * | | @@ -1557,6 +1711,34 @@ void process_main(void) * */ + OpCase(recv_mark_f): { + /* + * Save the current position in message buffer and the + * the label for the loop_rec/2 instruction for the + * the receive statement. + */ + c_p->msg.mark = (BeamInstr *) Arg(0); + c_p->msg.saved_last = c_p->msg.last; + Next(1); + } + + OpCase(i_recv_set): { + /* + * If the mark is valid (points to the loop_rec/2 + * instruction that follows), we know that the saved + * position points to the first message that could + * possibly be matched out. + * + * If the mark is invalid, we do nothing, meaning that + * we will look through all messages in the message queue. + */ + if (c_p->msg.mark == (BeamInstr *) (I+1)) { + c_p->msg.save = c_p->msg.saved_last; + } + I++; + /* Fall through to the loop_rec/2 instruction */ + } + /* * Pick up the next message and place it in x(0). * If no message, jump to a wait or wait_timeout instruction. @@ -1809,8 +1991,87 @@ void process_main(void) NextPF(0, next); } - OpCase(i_select_val_sfI): - GetArg1(0, tmp_arg1); + + { + Eterm select_val2; + + OpCase(i_select_tuple_arity2_yfAfAf): + select_val2 = yb(Arg(0)); + goto do_select_tuple_arity2; + + OpCase(i_select_tuple_arity2_xfAfAf): + select_val2 = xb(Arg(0)); + goto do_select_tuple_arity2; + + OpCase(i_select_tuple_arity2_rfAfAf): + select_val2 = r(0); + I--; + + do_select_tuple_arity2: + if (is_not_tuple(select_val2)) { + goto select_val2_fail; + } + select_val2 = *tuple_val(select_val2); + goto do_select_val2; + + OpCase(i_select_val2_yfcfcf): + select_val2 = yb(Arg(0)); + goto do_select_val2; + + OpCase(i_select_val2_xfcfcf): + select_val2 = xb(Arg(0)); + goto do_select_val2; + + OpCase(i_select_val2_rfcfcf): + select_val2 = r(0); + I--; + + do_select_val2: + if (select_val2 == Arg(2)) { + I += 2; + } else if (select_val2 == Arg(4)) { + I += 4; + } + + select_val2_fail: + SET_I((BeamInstr *) Arg(1)); + Goto(*I); + } + + { + Eterm select_val; + + OpCase(i_select_tuple_arity_xfI): + select_val = xb(Arg(0)); + goto do_select_tuple_arity; + + OpCase(i_select_tuple_arity_yfI): + select_val = yb(Arg(0)); + goto do_select_tuple_arity; + + OpCase(i_select_tuple_arity_rfI): + select_val = r(0); + I--; + + do_select_tuple_arity: + if (is_tuple(select_val)) { + select_val = *tuple_val(select_val); + goto do_binary_search; + } + SET_I((BeamInstr *) Arg(1)); + Goto(*I); + + OpCase(i_select_val_xfI): + select_val = xb(Arg(0)); + goto do_binary_search; + + OpCase(i_select_val_yfI): + select_val = yb(Arg(0)); + goto do_binary_search; + + OpCase(i_select_val_rfI): + select_val = r(0); + I--; do_binary_search: { @@ -1847,9 +2108,9 @@ void process_main(void) unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1); mid = (struct Pairs*)((char*)low + boffset); - if (tmp_arg1 < mid->val) { + if (select_val < mid->val) { high = mid; - } else if (tmp_arg1 > mid->val) { + } else if (select_val > mid->val) { low = mid + 1; } else { SET_I(mid->addr); @@ -1859,16 +2120,28 @@ void process_main(void) SET_I((BeamInstr *) Arg(1)); Goto(*I); } + } - OpCase(i_jump_on_val_zero_sfI): { - Eterm index; - - GetArg1(0, index); - if (is_small(index)) { - index = signed_val(index); - if (index < Arg(2)) { - SET_I((BeamInstr *) (&Arg(3))[index]); + Eterm jump_on_val_zero_index; + + OpCase(i_jump_on_val_zero_yfI): + jump_on_val_zero_index = yb(Arg(0)); + goto do_jump_on_val_zero_index; + + OpCase(i_jump_on_val_zero_xfI): + jump_on_val_zero_index = xb(Arg(0)); + goto do_jump_on_val_zero_index; + + OpCase(i_jump_on_val_zero_rfI): + jump_on_val_zero_index = r(0); + I--; + + do_jump_on_val_zero_index: + if (is_small(jump_on_val_zero_index)) { + jump_on_val_zero_index = signed_val(jump_on_val_zero_index); + if (jump_on_val_zero_index < Arg(2)) { + SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]); Goto(*I); } } @@ -1876,15 +2149,27 @@ void process_main(void) Goto(*I); } - OpCase(i_jump_on_val_sfII): { - Eterm index; + Eterm jump_on_val_index; - GetArg1(0, index); - if (is_small(index)) { - index = (Uint) (signed_val(index) - Arg(3)); - if (index < Arg(2)) { - SET_I((BeamInstr *) (&Arg(4))[index]); + + OpCase(i_jump_on_val_yfII): + jump_on_val_index = yb(Arg(0)); + goto do_jump_on_val_index; + + OpCase(i_jump_on_val_xfII): + jump_on_val_index = xb(Arg(0)); + goto do_jump_on_val_index; + + OpCase(i_jump_on_val_rfII): + jump_on_val_index = r(0); + I--; + + do_jump_on_val_index: + if (is_small(jump_on_val_index)) { + jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3)); + if (jump_on_val_index < Arg(2)) { + SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]); Goto(*I); } } @@ -1892,6 +2177,32 @@ void process_main(void) Goto(*I); } + do_put_tuple: { + Eterm* hp = HTOP; + + *hp++ = make_arityval(pt_arity); + + do { + Eterm term = *I++; + switch (term & _TAG_IMMED1_MASK) { + case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: + *hp++ = r(0); + break; + case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: + *hp++ = x(term >> _TAG_IMMED1_SIZE); + break; + case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: + *hp++ = y(term >> _TAG_IMMED1_SIZE); + break; + default: + *hp++ = term; + break; + } + } while (--pt_arity != 0); + HTOP = hp; + Goto(*I); + } + /* * All guards with zero arguments have special instructions: * self/0 @@ -1994,6 +2305,81 @@ void process_main(void) goto post_error_handling; } + OpCase(i_gc_bif2_jIId): /* Note, one less parameter than the i_gc_bif1 + and i_gc_bif3 */ + { + typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); + GcBifFunction bf; + Eterm result; + Uint live = (Uint) Arg(2); + + reg[0] = r(0); + reg[live++] = tmp_arg1; + reg[live] = tmp_arg2; + bf = (GcBifFunction) Arg(1); + c_p->fcalls = FCALLS; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + result = (*bf)(c_p, reg, live); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + r(0) = reg[0]; + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + if (is_value(result)) { + StoreBifResult(3, result); + } + if (Arg(0) != 0) { + SET_I((BeamInstr *) Arg(0)); + Goto(*I); + } + reg[0] = tmp_arg1; + reg[1] = tmp_arg2; + I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + goto post_error_handling; + } + + OpCase(i_gc_bif3_jIsId): + { + typedef Eterm (*GcBifFunction)(Process*, Eterm*, Uint); + GcBifFunction bf; + Eterm arg; + Eterm result; + Uint live = (Uint) Arg(3); + + GetArg1(2, arg); + reg[0] = r(0); + reg[live++] = arg; + reg[live++] = tmp_arg1; + reg[live] = tmp_arg2; + bf = (GcBifFunction) Arg(1); + c_p->fcalls = FCALLS; + SWAPOUT; + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + result = (*bf)(c_p, reg, live); + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + SWAPIN; + r(0) = reg[0]; + ERTS_HOLE_CHECK(c_p); + FCALLS = c_p->fcalls; + if (is_value(result)) { + StoreBifResult(4, result); + } + if (Arg(0) != 0) { + SET_I((BeamInstr *) Arg(0)); + Goto(*I); + } + reg[0] = arg; + reg[1] = tmp_arg1; + reg[2] = tmp_arg2; + I = handle_error(c_p, I, reg, translate_gc_bif((void *) bf)); + goto post_error_handling; + } + /* * Guards bifs and, or, xor in guards. */ @@ -2454,23 +2840,25 @@ void process_main(void) OpCase(i_int_bnot_jsId): { - GetArg1(1, tmp_arg1); - if (is_small(tmp_arg1)) { - tmp_arg1 = make_small(~signed_val(tmp_arg1)); + Eterm bnot_val; + + GetArg1(1, bnot_val); + if (is_small(bnot_val)) { + bnot_val = make_small(~signed_val(bnot_val)); } else { Uint live = Arg(2); SWAPOUT; reg[0] = r(0); - reg[live] = tmp_arg1; - tmp_arg1 = erts_gc_bnot(c_p, reg, live); + reg[live] = bnot_val; + bnot_val = erts_gc_bnot(c_p, reg, live); r(0) = reg[0]; SWAPIN; ERTS_HOLE_CHECK(c_p); - if (is_nil(tmp_arg1)) { + if (is_nil(bnot_val)) { goto lb_Cl_error; } } - StoreBifResult(3, tmp_arg1); + StoreBifResult(3, bnot_val); } badarith: @@ -2725,121 +3113,6 @@ void process_main(void) goto do_schedule1; } - OpCase(i_select_tuple_arity_sfI): - { - GetArg1(0, tmp_arg1); - - if (is_tuple(tmp_arg1)) { - tmp_arg1 = *tuple_val(tmp_arg1); - goto do_binary_search; - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - OpCase(i_select_big_sf): - { - Eterm* bigp; - Uint arity; - Eterm* given; - Uint given_arity; - Uint given_size; - - GetArg1(0, tmp_arg1); - if (is_big(tmp_arg1)) { - - /* - * The loader has sorted the bignumbers in descending order - * on the arity word. Therefore, we know that the search - * has failed as soon as we encounter an arity word less than - * the arity word of the given number. There is a zero word - * (less than any valid arity word) stored after the last bignumber. - */ - - given = big_val(tmp_arg1); - given_arity = given[0]; - given_size = thing_arityval(given_arity); - bigp = (Eterm *) &Arg(2); - while ((arity = bigp[0]) > given_arity) { - bigp += (TermWords(thing_arityval(arity) + 1) + 1) * (sizeof(BeamInstr)/sizeof(Eterm)); - } - while (bigp[0] == given_arity) { - if (memcmp(bigp+1, given+1, sizeof(Eterm)*given_size) == 0) { - BeamInstr *tmp = - ((BeamInstr *) (UWord) bigp) + TermWords(given_size + 1); - SET_I((BeamInstr *) *tmp); - Goto(*I); - } - bigp += (TermWords(thing_arityval(arity) + 1) + 1) * (sizeof(BeamInstr)/sizeof(Eterm)); - } - } - - /* - * Failed. - */ - - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - -#if defined(ARCH_64) && !HALFWORD_HEAP - OpCase(i_select_float_sfI): - { - Uint f; - int n; - struct ValLabel { - Uint f; - BeamInstr* addr; - }; - struct ValLabel* ptr; - - GetArg1(0, tmp_arg1); - ASSERT(is_float(tmp_arg1)); - f = float_val(tmp_arg1)[1]; - n = Arg(2); - ptr = (struct ValLabel *) &Arg(3); - while (n-- > 0) { - if (ptr->f == f) { - SET_I(ptr->addr); - Goto(*I); - } - ptr++; - } - SET_I((Eterm *) Arg(1)); - Goto(*I); - } -#else - OpCase(i_select_float_sfI): - { - Uint fpart1; - Uint fpart2; - int n; - struct ValLabel { - Uint fpart1; - Uint fpart2; - BeamInstr* addr; - }; - struct ValLabel* ptr; - - GetArg1(0, tmp_arg1); - ASSERT(is_float(tmp_arg1)); - fpart1 = float_val(tmp_arg1)[1]; - fpart2 = float_val(tmp_arg1)[2]; - - n = Arg(2); - ptr = (struct ValLabel *) &Arg(3); - while (n-- > 0) { - if (ptr->fpart1 == fpart1 && ptr->fpart2 == fpart2) { - SET_I(ptr->addr); - Goto(*I); - } - ptr++; - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } -#endif - OpCase(set_tuple_element_sdP): { Eterm element; Eterm tuple; @@ -2885,15 +3158,17 @@ void process_main(void) the first argument. We also handle atom tags in the first argument for backwards compatibility. */ - GetArg2(0, tmp_arg1, tmp_arg2); - c_p->fvalue = tmp_arg2; + Eterm raise_val1; + Eterm raise_val2; + GetArg2(0, raise_val1, raise_val2); + c_p->fvalue = raise_val2; if (c_p->freason == EXC_NULL) { /* a safety check for the R10-0 case; should not happen */ c_p->ftrace = NIL; c_p->freason = EXC_ERROR; } /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */ - switch (tmp_arg1) { + switch (raise_val1) { case am_throw: c_p->freason = EXC_THROWN & ~EXF_SAVETRACE; break; @@ -2909,8 +3184,8 @@ void process_main(void) passed from a user! Currently only expecting generated calls. */ struct StackTrace *s; - c_p->ftrace = tmp_arg1; - s = get_trace_from_exc(tmp_arg1); + c_p->ftrace = raise_val1; + s = get_trace_from_exc(raise_val1); if (s == NULL) { c_p->freason = EXC_ERROR; } else { @@ -2921,11 +3196,24 @@ void process_main(void) goto find_func_info; } - OpCase(badmatch_s): { - GetArg1(0, tmp_arg1); - c_p->fvalue = tmp_arg1; - c_p->freason = BADMATCH; - } + { + Eterm badmatch_val; + + OpCase(badmatch_y): + badmatch_val = yb(Arg(0)); + goto do_badmatch; + + OpCase(badmatch_x): + badmatch_val = xb(Arg(0)); + goto do_badmatch; + + OpCase(badmatch_r): + badmatch_val = r(0); + + do_badmatch: + c_p->fvalue = badmatch_val; + c_p->freason = BADMATCH; + } /* Fall through here */ find_func_info: { @@ -2948,12 +3236,11 @@ void process_main(void) */ SWAPOUT; reg[0] = r(0); - tmp_arg1 = call_error_handler(c_p, I-3, reg); + I = call_error_handler(c_p, I-3, reg, am_undefined_function); r(0) = reg[0]; SWAPIN; - if (tmp_arg1) { - SET_I(c_p->i); - Dispatch(); + if (I) { + Goto(*I); } /* Fall through */ @@ -2976,128 +3263,142 @@ void process_main(void) } } - OpCase(call_nif): - { - /* - * call_nif is always first instruction in function: - * - * I[-3]: Module - * I[-2]: Function - * I[-1]: Arity - * I[0]: &&call_nif - * I[1]: Function pointer to NIF function - * I[2]: Pointer to erl_module_nif - */ - BifFunction vbf; - - c_p->current = I-3; /* current and vbf set to please handle_error */ - SWAPOUT; - c_p->fcalls = FCALLS - 1; - PROCESS_MAIN_CHK_LOCKS(c_p); - tmp_arg2 = I[-1]; - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + { + Eterm nif_bif_result; + Eterm bif_nif_arity; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - { - typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; - struct enif_environment_t env; - erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); - reg[0] = r(0); - tmp_arg1 = (*fp)(&env, tmp_arg2, reg); - erts_post_nif(&env); - } - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); - goto apply_bif_or_nif_epilogue; - - OpCase(apply_bif): - /* - * At this point, I points to the code[3] in the export entry for - * the BIF: - * - * code[0]: Module - * code[1]: Function - * code[2]: Arity - * code[3]: &&apply_bif - * code[4]: Function pointer to BIF function - */ + OpCase(call_nif): + { + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + */ + BifFunction vbf; - c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ - c_p->i = I; /* In case we apply check_process_code/2. */ - c_p->arity = 0; /* To allow garbage collection on ourselves - * (check_process_code/2). - */ - SWAPOUT; - c_p->fcalls = FCALLS - 1; - vbf = (BifFunction) Arg(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - tmp_arg2 = I[-1]; - ASSERT(tmp_arg2 <= 3); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - switch (tmp_arg2) { - case 3: + c_p->current = I-3; /* current and vbf set to please handle_error */ + SWAPOUT; + c_p->fcalls = FCALLS - 1; + PROCESS_MAIN_CHK_LOCKS(c_p); + bif_nif_arity = I[-1]; + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, r(0), x(1), x(2), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); + typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); + NifF* fp = vbf = (NifF*) I[1]; + struct enif_environment_t env; + erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); + reg[0] = r(0); + nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + erts_post_nif(&env); } - break; - case 2: - { - Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, r(0), x(1), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); + PROCESS_MAIN_CHK_LOCKS(c_p); + goto apply_bif_or_nif_epilogue; + + OpCase(apply_bif): + /* + * At this point, I points to the code[3] in the export entry for + * the BIF: + * + * code[0]: Module + * code[1]: Function + * code[2]: Arity + * code[3]: &&apply_bif + * code[4]: Function pointer to BIF function + */ + + c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ + c_p->i = I; /* In case we apply check_process_code/2. */ + c_p->arity = 0; /* To allow garbage collection on ourselves + * (check_process_code/2). + */ + SWAPOUT; + c_p->fcalls = FCALLS - 1; + vbf = (BifFunction) Arg(0); + PROCESS_MAIN_CHK_LOCKS(c_p); + bif_nif_arity = I[-1]; + ASSERT(bif_nif_arity <= 3); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + switch (bif_nif_arity) { + case 3: + { + Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, r(0), x(1), x(2), I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + break; + case 2: + { + Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, r(0), x(1), I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + break; + case 1: + { + Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, r(0), I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + break; + case 0: + { + Eterm (*bf)(Process*, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + PROCESS_MAIN_CHK_LOCKS(c_p); + break; + } + default: + erl_exit(1, "apply_bif: invalid arity: %u\n", + bif_nif_arity); } - break; - case 1: - { - Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, r(0), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); + + apply_bif_or_nif_epilogue: + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_HOLE_CHECK(c_p); + if (c_p->mbuf) { + reg[0] = r(0); + nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result, + reg, bif_nif_arity); + r(0) = reg[0]; } - break; - case 0: - { - Eterm (*bf)(Process*, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); - break; + SWAPIN; /* There might have been a garbage collection. */ + FCALLS = c_p->fcalls; + if (is_value(nif_bif_result)) { + r(0) = nif_bif_result; + CHECK_TERM(r(0)); + SET_I(c_p->cp); + Goto(*I); + } else if (c_p->freason == TRAP) { + SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); + r(0) = c_p->def_arg_reg[0]; + x(1) = c_p->def_arg_reg[1]; + x(2) = c_p->def_arg_reg[2]; + Dispatch(); } - } -apply_bif_or_nif_epilogue: - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - ERTS_HOLE_CHECK(c_p); - if (c_p->mbuf) { reg[0] = r(0); - tmp_arg1 = erts_gc_after_bif_call(c_p, tmp_arg1, reg, tmp_arg2); - r(0) = reg[0]; + I = handle_error(c_p, c_p->cp, reg, vbf); + goto post_error_handling; } - SWAPIN; /* There might have been a garbage collection. */ - FCALLS = c_p->fcalls; - if (is_value(tmp_arg1)) { - r(0) = tmp_arg1; - CHECK_TERM(r(0)); - SET_I(c_p->cp); - Goto(*I); - } else if (c_p->freason == TRAP) { - SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); - r(0) = c_p->def_arg_reg[0]; - x(1) = c_p->def_arg_reg[1]; - x(2) = c_p->def_arg_reg[2]; - Dispatch(); - } - reg[0] = r(0); - I = handle_error(c_p, c_p->cp, reg, vbf); - goto post_error_handling; } OpCase(i_get_sd): @@ -3110,17 +3411,26 @@ apply_bif_or_nif_epilogue: StoreBifResult(1, result); } - OpCase(i_put_tuple_only_Ad): { - tmp_arg1 = make_tuple(HTOP); - *HTOP++ = Arg(0); - StoreBifResult(1, tmp_arg1); - } + { + Eterm case_end_val; - OpCase(case_end_s): - GetArg1(0, tmp_arg1); - c_p->fvalue = tmp_arg1; - c_p->freason = EXC_CASE_CLAUSE; - goto find_func_info; + OpCase(case_end_x): + case_end_val = xb(Arg(0)); + goto do_case_end; + + OpCase(case_end_y): + case_end_val = yb(Arg(0)); + goto do_case_end; + + OpCase(case_end_r): + case_end_val = r(0); + I--; + + do_case_end: + c_p->fvalue = case_end_val; + c_p->freason = EXC_CASE_CLAUSE; + goto find_func_info; + } OpCase(if_end): c_p->freason = EXC_IF_CLAUSE; @@ -3133,10 +3443,13 @@ apply_bif_or_nif_epilogue: } OpCase(try_case_end_s): - GetArg1(0, tmp_arg1); - c_p->fvalue = tmp_arg1; - c_p->freason = EXC_TRY_CLAUSE; - goto find_func_info; + { + Eterm try_case_end_val; + GetArg1(0, try_case_end_val); + c_p->fvalue = try_case_end_val; + c_p->freason = EXC_TRY_CLAUSE; + goto find_func_info; + } /* * Construction of binaries using new instructions. @@ -3283,12 +3596,12 @@ apply_bif_or_nif_epilogue: HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = num_bytes; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - MSO(c_p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); new_binary = make_binary(pb); goto do_bits_sub_bin; } @@ -3384,13 +3697,13 @@ apply_bif_or_nif_epilogue: HTOP += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = tmp_arg1; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - MSO(c_p).overhead += tmp_arg1 / sizeof(Eterm); + OH_OVERHEAD(&(MSO(c_p)), tmp_arg1 / sizeof(Eterm)); StoreBifResult(2, make_binary(pb)); } @@ -3426,42 +3739,6 @@ apply_bif_or_nif_epilogue: } } - OpCase(i_bs_bits_to_bytes_rjd): { - tmp_arg1 = r(0); - goto do_bits_to_bytes; - } - - OpCase(i_bs_bits_to_bytes_yjd): { - tmp_arg1 = yb(Arg(0)); - I++; - goto do_bits_to_bytes; - - OpCase(i_bs_bits_to_bytes_xjd): { - tmp_arg1 = xb(Arg(0)); - I++; - } - - do_bits_to_bytes: - { - if (is_valid_bit_size(tmp_arg1)) { - tmp_arg1 = make_small(unsigned_val(tmp_arg1) >> 3); - } else { - Uint bytes; - if (!term_to_Uint(tmp_arg1, &bytes)) { - goto badarg; - } - tmp_arg1 = bytes; - if ((tmp_arg1 & 0x07) != 0) { - goto badarg; - } - SWAPOUT; - tmp_arg1 = erts_make_integer(tmp_arg1 >> 3, c_p); - HTOP = HEAP_TOP(c_p); - } - StoreBifResult(1, tmp_arg1); - } - } - OpCase(i_bs_add_jId): { Uint Unit = Arg(1); if (is_both_small(tmp_arg1, tmp_arg2)) { @@ -3499,7 +3776,7 @@ apply_bif_or_nif_epilogue: /* * Now we know that one of the arguments is - * not at small. We must convert both arguments + * not a small. We must convert both arguments * to Uints and check for errors at the same time. * * Error checking is tricky. @@ -3720,19 +3997,20 @@ apply_bif_or_nif_epilogue: Eterm header; BeamInstr *next; Uint slots; + Eterm context; OpCase(i_bs_start_match2_rfIId): { - tmp_arg1 = r(0); + context = r(0); do_start_match: slots = Arg(2); - if (!is_boxed(tmp_arg1)) { + if (!is_boxed(context)) { ClauseFail(); } PreFetch(4, next); - header = *boxed_val(tmp_arg1); + header = *boxed_val(context); if (header_is_bin_matchstate(header)) { - ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1); + ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context); Uint actual_slots = HEADER_NUM_SLOTS(header); ms->save_offset[0] = ms->mb.offset; if (actual_slots < slots) { @@ -3740,8 +4018,8 @@ apply_bif_or_nif_epilogue: Uint live = Arg(1); Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - TestHeapPreserve(wordsneeded, live, tmp_arg1); - ms = (ErlBinMatchState *) boxed_val(tmp_arg1); + TestHeapPreserve(wordsneeded, live, context); + ms = (ErlBinMatchState *) boxed_val(context); dst = (ErlBinMatchState *) HTOP; *dst = *ms; *HTOP = HEADER_BIN_MATCHSTATE(slots); @@ -3753,12 +4031,12 @@ apply_bif_or_nif_epilogue: Eterm result; Uint live = Arg(1); Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - TestHeapPreserve(wordsneeded, live, tmp_arg1); + TestHeapPreserve(wordsneeded, live, context); HEAP_TOP(c_p) = HTOP; #ifdef DEBUG c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ #endif - result = erts_bs_start_match_2(c_p, tmp_arg1, slots); + result = erts_bs_start_match_2(c_p, context, slots); HTOP = HEAP_TOP(c_p); HEAP_SPACE_VERIFIED(0); if (is_non_value(result)) { @@ -3772,12 +4050,12 @@ apply_bif_or_nif_epilogue: NextPF(4, next); } OpCase(i_bs_start_match2_xfIId): { - tmp_arg1 = xb(Arg(0)); + context = xb(Arg(0)); I++; goto do_start_match; } OpCase(i_bs_start_match2_yfIId): { - tmp_arg1 = yb(Arg(0)); + context = yb(Arg(0)); I++; goto do_start_match; } @@ -3870,93 +4148,105 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } + { + Eterm bs_get_integer8_context; + OpCase(i_bs_get_integer_8_rfd): { - tmp_arg1 = r(0); - goto do_bs_get_integer_8; - } + bs_get_integer8_context = r(0); + goto do_bs_get_integer_8; + } OpCase(i_bs_get_integer_8_xfd): { - tmp_arg1 = xb(Arg(0)); - I++; - } + bs_get_integer8_context = xb(Arg(0)); + I++; + } do_bs_get_integer_8: { - ErlBinMatchBuffer *_mb; - Eterm _result; - _mb = ms_matchbuffer(tmp_arg1); - if (_mb->size - _mb->offset < 8) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 8, 0, _mb); - } else { - _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]); - _mb->offset += 8; + ErlBinMatchBuffer *_mb; + Eterm _result; + _mb = ms_matchbuffer(bs_get_integer8_context); + if (_mb->size - _mb->offset < 8) { + ClauseFail(); + } + if (BIT_OFFSET(_mb->offset) != 0) { + _result = erts_bs_get_integer_2(c_p, 8, 0, _mb); + } else { + _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]); + _mb->offset += 8; + } + StoreBifResult(1, _result); } - StoreBifResult(1, _result); } - OpCase(i_bs_get_integer_16_rfd): { - tmp_arg1 = r(0); + { + Eterm bs_get_integer_16_context; + + OpCase(i_bs_get_integer_16_rfd): + bs_get_integer_16_context = r(0); goto do_bs_get_integer_16; - } - OpCase(i_bs_get_integer_16_xfd): { - tmp_arg1 = xb(Arg(0)); + OpCase(i_bs_get_integer_16_xfd): + bs_get_integer_16_context = xb(Arg(0)); I++; - } - do_bs_get_integer_16: { - ErlBinMatchBuffer *_mb; - Eterm _result; - _mb = ms_matchbuffer(tmp_arg1); - if (_mb->size - _mb->offset < 16) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 16, 0, _mb); - } else { - _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset))); - _mb->offset += 16; + do_bs_get_integer_16: + { + ErlBinMatchBuffer *_mb; + Eterm _result; + _mb = ms_matchbuffer(bs_get_integer_16_context); + if (_mb->size - _mb->offset < 16) { + ClauseFail(); + } + if (BIT_OFFSET(_mb->offset) != 0) { + _result = erts_bs_get_integer_2(c_p, 16, 0, _mb); + } else { + _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset))); + _mb->offset += 16; + } + StoreBifResult(1, _result); } - StoreBifResult(1, _result); } - OpCase(i_bs_get_integer_32_rfId): { - tmp_arg1 = r(0); + { + Eterm bs_get_integer_32_context; + + OpCase(i_bs_get_integer_32_rfId): + bs_get_integer_32_context = r(0); goto do_bs_get_integer_32; - } + - OpCase(i_bs_get_integer_32_xfId): { - tmp_arg1 = xb(Arg(0)); + OpCase(i_bs_get_integer_32_xfId): + bs_get_integer_32_context = xb(Arg(0)); I++; - } - do_bs_get_integer_32: { - ErlBinMatchBuffer *_mb; - Uint32 _integer; - Eterm _result; - _mb = ms_matchbuffer(tmp_arg1); - if (_mb->size - _mb->offset < 32) { ClauseFail(); } - if (BIT_OFFSET(_mb->offset) != 0) { - _integer = erts_bs_get_unaligned_uint32(_mb); - } else { - _integer = get_int32(_mb->base + _mb->offset/8); - } - _mb->offset += 32; + + do_bs_get_integer_32: + { + ErlBinMatchBuffer *_mb; + Uint32 _integer; + Eterm _result; + _mb = ms_matchbuffer(bs_get_integer_32_context); + if (_mb->size - _mb->offset < 32) { ClauseFail(); } + if (BIT_OFFSET(_mb->offset) != 0) { + _integer = erts_bs_get_unaligned_uint32(_mb); + } else { + _integer = get_int32(_mb->base + _mb->offset/8); + } + _mb->offset += 32; #if !defined(ARCH_64) || HALFWORD_HEAP - if (IS_USMALL(0, _integer)) { + if (IS_USMALL(0, _integer)) { #endif - _result = make_small(_integer); + _result = make_small(_integer); #if !defined(ARCH_64) || HALFWORD_HEAP - } else { - TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); - _result = uint_to_big((Uint) _integer, HTOP); - HTOP += BIG_UINT_HEAP_SIZE; - HEAP_SPACE_VERIFIED(0); - } + } else { + TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); + _result = uint_to_big((Uint) _integer, HTOP); + HTOP += BIG_UINT_HEAP_SIZE; + HEAP_SPACE_VERIFIED(0); + } #endif - StoreBifResult(2, _result); + StoreBifResult(2, _result); + } } /* Operands: Size Live Fail Flags Dst */ @@ -4054,54 +4344,64 @@ apply_bif_or_nif_epilogue: StoreBifResult(3, result); } - /* Operands: MatchContext Fail Dst */ + { + Eterm get_utf8_context; + + /* Operands: MatchContext Fail Dst */ OpCase(i_bs_get_utf8_rfd): { - tmp_arg1 = r(0); - goto do_bs_get_utf8; - } + get_utf8_context = r(0); + goto do_bs_get_utf8; + } OpCase(i_bs_get_utf8_xfd): { - tmp_arg1 = xb(Arg(0)); - I++; - } + get_utf8_context = xb(Arg(0)); + I++; + } - /* - * tmp_arg1 = match_context - * Operands: Fail Dst - */ + /* + * get_utf8_context = match_context + * Operands: Fail Dst + */ - do_bs_get_utf8: { - Eterm result = erts_bs_get_utf8(ms_matchbuffer(tmp_arg1)); - if (is_non_value(result)) { - ClauseFail(); + do_bs_get_utf8: { + Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context)); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(1, result); } - StoreBifResult(1, result); } - /* Operands: MatchContext Fail Flags Dst */ + { + Eterm get_utf16_context; + + /* Operands: MatchContext Fail Flags Dst */ OpCase(i_bs_get_utf16_rfId): { - tmp_arg1 = r(0); - goto do_bs_get_utf16; - } + get_utf16_context = r(0); + goto do_bs_get_utf16; + } OpCase(i_bs_get_utf16_xfId): { - tmp_arg1 = xb(Arg(0)); - I++; - } + get_utf16_context = xb(Arg(0)); + I++; + } - /* - * tmp_arg1 = match_context - * Operands: Fail Flags Dst - */ - do_bs_get_utf16: { - Eterm result = erts_bs_get_utf16(ms_matchbuffer(tmp_arg1), Arg(1)); - if (is_non_value(result)) { - ClauseFail(); + /* + * get_utf16_context = match_context + * Operands: Fail Flags Dst + */ + do_bs_get_utf16: { + Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context), + Arg(1)); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(2, result); } - StoreBifResult(2, result); } { + Eterm context_to_binary_context; ErlBinMatchBuffer* mb; ErlSubBin* sb; Uint size; @@ -4110,27 +4410,29 @@ apply_bif_or_nif_epilogue: Uint hole_size; OpCase(bs_context_to_binary_r): { - tmp_arg1 = x0; + context_to_binary_context = x0; I -= 2; goto do_context_to_binary; } /* Unfortunately, inlining can generate this instruction. */ OpCase(bs_context_to_binary_y): { - tmp_arg1 = yb(Arg(0)); + context_to_binary_context = yb(Arg(0)); goto do_context_to_binary0; } OpCase(bs_context_to_binary_x): { - tmp_arg1 = xb(Arg(0)); + context_to_binary_context = xb(Arg(0)); do_context_to_binary0: I--; } do_context_to_binary: - if (is_boxed(tmp_arg1) && header_is_bin_matchstate(*boxed_val(tmp_arg1))) { - ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1); + if (is_boxed(context_to_binary_context) && + header_is_bin_matchstate(*boxed_val(context_to_binary_context))) { + ErlBinMatchState* ms; + ms = (ErlBinMatchState *) boxed_val(context_to_binary_context); mb = &ms->mb; offs = ms->save_offset[0]; size = mb->size - offs; @@ -4139,17 +4441,17 @@ apply_bif_or_nif_epilogue: Next(2); OpCase(i_bs_get_binary_all_reuse_rfI): { - tmp_arg1 = x0; + context_to_binary_context = x0; goto do_bs_get_binary_all_reuse; } OpCase(i_bs_get_binary_all_reuse_xfI): { - tmp_arg1 = xb(Arg(0)); + context_to_binary_context = xb(Arg(0)); I++; } do_bs_get_binary_all_reuse: - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(context_to_binary_context); size = mb->size - mb->offset; if (size % Arg(1) != 0) { ClauseFail(); @@ -4158,7 +4460,7 @@ apply_bif_or_nif_epilogue: do_bs_get_binary_all_reuse_common: orig = mb->orig; - sb = (ErlSubBin *) boxed_val(tmp_arg1); + sb = (ErlSubBin *) boxed_val(context_to_binary_context); hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; sb->thing_word = HEADER_SUB_BIN; sb->size = BYTE_OFFSET(size); @@ -4174,12 +4476,14 @@ apply_bif_or_nif_epilogue: } { + Eterm match_string_context; + OpCase(i_bs_match_string_rfII): { - tmp_arg1 = r(0); + match_string_context = r(0); goto do_bs_match_string; } OpCase(i_bs_match_string_xfII): { - tmp_arg1 = xb(Arg(0)); + match_string_context = xb(Arg(0)); I++; } @@ -4194,7 +4498,7 @@ apply_bif_or_nif_epilogue: PreFetch(3, next); bits = Arg(1); bytes = (byte *) Arg(2); - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(match_string_context); if (mb->size - mb->offset < bits) { ClauseFail(); } @@ -4307,7 +4611,7 @@ apply_bif_or_nif_epilogue: ASSERT(is_CP((BeamInstr)(ep->code))); ASSERT(is_internal_pid(c_p->tracer_proc) || is_internal_port(c_p->tracer_proc)); - E[2] = make_cp(c_p->cp); /* XXX:PaN - code in lower range on halfword */ + E[2] = make_cp(c_p->cp); /* Code in lower range on halfword */ E[1] = am_true; /* Process tracer */ E[0] = make_cp(ep->code); c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) @@ -4338,35 +4642,109 @@ apply_bif_or_nif_epilogue: OpCase(i_count_breakpoint): { BeamInstr real_I; - ErtsCountBreak((BeamInstr *) I, &real_I); + ErtsCountBreak(c_p, (BeamInstr *) I, &real_I); + ASSERT(VALID_INSTR(real_I)); + Goto(real_I); + } + + /* need to send mfa instead of bdt pointer + * the pointer might be deallocated. + */ + + OpCase(i_time_breakpoint): { + BeamInstr real_I; + BpData **bds = (BpData **) (I)[-4]; + BpDataTime *bdt = NULL; + Uint ix = 0; +#ifdef ERTS_SMP + ix = c_p->scheduler_data->no - 1; +#else + ix = 0; +#endif + bdt = (BpDataTime *)bds[ix]; + + ASSERT((I)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + ASSERT(bdt); + bdt = (BpDataTime *) bdt->next; + ASSERT(bdt); + bds[ix] = (BpData *) bdt; + real_I = bdt->orig_instr; ASSERT(VALID_INSTR(real_I)); + + if (IS_TRACED_FL(c_p, F_TRACE_CALLS) && !(bdt->pause)) { + if ( (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) || + (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) || + (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace))) { + /* This _IS_ a tail recursive call */ + SWAPOUT; + erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_TAIL_CALL); + SWAPIN; + } else { + SWAPOUT; + erts_trace_time_break(c_p, I, bdt, ERTS_BP_CALL_TIME_CALL); + + /* r register needs to be copied to the array + * for the garbage collector + */ + ASSERT(c_p->htop <= E && E <= c_p->hend); + if (E - 2 < HTOP) { + reg[0] = r(0); + PROCESS_MAIN_CHK_LOCKS(c_p); + FCALLS -= erts_garbage_collect(c_p, 2, reg, I[-1]); + PROCESS_MAIN_CHK_LOCKS(c_p); + r(0) = reg[0]; + } + SWAPIN; + + ASSERT(c_p->htop <= E && E <= c_p->hend); + + E -= 2; + E[0] = make_cp(I); + E[1] = make_cp(c_p->cp); /* original return address */ + c_p->cp = beam_return_time_trace; + } + } + Goto(real_I); } + OpCase(i_return_time_trace): { + BeamInstr *pc = (BeamInstr *) (UWord) E[0]; + SWAPOUT; + erts_trace_time_break(c_p, pc, NULL, ERTS_BP_CALL_TIME_RETURN); + SWAPIN; + c_p->cp = NULL; + SET_I((BeamInstr *) cp_val(E[1])); + E += 2; + Goto(*I); + } + OpCase(i_trace_breakpoint): if (! IS_TRACED_FL(c_p, F_TRACE_CALLS)) { BeamInstr real_I; - ErtsBreakSkip((BeamInstr *) I, &real_I); + ErtsBreakSkip(c_p, (BeamInstr *) I, &real_I); Goto(real_I); } /* Fall through to next case */ OpCase(i_mtrace_breakpoint): { - Uint real_I; + BeamInstr real_I; Uint32 flags; Eterm tracer_pid; - Uint *cpp; + Uint* cpp; int return_to_trace = 0, need = 0; flags = 0; SWAPOUT; reg[0] = r(0); if (*(c_p->cp) == (BeamInstr) OpCode(return_trace)) { - cpp = (Uint*)&E[2]; - } else if (*(c_p->cp) - == (BeamInstr) OpCode(i_return_to_trace)) { + cpp = &E[2]; + } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_to_trace)) { + return_to_trace = !0; + cpp = &E[0]; + } else if (*(c_p->cp) == (BeamInstr) OpCode(i_return_time_trace)) { return_to_trace = !0; - cpp = (Uint*)&E[0]; + cpp = &E[0]; } else { cpp = NULL; } @@ -4383,6 +4761,8 @@ apply_bif_or_nif_epilogue: } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_to_trace)) { return_to_trace = !0; cpp += 1; + } else if (*cp_val(*cpp) == (BeamInstr) OpCode(i_return_time_trace)) { + cpp += 2; } else break; } @@ -4581,7 +4961,7 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } - OpCase(fmove_new_ld): { + OpCase(fmove_ld): { Eterm fr = Arg(0); Eterm dest = make_float(HTOP); @@ -4611,11 +4991,6 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } - /* - * Old allocating fmove. - */ - - #ifdef NO_FPE_SIGNALS OpCase(fclearerror): OpCase(i_fcheckerror): @@ -4747,7 +5122,7 @@ apply_bif_or_nif_epilogue: neg_o_reds = -c_p->def_arg_reg[4]; FCALLS = c_p->fcalls; SWAPIN; - switch( c_p->def_arg_reg[3] ) { /* XXX:PaN - Halfword wont work with hipe yet... */ + switch( c_p->def_arg_reg[3] ) { /* Halfword wont work with hipe yet! */ case HIPE_MODE_SWITCH_RES_RETURN: ASSERT(is_value(reg[0])); MoveReturn(reg[0], r(0)); @@ -4827,12 +5202,11 @@ apply_bif_or_nif_epilogue: OpCase(i_debug_breakpoint): { SWAPOUT; reg[0] = r(0); - tmp_arg1 = call_breakpoint_handler(c_p, I-3, reg); + I = call_error_handler(c_p, I-3, reg, am_breakpoint); r(0) = reg[0]; SWAPIN; - if (tmp_arg1) { - SET_I(c_p->i); - Dispatch(); + if (I) { + Goto(*I); } goto no_error_handler; } @@ -4892,13 +5266,15 @@ apply_bif_or_nif_epilogue: em_call_error_handler = OpCode(call_error_handler); em_call_traced_function = OpCode(call_traced_function); em_apply_bif = OpCode(apply_bif); - beam_apply[0] = (BeamInstr) OpCode(i_apply); - beam_apply[1] = (BeamInstr) OpCode(normal_exit); - beam_exit[0] = (BeamInstr) OpCode(error_action_code); - beam_continue_exit[0] = (BeamInstr) OpCode(continue_exit); - beam_return_to_trace[0] = (BeamInstr) OpCode(i_return_to_trace); - beam_return_trace[0] = (BeamInstr) OpCode(return_trace); - beam_exception_trace[0] = (BeamInstr) OpCode(return_trace); /* UGLY */ + + beam_apply[0] = (BeamInstr) OpCode(i_apply); + beam_apply[1] = (BeamInstr) OpCode(normal_exit); + beam_exit[0] = (BeamInstr) OpCode(error_action_code); + beam_continue_exit[0] = (BeamInstr) OpCode(continue_exit); + beam_return_to_trace[0] = (BeamInstr) OpCode(i_return_to_trace); + beam_return_trace[0] = (BeamInstr) OpCode(return_trace); + beam_exception_trace[0] = (BeamInstr) OpCode(return_trace); /* UGLY */ + beam_return_time_trace[0] = (BeamInstr) OpCode(i_return_time_trace); /* * Enter all BIFs into the export table. @@ -4910,6 +5286,8 @@ apply_bif_or_nif_epilogue: bif_export[i] = ep; ep->code[3] = (BeamInstr) OpCode(apply_bif); ep->code[4] = (BeamInstr) bif_table[i].f; + /* XXX: set func info for bifs */ + ((BeamInstr*)ep->code + 3)[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; @@ -4954,6 +5332,10 @@ translate_gc_bif(void* gcf) return round_1; } else if (gcf == erts_gc_trunc_1) { return round_1; + } else if (gcf == erts_gc_binary_part_2) { + return binary_part_2; + } else if (gcf == erts_gc_binary_part_3) { + return binary_part_3; } else { erl_exit(1, "bad gc bif"); } @@ -5098,14 +5480,18 @@ next_catch(Process* c_p, Eterm *reg) { int active_catches = c_p->catches > 0; int have_return_to_trace = 0; Eterm *ptr, *prev, *return_to_trace_ptr = NULL; - BeamInstr i_return_trace = beam_return_trace[0]; - BeamInstr i_return_to_trace = beam_return_to_trace[0]; + + BeamInstr i_return_trace = beam_return_trace[0]; + BeamInstr i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_time_trace = beam_return_time_trace[0]; + ptr = prev = c_p->stop; ASSERT(is_CP(*ptr)); ASSERT(ptr <= STACK_START(c_p)); if (ptr == STACK_START(c_p)) return NULL; if ((is_not_CP(*ptr) || (*cp_val(*ptr) != i_return_trace && - *cp_val(*ptr) != i_return_to_trace)) + *cp_val(*ptr) != i_return_to_trace && + *cp_val(*ptr) != i_return_time_trace )) && c_p->cp) { /* Can not follow cp here - code may be unloaded */ BeamInstr *cpp = c_p->cp; @@ -5117,6 +5503,9 @@ next_catch(Process* c_p, Eterm *reg) { } else if (cpp == beam_return_trace) { /* Skip return_trace parameters */ ptr += 2; + } else if (cpp == beam_return_time_trace) { + /* Skip return_trace parameters */ + ptr += 1; } else if (cpp == beam_return_to_trace) { have_return_to_trace = !0; /* Record next cp */ } @@ -5146,6 +5535,13 @@ next_catch(Process* c_p, Eterm *reg) { } have_return_to_trace = !0; /* Record next cp */ return_to_trace_ptr = NULL; + } else if (*cp_val(*prev) == i_return_time_trace) { + /* Skip stack frame variables */ + while (++ptr, ptr < STACK_START(c_p) && is_not_CP(*ptr)) { + if (is_catch(*ptr) && active_catches) goto found_catch; + } + /* Skip return_trace parameters */ + ptr += 1; } else { if (have_return_to_trace) { /* Record this cp as possible return_to trace cp */ @@ -5560,8 +5956,8 @@ build_stacktrace(Process* c_p, Eterm exc) { } -static Eterm -call_error_handler(Process* p, BeamInstr* fi, Eterm* reg) +static BeamInstr* +call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) { Eterm* hp; Export* ep; @@ -5573,14 +5969,12 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg) /* * Search for the error_handler module. */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_undefined_function, 3); + ep = erts_find_function(erts_proc_get_error_handler(p), func, 3); if (ep == NULL) { /* No error handler */ p->current = fi; p->freason = EXC_UNDEF; return 0; } - p->i = ep->address; /* * Create a list with all arguments in the x registers. @@ -5600,63 +5994,14 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg) } /* - * Set up registers for call to error_handler:undefined_function/3. + * Set up registers for call to error_handler:<func>/3. */ reg[0] = fi[0]; reg[1] = fi[1]; reg[2] = args; - return 1; -} - -static Eterm -call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg) -{ - Eterm* hp; - Export* ep; - int arity; - Eterm args; - Uint sz; - int i; - - /* - * Search for error handler module. - */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_breakpoint, 3); - if (ep == NULL) { /* No error handler */ - p->current = fi; - p->freason = EXC_UNDEF; - return 0; - } - p->i = ep->address; - - /* - * Create a list with all arguments in the x registers. - */ - - arity = fi[2]; - sz = 2 * arity; - if (HeapWordsLeft(p) < sz) { - erts_garbage_collect(p, sz, reg, arity); - } - hp = HEAP_TOP(p); - HEAP_TOP(p) += sz; - args = NIL; - for (i = arity-1; i >= 0; i--) { - args = CONS(hp, reg[i], args); - hp += 2; - } - - /* - * Set up registers for call to error_handler:breakpoint/3. - */ - reg[0] = fi[0]; - reg[1] = fi[1]; - reg[2] = args; - return 1; + return ep->address; } - static Export* apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg) @@ -6159,8 +6504,8 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) erts_refc_inc(&fe->refc, 2); funp->thing_word = HEADER_FUN; #ifndef HYBRID /* FIND ME! */ - funp->next = MSO(p).funs; - MSO(p).funs = funp; + funp->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*) funp; #endif funp->fe = fe; funp->num_free = num_free; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 5e4375fc96..4e0d19dafa 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -89,13 +89,12 @@ typedef struct { } Label; /* - * Type for a operand for a generic instruction. + * Type for an operand for a generic instruction. */ typedef struct { unsigned type; /* Type of operand. */ - BeamInstr val; /* Value of operand. */ - Uint bigarity; /* Arity for bignumbers (only). */ + BeamInstr val; /* Value of operand. */ } GenOpArg; /* @@ -326,11 +325,6 @@ typedef struct { Literal* literals; /* Array of literals. */ LiteralPatch* literal_patches; /* Operands that need to be patched. */ Uint total_literal_size; /* Total heap size for all literals. */ - - /* - * Floating point. - */ - int new_float_instructions; /* New allocation scheme for floating point. */ } LoaderState; typedef struct { @@ -476,19 +470,18 @@ static int read_code_header(LoaderState* stp); static int load_code(LoaderState* stp); static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOpArg Tuple, GenOpArg Dst); -static GenOp* gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, +static GenOp* gen_split_values(LoaderState* stp, GenOpArg S, + GenOpArg TypeFail, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); static GenOp* gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); -static GenOp* gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail, - GenOpArg Size, GenOpArg* Rest); +static GenOp* gen_select_literals(LoaderState* stp, GenOpArg S, + GenOpArg Fail, GenOpArg Size, + GenOpArg* Rest); static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); static GenOp* gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg Func, GenOpArg arity, GenOpArg label); -static GenOp* -gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, - GenOpArg Src, GenOpArg Dst); static int freeze_code(LoaderState* stp); @@ -821,7 +814,6 @@ init_state(LoaderState* stp) stp->total_literal_size = 0; stp->literal_patches = 0; stp->string_patches = 0; - stp->new_float_instructions = 0; stp->may_load_nif = 0; stp->on_load = 0; } @@ -1382,8 +1374,10 @@ read_code_header(LoaderState* stp) stp->ci = MI_FUNCTIONS + stp->num_functions + 1; stp->code[MI_ATTR_PTR] = 0; + stp->code[MI_ATTR_SIZE] = 0; stp->code[MI_ATTR_SIZE_ON_HEAP] = 0; stp->code[MI_COMPILE_PTR] = 0; + stp->code[MI_COMPILE_SIZE] = 0; stp->code[MI_COMPILE_SIZE_ON_HEAP] = 0; stp->code[MI_NUM_BREAKPOINTS] = 0; @@ -1569,7 +1563,8 @@ load_code(LoaderState* stp) case 0: /* Floating point number */ { Eterm* hp; -#if !defined(ARCH_64) || HALFWORD_HEAP /* XXX:PaN - Should use ARCH_64 variant instead */ +/* XXX:PaN - Halfword should use ARCH_64 variant instead */ +#if !defined(ARCH_64) || HALFWORD_HEAP Uint high, low; # endif last_op->a[arg].val = new_literal(stp, &hp, @@ -1618,7 +1613,6 @@ load_code(LoaderState* stp) BeamInstr val; BeamInstr words = 0; - stp->new_float_instructions = 1; GetTagAndValue(stp, tag, n); VerifyTag(stp, tag, TAG_u); while (n-- > 0) { @@ -1772,7 +1766,7 @@ load_code(LoaderState* stp) } stp->specific_op = specific; - CodeNeed(opc[stp->specific_op].sz+2); /* Extra margin for packing */ + CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */ code[ci++] = BeamOpCode(stp->specific_op); } @@ -1936,7 +1930,8 @@ load_code(LoaderState* stp) } code[ci++] = (BeamInstr) stp->import[i].bf; break; - case 'P': /* Byte offset into tuple */ /* XXX:PaN - * sizeof(Eterm or Eterm *) ? */ + case 'P': /* Byte offset into tuple or stack */ + case 'Q': /* Like 'P', but packable */ VerifyTag(stp, tag, TAG_u); tmp = tmp_op->a[arg].val; code[ci++] = (BeamInstr) ((tmp_op->a[arg].val+1) * sizeof(Eterm)); @@ -1957,84 +1952,6 @@ load_code(LoaderState* stp) } /* - * Load any list arguments using the primitive tags. - */ - - for ( ; arg < tmp_op->arity; arg++) { - switch (tmp_op->a[arg].type) { - case TAG_i: - CodeNeed(1); - code[ci++] = make_small(tmp_op->a[arg].val); - break; - case TAG_u: - case TAG_a: - case TAG_v: - CodeNeed(1); - code[ci++] = tmp_op->a[arg].val; - break; - case TAG_f: - CodeNeed(1); - code[ci] = stp->labels[tmp_op->a[arg].val].patches; - stp->labels[tmp_op->a[arg].val].patches = ci; - ci++; - break; - case TAG_q: - { - Eterm lit; - - lit = stp->literals[tmp_op->a[arg].val].term; - if (is_big(lit)) { - Eterm* bigp; - Eterm *tmp; - Uint size; - Uint term_size; - - bigp = big_val(lit); - term_size = bignum_header_arity(*bigp); - size = TermWords(term_size + 1); - CodeNeed(size); - tmp = (Eterm *) (code + ci); - *tmp++ = *bigp++; - while (term_size-- > 0) { - *tmp++ = *bigp++; - } - ci +=size; - } else if (is_float(lit)) { -#if defined(ARCH_64) && !HALFWORD_HEAP - CodeNeed(1); - code[ci++] = float_val(stp->literals[tmp_op->a[arg].val].term)[1]; -#elif HALFWORD_HEAP - Eterm* fptr; - Uint size; - Eterm *tmp; - - fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1; - size = TermWords(2); - CodeNeed(size); - tmp = (Eterm *) (code + ci); - *tmp++ = *fptr++; - *tmp = *fptr; - ci += size; -#else - Eterm* fptr; - - fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1; - CodeNeed(2); - code[ci++] = *fptr++; - code[ci++] = *fptr; -#endif - } else { - LoadError0(stp, "literal is neither float nor big"); - } - } - break; - default: - LoadError1(stp, "unsupported primitive type '%c'", - tag_to_letter[tmp_op->a[arg].type]); - } - } - - /* * The packing engine. */ if (opc[stp->specific_op].pack[0]) { @@ -2057,6 +1974,11 @@ load_code(LoaderState* stp) case '6': /* Shift 16 steps */ packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci]; break; +#ifdef ARCH_64 + case 'w': /* Shift 32 steps */ + packed = (packed << BEAM_WIDE_SHIFT) | code[--ci]; + break; +#endif case 'p': /* Put instruction (from stack). */ code[ci++] = *--sp; break; @@ -2072,6 +1994,58 @@ load_code(LoaderState* stp) } /* + * Load any list arguments using the primitive tags. + */ + + for ( ; arg < tmp_op->arity; arg++) { + switch (tmp_op->a[arg].type) { + case TAG_i: + CodeNeed(1); + code[ci++] = make_small(tmp_op->a[arg].val); + break; + case TAG_u: + case TAG_a: + case TAG_v: + CodeNeed(1); + code[ci++] = tmp_op->a[arg].val; + break; + case TAG_f: + CodeNeed(1); + code[ci] = stp->labels[tmp_op->a[arg].val].patches; + stp->labels[tmp_op->a[arg].val].patches = ci; + ci++; + break; + case TAG_r: + CodeNeed(1); + code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) | + TAG_PRIMARY_HEADER; + break; + case TAG_x: + CodeNeed(1); + code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | + (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + break; + case TAG_y: + CodeNeed(1); + code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | + (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + break; + case TAG_n: + CodeNeed(1); + code[ci++] = NIL; + break; + case TAG_q: + CodeNeed(1); + new_literal_patch(stp, ci); + code[ci++] = tmp_op->a[arg].val; + break; + default: + LoadError1(stp, "unsupported primitive type '%c'", + tag_to_letter[tmp_op->a[arg].type]); + } + } + + /* * Handle a few special cases. */ switch (stp->specific_op) { @@ -2239,11 +2213,12 @@ use_jump_tab(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) } /* - * Predicate to test whether all values in a table are big numbers. + * Predicate to test whether all values in a table are either + * floats or bignums. */ static int -all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) +floats_or_bignums(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) { int i; @@ -2255,9 +2230,6 @@ all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) if (Rest[i].type != TAG_q) { return 0; } - if (is_not_big(stp->literals[Rest[i].val].term)) { - return 0; - } if (Rest[i+1].type != TAG_f) { return 0; } @@ -2317,6 +2289,14 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) return 0; } +static int +same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label) +{ + return Target.type = TAG_f && Label.type == TAG_u && + Target.val == Label.val; +} + + /* * Generate an instruction for element/2. */ @@ -2328,23 +2308,23 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_element_4; op->arity = 4; - op->a[0] = Fail; - op->a[1] = Index; - op->a[2] = Tuple; - op->a[3] = Dst; op->next = NULL; - /* - * If safe, generate a faster instruction. - */ - if (Index.type == TAG_i && Index.val > 0 && (Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) { op->op = genop_i_fast_element_4; - op->a[1].type = TAG_u; - op->a[1].val = Index.val; + op->a[0] = Tuple; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = Index.val; + op->a[3] = Dst; + } else { + op->op = genop_i_element_4; + op->a[0] = Tuple; + op->a[1] = Fail; + op->a[2] = Index; + op->a[3] = Dst; } return op; @@ -2595,8 +2575,6 @@ binary_too_big_bits(LoaderState* stp, GenOpArg Size) return Size.type == TAG_u && (((Size.val+7)/8) >> (8*sizeof(Uint)-3) != 0); } -#define new_float_allocation(Stp) ((Stp)->new_float_instructions) - static GenOp* gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size, GenOpArg Unit, GenOpArg Flags, GenOpArg Src) @@ -2809,6 +2787,52 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, return op; } +static GenOp* +gen_increment(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, + GenOpArg Live, GenOpArg Dst) +{ + GenOp* op; + + NEW_GENOP(stp, op); + op->op = genop_i_increment_4; + op->arity = 4; + op->next = NULL; + op->a[0] = Reg; + op->a[1].type = TAG_u; + op->a[1].val = Integer.val; + op->a[2] = Live; + op->a[3] = Dst; + return op; +} + +static GenOp* +gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, + GenOpArg Live, GenOpArg Dst) +{ + GenOp* op; + + NEW_GENOP(stp, op); + op->op = genop_i_increment_4; + op->arity = 4; + op->next = NULL; + op->a[0] = Reg; + op->a[1].type = TAG_u; + op->a[1].val = -Integer.val; + op->a[2] = Live; + op->a[3] = Dst; + return op; +} + +/* + * Test whether the negation of the given number is small. + */ +static int +negation_is_small(LoaderState* stp, GenOpArg Int) +{ + return Int.type == TAG_i && IS_SSMALL(-Int.val); +} + + static int smp(LoaderState* stp) { @@ -3000,6 +3024,21 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, ASSERT(op->a[i].val < op->a[i+2].val); } #endif + + /* + * Use a special-cased instruction if there are only two values. + */ + if (size == 2) { + op->op = genop_i_select_tuple_arity2_6; + op->arity--; + op->a[2].type = TAG_u; + op->a[2].val = arityval(op->a[3].val); + op->a[3] = op->a[4]; + op->a[4].type = TAG_u; + op->a[4].val = arityval(op->a[5].val); + op->a[5] = op->a[6]; + } + return op; } @@ -3009,18 +3048,24 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, */ static GenOp* -gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, - GenOpArg Size, GenOpArg* Rest) +gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail, + GenOpArg Fail, GenOpArg Size, GenOpArg* Rest) { GenOp* op1; GenOp* op2; GenOp* label; - Uint type; + GenOp* is_integer; int i; ASSERT(Size.val >= 2 && Size.val % 2 == 0); + NEW_GENOP(stp, is_integer); + is_integer->op = genop_is_integer_2; + is_integer->arity = 2; + is_integer->a[0] = TypeFail; + is_integer->a[1] = S; + NEW_GENOP(stp, label); label->op = genop_label_1; label->arity = 1; @@ -3046,15 +3091,13 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, op2->a[2].type = TAG_u; op2->a[2].val = 0; - op1->next = label; - label->next = op2; - op2->next = NULL; - - type = Rest[0].type; + /* + * Split the list. + */ ASSERT(Size.type == TAG_u); for (i = 0; i < Size.val; i += 2) { - GenOp* op = (Rest[i].type == type) ? op1 : op2; + GenOp* op = (Rest[i].type == TAG_q) ? op2 : op1; int dst = 3 + op->a[2].val; ASSERT(Rest[i+1].type == TAG_f); @@ -3063,13 +3106,36 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, op->arity += 2; op->a[2].val += 2; } + ASSERT(op1->a[2].val > 0); + ASSERT(op2->a[2].val > 0); /* - * None of the instructions should have zero elements in the list. + * Order the instruction sequence appropriately. */ - ASSERT(op1->a[2].val > 0); - ASSERT(op2->a[2].val > 0); + if (TypeFail.val == Fail.val) { + /* + * select_val L1 S ... (small numbers) + * label L1 + * is_integer Fail S + * select_val Fail S ... (bignums) + */ + op1->next = label; + label->next = is_integer; + is_integer->next = op2; + } else { + /* + * is_integer TypeFail S + * select_val L1 S ... (small numbers) + * label L1 + * select_val Fail S ... (bignums) + */ + is_integer->next = op1; + op1->next = label; + label->next = op2; + op1 = is_integer; + } + op2->next = NULL; return op1; } @@ -3091,6 +3157,29 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr ASSERT(Size.val >= 2 && Size.val % 2 == 0); /* + * If there is only one choice, don't generate a jump table. + */ + if (Size.val == 2) { + GenOp* jump; + + NEW_GENOP(stp, op); + op->arity = 3; + op->op = genop_is_ne_exact_3; + op->a[0] = Rest[1]; + op->a[1] = S; + op->a[2] = Rest[0]; + + NEW_GENOP(stp, jump); + jump->next = NULL; + jump->arity = 1; + jump->op = genop_jump_1; + jump->a[0] = Fail; + + op->next = jump; + return op; + } + + /* * Calculate the minimum and maximum values and size of jump table. */ @@ -3162,8 +3251,9 @@ genopargcompare(GenOpArg* a, GenOpArg* b) } /* - * Generate a select_val instruction. We know that a jump table is not suitable, - * and that all values are of the same type (integer, atoms, floats; never bignums). + * Generate a select_val instruction. We know that a jump table + * is not suitable, and that all values are of the same type + * (integer or atoms). */ static GenOp* @@ -3177,12 +3267,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, NEW_GENOP(stp, op); op->next = NULL; - if (Rest[0].type != TAG_q) { - op->op = genop_i_select_val_3; - } else { - ASSERT(is_float(stp->literals[Rest[0].val].term)); - op->op = genop_i_select_float_3; - } + op->op = genop_i_select_val_3; GENOP_ARITY(op, arity); op->a[0] = S; op->a[1] = Fail; @@ -3204,19 +3289,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, } #endif - return op; -} - -/* - * Compare function for qsort(). - */ + /* + * Use a special-cased instruction if there are only two values. + */ + if (size == 2) { + op->op = genop_i_select_val2_6; + op->arity--; + op->a[2] = op->a[3]; + op->a[3] = op->a[4]; + op->a[4] = op->a[5]; + op->a[5] = op->a[6]; + } -static int -genbigcompare(GenOpArg* a, GenOpArg* b) -{ - int val = (int)(b->bigarity - a->bigarity); - - return val != 0 ? val : ((int) (a->val - b->val)); + return op; } /* @@ -3224,37 +3309,35 @@ genbigcompare(GenOpArg* a, GenOpArg* b) */ static GenOp* -gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail, +gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest) { GenOp* op; - int arity = Size.val + 2 + 1; - int size = Size.val / 2; + GenOp* jump; + GenOp** prev_next = &op; + int i; - NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_select_big_2; - GENOP_ARITY(op, arity); - op->a[0] = S; - op->a[1] = Fail; for (i = 0; i < Size.val; i += 2) { + GenOp* op; ASSERT(Rest[i].type == TAG_q); - op->a[i+2] = Rest[i]; - op->a[i+2].bigarity = *big_val(stp->literals[op->a[i+2].val].term); - op->a[i+3] = Rest[i+1]; - } - ASSERT(i+2 == arity-1); - op->a[arity-1].type = TAG_u; - op->a[arity-1].val = 0; - - /* - * Sort the values in descending arity order. - */ - - qsort(op->a+2, size, 2*sizeof(GenOpArg), - (int (*)(const void *, const void *)) genbigcompare); + NEW_GENOP(stp, op); + op->op = genop_is_ne_exact_3; + op->arity = 3; + op->a[0] = Rest[i+1]; + op->a[1] = S; + op->a[2] = Rest[i]; + *prev_next = op; + prev_next = &op->next; + } + + NEW_GENOP(stp, jump); + jump->next = NULL; + jump->op = genop_jump_1; + jump->arity = 1; + jump->a[0] = Fail; + *prev_next = jump; return op; } @@ -3272,7 +3355,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, int i; ASSERT(Size.type == TAG_u); - ASSERT(S.type == TAG_q); NEW_GENOP(stp, op); op->next = NULL; @@ -3283,18 +3365,32 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, * Search for a literal matching the controlling expression. */ - if (S.type == TAG_q) { - Eterm expr = stp->literals[S.val].term; - for (i = 0; i < Size.val; i += 2) { - if (Rest[i].type == TAG_q) { - Eterm term = stp->literals[Rest[i].val].term; - if (eq(term, expr)) { - ASSERT(Rest[i+1].type == TAG_f); - op->a[0] = Rest[i+1]; - return op; + switch (S.type) { + case TAG_q: + { + Eterm expr = stp->literals[S.val].term; + for (i = 0; i < Size.val; i += 2) { + if (Rest[i].type == TAG_q) { + Eterm term = stp->literals[Rest[i].val].term; + if (eq(term, expr)) { + ASSERT(Rest[i+1].type == TAG_f); + op->a[0] = Rest[i+1]; + return op; + } } } } + break; + case TAG_i: + case TAG_a: + for (i = 0; i < Size.val; i += 2) { + if (Rest[i].val == S.val && Rest[i].type == S.type) { + ASSERT(Rest[i+1].type == TAG_f); + op->a[0] = Rest[i+1]; + return op; + } + } + break; } /* @@ -3358,9 +3454,15 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) op->next = NULL; return op; } - +/* + * Rewrite gc_bifs with one parameter (the common case). Utilized + * in ops.tab to rewrite instructions calling bif's in guards + * to use a garbage collecting implementation. The instructions + * are sometimes once again rewritten to handle literals (putting the + * parameter in the mostly unused r[0] before the instruction is executed). + */ static GenOp* -gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, +gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg Src, GenOpArg Dst) { GenOp* op; @@ -3372,6 +3474,8 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, op->a[0] = Fail; op->a[1].type = TAG_u; bf = stp->import[Bif.val].bf; + /* The translations here need to have a reverse counterpart in + beam_emu.c:translate_gc_bif for error handling to work properly. */ if (bf == length_1) { op->a[1].val = (BeamInstr) (void *) erts_gc_length_1; } else if (bf == size_1) { @@ -3398,6 +3502,127 @@ gen_guard_bif(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, return op; } +/* + * This is used by the ops.tab rule that rewrites gc_bifs with two parameters + * The instruction returned is then again rewritten to an i_load instruction + * folowed by i_gc_bif2_jIId, to handle literals properly. + * As opposed to the i_gc_bif1_jIsId, the instruction i_gc_bif2_jIId is + * always rewritten, regardless of if there actually are any literals. + */ +static GenOp* +gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, + GenOpArg S1, GenOpArg S2, GenOpArg Dst) +{ + GenOp* op; + BifFunction bf; + + NEW_GENOP(stp, op); + op->op = genop_ii_gc_bif2_6; + op->arity = 6; + op->a[0] = Fail; + op->a[1].type = TAG_u; + bf = stp->import[Bif.val].bf; + /* The translations here need to have a reverse counterpart in + beam_emu.c:translate_gc_bif for error handling to work properly. */ + if (bf == binary_part_2) { + op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2; + } else { + abort(); + } + op->a[2] = S1; + op->a[3] = S2; + op->a[4] = Live; + op->a[5] = Dst; + op->next = NULL; + return op; +} + +/* + * This is used by the ops.tab rule that rewrites gc_bifs with three parameters + * The instruction returned is then again rewritten to a move instruction that + * uses r[0] for temp storage, followed by an i_load instruction, + * folowed by i_gc_bif3_jIsId, to handle literals properly. Rewriting + * always occur, as with the gc_bif2 counterpart. + */ +static GenOp* +gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, + GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst) +{ + GenOp* op; + BifFunction bf; + + NEW_GENOP(stp, op); + op->op = genop_ii_gc_bif3_7; + op->arity = 7; + op->a[0] = Fail; + op->a[1].type = TAG_u; + bf = stp->import[Bif.val].bf; + /* The translations here need to have a reverse counterpart in + beam_emu.c:translate_gc_bif for error handling to work properly. */ + if (bf == binary_part_3) { + op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3; + } else { + abort(); + } + op->a[2] = S1; + op->a[3] = S2; + op->a[4] = S3; + op->a[5] = Live; + op->a[6] = Dst; + op->next = NULL; + return op; +} + +static GenOp* +tuple_append_put5(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, + GenOpArg* Puts, GenOpArg S1, GenOpArg S2, GenOpArg S3, + GenOpArg S4, GenOpArg S5) +{ + GenOp* op; + int arity = Arity.val; /* Arity of tuple, not the instruction */ + int i; + + NEW_GENOP(stp, op); + op->next = NULL; + GENOP_ARITY(op, arity+2+5); + op->op = genop_i_put_tuple_2; + op->a[0] = Dst; + op->a[1].type = TAG_u; + op->a[1].val = arity + 5; + for (i = 0; i < arity; i++) { + op->a[i+2] = Puts[i]; + } + op->a[arity+2] = S1; + op->a[arity+3] = S2; + op->a[arity+4] = S3; + op->a[arity+5] = S4; + op->a[arity+6] = S5; + return op; +} + +static GenOp* +tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, + GenOpArg* Puts, GenOpArg S) +{ + GenOp* op; + int arity = Arity.val; /* Arity of tuple, not the instruction */ + int i; + + NEW_GENOP(stp, op); + op->next = NULL; + GENOP_ARITY(op, arity+2+1); + op->op = genop_i_put_tuple_2; + op->a[0] = Dst; + op->a[1].type = TAG_u; + op->a[1].val = arity + 1; + for (i = 0; i < arity; i++) { + op->a[i+2] = Puts[i]; + } + op->a[arity+2] = S; + return op; +} + + /* * Freeze the code in memory, move the string table into place, @@ -3464,7 +3689,7 @@ freeze_code(LoaderState* stp) literal_end = (Uint *) (code+stp->ci); /* * Place the literal heap directly after the code and fix up all - * put_literal instructions that refer to it. + * instructions that refer to it. */ { Uint* ptr; @@ -3797,11 +4022,23 @@ transform_engine(LoaderState* st) if (i == 0) goto restart; break; +#if defined(TOP_is_eq) case TOP_is_eq: ASSERT(ap < instr->arity); if (*pc++ != instr->a[ap].val) goto restart; break; +#endif + case TOP_is_type_eq: + mask = *pc++; + + ASSERT(ap < instr->arity); + ASSERT(instr->a[ap].type < BEAM_NUM_TAGS); + if (((1 << instr->a[ap].type) & mask) == 0) + goto restart; + if (*pc++ != instr->a[ap].val) + goto restart; + break; case TOP_is_same_var: ASSERT(ap < instr->arity); i = *pc++; @@ -3922,14 +4159,17 @@ transform_engine(LoaderState* st) case TOP_rest_args: { int n = *pc++; + int formal_arity = gen_opc[instr->op].arity; + int num_vars = n + (instr->arity - formal_arity); + int j = formal_arity; + var = erts_alloc(ERTS_ALC_T_LOADER_TMP, - instr->arity * sizeof(GenOpArg)); + num_vars * sizeof(GenOpArg)); for (i = 0; i < n; i++) { var[i] = def_vars[i]; } - while (i < instr->arity) { - var[i] = instr->a[i]; - i++; + while (i < num_vars) { + var[i++] = instr->a[j++]; } } break; @@ -5122,8 +5362,10 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) code[MI_NUM_FUNCTIONS] = n; code[MI_ATTR_PTR] = 0; + code[MI_ATTR_SIZE] = 0; code[MI_ATTR_SIZE_ON_HEAP] = 0; code[MI_COMPILE_PTR] = 0; + code[MI_COMPILE_SIZE] = 0; code[MI_COMPILE_SIZE_ON_HEAP] = 0; code[MI_NUM_BREAKPOINTS] = 0; code[MI_ON_LOAD_FUNCTION_PTR] = 0; @@ -5234,6 +5476,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (state.lambdas != state.def_lambdas) { erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas); } + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels); + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom); + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export); if (bin != NULL) { driver_free_binary(bin); } @@ -5245,9 +5490,18 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (code != NULL) { erts_free(ERTS_ALC_T_CODE, code); } + if (state.labels != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels); + } if (state.lambdas != state.def_lambdas) { erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas); } + if (state.atom != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom); + } + if (state.export != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export); + } if (bin != NULL) { driver_free_binary(bin); } diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 85bf584337..bb237e378a 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -813,7 +813,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.min_heap_size = H_MIN_SIZE; so.min_vheap_size = BIN_VH_MIN_SIZE; so.priority = PRIORITY_NORMAL; - so.max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); + so.max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); so.scheduler = 0; /* @@ -1132,6 +1132,34 @@ BIF_RETTYPE error_2(Process* p, Eterm value, Eterm args) } /**********************************************************************/ +/* + * This is like exactly like error/1. The only difference is + * that Dialyzer thinks that it it will return an arbitrary term. + * It is useful in stub functions for NIFs. + */ + +BIF_RETTYPE nif_error_1(Process* p, Eterm term) +{ + p->fvalue = term; + BIF_ERROR(p, EXC_ERROR); +} + +/**********************************************************************/ +/* + * This is like exactly like error/2. The only difference is + * that Dialyzer thinks that it it will return an arbitrary term. + * It is useful in stub functions for NIFs. + */ + +BIF_RETTYPE nif_error_2(Process* p, Eterm value, Eterm args) +{ + Eterm* hp = HAlloc(p, 3); + + p->fvalue = TUPLE2(hp, value, args); + BIF_ERROR(p, EXC_ERROR_2); +} + +/**********************************************************************/ /* this is like throw/1 except that we set freason to EXC_EXIT */ BIF_RETTYPE exit_1(BIF_ALIST_1) @@ -1323,9 +1351,10 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) #ifdef ERTS_SMP if (rp == BIF_P) rp_locks &= ~ERTS_PROC_LOCK_MAIN; - else + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (rp != BIF_P) erts_smp_proc_dec_refc(rp); - erts_smp_proc_unlock(rp, rp_locks); #endif /* * We may have exited ourselves and may have to take action. @@ -3241,12 +3270,13 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_mtx_lock(&ports_snapshot_mtx); /* One snapshot at a time */ - erts_smp_atomic_set(&erts_dead_ports_ptr, (long) (port_buf + erts_max_ports)); + erts_smp_atomic_set(&erts_dead_ports_ptr, + (erts_aint_t) (port_buf + erts_max_ports)); next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot); if (erts_smp_atomic_read(&erts_ports_alive) > 0) { - long i; + erts_aint_t i; for (i = erts_max_ports-1; i >= 0; i--) { Port* prt = &erts_port[i]; erts_smp_port_state_lock(prt); @@ -3261,7 +3291,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) } dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr, - (long)NULL); + (erts_aint_t) NULL); erts_smp_mtx_unlock(&ports_snapshot_mtx); ASSERT(pp <= dead_ports); @@ -3272,7 +3302,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) ASSERT((alive+dead) <= erts_max_ports); if (alive+dead > 0) { - long i; + erts_aint_t i; Eterm *hp = HAlloc(BIF_P, (alive+dead)*2); for (i = 0; i < alive; i++) { @@ -3583,11 +3613,11 @@ BIF_RETTYPE list_to_pid_1(BIF_ALIST_1) etp = (ExternalThing *) HAlloc(BIF_P, EXTERNAL_THING_HEAD_SIZE + 1); etp->header = make_external_pid_header(1); - etp->next = MSO(BIF_P).externals; + etp->next = MSO(BIF_P).first; etp->node = enp; etp->data.ui[0] = make_pid_data(c, b); - MSO(BIF_P).externals = etp; + MSO(BIF_P).first = (struct erl_off_heap_header*) etp; erts_deref_dist_entry(dep); BIF_RET(make_external_pid(etp)); } @@ -3768,7 +3798,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) goto error; } nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n); - oval = (Uint) erts_smp_atomic_xchg(&erts_max_gen_gcs, (long) nval); + oval = (Uint) erts_smp_atomic32_xchg(&erts_max_gen_gcs, + (erts_aint32_t) nval); BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_min_heap_size) { int oval = H_MIN_SIZE; @@ -4111,7 +4142,7 @@ void erts_init_bif(void) erts_smp_spinlock_init(&make_ref_lock, "make_ref"); erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot"); - erts_smp_atomic_init(&erts_dead_ports_ptr, (long)NULL); + erts_smp_atomic_init(&erts_dead_ports_ptr, (erts_aint_t) NULL); /* * bif_return_trap/1 is a hidden BIF that bifs that need to @@ -4143,54 +4174,6 @@ void erts_init_bif(void) await_proc_exit_trap = erts_export_put(am_erlang,am_await_proc_exit,3); } -BIF_RETTYPE blocking_read_file_1(BIF_ALIST_1) -{ - Eterm bin; - Eterm* hp; - byte *buff; - int i, buff_size; - FILE *file; - struct stat file_info; - char *filename = NULL; - size_t size; - - i = list_length(BIF_ARG_1); - if (i < 0) { - BIF_ERROR(BIF_P, BADARG); - } - filename = erts_alloc(ERTS_ALC_T_TMP, i + 1); - if (intlist_to_buf(BIF_ARG_1, filename, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - filename[i] = '\0'; - - hp = HAlloc(BIF_P, 3); - - file = fopen(filename, "r"); - if(file == NULL){ - erts_free(ERTS_ALC_T_TMP, (void *) filename); - BIF_RET(TUPLE2(hp, am_error, am_nofile)); - } - - stat(filename, &file_info); - erts_free(ERTS_ALC_T_TMP, (void *) filename); - - buff_size = file_info.st_size; - buff = (byte *) erts_alloc_fnf(ERTS_ALC_T_TMP, buff_size); - if (!buff) { - fclose(file); - BIF_RET(TUPLE2(hp, am_error, am_allocator)); - } - size = fread(buff, 1, buff_size, file); - fclose(file); - if (size < 0) - size = 0; - else if (size > buff_size) - size = (size_t) buff_size; - bin = new_binary(BIF_P, buff, (int) size); - erts_free(ERTS_ALC_T_TMP, (void *) buff); - - BIF_RET(TUPLE2(hp, am_ok, bin)); -} #ifdef HARDDEBUG /* You'll need this line in bif.tab to be able to use this debug bif diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 50f5f4fbd6..a84ee7bb23 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -135,7 +135,6 @@ do { \ (Proc)->arity = 1; \ (Proc)->def_arg_reg[0] = (Eterm) (A0); \ *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ - (Proc)->def_arg_reg[3] = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -146,7 +145,6 @@ do { \ (Proc)->def_arg_reg[0] = (Eterm) (A0); \ (Proc)->def_arg_reg[1] = (Eterm) (A1); \ *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ - (Proc)->def_arg_reg[3] = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -158,7 +156,6 @@ do { \ (Proc)->def_arg_reg[1] = (Eterm) (A1); \ (Proc)->def_arg_reg[2] = (Eterm) (A2); \ *((UWord *) (UWord) ((Proc)->def_arg_reg + 3)) = (UWord) ((Trap)->address); \ - (Proc)->def_arg_reg[3] = (UWord) ((Trap)->address); \ (Proc)->freason = TRAP; \ (Ret) = THE_NON_VALUE; \ } while (0) diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index b6fa06354a..d9dd80fa8b 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -660,6 +660,7 @@ bif erts_debug:display/1 bif 'erl.system.debug':display/1 ebif_erts_debug_display_1 bif erts_debug:dist_ext_to_term/2 bif 'erl.system.debug':dist_ext_to_term/2 ebif_erts_debug_dist_ext_to_term_2 +bif erts_debug:instructions/0 # # Monitor testing bif's... @@ -686,8 +687,6 @@ bif 'erl.system.code':make_stub_module/3 ebif_code_make_stub_module_3 bif code:is_module_native/1 bif 'erl.system.code':is_native/1 ebif_code_is_native_1 code_is_module_native_1 -bif erlang:blocking_read_file/1 - # # New Bifs in R9C. # @@ -760,6 +759,50 @@ bif erlang:finish_after_on_load/2 bif erlang:binary_to_term/2 # +# The binary match bifs (New in R14A - EEP9) +# + +# +# The searching/splitting/substituting thingies +# +ubif erlang:binary_part/2 +ubif erlang:binary_part/3 + +bif binary:compile_pattern/1 +bif binary:match/2 +bif binary:match/3 +bif binary:matches/2 +bif binary:matches/3 +bif binary:longest_common_prefix/1 +bif binary:longest_common_suffix/1 +bif binary:first/1 +bif binary:last/1 +bif binary:at/2 +bif binary:part/2 binary_binary_part_2 +bif binary:part/3 binary_binary_part_3 +bif binary:bin_to_list/1 +bif binary:bin_to_list/2 +bif binary:bin_to_list/3 +bif binary:list_to_bin/1 +bif binary:copy/1 +bif binary:copy/2 +bif binary:referenced_byte_size/1 +bif binary:encode_unsigned/1 +bif binary:encode_unsigned/2 +bif binary:decode_unsigned/1 +bif binary:decode_unsigned/2 + +bif erlang:nif_error/1 +bif erlang:nif_error/2 + +# +# Helpers for unicode filenames +# +bif prim_file:internal_name2native/1 +bif prim_file:internal_native2name/1 +bif prim_file:internal_normalize_utf8/1 +bif file:native_name_encoding/0 +# # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 90d3a0304a..ff15d834ab 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1509,14 +1509,14 @@ Eterm erts_uint64_to_big(Uint64 x, Eterm **hpp) *hp = make_pos_bignum_header(2); BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff)); - *hpp += 2; + *hpp += 3; } else #endif { *hp = make_pos_bignum_header(1); BIG_DIGIT(hp, 0) = (Uint) x; - *hpp += 1; + *hpp += 2; } return make_big(hp); } @@ -1539,7 +1539,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) *hp = make_pos_bignum_header(2); BIG_DIGIT(hp, 0) = (Uint) (x & ((Uint) 0xffffffff)); BIG_DIGIT(hp, 1) = (Uint) ((x >> 32) & ((Uint) 0xffffffff)); - *hpp += 2; + *hpp += 3; } else #endif @@ -1549,7 +1549,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) else *hp = make_pos_bignum_header(1); BIG_DIGIT(hp, 0) = (Uint) x; - *hpp += 1; + *hpp += 2; } return make_big(hp); } @@ -1881,6 +1881,9 @@ term_to_Uint(Eterm term, Uint *up) int term_to_UWord(Eterm term, UWord *up) { +#if SIZEOF_VOID_P == ERTS_SIZEOF_ETERM + return term_to_Uint(term,up); +#else if (is_small(term)) { Sint i = signed_val(term); if (i < 0) { @@ -1903,7 +1906,47 @@ term_to_UWord(Eterm term, UWord *up) return 0; } while (xl-- > 0) { - uval |= ((Uint)(*xr++)) << n; + uval |= ((UWord)(*xr++)) << n; + n += D_EXP; + } + *up = uval; + return 1; + } else { + *up = BADARG; + return 0; + } +#endif +} + +int +term_to_Uint64(Eterm term, Uint64 *up) +{ +#if SIZEOF_VOID_P == 8 + return term_to_UWord(term,up); +#else + if (is_small(term)) { + Sint i = signed_val(term); + if (i < 0) { + *up = BADARG; + return 0; + } + *up = (Uint64) i; + return 1; + } else if (is_big(term)) { + ErtsDigit* xr = big_v(term); + dsize_t xl = big_size(term); + Uint64 uval = 0; + int n = 0; + + if (big_sign(term)) { + *up = BADARG; + return 0; + } else if (xl*D_EXP > sizeof(Uint64)*8) { + *up = SYSTEM_LIMIT; + return 0; + } + while (xl-- > 0) { + uval |= ((Uint64)(*xr++)) << n; n += D_EXP; } *up = uval; @@ -1912,8 +1955,10 @@ term_to_UWord(Eterm term, UWord *up) *up = BADARG; return 0; } +#endif } + int term_to_Sint(Eterm term, Sint *sp) { if (is_small(term)) { @@ -1948,6 +1993,47 @@ int term_to_Sint(Eterm term, Sint *sp) } } +#if HAVE_INT64 +int term_to_Sint64(Eterm term, Sint64 *sp) +{ +#if ERTS_SIZEOF_ETERM == 8 + return term_to_Sint(term, sp); +#else + if (is_small(term)) { + *sp = signed_val(term); + return 1; + } else if (is_big(term)) { + ErtsDigit* xr = big_v(term); + dsize_t xl = big_size(term); + int sign = big_sign(term); + Uint64 uval = 0; + int n = 0; + + if (xl*D_EXP > sizeof(Uint64)*8) { + return 0; + } + while (xl-- > 0) { + uval |= ((Uint64)(*xr++)) << n; + n += D_EXP; + } + if (sign) { + uval = -uval; + if ((Sint64)uval > 0) + return 0; + } else { + if ((Sint64)uval < 0) + return 0; + } + *sp = uval; + return 1; + } else { + return 0; + } +#endif +} +#endif /* HAVE_INT64 */ + + /* ** Add and subtract */ diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 56f3be372a..25466cd3c2 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -152,6 +152,10 @@ byte* big_to_bytes(Eterm, byte*); int term_to_Uint(Eterm, Uint*); int term_to_UWord(Eterm, UWord*); int term_to_Sint(Eterm, Sint*); +#if HAVE_INT64 +int term_to_Uint64(Eterm, Uint64*); +int term_to_Sint64(Eterm, Sint64*); +#endif Uint32 big_to_uint32(Eterm b); int term_equals_2pow32(Eterm); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 59c20398d5..4be869f269 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -88,8 +88,8 @@ new_binary(Process *p, byte *buf, int len) pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -97,7 +97,7 @@ new_binary(Process *p, byte *buf, int len) /* * Miscellanous updates. Return the tagged binary. */ - MSO(p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); return make_binary(pb); } @@ -127,8 +127,8 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len) pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; @@ -136,7 +136,7 @@ Eterm erts_new_mso_binary(Process *p, byte *buf, int len) /* * Miscellanous updates. Return the tagged binary. */ - MSO(p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); return make_binary(pb); } @@ -180,7 +180,7 @@ erts_realloc_binary(Eterm bin, size_t size) } byte* -erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra) +erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, ErtsAlcType_t allocator, unsigned extra) { byte* bytes; Eterm* real_bin; @@ -208,7 +208,7 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra) bytes = (byte *)(&(((ErlHeapBin *) real_bin)->data)) + offs; } if (bit_offs) { - byte* buf = (byte *) erts_alloc(ERTS_ALC_T_TMP, byte_size + extra); + byte* buf = (byte *) erts_alloc(allocator, byte_size + extra); *base_ptr = buf; buf += extra; erts_copy_bits(bytes, bit_offs, 1, buf, 0, 1, byte_size*8); @@ -217,8 +217,8 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, unsigned extra) return bytes; } -static Eterm -bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs) +Eterm +erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs) { if (bitoffs == 0) { while (size) { @@ -263,7 +263,7 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) Eterm* hp = HAlloc(BIF_P, 2 * size); byte* bytes = binary_bytes(real_bin)+offset; - BIF_RET(bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); + BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); } error: @@ -295,7 +295,7 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) } i = stop-start+1; hp = HAlloc(BIF_P, 2*i); - BIF_RET(bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs)); + BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs)); error: BIF_ERROR(BIF_P, BADARG); @@ -339,36 +339,47 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) previous = CONS(hp, make_binary(last), previous); hp += 2; } - BIF_RET(bin_bytes_to_list(previous, hp, bytes, size, bitoffs)); + BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs)); } /* Turn a possibly deep list of ints (and binaries) into */ /* One large binary object */ -BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) +/* + * This bif also exists in the binary module, under the name + * binary:list_to_bin/1, why it's divided into interface and + * implementation. Also the backend for iolist_to_binary_1. + */ + +BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg) { Eterm bin; int i; int offset; byte* bytes; - if (is_nil(BIF_ARG_1)) { - BIF_RET(new_binary(BIF_P,(byte*)"",0)); + if (is_nil(arg)) { + BIF_RET(new_binary(p,(byte*)"",0)); } - if (is_not_list(BIF_ARG_1)) { + if (is_not_list(arg)) { goto error; } - if ((i = io_list_len(BIF_ARG_1)) < 0) { + if ((i = io_list_len(arg)) < 0) { goto error; } - bin = new_binary(BIF_P, (byte *)NULL, i); + bin = new_binary(p, (byte *)NULL, i); bytes = binary_bytes(bin); - offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i); + offset = io_list_to_buf(arg, (char*) bytes, i); ASSERT(offset == 0); BIF_RET(bin); - error: - BIF_ERROR(BIF_P, BADARG); + error: + BIF_ERROR(p, BADARG); +} + +BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) +{ + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); } /* Turn a possibly deep list of ints (and binaries) into */ @@ -376,31 +387,10 @@ BIF_RETTYPE list_to_binary_1(BIF_ALIST_1) BIF_RETTYPE iolist_to_binary_1(BIF_ALIST_1) { - Eterm bin; - int i; - int offset; - byte* bytes; - if (is_binary(BIF_ARG_1)) { BIF_RET(BIF_ARG_1); } - if (is_nil(BIF_ARG_1)) { - BIF_RET(new_binary(BIF_P,(byte*)"",0)); - } - if (is_not_list(BIF_ARG_1)) { - goto error; - } - if ((i = io_list_len(BIF_ARG_1)) < 0) { - goto error; - } - bin = new_binary(BIF_P, (byte *)NULL, i); - bytes = binary_bytes(bin); - offset = io_list_to_buf(BIF_ARG_1, (char*) bytes, i); - ASSERT(offset == 0); - BIF_RET(bin); - - error: - BIF_ERROR(BIF_P, BADARG); + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); } BIF_RETTYPE list_to_bitstring_1(BIF_ALIST_1) @@ -497,16 +487,6 @@ BIF_RETTYPE split_binary_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } -void -erts_cleanup_mso(ProcBin* pb) -{ - while (pb != NULL) { - ProcBin* next = pb->next; - if (erts_refc_dectest(&pb->val->refc, 0) == 0) - erts_bin_free(pb->val); - pb = next; - } -} /* * Local functions. @@ -675,3 +655,4 @@ bitstr_list_len(Eterm obj) DESTROY_ESTACK(s); return (Sint) -1; } + diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 5cb1481a3a..d255cf3558 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -98,7 +98,7 @@ process_killer(void) switch(j) { case 'k': if (rp->status == P_WAITING) { - Uint32 rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; + ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; erts_smp_proc_inc_refc(rp); erts_smp_proc_lock(rp, rp_locks); (void) erts_send_exit_signal(NULL, @@ -258,12 +258,10 @@ print_process_info(int to, void *to_arg, Process *p) } { - long s = 0; int frags = 0; ErlHeapFragment *m = p->mbuf; while (m != NULL) { frags++; - s += m->size; m = m->next; } erts_print(to, to_arg, "Number of heap fragments: %d\n", frags); @@ -560,7 +558,7 @@ do_break(void) #endif #ifdef DEBUG case 't': - p_slpq(); + erts_p_slpq(); return; case 'b': bin_check(); @@ -613,29 +611,29 @@ static void bin_check(void) { Process *rp; - ProcBin *bp; - int i, printed; + struct erl_off_heap_header* hdr; + int i, printed = 0; for (i=0; i < erts_max_processes; i++) { if ((rp = process_tab[i]) == NULL) continue; - if (!(bp = rp->off_heap.mso)) - continue; - printed = 0; - while (bp) { - if (printed == 0) { - erts_printf("Process %T holding binary data \n", rp->id); - printed = 1; + for (hdr = rp->off_heap.first; hdr; hdr = hdr->next) { + if (hdr->thing_word == HEADER_PROC_BIN) { + ProcBin *bp = (ProcBin*) hdr; + if (!printed) { + erts_printf("Process %T holding binary data \n", rp->id); + printed = 1; + } + erts_printf("%p orig_size: %bpd, norefs = %bpd\n", + bp->val, + bp->val->orig_size, + erts_smp_atomic_read(&bp->val->refc)); } - erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n", - (unsigned long)bp->val, - (long)bp->val->orig_size, - erts_smp_atomic_read(&bp->val->refc)); - - bp = bp->next; } - if (printed == 1) + if (printed) { erts_printf("--------------------------------------\n"); + printed = 0; + } } /* db_bin_check() has to be rewritten for the AVL trees... */ /*db_bin_check();*/ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 0a5050b1fe..8bee47232e 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -37,6 +37,8 @@ MA_STACK_DECLARE(dst); MA_STACK_DECLARE(offset); #endif +static void move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap*); + void init_copy(void) { @@ -86,7 +88,7 @@ size_object(Eterm obj) obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); - } + } obj = *ptr; break; case TAG_PRIMARY_BOXED: @@ -99,7 +101,7 @@ size_object(Eterm obj) arity = header_arity(hdr); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ - goto size_common; + goto pop_next; } while (arity-- > 1) { obj = *++ptr; @@ -115,7 +117,6 @@ size_object(Eterm obj) ErlFunThing* funp = (ErlFunThing *) bptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); - sum += 1 /* header */ + sz + eterms; bptr += 1 /* header */ + sz; while (eterms-- > 1) { @@ -151,7 +152,7 @@ size_object(Eterm obj) } else { sum += heap_bin_size(binary_size(obj)+extra_bytes); } - goto size_common; + goto pop_next; } break; case BIN_MATCHSTATE_SUBTAG: @@ -159,18 +160,12 @@ size_object(Eterm obj) "size_object: matchstate term not allowed"); default: sum += thing_arityval(hdr) + 1; - /* Fall through */ - size_common: - if (ESTACK_ISEMPTY(s)) { - DESTROY_ESTACK(s); - return sum; - } - obj = ESTACK_POP(s); - break; + goto pop_next; } } break; case TAG_PRIMARY_IMMED1: + pop_next: if (ESTACK_ISEMPTY(s)) { DESTROY_ESTACK(s); return sum; @@ -319,10 +314,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *argp = make_binary(hbot); pb = (ProcBin*) hbot; erts_refc_inc(&pb->val->refc, 2); - pb->next = off_heap->mso; + pb->next = off_heap->first; pb->flags = 0; - off_heap->mso = pb; - off_heap->overhead += pb->size / sizeof(Eterm); + off_heap->first = (struct erl_off_heap_header*) pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); } break; case SUB_BINARY_SUBTAG: @@ -368,10 +363,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) to->val = from->val; erts_refc_inc(&to->val->refc, 2); to->bytes = from->bytes + offset; - to->next = off_heap->mso; + to->next = off_heap->first; to->flags = 0; - off_heap->mso = to; - off_heap->overhead += to->size / sizeof(Eterm); + off_heap->first = (struct erl_off_heap_header*) to; + OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); } *argp = make_binary(hbot); if (extra_bytes != 0) { @@ -401,8 +396,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } #ifndef HYBRID /* FIND ME! */ funp = (ErlFunThing *) tp; - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*) funp; erts_refc_inc(&funp->fe->refc, 2); #endif *argp = make_fun(tp); @@ -421,8 +416,8 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *htop++ = *objp++; } - etp->next = off_heap->externals; - off_heap->externals = etp; + etp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)etp; erts_refc_inc(&etp->node->refc, 2); *argp = make_external(tp); @@ -655,9 +650,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) *hp++ = *objp++; } erts_refc_inc(&pb->val->refc, 2); - pb->next = erts_global_offheap.mso; - erts_global_offheap.mso = pb; - erts_global_offheap.overhead += pb->size / sizeof(Eterm); + pb->next = erts_global_offheap.first; + erts_global_offheap.first = pb; + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); continue; } @@ -677,9 +672,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) while (i--) { *hp++ = *objp++; } -#ifndef HYBRID // FIND ME! - funp->next = erts_global_offheap.funs; - erts_global_offheap.funs = funp; +#ifndef HYBRID /* FIND ME! */ + funp->next = erts_global_offheap.first; + erts_global_offheap.first = funp; erts_refc_inc(&funp->fe->refc, 2); #endif for (i = k; i < j; i++) { @@ -723,8 +718,8 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) *hp++ = *objp++; } - etp->next = erts_global_offheap.externals; - erts_global_offheap.externals = etp; + etp->next = erts_global_offheap.first; + erts_global_offheap.first = etp; erts_refc_inc(&etp->node->refc, 2); continue; } @@ -780,9 +775,9 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) to_bin->size = real_size; to_bin->val = from_bin->val; to_bin->bytes = from_bin->bytes + sub_offset; - to_bin->next = erts_global_offheap.mso; - erts_global_offheap.mso = to_bin; - erts_global_offheap.overhead += to_bin->size / sizeof(Eterm); + to_bin->next = erts_global_offheap.first; + erts_global_offheap.first = to_bin; + OH_OVERHEAD(&erts_global_offheap, to_bin->size / sizeof(Eterm)); res_binary=make_binary(to_bin); hp += PROC_BIN_SIZE; } @@ -915,57 +910,43 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) break; case REFC_BINARY_SUBTAG: { - ProcBin* pb = (ProcBin *) (hp-1); - int tari = thing_arityval(val); - - sz -= tari; - while (tari--) { - *hp++ = *tp++; - } + ProcBin* pb = (ProcBin *) (tp-1); erts_refc_inc(&pb->val->refc, 2); - pb->next = off_heap->mso; - off_heap->mso = pb; - off_heap->overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(off_heap, pb->size / sizeof(Eterm)); } - break; + goto off_heap_common; + case FUN_SUBTAG: { -#ifndef HYBRID /* FIND ME! */ - ErlFunThing* funp = (ErlFunThing *) (hp-1); -#endif - int tari = thing_arityval(val); - - sz -= tari; - while (tari--) { - *hp++ = *tp++; - } -#ifndef HYBRID /* FIND ME! */ - funp->next = off_heap->funs; - off_heap->funs = funp; + ErlFunThing* funp = (ErlFunThing *) (tp-1); erts_refc_inc(&funp->fe->refc, 2); -#endif } - break; + goto off_heap_common; + case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: { - ExternalThing* etp = (ExternalThing *) (hp-1); + ExternalThing* etp = (ExternalThing *) (tp-1); + erts_refc_inc(&etp->node->refc, 2); + } + off_heap_common: + { + struct erl_off_heap_header* ohh = (struct erl_off_heap_header*)(hp-1); int tari = thing_arityval(val); - + sz -= tari; while (tari--) { *hp++ = *tp++; } - etp->next = off_heap->externals; - off_heap->externals = etp; - erts_refc_inc(&etp->node->refc, 2); + ohh->next = off_heap->first; + off_heap->first = ohh; } break; default: { int tari = header_arity(val); - + sz -= tari; while (tari--) { *hp++ = *tp++; @@ -979,3 +960,92 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) *hpp = hp; return make_tuple(ptr + offs); } + +/* Move all terms in heap fragments into heap. The terms must be guaranteed to + * be contained within the fragments. The source terms are destructed with + * move markers. + * Typically used to copy a multi-fragmented message (from NIF). + */ +void move_multi_frags(Eterm** hpp, ErlOffHeap* off_heap, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs) +{ + ErlHeapFragment* bp; + Eterm* hp_start = *hpp; + Eterm* hp_end; + Eterm* hp; + unsigned i; + + for (bp=first; bp!=NULL; bp=bp->next) { + move_one_frag(hpp, bp->mem, bp->used_size, off_heap); + OH_OVERHEAD(off_heap, bp->off_heap.overhead); + } + hp_end = *hpp; + for (hp=hp_start; hp<hp_end; ++hp) { + Eterm* ptr; + Eterm val; + Eterm gval = *hp; + switch (primary_tag(gval)) { + case TAG_PRIMARY_BOXED: + ptr = boxed_val(gval); + val = *ptr; + if (IS_MOVED_BOXED(val)) { + ASSERT(is_boxed(val)); + *hp = val; + } + break; + case TAG_PRIMARY_LIST: + ptr = list_val(gval); + val = *ptr; + if (IS_MOVED_CONS(val)) { + *hp = ptr[1]; + } + break; + case TAG_PRIMARY_HEADER: + if (header_is_thing(gval)) { + hp += thing_arityval(gval); + } + break; + } + } + for (i=0; i<nrefs; ++i) { + refs[i] = follow_moved(refs[i]); + } +} + +static void +move_one_frag(Eterm** hpp, Eterm* src, Uint src_sz, ErlOffHeap* off_heap) +{ + Eterm* ptr = src; + Eterm* end = ptr + src_sz; + Eterm dummy_ref; + Eterm* hp = *hpp; + + while (ptr != end) { + Eterm val; + ASSERT(ptr < end); + val = *ptr; + ASSERT(val != ERTS_HOLE_MARKER); + if (is_header(val)) { + struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; + ASSERT(ptr + header_arity(val) < end); + MOVE_BOXED(ptr, val, hp, &dummy_ref); + switch (val & _HEADER_SUBTAG_MASK) { + case REFC_BINARY_SUBTAG: + case FUN_SUBTAG: + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: + hdr->next = off_heap->first; + off_heap->first = hdr; + break; + } + } + else { /* must be a cons cell */ + ASSERT(ptr+1 < end); + MOVE_CONS(ptr, val, hp, &dummy_ref); + ptr += 2; + } + } + *hpp = hp; +} + diff --git a/erts/emulator/beam/decl.h b/erts/emulator/beam/decl.h deleted file mode 100644 index da1be29d53..0000000000 --- a/erts/emulator/beam/decl.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -#ifndef __DECL_H__ -#define __DECL_H__ - -#if defined(__STDC__) || defined(_MSC_VER) -#define EXTERN_FUNCTION(t, f, x) extern t f x -#define FUNCTION(t, f, x) t f x -#define _DOTS_ ... -#define _VOID_ void -#elif defined(__cplusplus) -#define EXTERN_FUNCTION(f, x) extern "C" { f x } -#define FUNCTION(t, f, x) t f x -#define _DOTS_ ... -#define _VOID_ void -#else -#define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/) -#define FUNCTION(t, f, x) t f (/*x*/) -#define _DOTS_ -#define _VOID_ -#endif - -/* -** Example of declarations -** -** EXTERN_FUNCTION(void, foo, (int, int, char)); -** FUNCTION(void, bar, (int, char)); -** -** struct funcs { -** FUNCTION(int*, (*f1), (int, int)); -** FUNCTION(void, (*f2), (int, char)); -** FUNCTION(void, (*f3), (_VOID_)); -** FUNCTION(int, (*f4), (char*, _DOTS_)); -** }; -** -*/ - -#endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d465017949..02910fad90 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -97,6 +97,8 @@ dist_msg_dbg(ErtsDistExternal *edep, char *what, byte *buf, int sz) #define PASS_THROUGH 'p' /* This code should go */ int erts_is_alive; /* System must be blocked on change */ +int erts_dist_buf_busy_limit; + /* distribution trap functions */ Export* dsend2_trap = NULL; @@ -160,7 +162,7 @@ Uint erts_dist_cache_size(void) static ErtsProcList * get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&dep->qlock)); + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&dep->qlock)); dep->qflgs &= ~unset_qflgs; if (dep->qflgs & ERTS_DE_QFLG_EXIT) { /* No resume when exit has been scheduled */ @@ -453,17 +455,17 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) if (dep->status & ERTS_DE_SFLG_EXITING) { #ifdef DEBUG - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qflgs & ERTS_DE_QFLG_EXIT); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); #endif } else { dep->status |= ERTS_DE_SFLG_EXITING; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); dep->qflgs |= ERTS_DE_QFLG_EXIT; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } erts_smp_de_links_lock(dep); @@ -577,7 +579,7 @@ static void clear_dist_entry(DistEntry *dep) erts_smp_de_links_unlock(dep); #endif - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); if (!dep->out_queue.last) obuf = dep->finalized_out_queue.first; @@ -593,7 +595,7 @@ static void clear_dist_entry(DistEntry *dep) dep->status = 0; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); erts_smp_atomic_set(&dep->dist_cmd_scheduled, 0); dep->send = NULL; erts_smp_de_rwunlock(dep); @@ -611,10 +613,10 @@ static void clear_dist_entry(DistEntry *dep) } if (obufsize) { - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } } @@ -915,6 +917,7 @@ int erts_net_message(Port *prt, Eterm token_size; ErtsMonitor *mon; ErtsLink *lnk; + Uint tuple_arity; int res; #ifdef ERTS_DIST_MSG_DBG int orig_len = len; @@ -922,12 +925,8 @@ int erts_net_message(Port *prt, UseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); /* Thanks to Luke Gorrie */ - off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - off_heap.funs = NULL; -#endif + off_heap.first = NULL; off_heap.overhead = 0; - off_heap.externals = NULL; ERTS_SMP_CHK_NO_PROC_LOCKS; @@ -1005,29 +1004,23 @@ int erts_net_message(Port *prt, #endif if (is_not_tuple(arg) || - (tuple = tuple_val(arg), arityval(*tuple) < 1) || + (tuple = tuple_val(arg), (tuple_arity = arityval(*tuple)) < 1) || is_not_small(tuple[1])) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg); - erts_send_error_to_logger_nogl(dsbufp); - goto data_error; + goto invalid_message; } token_size = 0; switch (type = unsigned_val(tuple[1])) { case DOP_LINK: + if (tuple_arity != 3) { + goto invalid_message; + } from = tuple[2]; to = tuple[3]; /* local proc to link to */ if (is_not_pid(from) || is_not_pid(to)) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - PURIFY_MSG("data error"); - erts_dsprintf(dsbufp, - "Invalid DOP_LINK distribution message: %.200T", - arg); - erts_send_error_to_logger_nogl(dsbufp); - goto data_error; + goto invalid_message; } rp = erts_pid2proc_opt(NULL, 0, @@ -1066,8 +1059,14 @@ int erts_net_message(Port *prt, case DOP_UNLINK: { ErtsDistLinkData dld; + if (tuple_arity != 3) { + goto invalid_message; + } from = tuple[2]; to = tuple[3]; + if (is_not_pid(from) || is_not_pid(to)) { + goto invalid_message; + } rp = erts_pid2proc_opt(NULL, 0, to, ERTS_PROC_LOCK_LINK, @@ -1094,11 +1093,19 @@ int erts_net_message(Port *prt, /* A remote process wants to monitor us, we get: {DOP_MONITOR_P, Remote pid, local pid or name, ref} */ Eterm name; + + if (tuple_arity != 4) { + goto invalid_message; + } watcher = tuple[2]; watched = tuple[3]; /* local proc to monitor */ ref = tuple[4]; + if (is_not_ref(ref)) { + goto invalid_message; + } + if (is_atom(watched)) { name = watched; rp = erts_whereis_process(NULL, 0, @@ -1140,10 +1147,17 @@ int erts_net_message(Port *prt, We get {DOP_DEMONITOR_P, Remote pid, Local pid or name, ref}, We need only the ref of course */ + if (tuple_arity != 4) { + goto invalid_message; + } /* watcher = tuple[2]; */ /* watched = tuple[3]; May be an atom in case of monitor name */ ref = tuple[4]; + if(is_not_ref(ref)) { + goto invalid_message; + } + erts_smp_de_links_lock(dep); mon = erts_remove_monitor(&(dep->monitors),ref); erts_smp_de_links_unlock(dep); @@ -1168,10 +1182,11 @@ int erts_net_message(Port *prt, erts_destroy_monitor(mon); break; - case DOP_NODE_LINK: /* XXX never sent ?? */ - break; - case DOP_REG_SEND_TT: + if (tuple_arity != 5) { + goto invalid_message; + } + token_size = size_object(tuple[5]); /* Fall through ... */ case DOP_REG_SEND: @@ -1182,12 +1197,19 @@ int erts_net_message(Port *prt, * There is intentionally no testing of the cookie (it is always '') * from R9B and onwards. */ + if (type != DOP_REG_SEND_TT && tuple_arity != 4) { + goto invalid_message; + } + #ifdef ERTS_DIST_MSG_DBG dist_msg_dbg(&ede, "MSG", buf, orig_len); #endif from = tuple[2]; to = tuple[4]; + if (is_not_pid(from) || is_not_atom(to)){ + goto invalid_message; + } rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); if (rp) { Uint xsize = (type == DOP_REG_SEND @@ -1219,6 +1241,10 @@ int erts_net_message(Port *prt, break; case DOP_SEND_TT: + if (tuple_arity != 4) { + goto invalid_message; + } + token_size = size_object(tuple[4]); /* Fall through ... */ case DOP_SEND: @@ -1229,8 +1255,13 @@ int erts_net_message(Port *prt, #ifdef ERTS_DIST_MSG_DBG dist_msg_dbg(&ede, "MSG", buf, orig_len); #endif - + if (type != DOP_SEND_TT && tuple_arity != 3) { + goto invalid_message; + } to = tuple[3]; + if (is_not_pid(to)) { + goto invalid_message; + } rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); if (rp) { Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size); @@ -1268,11 +1299,19 @@ int erts_net_message(Port *prt, Eterm sysname; ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK; + if (tuple_arity != 5) { + goto invalid_message; + } + /* watched = tuple[2]; */ /* remote proc which died */ /* watcher = tuple[3]; */ ref = tuple[4]; reason = tuple[5]; + if(is_not_ref(ref)) { + goto invalid_message; + } + erts_smp_de_links_lock(dep); sysname = dep->sysname; mon = erts_remove_monitor(&(dep->monitors), ref); @@ -1319,24 +1358,25 @@ int erts_net_message(Port *prt, ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND; /* 'from', which 'to' is linked to, died */ if (type == DOP_EXIT) { - from = tuple[2]; - to = tuple[3]; - reason = tuple[4]; - token = NIL; + if (tuple_arity != 4) { + goto invalid_message; + } + + from = tuple[2]; + to = tuple[3]; + reason = tuple[4]; + token = NIL; } else { - from = tuple[2]; - to = tuple[3]; - token = tuple[4]; - reason = tuple[5]; + if (tuple_arity != 5) { + goto invalid_message; + } + from = tuple[2]; + to = tuple[3]; + token = tuple[4]; + reason = tuple[5]; } - if (is_not_internal_pid(to)) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - PURIFY_MSG("data error"); - erts_dsprintf(dsbufp, - "Invalid DOP_EXIT distribution message: %.200T", - arg); - erts_send_error_to_logger_nogl(dsbufp); - goto data_error; + if (is_not_pid(from) || is_not_internal_pid(to)) { + goto invalid_message; } rp = erts_pid2proc(NULL, 0, to, rp_locks); @@ -1383,15 +1423,24 @@ int erts_net_message(Port *prt, ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; /* 'from' is send an exit signal to 'to' */ if (type == DOP_EXIT2) { - from = tuple[2]; - to = tuple[3]; - reason = tuple[4]; - token = NIL; + if (tuple_arity != 4) { + goto invalid_message; + } + from = tuple[2]; + to = tuple[3]; + reason = tuple[4]; + token = NIL; } else { - from = tuple[2]; - to = tuple[3]; - token = tuple[4]; - reason = tuple[5]; + if (tuple_arity != 5) { + goto invalid_message; + } + from = tuple[2]; + to = tuple[3]; + token = tuple[4]; + reason = tuple[5]; + } + if (is_not_pid(from) || is_not_internal_pid(to)) { + goto invalid_message; } rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); @@ -1410,10 +1459,14 @@ int erts_net_message(Port *prt, break; } case DOP_GROUP_LEADER: + if (tuple_arity != 3) { + goto invalid_message; + } from = tuple[2]; /* Group leader */ to = tuple[3]; /* new member */ - if (is_not_pid(from)) - break; + if (is_not_pid(from) || is_not_pid(to)) { + goto invalid_message; + } rp = erts_pid2proc(NULL, 0, to, ERTS_PROC_LOCK_MAIN); if (!rp) @@ -1422,28 +1475,12 @@ int erts_net_message(Port *prt, erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); break; - default: { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Illegal value in distribution dispatch switch: " - "%.200T", - arg); - erts_send_error_to_logger_nogl(dsbufp); - PURIFY_MSG("data error"); - goto data_error; - } + default: + goto invalid_message; } - if (off_heap.mso) { - erts_cleanup_mso(off_heap.mso); - } - if (off_heap.externals) { - erts_cleanup_externals(off_heap.externals); - } + erts_cleanup_offheap(&off_heap); #ifndef HYBRID /* FIND ME! */ - if (off_heap.funs) { - erts_cleanup_funs(off_heap.funs); - } if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } @@ -1451,18 +1488,16 @@ int erts_net_message(Port *prt, UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); ERTS_SMP_CHK_NO_PROC_LOCKS; return 0; - - data_error: - if (off_heap.mso) { - erts_cleanup_mso(off_heap.mso); - } - if (off_heap.externals) { - erts_cleanup_externals(off_heap.externals); + invalid_message: + { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg); + erts_send_error_to_logger_nogl(dsbufp); } + data_error: + PURIFY_MSG("data error"); + erts_cleanup_offheap(&off_heap); #ifndef HYBRID /* FIND ME! */ - if (off_heap.funs) { - erts_cleanup_funs(off_heap.funs); - } if (ctl != ctl_default) { erts_free(ERTS_ALC_T_DCTRL_BUF, (void *) ctl); } @@ -1473,8 +1508,6 @@ int erts_net_message(Port *prt, return -1; } -#define ERTS_DE_BUSY_LIMIT (128*1024) - static int dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) { @@ -1558,18 +1591,18 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) } else { ErtsProcList *plp = NULL; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); dep->qsize += size_obuf(obuf); - if (dep->qsize >= ERTS_DE_BUSY_LIMIT) + if (dep->qsize >= erts_dist_buf_busy_limit) dep->qflgs |= ERTS_DE_QFLG_BUSY; if (!force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) { - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); plp = erts_proclist_create(c_p); plp->next = NULL; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); suspended = 1; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); } /* Enqueue obuf on dist entry */ @@ -1595,7 +1628,7 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) } } - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); erts_schedule_dist_command(NULL, dep); erts_smp_de_runlock(dep); @@ -1728,10 +1761,8 @@ erts_dist_command(Port *prt, int reds_limit) { Sint reds = ERTS_PORT_REDS_DIST_CMD_START; int prt_busy; - int de_busy; Uint32 status; Uint32 flags; - Uint32 qflgs; Sint obufsize = 0; ErtsDistOutputQueue oq, foq; DistEntry *dep = prt->dist_entry; @@ -1766,13 +1797,12 @@ erts_dist_command(Port *prt, int reds_limit) * a mess. */ - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); oq.first = dep->out_queue.first; oq.last = dep->out_queue.last; dep->out_queue.first = NULL; dep->out_queue.last = NULL; - qflgs = dep->qflgs; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); foq.first = dep->finalized_out_queue.first; foq.last = dep->finalized_out_queue.last; @@ -1783,17 +1813,8 @@ erts_dist_command(Port *prt, int reds_limit) goto preempted; prt_busy = (int) (prt->status & ERTS_PORT_SFLG_PORT_BUSY); - de_busy = (int) (qflgs & ERTS_DE_QFLG_BUSY); - if (prt_busy) { - if (!de_busy) { - erts_smp_spin_lock(&dep->qlock); - dep->qflgs |= ERTS_DE_QFLG_BUSY; - erts_smp_spin_unlock(&dep->qlock); - de_busy = 1; - } - } - else if (foq.first) { + if (!prt_busy && foq.first) { int preempt = 0; do { Uint size; @@ -1811,10 +1832,7 @@ erts_dist_command(Port *prt, int reds_limit) free_dist_obuf(fob); preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - erts_smp_spin_lock(&dep->qlock); - dep->qflgs |= ERTS_DE_QFLG_BUSY; - erts_smp_spin_unlock(&dep->qlock); - de_busy = prt_busy = 1; + prt_busy = 1; break; } } while (foq.first && !preempt); @@ -1897,10 +1915,7 @@ erts_dist_command(Port *prt, int reds_limit) free_dist_obuf(fob); preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - erts_smp_spin_lock(&dep->qlock); - dep->qflgs |= ERTS_DE_QFLG_BUSY; - erts_smp_spin_unlock(&dep->qlock); - de_busy = prt_busy = 1; + prt_busy = 1; if (oq.first && !preempt) goto finalize_only; } @@ -1927,22 +1942,23 @@ erts_dist_command(Port *prt, int reds_limit) * dist entry in a non-busy state and resume suspended * processes. */ - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; obufsize = 0; - if (de_busy && !prt_busy && dep->qsize < ERTS_DE_BUSY_LIMIT) { + if (!prt_busy + && (dep->qflgs & ERTS_DE_QFLG_BUSY) + && dep->qsize < erts_dist_buf_busy_limit) { ErtsProcList *suspendees; int resumed; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); resumed = erts_resume_processes(suspendees); reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; - de_busy = 0; } else - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } ASSERT(!oq.first && !oq.last); @@ -1951,10 +1967,10 @@ erts_dist_command(Port *prt, int reds_limit) if (obufsize != 0) { ASSERT(obufsize > 0); - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } ASSERT(foq.first || !foq.last); @@ -2004,9 +2020,9 @@ erts_dist_command(Port *prt, int reds_limit) foq.last = NULL; #ifdef DEBUG - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize == obufsize); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); #endif } else { @@ -2015,14 +2031,14 @@ erts_dist_command(Port *prt, int reds_limit) * Unhandle buffers need to be put back first * in out_queue. */ - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); dep->qsize -= obufsize; obufsize = 0; oq.last->next = dep->out_queue.first; dep->out_queue.first = oq.first; if (!dep->out_queue.last) dep->out_queue.last = oq.last; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } erts_schedule_dist_command(prt, NULL); @@ -2046,10 +2062,10 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) dep->status |= ERTS_DE_SFLG_EXITING; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); dep->qflgs |= ERTS_DE_QFLG_EXIT; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); erts_schedule_dist_command(NULL, dep); } @@ -2420,13 +2436,13 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) ErtsProcList *plp = erts_proclist_create(BIF_P); plp->next = NULL; erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); if (dep->suspended.last) dep->suspended.last->next = plp; else dep->suspended.first = plp; dep->suspended.last = plp; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); goto yield; } @@ -2454,9 +2470,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) ASSERT(dep->send); #ifdef DEBUG - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize == 0); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); #endif erts_set_dist_entry_connected(dep, BIF_ARG_2, flags); diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index fa19c7fb45..695a4fc3fe 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -38,6 +38,7 @@ #define DFLAG_UNICODE_IO 0x1000 #define DFLAG_DIST_HDR_ATOM_CACHE 0x2000 #define DFLAG_SMALL_ATOM_TAGS 0x4000 +#define DFLAGS_INTERNAL_TAGS 0x8000 /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ @@ -51,7 +52,7 @@ #define DOP_SEND 2 #define DOP_EXIT 3 #define DOP_UNLINK 4 -#define DOP_NODE_LINK 5 +/* Ancient DOP_NODE_LINK (5) was here, can be reused */ #define DOP_REG_SEND 6 #define DOP_GROUP_LEADER 7 #define DOP_EXIT2 8 @@ -68,7 +69,6 @@ /* distribution trap functions */ extern Export* dsend2_trap; extern Export* dsend3_trap; -/*extern Export* dsend_nosuspend_trap;*/ extern Export* dlink_trap; extern Export* dunlink_trap; extern Export* dmonitor_node_trap; @@ -99,7 +99,8 @@ typedef struct { #define ERTS_DE_IS_CONNECTED(DEP) \ (!ERTS_DE_IS_NOT_CONNECTED((DEP))) - +#define ERTS_DE_BUSY_LIMIT (1024*1024) +extern int erts_dist_buf_busy_limit; extern int erts_is_alive; /* @@ -153,10 +154,10 @@ erts_dsig_prepare(ErtsDSigData *dsdp, } if (no_suspend) { failure = ERTS_DSIG_PREP_CONNECTED; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); if (dep->qflgs & ERTS_DE_QFLG_BUSY) failure = ERTS_DSIG_PREP_WOULD_SUSPEND; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND) goto fail; } diff --git a/erts/emulator/beam/elib_malloc.c b/erts/emulator/beam/elib_malloc.c deleted file mode 100644 index b18c48d8d6..0000000000 --- a/erts/emulator/beam/elib_malloc.c +++ /dev/null @@ -1,2334 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* -** Description: Faster malloc(). -*/ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" - -#ifdef ENABLE_ELIB_MALLOC - -#undef THREAD_SAFE_ELIB_MALLOC -#ifdef USE_THREADS -#define THREAD_SAFE_ELIB_MALLOC 1 -#else -#define THREAD_SAFE_ELIB_MALLOC 0 -#endif - -#include "erl_driver.h" -#include "erl_threads.h" -#include "elib_stat.h" -#include <stdio.h> -#include <stdlib.h> - -/* To avoid clobbering of names becaure of reclaim on VxWorks, - we undefine all possible malloc, calloc etc. */ -#undef malloc -#undef calloc -#undef free -#undef realloc - -#define ELIB_INLINE /* inline all possible functions */ - -#ifndef ELIB_ALIGN -#define ELIB_ALIGN sizeof(double) -#endif - -#ifndef ELIB_HEAP_SIZE -#define ELIB_HEAP_SIZE (64*1024) /* Default 64K */ -#endif - -#ifndef ELIB_HEAP_INCREAMENT -#define ELIB_HEAP_INCREAMENT (32*1024) /* Default 32K */ -#endif - -#ifndef ELIB_FAILURE -#define ELIB_FAILURE abort() -#endif - -#undef ASSERT -#ifdef DEBUG -#define ASSERT(B) \ - ((void) ((B) ? 1 : (fprintf(stderr, "%s:%d: Assertion failed: %s\n", \ - __FILE__, __LINE__, #B), abort(), 0))) -#else -#define ASSERT(B) ((void) 1) -#endif - -#ifndef USE_RECURSIVE_MALLOC_MUTEX -#define USE_RECURSIVE_MALLOC_MUTEX 0 -#endif - -#if USE_RECURSIVE_MALLOC_MUTEX -static erts_mtx_t malloc_mutex = ERTS_REC_MTX_INITER; -#else /* #if USE_RECURSIVE_MALLOC_MUTEX */ -static erts_mtx_t malloc_mutex = ERTS_MTX_INITER; -#if THREAD_SAFE_ELIB_MALLOC -static erts_cnd_t malloc_cond = ERTS_CND_INITER; -#endif -#endif /* #if USE_RECURSIVE_MALLOC_MUTEX */ - -typedef unsigned long EWord; /* Assume 32-bit in this implementation */ -typedef unsigned short EHalfWord; /* Assume 16-bit in this implementation */ -typedef unsigned char EByte; /* Assume 8-bit byte */ - - -#define elib_printf fprintf -#define elib_putc fputc - - -#if defined(__STDC__) || defined(__WIN32__) -#define CONCAT(x,y) x##y -#else -#define CONCAT(x,y) x/**/y -#endif - - -#ifdef ELIB_DEBUG -#define ELIB_PREFIX(fun, args) CONCAT(elib__,fun) args -#else -#define ELIB_PREFIX(fun, args) CONCAT(elib_,fun) args -#endif - -#if defined(__STDC__) -void *ELIB_PREFIX(malloc, (size_t)); -void *ELIB_PREFIX(calloc, (size_t, size_t)); -void ELIB_PREFIX(cfree, (EWord *)); -void ELIB_PREFIX(free, (EWord *)); -void *ELIB_PREFIX(realloc, (EWord *, size_t)); -void* ELIB_PREFIX(memresize, (EWord *, int)); -void* ELIB_PREFIX(memalign, (int, int)); -void* ELIB_PREFIX(valloc, (int)); -void* ELIB_PREFIX(pvalloc, (int)); -int ELIB_PREFIX(memsize, (EWord *)); -/* Extern interfaces used by VxWorks */ -size_t elib_sizeof(void *); -void elib_init(EWord *, EWord); -void elib_force_init(EWord *, EWord); -#endif - -#if defined(__STDC__) -/* define prototypes for missing */ -void* memalign(size_t a, size_t s); -void* pvalloc(size_t nb); -void* memresize(void *p, int nb); -int memsize(void *p); -#endif - -/* bytes to pages */ -#define PAGES(x) (((x)+page_size-1) / page_size) -#define PAGE_ALIGN(p) ((char*)((((EWord)(p))+page_size-1)&~(page_size-1))) - -/* bytes to words */ -#define WORDS(x) (((x)+sizeof(EWord)-1) / sizeof(EWord)) - -/* Align an address */ -#define ALIGN(p) ((EWord*)((((EWord)(p)+ELIB_ALIGN-1)&~(ELIB_ALIGN-1)))) - -/* Calculate the size needed to keep alignment */ - -#define ALIGN_BSZ(nb) ((nb+sizeof(EWord)+ELIB_ALIGN-1) & ~(ELIB_ALIGN-1)) - -#define ALIGN_WSZ(nb) WORDS(ALIGN_BSZ(nb)) - -#define ALIGN_SIZE(nb) (ALIGN_WSZ(nb) - 1) - - -/* PARAMETERS */ - -#if defined(ELIB_HEAP_SBRK) - -#undef PAGE_SIZE - -/* Get the system page size (NEED MORE DEFINES HERE) */ -#ifdef _SC_PAGESIZE -#define PAGE_SIZE sysconf(_SC_PAGESIZE) -#elif defined(_MSC_VER) -# ifdef _M_ALPHA -# define PAGE_SIZE 0x2000 -# else -# define PAGE_SIZE 0x1000 -# endif -#else -#define PAGE_SIZE getpagesize() -#endif - -#define ELIB_EXPAND(need) expand_sbrk(need) -static FUNCTION(int, expand_sbrk, (EWord)); - -#elif defined(ELIB_HEAP_FIXED) - -#define PAGE_SIZE 1024 -#define ELIB_EXPAND(need) -1 -static EWord fix_heap[WORDS(ELIB_HEAP_SIZE)]; - -#elif defined(ELIB_HEAP_USER) - -#define PAGE_SIZE 1024 -#define ELIB_EXPAND(need) -1 - -#else - -#error "ELIB HEAP TYPE NOT SET" - -#endif - - -#define STAT_ALLOCED_BLOCK(SZ) \ -do { \ - tot_allocated += (SZ); \ - if (max_allocated < tot_allocated) \ - max_allocated = tot_allocated; \ -} while (0) - -#define STAT_FREED_BLOCK(SZ) \ -do { \ - tot_allocated -= (SZ); \ -} while (0) - -static int max_allocated = 0; -static int tot_allocated = 0; -static EWord* eheap; /* Align heap start */ -static EWord* eheap_top; /* Point to end of heap */ -EWord page_size = 0; /* Set by elib_init */ - -#if defined(ELIB_DEBUG) || defined(DEBUG) -#define ALIGN_CHECK(a, p) \ - do { \ - if ((EWord)(p) & (a-1)) { \ - elib_printf(stderr, \ - "RUNTIME ERROR: bad alignment (0x%lx:%d:%d)\n", \ - (unsigned long) (p), (int) a, __LINE__); \ - ELIB_FAILURE; \ - } \ - } while(0) -#define ELIB_ALIGN_CHECK(p) ALIGN_CHECK(ELIB_ALIGN, p) -#else -#define ALIGN_CHECK(a, p) -#define ELIB_ALIGN_CHECK(p) -#endif - -#define DYNAMIC 32 - -/* -** Free block layout -** 1 1 30 -** +--------------------------+ -** |F|P| Size | -** +--------------------------+ -** -** Where F is the free bit -** P is the free above bit -** Size is messured in words and does not include the hdr word -** -** If block is on the free list the size is also stored last in the block. -** -*/ -typedef struct _free_block FreeBlock; -struct _free_block { - EWord hdr; - Uint flags; - FreeBlock* parent; - FreeBlock* left; - FreeBlock* right; - EWord v[1]; -}; - -typedef struct _allocated_block { - EWord hdr; - EWord v[5]; -} AllocatedBlock; - - -/* - * Interface to tree routines. - */ -typedef Uint Block_t; - -static Block_t* get_free_block(Uint); -static void link_free_block(Block_t *); -static void unlink_free_block(Block_t *del); - -#define FREE_BIT 0x80000000 -#define FREE_ABOVE_BIT 0x40000000 -#define SIZE_MASK 0x3fffffff /* 2^30 words = 2^32 bytes */ - -/* Work on both FreeBlock and AllocatedBlock */ -#define SIZEOF(p) ((p)->hdr & SIZE_MASK) -#define IS_FREE(p) (((p)->hdr & FREE_BIT) != 0) -#define IS_FREE_ABOVE(p) (((p)->hdr & FREE_ABOVE_BIT) != 0) - -/* Given that we have a free block above find its size */ -#define SIZEOF_ABOVE(p) *(((EWord*) (p)) - 1) - -#define MIN_BLOCK_SIZE (sizeof(FreeBlock)/sizeof(EWord)) -#define MIN_WORD_SIZE (MIN_BLOCK_SIZE-1) -#define MIN_BYTE_SIZE (sizeof(FreeBlock)-sizeof(EWord)) - -#define MIN_ALIGN_SIZE ALIGN_SIZE(MIN_BYTE_SIZE) - - -static AllocatedBlock* heap_head = 0; -static AllocatedBlock* heap_tail = 0; -static EWord eheap_size = 0; - -static int heap_locked; - -static int elib_need_init = 1; -#if THREAD_SAFE_ELIB_MALLOC -static int elib_is_initing = 0; -#endif - -typedef FreeBlock RBTree_t; - -static RBTree_t* root = NULL; - - -static FUNCTION(void, deallocate, (AllocatedBlock*, int)); - -/* - * Unlink a free block - */ - -#define mark_allocated(p, szp) do { \ - (p)->hdr = ((p)->hdr & FREE_ABOVE_BIT) | (szp); \ - (p)->v[szp] &= ~FREE_ABOVE_BIT; \ - } while(0) - -#define mark_free(p, szp) do { \ - (p)->hdr = FREE_BIT | (szp); \ - ((FreeBlock *)p)->v[szp-sizeof(FreeBlock)/sizeof(EWord)+1] = (szp); \ - } while(0) - -#if 0 -/* Help macros to log2 */ -#define LOG_1(x) (((x) > 1) ? 1 : 0) -#define LOG_2(x) (((x) > 3) ? 2+LOG_1((x) >> 2) : LOG_1(x)) -#define LOG_4(x) (((x) > 15) ? 4+LOG_2((x) >> 4) : LOG_2(x)) -#define LOG_8(x) (((x) > 255) ? 8+LOG_4((x)>>8) : LOG_4(x)) -#define LOG_16(x) (((x) > 65535) ? 16+LOG_8((x)>>16) : LOG_8(x)) - -#define log2(x) LOG_16(x) -#endif - -/* - * Split a block to be allocated. - * Mark block as ALLOCATED and clear - * FREE_ABOVE_BIT on next block - * - * nw is SIZE aligned and szp is SIZE aligned + 1 - */ -static void -split_block(FreeBlock* p, EWord nw, EWord szp) -{ - EWord szq; - FreeBlock* q; - - szq = szp - nw; - /* Preserve FREE_ABOVE bit in p->hdr !!! */ - - if (szq >= MIN_ALIGN_SIZE+1) { - szq--; - p->hdr = (p->hdr & FREE_ABOVE_BIT) | nw; - - q = (FreeBlock*) (((EWord*) p) + nw + 1); - mark_free(q, szq); - link_free_block((Block_t *) q); - - q = (FreeBlock*) (((EWord*) q) + szq + 1); - q->hdr |= FREE_ABOVE_BIT; - } - else { - mark_allocated((AllocatedBlock*)p, szp); - } -} - -/* - * Find a free block - */ -static FreeBlock* -alloc_block(EWord nw) -{ - for (;;) { - FreeBlock* p = (FreeBlock *) get_free_block(nw); - - if (p != NULL) { - return p; - } else if (ELIB_EXPAND(nw+MIN_WORD_SIZE)) { - return 0; - } - } -} - - -size_t elib_sizeof(void *p) -{ - AllocatedBlock* pp; - - if (p != 0) { - pp = (AllocatedBlock*) (((char *)p)-1); - return SIZEOF(pp); - } - return 0; -} - -static void locked_elib_init(EWord*, EWord); -static void init_elib_malloc(EWord*, EWord); - -/* -** Initialize the elib -** The addr and sz is only used when compiled with EXPAND_ADDR -*/ -/* Not static, this is used by VxWorks */ -void elib_init(EWord* addr, EWord sz) -{ - if (!elib_need_init) - return; - erts_mtx_lock(&malloc_mutex); - locked_elib_init(addr, sz); - erts_mtx_unlock(&malloc_mutex); -} - -static void locked_elib_init(EWord* addr, EWord sz) -{ - if (!elib_need_init) - return; - -#if THREAD_SAFE_ELIB_MALLOC - -#if !USE_RECURSIVE_MALLOC_MUTEX - { - static erts_tid_t initer_tid; - - if(elib_is_initing) { - - if(erts_equal_tids(initer_tid, erts_thr_self())) - return; - - /* Wait until initializing thread is done with initialization */ - - while(elib_need_init) - erts_cnd_wait(&malloc_cond, &malloc_mutex); - - return; - } - else { - initer_tid = erts_thr_self(); - elib_is_initing = 1; - } - } -#else - if(elib_is_initing) - return; - elib_is_initing = 1; -#endif - -#endif /* #if THREAD_SAFE_ELIB_MALLOC */ - - /* Do the actual initialization of the malloc implementation */ - init_elib_malloc(addr, sz); - -#if THREAD_SAFE_ELIB_MALLOC - -#if !USE_RECURSIVE_MALLOC_MUTEX - erts_mtx_unlock(&malloc_mutex); -#endif - - /* Recursive calls to malloc are allowed here... */ - erts_mtx_set_forksafe(&malloc_mutex); - -#if !USE_RECURSIVE_MALLOC_MUTEX - erts_mtx_lock(&malloc_mutex); - elib_is_initing = 0; -#endif - -#endif /* #if THREAD_SAFE_ELIB_MALLOC */ - - elib_need_init = 0; - -#if THREAD_SAFE_ELIB_MALLOC && !USE_RECURSIVE_MALLOC_MUTEX - erts_cnd_broadcast(&malloc_cond); -#endif - -} - -static void init_elib_malloc(EWord* addr, EWord sz) -{ - int i; - FreeBlock* freep; - EWord tmp_sz; -#ifdef ELIB_HEAP_SBRK - char* top; - EWord n; -#endif - - max_allocated = 0; - tot_allocated = 0; - root = NULL; - - /* Get the page size (may involve system call!!!) */ - page_size = PAGE_SIZE; - -#if defined(ELIB_HEAP_SBRK) - sz = PAGES(ELIB_HEAP_SIZE)*page_size; - - if ((top = (char*) sbrk(0)) == (char*)-1) { - elib_printf(stderr, "could not initialize elib, sbrk(0)"); - ELIB_FAILURE; - } - n = PAGE_ALIGN(top) - top; - if ((top = (char*) sbrk(n)) == (char*)-1) { - elib_printf(stderr, "could not initialize elib, sbrk(n)"); - ELIB_FAILURE; - } - if ((eheap = (EWord*) sbrk(sz)) == (EWord*)-1) { - elib_printf(stderr, "could not initialize elib, sbrk(SIZE)"); - ELIB_FAILURE; - } - sz = WORDS(ELIB_HEAP_SIZE); -#elif defined(ELIB_HEAP_FIXED) - eheap = fix_heap; - sz = WORDS(ELIB_HEAP_SIZE); -#elif defined(ELIB_HEAP_USER) - eheap = addr; - sz = WORDS(sz); -#else - return -1; -#endif - eheap_size = 0; - - /* Make sure that the first word of the heap_head is aligned */ - addr = ALIGN(eheap+1); - sz -= ((addr - 1) - eheap); /* Subtract unusable size */ - eheap_top = eheap = addr - 1; /* Set new aligned heap start */ - - eheap_top[sz-1] = 0; /* Heap stop mark */ - - addr = eheap; - heap_head = (AllocatedBlock*) addr; - heap_head->hdr = MIN_ALIGN_SIZE; - for (i = 0; i < MIN_ALIGN_SIZE; i++) - heap_head->v[i] = 0; - - addr += (MIN_ALIGN_SIZE+1); - freep = (FreeBlock*) addr; - tmp_sz = sz - (((MIN_ALIGN_SIZE+1) + MIN_BLOCK_SIZE) + 1 + 1); - mark_free(freep, tmp_sz); - link_free_block((Block_t *) freep); - - /* No need to align heap tail */ - heap_tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1]; - heap_tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE; - heap_tail->v[0] = 0; - heap_tail->v[1] = 0; - heap_tail->v[2] = 0; - - eheap_top += sz; - eheap_size += sz; - - heap_locked = 0; -} - -#ifdef ELIB_HEAP_USER -void elib_force_init(EWord* addr, EWord sz) -{ - elib_need_init = 1; - elib_init(addr,sz); -} -#endif - -#ifdef ELIB_HEAP_SBRK - -/* -** need in number of words (should include head and tail words) -*/ -static int expand_sbrk(EWord sz) -{ - EWord* p; - EWord bytes = sz * sizeof(EWord); - EWord size; - AllocatedBlock* tail; - - if (bytes < ELIB_HEAP_SIZE) - size = PAGES(ELIB_HEAP_INCREAMENT)*page_size; - else - size = PAGES(bytes)*page_size; - - if ((p = (EWord*) sbrk(size)) == ((EWord*) -1)) - return -1; - - if (p != eheap_top) { - elib_printf(stderr, "panic: sbrk moved\n"); - ELIB_FAILURE; - } - - sz = WORDS(size); - - /* Set new endof heap marker and a new heap tail */ - eheap_top[sz-1] = 0; - - tail = (AllocatedBlock*) &eheap_top[sz-MIN_BLOCK_SIZE-1]; - tail->hdr = FREE_ABOVE_BIT | MIN_WORD_SIZE; - tail->v[0] = 0; - tail->v[1] = 0; - tail->v[2] = 0; - - /* Patch old tail with new appended size */ - heap_tail->hdr = (heap_tail->hdr & FREE_ABOVE_BIT) | - (MIN_WORD_SIZE+1+(sz-MIN_BLOCK_SIZE-1)); - deallocate(heap_tail, 0); - - heap_tail = tail; - - eheap_size += sz; - eheap_top += sz; - - return 0; -} - -#endif /* ELIB_HEAP_SBRK */ - - -/* -** Scan heap and check for corrupted heap -*/ -int elib_check_heap(void) -{ - AllocatedBlock* p = heap_head; - EWord sz; - - if (heap_locked) { - elib_printf(stderr, "heap is locked no info avaiable\n"); - return 0; - } - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - if (p->v[sz-1] != sz) { - elib_printf(stderr, "panic: heap corrupted\r\n"); - ELIB_FAILURE; - } - p = (AllocatedBlock*) (p->v + sz); - if (!IS_FREE_ABOVE(p)) { - elib_printf(stderr, "panic: heap corrupted\r\n"); - ELIB_FAILURE; - } - } - else - p = (AllocatedBlock*) (p->v + sz); - } - return 1; -} - -/* -** Load the byte vector pointed to by v of length vsz -** with a heap image -** The scale is defined by vsz and the current heap size -** free = 0, full = 255 -** -** -*/ -int elib_heap_map(EByte* v, int vsz) -{ - AllocatedBlock* p = heap_head; - EWord sz; - int gsz = eheap_size / vsz; /* The granuality used */ - int fsz = 0; - int usz = 0; - - if (gsz == 0) - return -1; /* too good reolution */ - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - fsz += sz; - if ((fsz + usz) > gsz) { - *v++ = (255*usz)/gsz; - fsz -= (gsz - usz); - usz = 0; - while(fsz >= gsz) { - *v++ = 0; - fsz -= gsz; - } - } - } - else { - usz += sz; - if ((fsz + usz) > gsz) { - *v++ = 255 - (255*fsz)/gsz; - usz -= (gsz - fsz); - fsz = 0; - while(usz >= gsz) { - *v++ = 255; - usz -= gsz; - } - } - } - p = (AllocatedBlock*) (p->v + sz); - } - return 0; -} - -/* -** Generate a histogram of free/allocated blocks -** Count granuality of 10 gives -** (0-10],(10-100],(100-1000],(1000-10000] ... -** (0-2], (2-4], (4-8], (8-16], .... -*/ -static int i_logb(EWord size, int base) -{ - int lg = 0; - while(size >= base) { - size /= base; - lg++; - } - return lg; -} - -int elib_histo(EWord* vf, EWord* va, int vsz, int base) -{ - AllocatedBlock* p = heap_head; - EWord sz; - int i; - int linear; - - if ((vsz <= 1) || (vf == 0 && va == 0)) - return -1; - - if (base < 0) { - linear = 1; - base = -base; - } - else - linear = 0; - - if (base <= 1) - return -1; - - if (vf != 0) { - for (i = 0; i < vsz; i++) - vf[i] = 0; - } - if (va != 0) { - for (i = 0; i < vsz; i++) - va[i] = 0; - } - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - if (vf != 0) { - int val; - if (linear) - val = sz / base; - else - val = i_logb(sz, base); - if (val >= vsz) - vf[vsz-1]++; - else - vf[val]++; - } - } - else { - if (va != 0) { - int val; - if (linear) - val = sz / base; - else - val = i_logb(sz, base); - if (val >= vsz) - va[vsz-1]++; - else - va[val]++; - } - } - p = (AllocatedBlock*) (p->v + sz); - } - return 0; -} - -/* -** Fill the info structure with actual values -** Total -** Allocated -** Free -** maxMaxFree -*/ -void elib_stat(struct elib_stat* info) -{ - EWord blks = 0; - EWord sz_free = 0; - EWord sz_alloc = 0; - EWord sz_max_free = 0; - EWord sz_min_used = 0x7fffffff; - EWord sz; - EWord num_free = 0; - AllocatedBlock* p = heap_head; - - info->mem_total = eheap_size; - - p = (AllocatedBlock*) (p->v + SIZEOF(p)); - - while((sz = SIZEOF(p)) != 0) { - blks++; - if (IS_FREE(p)) { - if (sz > sz_max_free) - sz_max_free = sz; - sz_free += sz; - ++num_free; - } - else { - if (sz < sz_min_used) - sz_min_used = sz; - sz_alloc += sz; - } - p = (AllocatedBlock*) (p->v + sz); - } - info->mem_blocks = blks; - info->free_blocks = num_free; - info->mem_alloc = sz_alloc; - info->mem_free = sz_free; - info->min_used = sz_min_used; - info->max_free = sz_max_free; - info->mem_max_alloc = max_allocated; - ASSERT(sz_alloc == tot_allocated); -} - -/* -** Dump the heap -*/ -void elib_heap_dump(char* label) -{ - AllocatedBlock* p = heap_head; - EWord sz; - - elib_printf(stderr, "HEAP DUMP (%s)\n", label); - if (!elib_check_heap()) - return; - - while((sz = SIZEOF(p)) != 0) { - if (IS_FREE(p)) { - elib_printf(stderr, "%p: FREE, size = %d\n", p, (int) sz); - } - else { - elib_printf(stderr, "%p: USED, size = %d %s\n", p, (int) sz, - IS_FREE_ABOVE(p)?"(FREE ABOVE)":""); - } - p = (AllocatedBlock*) (p->v + sz); - } -} - -/* -** Scan heaps and count: -** free_size, allocated_size, max_free_block -*/ -void elib_statistics(void* to) -{ - struct elib_stat info; - EWord frag; - - if (!elib_check_heap()) - return; - - elib_stat(&info); - - frag = 1000 - ((1000 * info.max_free) / info.mem_free); - - elib_printf(to, "Heap Statistics: total(%d), blocks(%d), frag(%d.%d%%)\n", - info.mem_total, info.mem_blocks, - (int) frag/10, (int) frag % 10); - - elib_printf(to, " allocated(%d), free(%d), " - "free_blocks(%d)\n", - info.mem_alloc, info.mem_free,info.free_blocks); - elib_printf(to, " max_free(%d), min_used(%d)\n", - info.max_free, info.min_used); -} - -/* -** Allocate a least nb bytes with alignment a -** Algorithm: -** 1) Try locate a block which match exacly among the by direct index. -** 2) Try using a fix block of greater size -** 3) Try locate a block by searching in lists where block sizes -** X may vary between 2^i < X <= 2^(i+1) -** -** Reset memory to zero if clear is true -*/ -static AllocatedBlock* allocate(EWord nb, EWord a, int clear) -{ - FreeBlock* p; - EWord nw; - - if (a == ELIB_ALIGN) { - /* - * Common case: Called by malloc(), realloc(), calloc(). - */ - nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb); - - if ((p = alloc_block(nw)) == 0) - return NULL; - } else { - /* - * Special case: Called by memalign(). - */ - EWord asz, szp, szq, tmpsz; - FreeBlock *q; - - if ((p = alloc_block((1+MIN_ALIGN_SIZE)*sizeof(EWord)+a-1+nb)) == 0) - return NULL; - - asz = a - ((EWord) ((AllocatedBlock *)p)->v) % a; - - if (asz != a) { - /* Enforce the alignment requirement by cutting of a free - block at the beginning of the block. */ - - if (asz < (1+MIN_ALIGN_SIZE)*sizeof(EWord) && !IS_FREE_ABOVE(p)) { - /* Not enough room to cut of a free block; - increase align size */ - asz += (((1+MIN_ALIGN_SIZE)*sizeof(EWord) + a - 1)/a)*a; - } - - szq = ALIGN_SIZE(asz - sizeof(EWord)); - szp = SIZEOF(p) - szq - 1; - - q = p; - p = (FreeBlock*) (((EWord*) q) + szq + 1); - p->hdr = FREE_ABOVE_BIT | FREE_BIT | szp; - - if (IS_FREE_ABOVE(q)) { /* This should not be possible I think, - but just in case... */ - tmpsz = SIZEOF_ABOVE(q) + 1; - szq += tmpsz; - q = (FreeBlock*) (((EWord*) q) - tmpsz); - unlink_free_block((Block_t *) q); - q->hdr = (q->hdr & FREE_ABOVE_BIT) | FREE_BIT | szq; - } - mark_free(q, szq); - link_free_block((Block_t *) q); - - } /* else already had the correct alignment */ - - nw = nb < MIN_BYTE_SIZE ? MIN_ALIGN_SIZE : ALIGN_SIZE(nb); - } - - split_block(p, nw, SIZEOF(p)); - - STAT_ALLOCED_BLOCK(SIZEOF(p)); - - if (clear) { - EWord* pp = ((AllocatedBlock*)p)->v; - - while(nw--) - *pp++ = 0; - } - - return (AllocatedBlock*) p; -} - - -/* -** Deallocate memory pointed to by p -** 1. Merge with block above if this block is free -** 2. Merge with block below if this block is free -** Link the block to the correct free list -** -** p points to the block header! -** -*/ -static void deallocate(AllocatedBlock* p, int stat_count) -{ - FreeBlock* q; - EWord szq; - EWord szp; - - szp = SIZEOF(p); - - if (stat_count) - STAT_FREED_BLOCK(SIZEOF(p)); - - if (IS_FREE_ABOVE(p)) { - szq = SIZEOF_ABOVE(p); - q = (FreeBlock*) ( ((EWord*) p) - szq - 1); - unlink_free_block((Block_t *) q); - - p = (AllocatedBlock*) q; - szp += (szq + 1); - } - q = (FreeBlock*) (p->v + szp); - if (IS_FREE(q)) { - szq = SIZEOF(q); - unlink_free_block((Block_t *) q); - szp += (szq + 1); - } - else - q->hdr |= FREE_ABOVE_BIT; - - /* The block above p can NEVER be free !!! */ - p->hdr = FREE_BIT | szp; - p->v[szp-1] = szp; - - link_free_block((Block_t *) p); -} - -/* -** Reallocate memory -** If preserve is true then data is moved if neccesary -*/ -static AllocatedBlock* reallocate(AllocatedBlock* p, EWord nb, int preserve) -{ - EWord szp; - EWord szq; - EWord sz; - EWord nw; - FreeBlock* q; - - if (nb < MIN_BYTE_SIZE) - nw = MIN_ALIGN_SIZE; - else - nw = ALIGN_SIZE(nb); - - sz = szp = SIZEOF(p); - - STAT_FREED_BLOCK(szp); - - /* Merge with block below */ - q = (FreeBlock*) (p->v + szp); - if (IS_FREE(q)) { - szq = SIZEOF(q); - unlink_free_block((Block_t *) q); - szp += (szq + 1); - } - - if (nw <= szp) { - split_block((FreeBlock *) p, nw, szp); - STAT_ALLOCED_BLOCK(SIZEOF(p)); - return p; - } - else { - EWord* dp = p->v; - AllocatedBlock* npp; - - if (IS_FREE_ABOVE(p)) { - szq = SIZEOF_ABOVE(p); - if (szq + szp + 1 >= nw) { - q = (FreeBlock*) (((EWord*) p) - szq - 1); - unlink_free_block((Block_t * )q); - szp += (szq + 1); - p = (AllocatedBlock*) q; - - if (preserve) { - EWord* pp = p->v; - while(sz--) - *pp++ = *dp++; - } - split_block((FreeBlock *) p, nw, szp); - STAT_ALLOCED_BLOCK(SIZEOF(p)); - return p; - } - } - - /* - * Update p so that allocate() and deallocate() works. - * (Note that allocate() may call expand_sbrk(), which in - * in turn calls deallocate().) - */ - - p->hdr = (p->hdr & FREE_ABOVE_BIT) | szp; - p->v[szp] &= ~FREE_ABOVE_BIT; - - npp = allocate(nb, ELIB_ALIGN, 0); - if(npp == NULL) - return NULL; - if (preserve) { - EWord* pp = npp->v; - while(sz--) - *pp++ = *dp++; - } - deallocate(p, 0); - return npp; - } -} - -/* -** What malloc() and friends should do (and return) when the heap is -** exhausted. [sverkerw] -*/ -static void* heap_exhausted(void) -{ - /* Choose behaviour */ -#if 0 - /* Crash-and-burn --- leave a usable corpse (hopefully) */ - abort(); -#endif - /* The usual ANSI-compliant behaviour */ - return NULL; -} - -/* -** Allocate size bytes of memory -*/ -void* ELIB_PREFIX(malloc, (size_t nb)) -{ - void *res; - AllocatedBlock* p; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (nb == 0) - res = NULL; - else if ((p = allocate(nb, ELIB_ALIGN, 0)) != 0) { - ELIB_ALIGN_CHECK(p->v); - res = p->v; - } - else - res = heap_exhausted(); - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - - -void* ELIB_PREFIX(calloc, (size_t nelem, size_t size)) -{ - void *res; - int nb; - AllocatedBlock* p; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if ((nb = nelem * size) == 0) - res = NULL; - else if ((p = allocate(nb, ELIB_ALIGN, 1)) != 0) { - ELIB_ALIGN_CHECK(p->v); - res = p->v; - } - else - res = heap_exhausted(); - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - -/* -** Free memory allocated by malloc -*/ - -void ELIB_PREFIX(free, (EWord* p)) -{ - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (p != 0) - deallocate((AllocatedBlock*)(p-1), 1); - - erts_mtx_unlock(&malloc_mutex); -} - -void ELIB_PREFIX(cfree, (EWord* p)) -{ - ELIB_PREFIX(free, (p)); -} - - -/* -** Realloc the memory allocated in p to nb number of bytes -** -*/ - -void* ELIB_PREFIX(realloc, (EWord* p, size_t nb)) -{ - void *res = NULL; - AllocatedBlock* pp; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (p != 0) { - pp = (AllocatedBlock*) (p-1); - if (nb > 0) { - if ((pp = reallocate(pp, nb, 1)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - } - else - deallocate(pp, 1); - } - else if (nb > 0) { - if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - else - res = heap_exhausted(); - } - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - -/* -** Resize the memory area pointed to by p with nb number of bytes -*/ -void* ELIB_PREFIX(memresize, (EWord* p, int nb)) -{ - void *res = NULL; - AllocatedBlock* pp; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (p != 0) { - pp = (AllocatedBlock*) (p-1); - if (nb > 0) { - if ((pp = reallocate(pp, nb, 0)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - } - else - deallocate(pp, 1); - } - else if (nb > 0) { - if ((pp = allocate(nb, ELIB_ALIGN, 0)) != 0) { - ELIB_ALIGN_CHECK(pp->v); - res = pp->v; - } - else - res = heap_exhausted(); - } - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - - -/* Create aligned memory a must be a power of 2 !!! */ - -void* ELIB_PREFIX(memalign, (int a, int nb)) -{ - void *res; - AllocatedBlock* p; - - erts_mtx_lock(&malloc_mutex); - if (elib_need_init) - locked_elib_init(NULL,(EWord)0); - - if (nb == 0 || a <= 0) - res = NULL; - else if ((p = allocate(nb, a, 0)) != 0) { - ALIGN_CHECK(a, p->v); - res = p->v; - } - else - res = heap_exhausted(); - - erts_mtx_unlock(&malloc_mutex); - - return res; -} - -void* ELIB_PREFIX(valloc, (int nb)) -{ - return ELIB_PREFIX(memalign, (page_size, nb)); -} - - -void* ELIB_PREFIX(pvalloc, (int nb)) -{ - return ELIB_PREFIX(memalign, (page_size, PAGES(nb)*page_size)); -} -/* Return memory size for pointer p in bytes */ - -int ELIB_PREFIX(memsize, (p)) -EWord* p; -{ - return SIZEOF((AllocatedBlock*)(p-1))*4; -} - - -/* -** -------------------------------------------------------------------------- -** DEBUG LIBRARY -** -------------------------------------------------------------------------- -*/ - -#ifdef ELIB_DEBUG - -#define IN_HEAP(p) (((p) >= (char*) eheap) && (p) < (char*) eheap_top) -/* -** ptr_to_block: return the pointer to heap block pointed into by ptr -** Returns 0 if not pointing into a block -*/ - -static EWord* ptr_to_block(char* ptr) -{ - AllocatedBlock* p = heap_head; - EWord sz; - - while((sz = SIZEOF(p)) != 0) { - if ((ptr >= (char*) p->v) && (ptr < (char*)(p->v+sz))) - return p->v; - p = (AllocatedBlock*) (p->v + sz); - } - return 0; -} - -/* -** Validate a pointer -** returns: -** 0 - if points to start of a block -** 1 - if points outsize heap -** -1 - if points inside block -** -*/ -static int check_pointer(char* ptr) -{ - if (IN_HEAP(ptr)) { - if (ptr_to_block(ptr) == 0) - return 1; - return 0; - } - return -1; -} - -/* -** Validate a memory area -** returns: -** 0 - if area is included in a block -** -1 - if area overlap a heap block -** 1 - if area is outside heap -*/ -static int check_area(char* ptr, int n) -{ - if (IN_HEAP(ptr)) { - if (IN_HEAP(ptr+n-1)) { - EWord* p1 = ptr_to_block(ptr); - EWord* p2 = ptr_to_block(ptr+n-1); - - if (p1 == p2) - return (p1 == 0) ? -1 : 0; - return -1; - } - } - else if (IN_HEAP(ptr+n-1)) - return -1; - return 1; -} - -/* -** Check if a block write will overwrite heap block -*/ -static void check_write(char* ptr, int n, char* file, int line, char* fun) -{ - if (check_area(ptr, n) == -1) { - elib_printf(stderr, "RUNTIME ERROR: %s heap overwrite\n", fun); - elib_printf(stderr, "File: %s Line: %d\n", file, line); - ELIB_FAILURE; - } -} - -/* -** Check if a pointer is an allocated object -*/ -static void check_allocated_block(char* ptr, char* file, int line, char* fun) -{ - EWord* q; - - if (!IN_HEAP(ptr) || ((q=ptr_to_block(ptr)) == 0) || (ptr != (char*) q)) { - elib_printf(stderr, "RUNTIME ERROR: %s non heap pointer\n", fun); - elib_printf(stderr, "File: %s Line: %d\n", file, line); - ELIB_FAILURE; - } - - if (IS_FREE((AllocatedBlock*)(q-1))) { - elib_printf(stderr, "RUNTIME ERROR: %s free pointer\n", fun); - elib_printf(stderr, "File: %s Line: %d\n", file, line); - ELIB_FAILURE; - } - -} - -/* -** -------------------------------------------------------------------------- -** DEBUG VERSIONS (COMPILED WITH THE ELIB.H) -** -------------------------------------------------------------------------- -*/ - -void* elib_dbg_malloc(int n, char* file, int line) -{ - return elib__malloc(n); -} - -void* elib_dbg_calloc(int n, int s, char* file, int line) -{ - return elib__calloc(n, s); -} - -void* elib_dbg_realloc(EWord* p, int n, char* file, int line) -{ - if (p == 0) - return elib__malloc(n); - check_allocated_block(p, file, line, "elib_realloc"); - return elib__realloc(p, n); -} - -void elib_dbg_free(EWord* p, char* file, int line) -{ - if (p == 0) - return; - check_allocated_block(p, file, line, "elib_free"); - elib__free(p); -} - -void elib_dbg_cfree(EWord* p, char* file, int line) -{ - if (p == 0) - return; - check_allocated_block(p, file, line, "elib_free"); - elib__cfree(p); -} - -void* elib_dbg_memalign(int a, int n, char* file, int line) -{ - return elib__memalign(a, n); -} - -void* elib_dbg_valloc(int n, char* file, int line) -{ - return elib__valloc(n); -} - -void* elib_dbg_pvalloc(int n, char* file, int line) -{ - return elib__pvalloc(n); -} - -void* elib_dbg_memresize(EWord* p, int n, char* file, int line) -{ - if (p == 0) - return elib__malloc(n); - check_allocated_block(p, file, line, "elib_memresize"); - return elib__memresize(p, n); -} - -int elib_dbg_memsize(void* p, char* file, int line) -{ - check_allocated_block(p, file, line, "elib_memsize"); - return elib__memsize(p); -} - -/* -** -------------------------------------------------------------------------- -** LINK TIME FUNCTIONS (NOT COMPILED CALLS) -** -------------------------------------------------------------------------- -*/ - -void* elib_malloc(int n) -{ - return elib_dbg_malloc(n, "", -1); -} - -void* elib_calloc(int n, int s) -{ - return elib_dbg_calloc(n, s, "", -1); -} - -void* elib_realloc(EWord* p, int n) -{ - return elib_dbg_realloc(p, n, "", -1); -} - -void elib_free(EWord* p) -{ - elib_dbg_free(p, "", -1); -} - -void elib_cfree(EWord* p) -{ - elib_dbg_cfree(p, "", -1); -} - -void* elib_memalign(int a, int n) -{ - return elib_dbg_memalign(a, n, "", -1); -} - -void* elib_valloc(int n) -{ - return elib_dbg_valloc(n, "", -1); -} - -void* elib_pvalloc(int n) -{ - return elib_dbg_pvalloc(n, "", -1); -} - -void* elib_memresize(EWord* p, int n) -{ - return elib_dbg_memresize(p, n, "", -1); -} - - -int elib_memsize(EWord* p) -{ - return elib_dbg_memsize(p, "", -1); -} - -#endif /* ELIB_DEBUG */ - -/* -** -------------------------------------------------------------------------- -** Map c library functions to elib -** -------------------------------------------------------------------------- -*/ - -#if defined(ELIB_ALLOC_IS_CLIB) -void* malloc(size_t nb) -{ - return elib_malloc(nb); -} - -void* calloc(size_t nelem, size_t size) -{ - return elib_calloc(nelem, size); -} - - -void free(void *p) -{ - elib_free(p); -} - -void cfree(void *p) -{ - elib_cfree(p); -} - -void* realloc(void* p, size_t nb) -{ - return elib_realloc(p, nb); -} - - -void* memalign(size_t a, size_t s) -{ - return elib_memalign(a, s); -} - -void* valloc(size_t nb) -{ - return elib_valloc(nb); -} - -void* pvalloc(size_t nb) -{ - return elib_pvalloc(nb); -} - -#if 0 -void* memresize(void* p, int nb) -{ - return elib_memresize(p, nb); -} - -int memsize(void* p) -{ - return elib_memsize(p); -} -#endif -#endif /* ELIB_ALLOC_IS_CLIB */ - -#endif /* ENABLE_ELIB_MALLOC */ - -void elib_ensure_initialized(void) -{ -#ifdef ENABLE_ELIB_MALLOC -#ifndef ELIB_DONT_INITIALIZE - elib_init(NULL, 0); -#endif -#endif -} - -#ifdef ENABLE_ELIB_MALLOC -/** - ** A Slightly modified version of the "address order best fit" algorithm - ** used in erl_bestfit_alloc.c. Comments refer to that implementation. - **/ - -/* - * Description: A combined "address order best fit"/"best fit" allocator - * based on a Red-Black (binary search) Tree. The search, - * insert, and delete operations are all O(log n) operations - * on a Red-Black Tree. In the "address order best fit" case - * n equals number of free blocks, and in the "best fit" case - * n equals number of distinct sizes of free blocks. Red-Black - * Trees are described in "Introduction to Algorithms", by - * Thomas H. Cormen, Charles E. Leiserson, and - * Ronald L. Riverest. - * - * This module is a callback-module for erl_alloc_util.c - * - * Author: Rickard Green - */ - -#ifdef DEBUG -#if 0 -#define HARD_DEBUG -#endif -#else -#undef HARD_DEBUG -#endif - -#define SZ_MASK SIZE_MASK -#define FLG_MASK (~(SZ_MASK)) - -#define BLK_SZ(B) (*((Block_t *) (B)) & SZ_MASK) - -#define TREE_NODE_FLG (((Uint) 1) << 0) -#define RED_FLG (((Uint) 1) << 1) -#ifdef HARD_DEBUG -# define LEFT_VISITED_FLG (((Uint) 1) << 2) -# define RIGHT_VISITED_FLG (((Uint) 1) << 3) -#endif - -#define IS_TREE_NODE(N) (((RBTree_t *) (N))->flags & TREE_NODE_FLG) -#define IS_LIST_ELEM(N) (!IS_TREE_NODE(((RBTree_t *) (N)))) - -#define SET_TREE_NODE(N) (((RBTree_t *) (N))->flags |= TREE_NODE_FLG) -#define SET_LIST_ELEM(N) (((RBTree_t *) (N))->flags &= ~TREE_NODE_FLG) - -#define IS_RED(N) (((RBTree_t *) (N)) \ - && ((RBTree_t *) (N))->flags & RED_FLG) -#define IS_BLACK(N) (!IS_RED(((RBTree_t *) (N)))) - -#define SET_RED(N) (((RBTree_t *) (N))->flags |= RED_FLG) -#define SET_BLACK(N) (((RBTree_t *) (N))->flags &= ~RED_FLG) - -#undef ASSERT -#define ASSERT ASSERT_EXPR - -#if 1 -#define RBT_ASSERT ASSERT -#else -#define RBT_ASSERT(x) -#endif - - -#ifdef HARD_DEBUG -static RBTree_t * check_tree(Uint); -#endif - -#ifdef ERTS_INLINE -# ifndef ERTS_CAN_INLINE -# define ERTS_CAN_INLINE 1 -# endif -#else -# if defined(__GNUC__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline__ -# elif defined(__WIN32__) -# define ERTS_CAN_INLINE 1 -# define ERTS_INLINE __inline -# else -# define ERTS_CAN_INLINE 0 -# define ERTS_INLINE -# endif -#endif - -/* Types... */ -#if 0 -typedef struct RBTree_t_ RBTree_t; - -struct RBTree_t_ { - Block_t hdr; - Uint flags; - RBTree_t *parent; - RBTree_t *left; - RBTree_t *right; -}; -#endif - -#if 0 -typedef struct { - RBTree_t t; - RBTree_t *next; -} RBTreeList_t; - -#define LIST_NEXT(N) (((RBTreeList_t *) (N))->next) -#define LIST_PREV(N) (((RBTreeList_t *) (N))->t.parent) -#endif - -#ifdef DEBUG - -/* Destroy all tree fields */ -#define DESTROY_TREE_NODE(N) \ - sys_memset((void *) (((Block_t *) (N)) + 1), \ - 0xff, \ - (sizeof(RBTree_t) - sizeof(Block_t))) - -/* Destroy all tree and list fields */ -#define DESTROY_LIST_ELEM(N) \ - sys_memset((void *) (((Block_t *) (N)) + 1), \ - 0xff, \ - (sizeof(RBTreeList_t) - sizeof(Block_t))) - -#else - -#define DESTROY_TREE_NODE(N) -#define DESTROY_LIST_ELEM(N) - -#endif - - -/* - * Red-Black Tree operations needed - */ - -static ERTS_INLINE void -left_rotate(RBTree_t **root, RBTree_t *x) -{ - RBTree_t *y = x->right; - x->right = y->left; - if (y->left) - y->left->parent = x; - y->parent = x->parent; - if (!y->parent) { - RBT_ASSERT(*root == x); - *root = y; - } - else if (x == x->parent->left) - x->parent->left = y; - else { - RBT_ASSERT(x == x->parent->right); - x->parent->right = y; - } - y->left = x; - x->parent = y; -} - -static ERTS_INLINE void -right_rotate(RBTree_t **root, RBTree_t *x) -{ - RBTree_t *y = x->left; - x->left = y->right; - if (y->right) - y->right->parent = x; - y->parent = x->parent; - if (!y->parent) { - RBT_ASSERT(*root == x); - *root = y; - } - else if (x == x->parent->right) - x->parent->right = y; - else { - RBT_ASSERT(x == x->parent->left); - x->parent->left = y; - } - y->right = x; - x->parent = y; -} - - -/* - * Replace node x with node y - * NOTE: block header of y is not changed - */ -static ERTS_INLINE void -replace(RBTree_t **root, RBTree_t *x, RBTree_t *y) -{ - - if (!x->parent) { - RBT_ASSERT(*root == x); - *root = y; - } - else if (x == x->parent->left) - x->parent->left = y; - else { - RBT_ASSERT(x == x->parent->right); - x->parent->right = y; - } - if (x->left) { - RBT_ASSERT(x->left->parent == x); - x->left->parent = y; - } - if (x->right) { - RBT_ASSERT(x->right->parent == x); - x->right->parent = y; - } - - y->flags = x->flags; - y->parent = x->parent; - y->right = x->right; - y->left = x->left; - - DESTROY_TREE_NODE(x); - -} - -static void -tree_insert_fixup(RBTree_t *blk) -{ - RBTree_t *x = blk, *y; - - /* - * Rearrange the tree so that it satisfies the Red-Black Tree properties - */ - - RBT_ASSERT(x != root && IS_RED(x->parent)); - do { - - /* - * x and its parent are both red. Move the red pair up the tree - * until we get to the root or until we can separate them. - */ - - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(x->parent->parent); - - if (x->parent == x->parent->parent->left) { - y = x->parent->parent->right; - if (IS_RED(y)) { - SET_BLACK(y); - x = x->parent; - SET_BLACK(x); - x = x->parent; - SET_RED(x); - } - else { - - if (x == x->parent->right) { - x = x->parent; - left_rotate(&root, x); - } - - RBT_ASSERT(x == x->parent->parent->left->left); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(IS_BLACK(y)); - - SET_BLACK(x->parent); - SET_RED(x->parent->parent); - right_rotate(&root, x->parent->parent); - - RBT_ASSERT(x == x->parent->left); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent->right)); - RBT_ASSERT(IS_BLACK(x->parent)); - break; - } - } - else { - RBT_ASSERT(x->parent == x->parent->parent->right); - y = x->parent->parent->left; - if (IS_RED(y)) { - SET_BLACK(y); - x = x->parent; - SET_BLACK(x); - x = x->parent; - SET_RED(x); - } - else { - - if (x == x->parent->left) { - x = x->parent; - right_rotate(&root, x); - } - - RBT_ASSERT(x == x->parent->parent->right->right); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent)); - RBT_ASSERT(IS_BLACK(x->parent->parent)); - RBT_ASSERT(IS_BLACK(y)); - - SET_BLACK(x->parent); - SET_RED(x->parent->parent); - left_rotate(&root, x->parent->parent); - - RBT_ASSERT(x == x->parent->right); - RBT_ASSERT(IS_RED(x)); - RBT_ASSERT(IS_RED(x->parent->left)); - RBT_ASSERT(IS_BLACK(x->parent)); - break; - } - } - } while (x != root && IS_RED(x->parent)); - - SET_BLACK(root); -} - -static void -unlink_free_block(Block_t *del) -{ - Uint spliced_is_black; - RBTree_t *x, *y, *z = (RBTree_t *) del; - RBTree_t null_x; /* null_x is used to get the fixup started when we - splice out a node without children. */ - - null_x.parent = NULL; - -#ifdef HARD_DEBUG - check_tree(0); -#endif - - /* Remove node from tree... */ - - /* Find node to splice out */ - if (!z->left || !z->right) - y = z; - else - /* Set y to z:s successor */ - for(y = z->right; y->left; y = y->left); - /* splice out y */ - x = y->left ? y->left : y->right; - spliced_is_black = IS_BLACK(y); - if (x) { - x->parent = y->parent; - } - else if (!x && spliced_is_black) { - x = &null_x; - x->flags = 0; - SET_BLACK(x); - x->right = x->left = NULL; - x->parent = y->parent; - y->left = x; - } - - if (!y->parent) { - RBT_ASSERT(root == y); - root = x; - } - else if (y == y->parent->left) - y->parent->left = x; - else { - RBT_ASSERT(y == y->parent->right); - y->parent->right = x; - } - if (y != z) { - /* We spliced out the successor of z; replace z by the successor */ - replace(&root, z, y); - } - - if (spliced_is_black) { - /* We removed a black node which makes the resulting tree - violate the Red-Black Tree properties. Fixup tree... */ - - while (IS_BLACK(x) && x->parent) { - - /* - * x has an "extra black" which we move up the tree - * until we reach the root or until we can get rid of it. - * - * y is the sibbling of x - */ - - if (x == x->parent->left) { - y = x->parent->right; - RBT_ASSERT(y); - if (IS_RED(y)) { - RBT_ASSERT(y->right); - RBT_ASSERT(y->left); - SET_BLACK(y); - RBT_ASSERT(IS_BLACK(x->parent)); - SET_RED(x->parent); - left_rotate(&root, x->parent); - y = x->parent->right; - } - RBT_ASSERT(y); - RBT_ASSERT(IS_BLACK(y)); - if (IS_BLACK(y->left) && IS_BLACK(y->right)) { - SET_RED(y); - x = x->parent; - } - else { - if (IS_BLACK(y->right)) { - SET_BLACK(y->left); - SET_RED(y); - right_rotate(&root, y); - y = x->parent->right; - } - RBT_ASSERT(y); - if (IS_RED(x->parent)) { - - SET_BLACK(x->parent); - SET_RED(y); - } - RBT_ASSERT(y->right); - SET_BLACK(y->right); - left_rotate(&root, x->parent); - x = root; - break; - } - } - else { - RBT_ASSERT(x == x->parent->right); - y = x->parent->left; - RBT_ASSERT(y); - if (IS_RED(y)) { - RBT_ASSERT(y->right); - RBT_ASSERT(y->left); - SET_BLACK(y); - RBT_ASSERT(IS_BLACK(x->parent)); - SET_RED(x->parent); - right_rotate(&root, x->parent); - y = x->parent->left; - } - RBT_ASSERT(y); - RBT_ASSERT(IS_BLACK(y)); - if (IS_BLACK(y->right) && IS_BLACK(y->left)) { - SET_RED(y); - x = x->parent; - } - else { - if (IS_BLACK(y->left)) { - SET_BLACK(y->right); - SET_RED(y); - left_rotate(&root, y); - y = x->parent->left; - } - RBT_ASSERT(y); - if (IS_RED(x->parent)) { - SET_BLACK(x->parent); - SET_RED(y); - } - RBT_ASSERT(y->left); - SET_BLACK(y->left); - right_rotate(&root, x->parent); - x = root; - break; - } - } - } - SET_BLACK(x); - - if (null_x.parent) { - if (null_x.parent->left == &null_x) - null_x.parent->left = NULL; - else { - RBT_ASSERT(null_x.parent->right == &null_x); - null_x.parent->right = NULL; - } - RBT_ASSERT(!null_x.left); - RBT_ASSERT(!null_x.right); - } - else if (root == &null_x) { - root = NULL; - RBT_ASSERT(!null_x.left); - RBT_ASSERT(!null_x.right); - } - } - - - DESTROY_TREE_NODE(del); - -#ifdef HARD_DEBUG - check_tree(0); -#endif - -} - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * "Address order best fit" specific callbacks. * -\* */ - -static void -link_free_block(Block_t *block) -{ - RBTree_t *blk = (RBTree_t *) block; - Uint blk_sz = BLK_SZ(blk); - - blk->flags = 0; - blk->left = NULL; - blk->right = NULL; - - if (!root) { - blk->parent = NULL; - SET_BLACK(blk); - root = blk; - } else { - RBTree_t *x = root; - while (1) { - Uint size; - - size = BLK_SZ(x); - - if (blk_sz < size || (blk_sz == size && blk < x)) { - if (!x->left) { - blk->parent = x; - x->left = blk; - break; - } - x = x->left; - } - else { - if (!x->right) { - blk->parent = x; - x->right = blk; - break; - } - x = x->right; - } - - } - - /* Insert block into size tree */ - RBT_ASSERT(blk->parent); - - SET_RED(blk); - if (IS_RED(blk->parent)) { - tree_insert_fixup(blk); - } - } - -#ifdef HARD_DEBUG - check_tree(0); -#endif -} - - -static Block_t * -get_free_block(Uint size) -{ - RBTree_t *x = root; - RBTree_t *blk = NULL; - Uint blk_sz; - - while (x) { - blk_sz = BLK_SZ(x); - if (blk_sz < size) { - x = x->right; - } - else { - blk = x; - x = x->left; - } - } - - if (!blk) - return NULL; - -#ifdef HARD_DEBUG - ASSERT(blk == check_tree(size)); -#endif - - unlink_free_block((Block_t *) blk); - - return (Block_t *) blk; -} - - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * Debug functions * -\* */ - - -#ifdef HARD_DEBUG - -#define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) -#define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) - -#define SET_LEFT_VISITED(FB) ((FB)->flags |= LEFT_VISITED_FLG) -#define SET_RIGHT_VISITED(FB) ((FB)->flags |= RIGHT_VISITED_FLG) - -#define UNSET_LEFT_VISITED(FB) ((FB)->flags &= ~LEFT_VISITED_FLG) -#define UNSET_RIGHT_VISITED(FB) ((FB)->flags &= ~RIGHT_VISITED_FLG) - - -#if 0 -# define PRINT_TREE -#else -# undef PRINT_TREE -#endif - -#ifdef PRINT_TREE -static void print_tree(void); -#endif - -/* - * Checks that the order between parent and children are correct, - * and that the Red-Black Tree properies are satisfied. if size > 0, - * check_tree() returns a node that satisfies "best fit" resp. - * "address order best fit". - * - * The Red-Black Tree properies are: - * 1. Every node is either red or black. - * 2. Every leaf (NIL) is black. - * 3. If a node is red, then both its children are black. - * 4. Every simple path from a node to a descendant leaf - * contains the same number of black nodes. - */ - -static RBTree_t * -check_tree(Uint size) -{ - RBTree_t *res = NULL; - Sint blacks; - Sint curr_blacks; - RBTree_t *x; - -#ifdef PRINT_TREE - print_tree(); -#endif - - if (!root) - return res; - - x = root; - ASSERT(IS_BLACK(x)); - ASSERT(!x->parent); - curr_blacks = 1; - blacks = -1; - - while (x) { - if (!IS_LEFT_VISITED(x)) { - SET_LEFT_VISITED(x); - if (x->left) { - x = x->left; - if (IS_BLACK(x)) - curr_blacks++; - continue; - } - else { - if (blacks < 0) - blacks = curr_blacks; - ASSERT(blacks == curr_blacks); - } - } - - if (!IS_RIGHT_VISITED(x)) { - SET_RIGHT_VISITED(x); - if (x->right) { - x = x->right; - if (IS_BLACK(x)) - curr_blacks++; - continue; - } - else { - if (blacks < 0) - blacks = curr_blacks; - ASSERT(blacks == curr_blacks); - } - } - - - if (IS_RED(x)) { - ASSERT(IS_BLACK(x->right)); - ASSERT(IS_BLACK(x->left)); - } - - ASSERT(x->parent || x == root); - - if (x->left) { - ASSERT(x->left->parent == x); - ASSERT(BLK_SZ(x->left) < BLK_SZ(x) - || (BLK_SZ(x->left) == BLK_SZ(x) && x->left < x)); - } - - if (x->right) { - ASSERT(x->right->parent == x); - ASSERT(BLK_SZ(x->right) > BLK_SZ(x) - || (BLK_SZ(x->right) == BLK_SZ(x) && x->right > x)); - } - - if (size && BLK_SZ(x) >= size) { - if (!res - || BLK_SZ(x) < BLK_SZ(res) - || (BLK_SZ(x) == BLK_SZ(res) && x < res)) - res = x; - } - - UNSET_LEFT_VISITED(x); - UNSET_RIGHT_VISITED(x); - if (IS_BLACK(x)) - curr_blacks--; - x = x->parent; - - } - - ASSERT(curr_blacks == 0); - - UNSET_LEFT_VISITED(root); - UNSET_RIGHT_VISITED(root); - - return res; - -} - - -#ifdef PRINT_TREE -#define INDENT_STEP 2 - -#include <stdio.h> - -static void -print_tree_aux(RBTree_t *x, int indent) -{ - int i; - - if (!x) { - for (i = 0; i < indent; i++) { - putc(' ', stderr); - } - fprintf(stderr, "BLACK: nil\r\n"); - } - else { - print_tree_aux(x->right, indent + INDENT_STEP); - for (i = 0; i < indent; i++) { - putc(' ', stderr); - } - fprintf(stderr, "%s: sz=%lu addr=0x%lx\r\n", - IS_BLACK(x) ? "BLACK" : "RED", - BLK_SZ(x), - (Uint) x); - print_tree_aux(x->left, indent + INDENT_STEP); - } -} - - -static void -print_tree(void) -{ - fprintf(stderr, " --- Size-Adress tree begin ---\r\n"); - print_tree_aux(root, 0); - fprintf(stderr, " --- Size-Adress tree end ---\r\n"); -} - -#endif - -#endif - -#endif /* ENABLE_ELIB_MALLOC */ diff --git a/erts/emulator/beam/elib_stat.h b/erts/emulator/beam/elib_stat.h deleted file mode 100644 index d8c7f31737..0000000000 --- a/erts/emulator/beam/elib_stat.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* -** Interface to elib statistics -** -*/ -#ifndef __ELIB_STAT_H__ -#define __ELIB_STAT_H__ - -struct elib_stat { - int mem_total; /* Number of heap words */ - int mem_blocks; /* Number of block */ - int mem_alloc; /* Number of words in use */ - int mem_free; /* Number of words free */ - int min_used; /* Size of the smallest block used */ - int max_free; /* Size of the largest free block */ - int free_blocks; /* Number of fragments in free list */ - int mem_max_alloc;/* Max number of words in use */ -}; - -EXTERN_FUNCTION(void, elib_statistics, (void*)); -EXTERN_FUNCTION(int, elib_check_heap, (_VOID_)); -EXTERN_FUNCTION(void, elib_heap_dump, (char*)); -EXTERN_FUNCTION(void, elib_stat, (struct elib_stat*)); -EXTERN_FUNCTION(int, elib_heap_map, (unsigned char*, int)); -EXTERN_FUNCTION(int, elib_histo, (unsigned long*, unsigned long*, int, int)); - -#endif diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index f8823b85fe..e85e2d7e3f 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -38,9 +38,6 @@ #include "erl_bits.h" #include "erl_instrument.h" #include "erl_mseg.h" -#ifdef ELIB_ALLOC_IS_CLIB -#include "erl_version.h" -#endif #include "erl_monitors.h" #include "erl_bif_timer.h" #if defined(ERTS_ALC_T_DRV_SEL_D_STATE) || defined(ERTS_ALC_T_DRV_EV_D_STATE) @@ -64,8 +61,15 @@ #ifdef DEBUG static Uint install_debug_functions(void); +#if 0 +#define HARD_DEBUG +#ifdef __GNUC__ +#warning "* * * * * * * * * * * * * *" +#warning "* HARD DEBUG IS ENABLED! *" +#warning "* * * * * * * * * * * * * *" +#endif +#endif #endif -extern void elib_ensure_initialized(void); ErtsAllocatorFunctions_t erts_allctrs[ERTS_ALC_A_MAX+1]; ErtsAllocatorInfo_t erts_allctrs_info[ERTS_ALC_A_MAX+1]; @@ -391,6 +395,10 @@ refuse_af_strategy(struct au_init *init) static void init_thr_ix(int static_ixs); +#ifdef HARD_DEBUG +static void hdbg_init(void); +#endif + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { @@ -406,6 +414,10 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) ERTS_DEFAULT_ALCU_INIT }; +#ifdef HARD_DEBUG + hdbg_init(); +#endif + erts_sys_alloc_init(); init_thr_ix(erts_no_schedulers); erts_init_utils_mem(); @@ -1336,6 +1348,13 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) argv[j++] = argv[i]; } *argc = j; +#if HALFWORD_HEAP + /* If halfword heap, silently ignore any disabling of internal + allocators */ + for (i = 0; i < aui_sz; ++i) + aui[i]->enable = 1; +#endif + } @@ -1549,7 +1568,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) Eterm atoms[sizeof(size)/sizeof(Uint)]; Uint *uintps[sizeof(size)/sizeof(Uint)]; Eterm euints[sizeof(size)/sizeof(Uint)]; - int need_atom; int want_tot_or_sys; int length; Eterm res = THE_NON_VALUE; @@ -1737,7 +1755,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) /* Calculate values needed... */ want_tot_or_sys = want.total || want.system; - need_atom = ERTS_MEM_NEED_ALL_ALCU || want.atom; if (ERTS_MEM_NEED_ALL_ALCU) { size.total = 0; @@ -2305,13 +2322,8 @@ erts_allocator_info_term(void *proc, Eterm which_alloc, int only_sz) l = 0; as[l] = am_atom_put("e", 1); ts[l++] = am_true; -#ifdef ELIB_ALLOC_IS_CLIB - as[l] = am_atom_put("m", 1); - ts[l++] = am_atom_put("elib", 4); -#else as[l] = am_atom_put("m", 1); ts[l++] = am_atom_put("libc", 4); -#endif if(sas.trim_threshold >= 0) { as[l] = am_atom_put("tt", 2); ts[l++] = erts_bld_uint(hpp, szp, @@ -2465,11 +2477,7 @@ erts_allocator_info(int to, void *arg) case ERTS_ALC_A_SYSTEM: { SysAllocStat sas; erts_print(to, arg, "option e: true\n"); -#ifdef ELIB_ALLOC_IS_CLIB - erts_print(to, arg, "option m: elib\n"); -#else erts_print(to, arg, "option m: libc\n"); -#endif sys_alloc_stat(&sas); if(sas.trim_threshold >= 0) erts_print(to, arg, "option tt: %d\n", sas.trim_threshold); @@ -2573,13 +2581,8 @@ erts_allocator_options(void *proc) switch (a) { case ERTS_ALC_A_SYSTEM: -#ifdef ELIB_ALLOC_IS_CLIB - as[l] = am_atom_put("m", 1); - ts[l++] = am_atom_put("elib", 4); -#else as[l] = am_atom_put("m", 1); ts[l++] = am_atom_put("libc", 4); -#endif if(sas.trim_threshold >= 0) { as[l] = am_atom_put("tt", 2); ts[l++] = erts_bld_uint(hpp, szp, @@ -2650,23 +2653,7 @@ erts_allocator_options(void *proc) features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; -#if defined(ELIB_ALLOC_IS_CLIB) - { - Eterm version; - int i; - int ver[5]; - i = sscanf(ERLANG_VERSION, - "%d.%d.%d.%d.%d", - &ver[0], &ver[1], &ver[2], &ver[3], &ver[4]); - - version = NIL; - for(i--; i >= 0; i--) - version = erts_bld_cons(hpp, szp, make_small(ver[i]), version); - - res = erts_bld_tuple(hpp, szp, 4, - am_elib_malloc, version, features, settings); - } -#elif defined(__GLIBC__) +#if defined(__GLIBC__) { Eterm AM_glibc = am_atom_put("glibc", 5); Eterm version; @@ -2862,12 +2849,10 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0a: - if (ethr_mutex_lock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_lock((ethr_mutex *) a1); break; case 0xf0b: - if (ethr_mutex_unlock((ethr_mutex *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_mutex_unlock((ethr_mutex *) a1); break; case 0xf0c: { ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond)); @@ -2883,31 +2868,21 @@ unsigned long erts_alc_test(unsigned long op, break; } case 0xf0e: - if (ethr_cond_broadcast((ethr_cond *) a1) != 0) - ERTS_ALC_TEST_ABORT; + ethr_cond_broadcast((ethr_cond *) a1); break; case 0xf0f: { int res; do { res = ethr_cond_wait((ethr_cond *) a1, (ethr_mutex *) a2); } while (res == EINTR); - if (res != 0) - ERTS_ALC_TEST_ABORT; break; } case 0xf10: { ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid)); -#ifdef ERTS_ENABLE_LOCK_COUNT - if (erts_lcnt_thr_create(tid, - (void * (*)(void *)) a1, - (void *) a2, - NULL) != 0) -#else if (ethr_thr_create(tid, (void * (*)(void *)) a1, (void *) a2, NULL) != 0) -#endif ERTS_ALC_TEST_ABORT; return (unsigned long) tid; } @@ -2946,8 +2921,11 @@ unsigned long erts_alc_test(unsigned long op, #undef PRINT_OPS #endif - +#ifdef HARD_DEBUG +#define FENCE_SZ (4*sizeof(UWord)) +#else #define FENCE_SZ (3*sizeof(UWord)) +#endif #if defined(ARCH_64) #define FENCE_PATTERN 0xABCDEF97ABCDEF97 @@ -2969,6 +2947,104 @@ unsigned long erts_alc_test(unsigned long op, #define GET_TYPE_OF_PATTERN(P) \ (((P) >> TYPE_PATTERN_SHIFT) & TYPE_PATTERN_MASK) +#ifdef HARD_DEBUG + +#define ERL_ALC_HDBG_MAX_MBLK 100000 +#define ERTS_ALC_O_CHECK -1 + +typedef struct hdbg_mblk_ hdbg_mblk; +struct hdbg_mblk_ { + hdbg_mblk *next; + hdbg_mblk *prev; + void *p; + Uint s; + ErtsAlcType_t n; +}; + +static hdbg_mblk hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK]; + +static hdbg_mblk *free_hdbg_mblks; +static hdbg_mblk *used_hdbg_mblks; +static erts_mtx_t hdbg_mblk_mtx; + +static void +hdbg_init(void) +{ + int i; + for (i = 0; i < ERL_ALC_HDBG_MAX_MBLK-1; i++) + hdbg_mblks[i].next = &hdbg_mblks[i+1]; + hdbg_mblks[ERL_ALC_HDBG_MAX_MBLK-1].next = NULL; + free_hdbg_mblks = &hdbg_mblks[0]; + used_hdbg_mblks = NULL; + erts_mtx_init(&hdbg_mblk_mtx, "erts_alloc_hard_debug"); +} + +static void *check_memory_fence(void *ptr, + Uint *size, + ErtsAlcType_t n, + int func); +void erts_hdbg_chk_blks(void); + +void +erts_hdbg_chk_blks(void) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + for (mblk = used_hdbg_mblks; mblk; mblk = mblk->next) { + Uint sz; + check_memory_fence(mblk->p, &sz, mblk->n, ERTS_ALC_O_CHECK); + ASSERT(sz == mblk->s); + } + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +static hdbg_mblk * +hdbg_alloc(void *p, Uint s, ErtsAlcType_t n) +{ + hdbg_mblk *mblk; + + erts_mtx_lock(&hdbg_mblk_mtx); + mblk = free_hdbg_mblks; + if (!mblk) { + erts_fprintf(stderr, + "Ran out of debug blocks; please increase " + "ERL_ALC_HDBG_MAX_MBLK=%d and recompile!\n", + ERL_ALC_HDBG_MAX_MBLK); + abort(); + } + free_hdbg_mblks = mblk->next; + + mblk->p = p; + mblk->s = s; + mblk->n = n; + + mblk->next = used_hdbg_mblks; + mblk->prev = NULL; + if (used_hdbg_mblks) + used_hdbg_mblks->prev = mblk; + used_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); + return (void *) mblk; +} + +static void +hdbg_free(hdbg_mblk *mblk) +{ + erts_mtx_lock(&hdbg_mblk_mtx); + if (mblk->next) + mblk->next->prev = mblk->prev; + if (mblk->prev) + mblk->prev->next = mblk->next; + else + used_hdbg_mblks = mblk->next; + + mblk->next = free_hdbg_mblks; + free_hdbg_mblks = mblk; + erts_mtx_unlock(&hdbg_mblk_mtx); +} + +#endif #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void *check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func); @@ -3014,17 +3090,28 @@ set_memory_fence(void *ptr, Uint sz, ErtsAlcType_t n) { UWord *ui_ptr; UWord pattern; +#ifdef HARD_DEBUG + hdbg_mblk **mblkpp; +#endif if (!ptr) return NULL; ui_ptr = (UWord *) ptr; pattern = MK_PATTERN(n); - + +#ifdef HARD_DEBUG + mblkpp = (hdbg_mblk **) ui_ptr++; +#endif + *(ui_ptr++) = sz; *(ui_ptr++) = pattern; memcpy((void *) (((char *) ui_ptr)+sz), (void *) &pattern, sizeof(UWord)); +#ifdef HARD_DEBUG + *mblkpp = hdbg_alloc((void *) ui_ptr, sz, n); +#endif + return (void *) ui_ptr; } @@ -3036,6 +3123,9 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) UWord pre_pattern; UWord post_pattern; UWord *ui_ptr; +#ifdef HARD_DEBUG + hdbg_mblk *mblk; +#endif if (!ptr) return NULL; @@ -3043,6 +3133,9 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) ui_ptr = (UWord *) ptr; pre_pattern = *(--ui_ptr); *size = sz = *(--ui_ptr); +#ifdef HARD_DEBUG + mblk = (hdbg_mblk *) *(--ui_ptr); +#endif found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { @@ -3098,6 +3191,17 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) (unsigned long) ptr, (unsigned long) sz, ftype, op_str, otype); } +#ifdef HARD_DEBUG + switch (func) { + case ERTS_ALC_O_REALLOC: + case ERTS_ALC_O_FREE: + hdbg_free(mblk); + break; + default: + break; + } +#endif + return (void *) ui_ptr; } @@ -3110,6 +3214,10 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint dsize; void *res; +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; res = (*real_af->alloc)(n, real_af->extra, dsize); @@ -3139,13 +3247,17 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) dsize = size + FENCE_SZ; dptr = check_memory_fence(ptr, &old_size, n, ERTS_ALC_O_REALLOC); +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + if (old_size > size) sys_memset((void *) (((char *) ptr) + size), 0xf, sizeof(Uint) + old_size - size); res = (*real_af->realloc)(n, real_af->extra, dptr, dsize); - + res = set_memory_fence(res, size, n); #ifdef PRINT_OPS @@ -3175,6 +3287,10 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr); #endif +#ifdef HARD_DEBUG + erts_hdbg_chk_blks(); +#endif + } static Uint diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 5d2872a4e3..b7b9c6a133 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -212,7 +212,8 @@ type LOGGER_DSBUF TEMPORARY SYSTEM logger_dsbuf type TMP_DSBUF TEMPORARY SYSTEM tmp_dsbuf type INFO_DSBUF SYSTEM SYSTEM info_dsbuf # INFO_DSBUF have to use the SYSTEM allocator; otherwise, a deadlock might occur -type SCHDLR_DATA LONG_LIVED PROCESSES scheduler_data +type SCHDLR_DATA LONG_LIVED SYSTEM scheduler_data +type SCHDLR_SLP_INFO LONG_LIVED SYSTEM scheduler_sleep_info type RUNQS LONG_LIVED SYSTEM run_queues type DDLL_PROCESS STANDARD SYSTEM ddll_processes type DDLL_HANDLE STANDARD SYSTEM ddll_handle @@ -232,6 +233,7 @@ type RE_SUBJECT SHORT_LIVED SYSTEM re_subject type RE_HEAP STANDARD SYSTEM re_heap type RE_STACK SHORT_LIVED SYSTEM re_stack type UNICODE_BUFFER SHORT_LIVED SYSTEM unicode_buffer +type BINARY_BUFFER SHORT_LIVED SYSTEM binary_buffer type PRE_ALLOC_DATA LONG_LIVED SYSTEM pre_alloc_data type DRV_THR_OPTS DRIVER SYSTEM driver_thread_opts type DRV_TID DRIVER SYSTEM driver_tid @@ -245,6 +247,7 @@ type CPUDATA LONG_LIVED SYSTEM cpu_data type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data type ZLIB STANDARD SYSTEM zlib +type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map +if smp type ASYNC SHORT_LIVED SYSTEM async @@ -260,6 +263,8 @@ type XPORTS_LIST SHORT_LIVED SYSTEM extra_port_list type PROC_LCK_WTR LONG_LIVED SYSTEM proc_lock_waiter type PROC_LCK_QS LONG_LIVED SYSTEM proc_lock_queues type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing +type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q +type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work +endif # @@ -268,7 +273,9 @@ type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing +if threads -type ETHR_INTERNAL SYSTEM SYSTEM ethread_internal +type ETHR_STD STANDARD SYSTEM ethread_standard +type ETHR_SL SHORT_LIVED SYSTEM ethread_short_lived +type ETHR_LL LONG_LIVED SYSTEM ethread_long_lived +ifnot smp diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index be691317ee..12c7631448 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -70,7 +70,6 @@ static ErlAsync* async_ready_list = NULL; /* Detach from driver */ static void async_detach(DE_Handle* dh) { - /* XXX:PaN what should happen here? we want to unload the driver or??? */ return; } @@ -176,7 +175,6 @@ int exit_async() static void async_add(ErlAsync* a, AsyncQueue* q) { - /* XXX:PaN Is this still necessary when ports lock drivers? */ if (is_internal_port(a->port)) { ERTS_LC_ASSERT(erts_drvportid2port(a->port)); /* make sure the driver will stay around */ diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c new file mode 100644 index 0000000000..684fa5d12f --- /dev/null +++ b/erts/emulator/beam/erl_bif_binary.c @@ -0,0 +1,2939 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * NOTE: This file contains the BIF's for the *module* binary in stdlib. + * other BIF's concerning binaries are in binary.c. + */ + + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "erl_vm.h" +#include "global.h" +#include "erl_process.h" +#include "error.h" +#include "bif.h" +#include "big.h" +#include "erl_binary.h" +#include "erl_bits.h" + + +/* + * The native implementation functions for the module binary. + * Searching is implemented using either Boyer-Moore or Aho-Corasick + * depending on number of searchstrings (BM if one, AC if more than one). + * Native implementation is mostly for efficiency, nothing + * (except binary:referenced_byte_size) really *needs* to be implemented + * in native code. + */ + +/* #define HARDDEBUG */ + +/* Init and local variables */ + +static Export binary_match_trap_export; +static BIF_RETTYPE binary_match_trap(BIF_ALIST_3); +static Export binary_matches_trap_export; +static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3); +static Export binary_longest_prefix_trap_export; +static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3); +static Export binary_longest_suffix_trap_export; +static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3); +static Export binary_bin_to_list_trap_export; +static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3); +static Export binary_copy_trap_export; +static BIF_RETTYPE binary_copy_trap(BIF_ALIST_2); +static Uint max_loop_limit; + + +void erts_init_bif_binary(void) +{ + sys_memset((void *) &binary_match_trap_export, 0, sizeof(Export)); + binary_match_trap_export.address = &binary_match_trap_export.code[3]; + binary_match_trap_export.code[0] = am_erlang; + binary_match_trap_export.code[1] = am_binary_match_trap; + binary_match_trap_export.code[2] = 3; + binary_match_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_match_trap_export.code[4] = (BeamInstr) &binary_match_trap; + + sys_memset((void *) &binary_matches_trap_export, 0, sizeof(Export)); + binary_matches_trap_export.address = &binary_matches_trap_export.code[3]; + binary_matches_trap_export.code[0] = am_erlang; + binary_matches_trap_export.code[1] = am_binary_matches_trap; + binary_matches_trap_export.code[2] = 3; + binary_matches_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_matches_trap_export.code[4] = (BeamInstr) &binary_matches_trap; + + sys_memset((void *) &binary_longest_prefix_trap_export, 0, sizeof(Export)); + binary_longest_prefix_trap_export.address = &binary_longest_prefix_trap_export.code[3]; + binary_longest_prefix_trap_export.code[0] = am_erlang; + binary_longest_prefix_trap_export.code[1] = am_binary_longest_prefix_trap; + binary_longest_prefix_trap_export.code[2] = 3; + binary_longest_prefix_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_longest_prefix_trap_export.code[4] = (BeamInstr) &binary_longest_prefix_trap; + + sys_memset((void *) &binary_longest_suffix_trap_export, 0, sizeof(Export)); + binary_longest_suffix_trap_export.address = &binary_longest_suffix_trap_export.code[3]; + binary_longest_suffix_trap_export.code[0] = am_erlang; + binary_longest_suffix_trap_export.code[1] = am_binary_longest_suffix_trap; + binary_longest_suffix_trap_export.code[2] = 3; + binary_longest_suffix_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_longest_suffix_trap_export.code[4] = (BeamInstr) &binary_longest_suffix_trap; + + sys_memset((void *) &binary_bin_to_list_trap_export, 0, sizeof(Export)); + binary_bin_to_list_trap_export.address = &binary_bin_to_list_trap_export.code[3]; + binary_bin_to_list_trap_export.code[0] = am_erlang; + binary_bin_to_list_trap_export.code[1] = am_binary_bin_to_list_trap; + binary_bin_to_list_trap_export.code[2] = 3; + binary_bin_to_list_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_bin_to_list_trap_export.code[4] = (BeamInstr) &binary_bin_to_list_trap; + sys_memset((void *) &binary_copy_trap_export, 0, sizeof(Export)); + binary_copy_trap_export.address = &binary_copy_trap_export.code[3]; + binary_copy_trap_export.code[0] = am_erlang; + binary_copy_trap_export.code[1] = am_binary_copy_trap; + binary_copy_trap_export.code[2] = 2; + binary_copy_trap_export.code[3] = (BeamInstr) em_apply_bif; + binary_copy_trap_export.code[4] = (BeamInstr) &binary_copy_trap; + + max_loop_limit = 0; + return; +} + +/* + * Setting the loop_limit for searches for debugging + */ +Sint erts_binary_set_loop_limit(Sint limit) +{ + Sint save = (Sint) max_loop_limit; + if (limit <= 0) { + max_loop_limit = 0; + } else { + max_loop_limit = (Uint) limit; + } + + return save; +} + +static Uint get_reds(Process *p, int loop_factor) +{ + Uint reds = ERTS_BIF_REDS_LEFT(p) * loop_factor; + Uint tmp = max_loop_limit; + if (tmp != 0 && tmp < reds) { + return tmp; + } + if (!reds) { + reds = 1; + } + return reds; +} + +/* + * A micro allocator used when building search structures, just a convenience + * for building structures inside a pre-allocated magic binary using + * conventional malloc-like interface. + */ + +#define MYALIGN(Size) (SIZEOF_VOID_P * (((Size) / SIZEOF_VOID_P) + \ + !!(((Size) % SIZEOF_VOID_P)))) + +#ifdef DEBUG +#define CHECK_ALLOCATOR(My) ASSERT((My).current <= ((My).mem + (My).size)) +#else +#define CHECK_ALLOCATOR(My) /* nothing */ +#endif + +typedef struct _my_allocator { + Uint size; + byte *current; + byte *mem; +} MyAllocator; + +static void init_my_allocator(MyAllocator *my, Uint siz, byte *array) +{ + ASSERT((siz % SIZEOF_VOID_P) == 0); + my->size = siz; + my->mem = array; + my->current = my->mem; +} + +static void *my_alloc(MyAllocator *my, Uint size) +{ + void *ptr = my->current; + my->current += MYALIGN(size); + return ptr; +} + +/* + * The search functionality. + * + * The search is byte oriented, which works nicely for UTF-8 as well as + * latin1 data + */ + +#define ALPHABET_SIZE 256 + +typedef struct _ac_node { +#ifdef HARDDEBUG + Uint32 id; /* To identify h pointer targets when + dumping */ +#endif + Uint32 d; /* Depth in trie, also represents the + length (-1) of the matched string if + in final set */ + Sint32 final; /* Members in final set represent + * matches. + * The set representation is scattered + * among the nodes in this way: + * >0 -> this represents a member of + * the final set, <0 -> member of + * final set somewhere in the failure + * chain, + * 0 -> not member of the final set */ + struct _ac_node *h; /* h(Hode) is the failure function */ + struct _ac_node *g[ALPHABET_SIZE]; /* g(Node,Character) is the + transition function */ +} ACNode; + +typedef struct _ac_trie { +#ifdef HARDDEBUG + Uint32 idc; +#endif + Uint32 counter; /* Number of added patterns */ + ACNode *root; /* pointer to the root state */ +} ACTrie; + +typedef struct _bm_data { + byte *x; + Sint len; + Sint *goodshift; + Sint badshift[ALPHABET_SIZE]; +} BMData; + +#ifdef HARDDEBUG +static void dump_bm_data(BMData *bm); +static void dump_ac_trie(ACTrie *act); +static void dump_ac_node(ACNode *node, int indent, int ch); +#endif + +/* + * The needed size of binary data for a search structure - given the + * accumulated string lengths. + */ +#define BM_SIZE(StrLen) /* StrLen: length of searchstring */ \ +((MYALIGN(sizeof(Sint) * (StrLen))) + /* goodshift array */ \ + MYALIGN(StrLen) + /* searchstring saved */ \ + (MYALIGN(sizeof(BMData)))) /* Structure */ + +#define AC_SIZE(StrLens) /* StrLens: sum of all searchstring lengths */ \ +((MYALIGN(sizeof(ACNode)) * \ +((StrLens)+1)) + /* The actual nodes (including rootnode) */ \ + MYALIGN(sizeof(ACTrie))) /* Structure */ + + +#ifndef MAX +#define MAX(A,B) (((A) > (B)) ? (A) : (B)) +#endif + +#ifndef MIN +#define MIN(A,B) (((A) > (B)) ? (B) : (A)) +#endif +/* + * Callback for the magic binary + */ +static void cleanup_my_data_ac(Binary *bp) +{ + return; +} +static void cleanup_my_data_bm(Binary *bp) +{ + return; +} + +/* + * Initiate a (allocated) micro allocator and fill in the base + * for an Aho-Corasick search trie, given the accumulated length of the search + * strings. + */ +static ACTrie *create_acdata(MyAllocator *my, Uint len, + ACNode ***qbuff /* out */, + Binary **the_bin /* out */) +{ + Uint datasize = AC_SIZE(len); + ACTrie *act; + ACNode *acn; + Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_ac); + byte *data = ERTS_MAGIC_BIN_DATA(mb); + + init_my_allocator(my, datasize, data); + act = my_alloc(my, sizeof(ACTrie)); /* Important that this is the first + allocation */ + act->counter = 0; + act->root = acn = my_alloc(my, sizeof(ACNode)); + acn->d = 0; + acn->final = 0; + acn->h = NULL; + memset(acn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); +#ifdef HARDDEBUG + act->idc = 0; + acn->id = 0; +#endif + *qbuff = erts_alloc(ERTS_ALC_T_TMP, sizeof(ACNode *) * len); + *the_bin = mb; + return act; +} + +/* + * The same initialization of allocator and basic data for Boyer-Moore. + */ +static BMData *create_bmdata(MyAllocator *my, byte *x, Uint len, + Binary **the_bin /* out */) +{ + Uint datasize = BM_SIZE(len); + BMData *bmd; + Binary *mb = erts_create_magic_binary(datasize,cleanup_my_data_bm); + byte *data = ERTS_MAGIC_BIN_DATA(mb); + init_my_allocator(my, datasize, data); + bmd = my_alloc(my, sizeof(BMData)); + bmd->x = my_alloc(my,len); + memcpy(bmd->x,x,len); + bmd->len = len; + bmd->goodshift = my_alloc(my,sizeof(Uint) * len); + *the_bin = mb; + return bmd; +} + +/* + * Compilation of search structures + */ + +/* + * Aho Corasick - Build a Trie and fill in the failure functions + * when all strings are added. + * The algorithm is nicely described by Dieter B�hler of University of + * T�bingen: + * http://www-sr.informatik.uni-tuebingen.de/~buehler/AC/AC.html + */ + +/* + * Helper called once for each search pattern + */ +static void ac_add_one_pattern(MyAllocator *my, ACTrie *act, byte *x, Uint len) +{ + ACNode *acn = act->root; + Uint32 n = ++act->counter; /* Always increase counter, even if it's a + duplicate as this may identify the pattern + in the final set (not in current interface + though) */ + Uint i = 0; + + while(i < len) { + if (acn->g[x[i]] != NULL) { + /* node exists, continue */ + acn = acn->g[x[i]]; + ++i; + } else { + /* allocate a new node */ + ACNode *nn = my_alloc(my,sizeof(ACNode)); +#ifdef HARDDEBUG + nn->id = ++(act->idc); +#endif + nn->d = i+1; + nn->h = act->root; + nn->final = 0; + memset(nn->g, 0, sizeof(ACNode *) * ALPHABET_SIZE); + acn->g[x[i]] = nn; + ++i; + acn = nn; + } + } + if (acn->final == 0) { /* New pattern, add to final set */ + acn->final = n; + } +} + +/* + * Called when all search patterns are added. + */ +static void ac_compute_failure_functions(ACTrie *act, ACNode **qbuff) +{ + ACNode *root = act->root; + ACNode *parent; + int i; + int qh = 0,qt = 0; + ACNode *child, *r; + + /* Set all children of the root to have the root as failure function */ + for (i = 0; i < ALPHABET_SIZE; ++i) { + if (root->g[i] != NULL) { + root->g[i]->h = root; + /* Add to que for later traversal */ + qbuff[qt++] = root->g[i]; + } + } + + /* So, now we've handled children of the root state, traverse the + rest of the trie BF... */ + while (qh < qt) { + parent = qbuff[qh++]; + for (i = 0; i < ALPHABET_SIZE; ++ i) { + if ((child = parent->g[i]) != NULL) { + /* Visit this node to */ + qbuff[qt++] = child; + /* Search for correct failure function, follow the parent's + failure function until you find a similar transition + funtion to this child's */ + r = parent->h; + while (r != NULL && r->g[i] == NULL) { + r = r->h; + } + if (r == NULL) { + /* Replace NULL failures with the root as we go */ + child->h = (root->g[i] == NULL) ? root : root->g[i]; + } else { + child->h = r->g[i]; + /* + * The "final" set is scattered among the nodes. When + * the failure function points to a member of the final + * set, we have a match, but we might not see it in the + * current node if we dont mark it as a special type of + * final, i.e. foolow the failure function and you will + * find a real member of final set. This is marked with + * a negative string id and only done if this node does + * not represent a member in the final set. + */ + if (!(child->final) && (child->h->final)) { + child->final = -1; + } + } + } + } + } + /* Finally the failure function of the root should point to itself */ + root->h = root; +} + + +/* + * The actual searching for needles in the haystack... + * Find first match using Aho-Coracick Trie + * return pattern number and fill in mpos + mlen if found, otherwise return 0 + * Return the matching pattern that *starts* first, and ends + * last (difference when overlapping), hence the candidate thing. + * Basic AC finds the first end before the first start... + * + */ +typedef struct { + ACNode *q; + Uint pos; + Uint len; + ACNode *candidate; + Uint candidate_start; +} ACFindFirstState; + + +static void ac_init_find_first_match(ACFindFirstState *state, ACTrie *act, Sint startpos, Uint len) +{ + state->q = act->root; + state->pos = startpos; + state->len = len; + state->candidate = NULL; + state->candidate_start = 0; +} +#define AC_OK 0 +#define AC_NOT_FOUND -1 +#define AC_RESTART -2 + +#define AC_LOOP_FACTOR 10 + +static int ac_find_first_match(ACFindFirstState *state, byte *haystack, + Uint *mpos, Uint *mlen, Uint *reductions) +{ + ACNode *q = state->q; + Uint i = state->pos; + ACNode *candidate = state->candidate, *r; + Uint len = state->len; + Uint candidate_start = state->candidate_start; + Uint rstart; + register Uint reds = *reductions; + + while (i < len) { + if (--reds == 0) { + state->q = q; + state->pos = i; + state->len = len; + state->candidate = candidate; + state->candidate_start = candidate_start; + return AC_RESTART; + } + + while (q->g[haystack[i]] == NULL && q->h != q) { + q = q->h; + } + if (q->g[haystack[i]] != NULL) { + q = q->g[haystack[i]]; + } +#ifdef HARDDEBUG + erts_printf("ch = %c, Current: %u\n", (int) haystack[i], (unsigned) q->id); +#endif + ++i; + if (candidate != NULL && (i - q->d) > candidate_start) { + break; + } + if (q->final) { + r = q; + while (r->final < 0) + r = r->h; + rstart = i - r->d; + if (candidate == NULL || rstart < candidate_start || + (rstart == candidate_start && candidate->d < q->d)) { + candidate_start = rstart; + candidate = r; + } + } + } + *reductions = reds; + if (!candidate) { + return AC_NOT_FOUND; + } +#ifdef HARDDEBUG + dump_ac_node(candidate,0,'?'); +#endif + *mpos = candidate_start; + *mlen = candidate->d; + return AC_OK; +} + +typedef struct _findall_data { + Uint pos; + Uint len; +#ifdef HARDDEBUG + Uint id; +#endif + Eterm epos; + Eterm elen; +} FindallData; + +typedef struct { + ACNode *q; + Uint pos; + Uint len; + Uint m; + Uint allocated; + FindallData *out; +} ACFindAllState; + +static void ac_init_find_all(ACFindAllState *state, ACTrie *act, Sint startpos, Uint len) +{ + state->q = act->root; + state->pos = startpos; + state->len = len; + state->m = 0; + state->allocated = 0; + state->out = NULL; +} + +static void ac_restore_find_all(ACFindAllState *state, char *buff) +{ + memcpy(state,buff,sizeof(ACFindAllState)); + if (state->allocated > 0) { + state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * (state->allocated)); + memcpy(state->out,buff+sizeof(ACFindAllState),sizeof(FindallData)*state->m); + } else { + state->out = NULL; + } +} + +static void ac_serialize_find_all(ACFindAllState *state, char *buff) +{ + memcpy(buff,state,sizeof(ACFindAllState)); + memcpy(buff+sizeof(ACFindAllState),state->out,sizeof(FindallData)*state->m); +} + +static void ac_clean_find_all(ACFindAllState *state) +{ + if (state->out != NULL) { + erts_free(ERTS_ALC_T_TMP, state->out); + } +#ifdef HARDDEBUG + state->out = NULL; + state->allocated = 0; +#endif +} + +#define SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(S) \ + (sizeof(ACFindAllState)+(sizeof(FindallData)*(S).m)) + +/* + * Differs to the find_first function in that it stores all matches and the values + * arte returned only in the state. + */ +static int ac_find_all_non_overlapping(ACFindAllState *state, byte *haystack, + Uint *reductions) +{ + ACNode *q = state->q; + Uint i = state->pos; + Uint rstart; + ACNode *r; + Uint len = state->len; + Uint m = state->m, save_m; + Uint allocated = state->allocated; + FindallData *out = state->out; + register Uint reds = *reductions; + + + while (i < len) { + if (--reds == 0) { + state->q = q; + state->pos = i; + state->len = len; + state->m = m; + state->allocated = allocated; + state->out = out; + return AC_RESTART; + } + while (q->g[haystack[i]] == NULL && q->h != q) { + q = q->h; + } + if (q->g[haystack[i]] != NULL) { + q = q->g[haystack[i]]; + } + ++i; + if (q->final) { + r = q; + while (r->final) { + while (r->final < 0) + r = r->h; +#ifdef HARDDEBUG + erts_printf("Trying to add %u\n",(unsigned) r->final); +#endif + rstart = i - r->d; + save_m = m; + while (m > 0 && (out[m-1].pos > rstart || + (out[m-1].pos == rstart && + out[m-1].len < r->d))) { +#ifdef HARDDEBUG + erts_printf("Popping %u\n",(unsigned) out[m-1].id); +#endif + --m; + } +#ifdef HARDDEBUG + if (m > 0) { + erts_printf("Pos %u\n",out[m-1].pos); + erts_printf("Len %u\n",out[m-1].len); + } + erts_printf("Rstart %u\n",rstart); +#endif + if (m == 0 || out[m-1].pos + out[m-1].len <= rstart) { + if (m >= allocated) { + if (!allocated) { + allocated = 10; + out = erts_alloc(ERTS_ALC_T_TMP, + sizeof(FindallData) * allocated); + } else { + allocated *= 2; + out = erts_realloc(ERTS_ALC_T_TMP, out, + sizeof(FindallData) * + allocated); + } + } + out[m].pos = rstart; + out[m].len = r->d; +#ifdef HARDDEBUG + out[m].id = r->final; +#endif + ++m; +#ifdef HARDDEBUG + erts_printf("Pushing %u\n",(unsigned) out[m-1].id); +#endif + } else { +#ifdef HARDDEBUG + erts_printf("Backtracking %d steps\n",save_m - m); +#endif + m = save_m; + } + r = r->h; + } + } + } + *reductions = reds; + state->m = m; + state->out = out; + return (m == 0) ? AC_NOT_FOUND : AC_OK; +} + +/* + * Boyer Moore - most obviously implemented more or less exactly as + * Christian Charras and Thierry Lecroq describe it in "Handbook of + * Exact String-Matching Algorithms" + * http://www-igm.univ-mlv.fr/~lecroq/string/ + */ + +/* + * Call this to compute badshifts array + */ +static void compute_badshifts(BMData *bmd) +{ + Sint i; + Sint m = bmd->len; + + for (i = 0; i < ALPHABET_SIZE; ++i) { + bmd->badshift[i] = m; + } + for (i = 0; i < m - 1; ++i) { + bmd->badshift[bmd->x[i]] = m - i - 1; + } +} + +/* Helper for "compute_goodshifts" */ +static void compute_suffixes(byte *x, Sint m, Sint *suffixes) +{ + int f,g,i; + + suffixes[m - 1] = m; + + f = 0; /* To avoid use before set warning */ + + g = m - 1; + + for (i = m - 2; i >= 0; --i) { + if (i > g && suffixes[i + m - 1 - f] < i - g) { + suffixes[i] = suffixes[i + m - 1 - f]; + } else { + if (i < g) { + g = i; + } + f = i; + while ( g >= 0 && x[g] == x[g + m - 1 - f] ) { + --g; + } + suffixes[i] = f - g; + } + } +} + +/* + * Call this to compute goodshift array + */ +static void compute_goodshifts(BMData *bmd) +{ + Sint m = bmd->len; + byte *x = bmd->x; + Sint i, j; + Sint *suffixes = erts_alloc(ERTS_ALC_T_TMP, m * sizeof(Sint)); + + compute_suffixes(x, m, suffixes); + + for (i = 0; i < m; ++i) { + bmd->goodshift[i] = m; + } + + j = 0; + + for (i = m - 1; i >= -1; --i) { + if (i == -1 || suffixes[i] == i + 1) { + while (j < m - 1 - i) { + if (bmd->goodshift[j] == m) { + bmd->goodshift[j] = m - 1 - i; + } + ++j; + } + } + } + for (i = 0; i <= m - 2; ++i) { + bmd->goodshift[m - 1 - suffixes[i]] = m - 1 - i; + } + erts_free(ERTS_ALC_T_TMP, suffixes); +} + +typedef struct { + Sint pos; + Sint len; +} BMFindFirstState; + +#define BM_OK 0 /* used only for find_all */ +#define BM_NOT_FOUND -1 +#define BM_RESTART -2 +#define BM_LOOP_FACTOR 10 /* Should we have a higher value? */ + +static void bm_init_find_first_match(BMFindFirstState *state, Sint startpos, + Uint len) +{ + state->pos = startpos; + state->len = (Sint) len; +} + + +static Sint bm_find_first_match(BMFindFirstState *state, BMData *bmd, + byte *haystack, Uint *reductions) +{ + Sint blen = bmd->len; + Sint len = state->len; + Sint *gs = bmd->goodshift; + Sint *bs = bmd->badshift; + byte *needle = bmd->x; + Sint i; + Sint j = state->pos; + register Uint reds = *reductions; + + while (j <= len - blen) { + if (--reds == 0) { + state->pos = j; + return BM_RESTART; + } + for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) + ; + if (i < 0) { /* found */ + *reductions = reds; + return j; + } + j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); + } + *reductions = reds; + return BM_NOT_FOUND; +} + +typedef struct { + Sint pos; + Sint len; + Uint m; + Uint allocated; + FindallData *out; +} BMFindAllState; + +static void bm_init_find_all(BMFindAllState *state, Sint startpos, Uint len) +{ + state->pos = startpos; + state->len = (Sint) len; + state->m = 0; + state->allocated = 0; + state->out = NULL; +} + +static void bm_restore_find_all(BMFindAllState *state, char *buff) +{ + memcpy(state,buff,sizeof(BMFindAllState)); + if (state->allocated > 0) { + state->out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * + (state->allocated)); + memcpy(state->out,buff+sizeof(BMFindAllState), + sizeof(FindallData)*state->m); + } else { + state->out = NULL; + } +} + +static void bm_serialize_find_all(BMFindAllState *state, char *buff) +{ + memcpy(buff,state,sizeof(BMFindAllState)); + memcpy(buff+sizeof(BMFindAllState),state->out, + sizeof(FindallData)*state->m); +} + +static void bm_clean_find_all(BMFindAllState *state) +{ + if (state->out != NULL) { + erts_free(ERTS_ALC_T_TMP, state->out); + } +#ifdef HARDDEBUG + state->out = NULL; + state->allocated = 0; +#endif +} + +#define SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(S) \ + (sizeof(BMFindAllState)+(sizeof(FindallData)*(S).m)) + +/* + * Differs to the find_first function in that it stores all matches and the + * values are returned only in the state. + */ +static Sint bm_find_all_non_overlapping(BMFindAllState *state, + BMData *bmd, byte *haystack, + Uint *reductions) +{ + Sint blen = bmd->len; + Sint len = state->len; + Sint *gs = bmd->goodshift; + Sint *bs = bmd->badshift; + byte *needle = bmd->x; + Sint i; + Sint j = state->pos; + Uint m = state->m; + Uint allocated = state->allocated; + FindallData *out = state->out; + register Uint reds = *reductions; + + while (j <= len - blen) { + if (--reds == 0) { + state->pos = j; + state->m = m; + state->allocated = allocated; + state->out = out; + return BM_RESTART; + } + for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) + ; + if (i < 0) { /* found */ + if (m >= allocated) { + if (!allocated) { + allocated = 10; + out = erts_alloc(ERTS_ALC_T_TMP, sizeof(FindallData) * allocated); + } else { + allocated *= 2; + out = erts_realloc(ERTS_ALC_T_TMP, out, + sizeof(FindallData) * allocated); + } + } + out[m].pos = j; + out[m].len = blen; + ++m; + j += blen; + } else { + j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); + } + } + state->m = m; + state->out = out; + *reductions = reds; + return (m == 0) ? BM_NOT_FOUND : BM_OK; +} + +/* + * Interface functions (i.e. "bif's") + */ + +/* + * Search functionality interfaces + */ + +static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp) +{ + Eterm t, b, comp_term = NIL; + Uint characters; + Uint words; + + characters = 0; + words = 0; + + if (is_list(argument)) { + t = argument; + while (is_list(t)) { + b = CAR(list_val(t)); + t = CDR(list_val(t)); + if (!is_binary(b)) { + goto badarg; + } + if (binary_bitsize(b) != 0) { + goto badarg; + } + ++words; + characters += binary_size(b); + } + if (is_not_nil(t)) { + goto badarg; + } + if (words > 1) { + comp_term = argument; + } else { + comp_term = CAR(list_val(argument)); + } + } else if (is_binary(argument)) { + if (binary_bitsize(argument) != 0) { + goto badarg; + } + words = 1; + comp_term = argument; + characters = binary_size(argument); + } + + if (characters == 0) { + goto badarg; + } + ASSERT(words > 0); + + if (words == 1) { + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + MyAllocator my; + BMData *bmd; + Binary *bin; + + ERTS_GET_BINARY_BYTES(comp_term, bytes, bitoffs, bitsize); + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(comp_term, &temp_alloc); + } + bmd = create_bmdata(&my, bytes, characters, &bin); + compute_badshifts(bmd); + compute_goodshifts(bmd); + erts_free_aligned_binary_bytes(temp_alloc); + CHECK_ALLOCATOR(my); + *tag = am_bm; + *binp = bin; + return 0; + } else { + ACTrie *act; + MyAllocator my; + ACNode **qbuff; + Binary *bin; + + act = create_acdata(&my, characters, &qbuff, &bin); + t = comp_term; + while (is_list(t)) { + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + b = CAR(list_val(t)); + t = CDR(list_val(t)); + ERTS_GET_BINARY_BYTES(b, bytes, bitoffs, bitsize); + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(b, &temp_alloc); + } + ac_add_one_pattern(&my,act,bytes,binary_size(b)); + erts_free_aligned_binary_bytes(temp_alloc); + } + ac_compute_failure_functions(act,qbuff); + CHECK_ALLOCATOR(my); + erts_free(ERTS_ALC_T_TMP,qbuff); + *tag = am_ac; + *binp = bin; + return 0; + } + badarg: + return -1; +} + +BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1) +{ + Binary *bin; + Eterm tag, ret; + Eterm *hp; + + if (do_binary_match_compile(BIF_ARG_1,&tag,&bin)) { + BIF_ERROR(BIF_P,BADARG); + } + hp = HAlloc(BIF_P, PROC_BIN_SIZE+3); + ret = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + ret = TUPLE2(hp, tag, ret); + BIF_RET(ret); +} + +#define DO_BIN_MATCH_OK 0 +#define DO_BIN_MATCH_BADARG -1 +#define DO_BIN_MATCH_RESTART -2 + +static int do_binary_match(Process *p, Eterm subject, Uint hsstart, Uint hsend, + Eterm type, Binary *bin, Eterm state_term, + Eterm *res_term) +{ + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } + if (state_term != NIL) { + Eterm *ptr = big_val(state_term); + type = ptr[1]; + } + + if (type == am_bm) { + BMData *bm; + Sint pos; + Eterm ret; + Eterm *hp; + BMFindFirstState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_first_match(&state, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + memcpy(&state,ptr+2,sizeof(state)); + } +#ifdef HARDDEBUG + erts_printf("(bm) state->pos = %ld, state->len = %lu\n",state.pos, + state.len); +#endif + pos = bm_find_first_match(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + ret = am_nomatch; + } else if (pos == BM_RESTART) { + int x = (sizeof(BMFindFirstState) / sizeof(Eterm)) + + !!(sizeof(BMFindFirstState) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + memcpy(hp+2,&state,sizeof(state)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + Eterm erlen = erts_make_integer((Uint) bm->len, p); + ret = erts_make_integer(pos,p); + hp = HAlloc(p,3); + ret = TUPLE2(hp, ret, erlen); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } else if (type == am_ac) { + ACTrie *act; + Uint pos, rlen; + int acr; + ACFindFirstState state; + Eterm ret; + Eterm *hp; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_first_match(&state, act, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + memcpy(&state,ptr+2,sizeof(state)); + } + acr = ac_find_first_match(&state, bytes, &pos, &rlen, &reds); + if (acr == AC_NOT_FOUND) { + ret = am_nomatch; + } else if (acr == AC_RESTART) { + int x = (sizeof(state) / sizeof(Eterm)) + + !!(sizeof(ACFindFirstState) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + memcpy(hp+2,&state,sizeof(state)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + return DO_BIN_MATCH_RESTART; + } else { + Eterm epos = erts_make_integer(pos+hsstart,p); + Eterm erlen = erts_make_integer(rlen,p); + hp = HAlloc(p,3); + ret = TUPLE2(hp, epos, erlen); + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } + badarg: + return DO_BIN_MATCH_BADARG; +} + +static int do_binary_matches(Process *p, Eterm subject, Uint hsstart, + Uint hsend, Eterm type, Binary *bin, + Eterm state_term, Eterm *res_term) +{ + byte *bytes; + Uint bitoffs, bitsize; + byte *temp_alloc = NULL; + + ERTS_GET_BINARY_BYTES(subject, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + if (bitoffs != 0) { + bytes = erts_get_aligned_binary_bytes(subject, &temp_alloc); + } + if (state_term != NIL) { + Eterm *ptr = big_val(state_term); + type = ptr[1]; + } + + if (type == am_bm) { + BMData *bm; + Sint pos; + Eterm ret,tpl; + Eterm *hp; + BMFindAllState state; + Uint reds = get_reds(p, BM_LOOP_FACTOR); + Uint save_reds = reds; + + bm = (BMData *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_bm_data(bm); +#endif + if (state_term == NIL) { + bm_init_find_all(&state, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + bm_restore_find_all(&state,(char *) (ptr+2)); + } + + pos = bm_find_all_non_overlapping(&state, bm, bytes, &reds); + if (pos == BM_NOT_FOUND) { + ret = NIL; + } else if (pos == BM_RESTART) { + int x = + (SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_BM_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap bm!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + bm_serialize_find_all(&state, (char *) (hp+2)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + FindallData *fad = state.out; + int i; + for (i = 0; i < state.m; ++i) { + fad[i].epos = erts_make_integer(fad[i].pos,p); + fad[i].elen = erts_make_integer(fad[i].len,p); + } + hp = HAlloc(p,state.m * (3 + 2)); + ret = NIL; + for (i = state.m - 1; i >= 0; --i) { + tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); + hp +=3; + ret = CONS(hp,tpl,ret); + hp += 2; + } + } + erts_free_aligned_binary_bytes(temp_alloc); + bm_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / BM_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } else if (type == am_ac) { + ACTrie *act; + int acr; + ACFindAllState state; + Eterm ret,tpl; + Eterm *hp; + Uint reds = get_reds(p, AC_LOOP_FACTOR); + Uint save_reds = reds; + + act = (ACTrie *) ERTS_MAGIC_BIN_DATA(bin); +#ifdef HARDDEBUG + dump_ac_trie(act); +#endif + if (state_term == NIL) { + ac_init_find_all(&state, act, hsstart, hsend); + } else { + Eterm *ptr = big_val(state_term); + ac_restore_find_all(&state,(char *) (ptr+2)); + } + acr = ac_find_all_non_overlapping(&state, bytes, &reds); + if (acr == AC_NOT_FOUND) { + ret = NIL; + } else if (acr == AC_RESTART) { + int x = + (SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) / sizeof(Eterm)) + + !!(SIZEOF_AC_SERIALIZED_FIND_ALL_STATE(state) % sizeof(Eterm)); +#ifdef HARDDEBUG + erts_printf("Trap ac!\n"); +#endif + hp = HAlloc(p,x+2); + hp[0] = make_pos_bignum_header(x+1); + hp[1] = type; + ac_serialize_find_all(&state, (char *) (hp+2)); + *res_term = make_big(hp); + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + return DO_BIN_MATCH_RESTART; + } else { + FindallData *fad = state.out; + int i; + for (i = 0; i < state.m; ++i) { + fad[i].epos = erts_make_integer(fad[i].pos,p); + fad[i].elen = erts_make_integer(fad[i].len,p); + } + hp = HAlloc(p,state.m * (3 + 2)); + ret = NIL; + for (i = state.m - 1; i >= 0; --i) { + tpl = TUPLE2(hp, fad[i].epos, fad[i].elen); + hp +=3; + ret = CONS(hp,tpl,ret); + hp += 2; + } + } + erts_free_aligned_binary_bytes(temp_alloc); + ac_clean_find_all(&state); + BUMP_REDS(p, (save_reds - reds) / AC_LOOP_FACTOR); + *res_term = ret; + return DO_BIN_MATCH_OK; + } + badarg: + return DO_BIN_MATCH_BADARG; +} + +static int parse_match_opts_list(Eterm l, Eterm bin, Uint *posp, Uint *endp) +{ + Eterm *tp; + Uint pos; + Sint len; + if (l == ((Eterm) 0) || l == NIL) { + /* Invalid term or NIL, we're called from binary_match(es)_2 or + have no options*/ + *posp = 0; + *endp = binary_size(bin); + return 0; + } else if (is_list(l)) { + while(is_list(l)) { + Eterm t = CAR(list_val(l)); + Uint orig_size; + if (!is_tuple(t)) { + goto badarg; + } + tp = tuple_val(t); + if (arityval(*tp) != 2) { + goto badarg; + } + if (tp[1] != am_scope || is_not_tuple(tp[2])) { + goto badarg; + } + tp = tuple_val(tp[2]); + if (arityval(*tp) != 2) { + goto badarg; + } + if (!term_to_Uint(tp[1], &pos)) { + goto badarg; + } + if (!term_to_Sint(tp[2], &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + *endp = len + pos; + *posp = pos; + if ((orig_size = binary_size(bin)) < pos || + orig_size < (*endp)) { + goto badarg; + } + l = CDR(list_val(l)); + } + return 0; + } else { + badarg: + return 1; + } +} + +static BIF_RETTYPE binary_match_trap(BIF_ALIST_3) +{ + int runres; + Eterm result; + Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; + runres = do_binary_match(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); + if (runres == DO_BIN_MATCH_OK) { + BIF_RET(result); + } else { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, + BIF_ARG_3); + } +} + +static BIF_RETTYPE binary_matches_trap(BIF_ALIST_3) +{ + int runres; + Eterm result; + Binary *bin = ((ProcBin *) binary_val(BIF_ARG_3))->val; + runres = do_binary_matches(BIF_P,BIF_ARG_1,0,0,NIL,bin,BIF_ARG_2,&result); + if (runres == DO_BIN_MATCH_OK) { + BIF_RET(result); + } else { + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, + BIF_ARG_3); + } +} + +BIF_RETTYPE binary_match_3(BIF_ALIST_3) +{ + Uint hsstart; + Uint hsend; + Eterm *tp; + Eterm type; + Binary *bin; + Eterm bin_term = NIL; + int runres; + Eterm result; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + goto badarg; + } + if (hsend == 0) { + BIF_RET(am_nomatch); + } + if (is_tuple(BIF_ARG_2)) { + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2 || is_not_atom(tp[1])) { + goto badarg; + } + if (((tp[1] != am_bm) && (tp[1] != am_ac)) || + !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { + goto badarg; + } + type = tp[1]; + bin = ((ProcBin *) binary_val(tp[2]))->val; + if (type == am_bm && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { + goto badarg; + } + if (type == am_ac && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { + goto badarg; + } + bin_term = tp[2]; + } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + goto badarg; + } + runres = do_binary_match(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin,NIL,&result); + if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { + Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + } else if (bin_term == NIL) { + erts_bin_free(bin); + } + switch (runres) { + case DO_BIN_MATCH_OK: + BIF_RET(result); + case DO_BIN_MATCH_RESTART: + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_match_trap_export, BIF_P, BIF_ARG_1, result, bin_term); + default: + goto badarg; + } + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_matches_3(BIF_ALIST_3) +{ + Uint hsstart, hsend; + Eterm *tp; + Eterm type; + Binary *bin; + Eterm bin_term = NIL; + int runres; + Eterm result; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + if (parse_match_opts_list(BIF_ARG_3,BIF_ARG_1,&hsstart,&hsend)) { + goto badarg; + } + if (hsend == 0) { + BIF_RET(NIL); + } + if (is_tuple(BIF_ARG_2)) { + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2 || is_not_atom(tp[1])) { + goto badarg; + } + if (((tp[1] != am_bm) && (tp[1] != am_ac)) || + !ERTS_TERM_IS_MAGIC_BINARY(tp[2])) { + goto badarg; + } + type = tp[1]; + bin = ((ProcBin *) binary_val(tp[2]))->val; + if (type == am_bm && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_bm) { + goto badarg; + } + if (type == am_ac && + ERTS_MAGIC_BIN_DESTRUCTOR(bin) != cleanup_my_data_ac) { + goto badarg; + } + bin_term = tp[2]; + } else if (do_binary_match_compile(BIF_ARG_2,&type,&bin)) { + goto badarg; + } + runres = do_binary_matches(BIF_P,BIF_ARG_1,hsstart,hsend,type,bin, + NIL,&result); + if (runres == DO_BIN_MATCH_RESTART && bin_term == NIL) { + Eterm *hp = HAlloc(BIF_P, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), bin); + } else if (bin_term == NIL) { + erts_bin_free(bin); + } + switch (runres) { + case DO_BIN_MATCH_OK: + BIF_RET(result); + case DO_BIN_MATCH_RESTART: + BUMP_ALL_REDS(BIF_P); + BIF_TRAP3(&binary_matches_trap_export, BIF_P, BIF_ARG_1, result, + bin_term); + default: + goto badarg; + } + badarg: + BIF_ERROR(BIF_P,BADARG); +} + + +BIF_RETTYPE binary_match_2(BIF_ALIST_2) +{ + return binary_match_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); +} + + +BIF_RETTYPE binary_matches_2(BIF_ALIST_2) +{ + return binary_matches_3(BIF_P,BIF_ARG_1,BIF_ARG_2,((Eterm) 0)); +} + + +BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen) +{ + Uint pos; + Sint len; + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + Eterm* hp; + ErlSubBin* sb; + + if (is_not_binary(binary)) { + goto badarg; + } + if (!term_to_Uint(epos, &pos)) { + goto badarg; + } + if (!term_to_Sint(elen, &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + if (len > pos) { + goto badarg; + } + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)){ + goto badarg; + } + if ((orig_size = binary_size(binary)) < pos || + orig_size < (pos + len)) { + goto badarg; + } + + + + hp = HAlloc(p, ERL_SUB_BIN_SIZE); + + ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size); + sb = (ErlSubBin *) hp; + sb->thing_word = HEADER_SUB_BIN; + sb->size = len; + sb->offs = offset + pos; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + + BIF_RET(make_binary(sb)); + + badarg: + BIF_ERROR(p, BADARG); +} + +#define ERTS_NEED_GC(p, need) ((HEAP_LIMIT((p)) - HEAP_TOP((p))) <= (need)) + +BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple) +{ + Uint pos; + Sint len; + size_t orig_size; + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + Eterm* hp; + ErlSubBin* sb; + Eterm binary; + Eterm *tp; + Eterm epos, elen; + int extra_args; + + + if (range_is_tuple) { + Eterm tpl = reg[live]; + extra_args = 1; + if (is_not_tuple(tpl)) { + goto badarg; + } + tp = tuple_val(tpl); + if (arityval(*tp) != 2) { + goto badarg; + } + + epos = tp[1]; + elen = tp[2]; + } else { + extra_args = 2; + epos = reg[live-1]; + elen = reg[live]; + } + binary = reg[live-extra_args]; + + if (is_not_binary(binary)) { + goto badarg; + } + if (!term_to_Uint(epos, &pos)) { + goto badarg; + } + if (!term_to_Sint(elen, &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + if (len > pos) { + goto badarg; + } + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + if ((orig_size = binary_size(binary)) < pos || + orig_size < (pos + len)) { + goto badarg; + } + + if (ERTS_NEED_GC(p, ERL_SUB_BIN_SIZE)) { + erts_garbage_collect(p, ERL_SUB_BIN_SIZE, reg, live+1-extra_args); /* I don't need the tuple + or indices any more */ + binary = reg[live-extra_args]; + } + + hp = p->htop; + p->htop += ERL_SUB_BIN_SIZE; + + ERTS_GET_REAL_BIN(binary, orig, offset, bit_offset, bit_size); + + sb = (ErlSubBin *) hp; + sb->thing_word = HEADER_SUB_BIN; + sb->size = len; + sb->offs = offset + pos; + sb->orig = orig; + sb->bitoffs = bit_offset; + sb->bitsize = 0; + sb->is_writable = 0; + + BIF_RET(make_binary(sb)); + + badarg: + BIF_ERROR(p, BADARG); +} +/************************************************************* + * The actual guard BIFs are in erl_bif_guard.c + * but the implementation of both the non-gc and the gc + * variants are here. Note that the functions are named so that they do + * not clash with the guard bif's erlang:binary_part/2,3 + *************************************************************/ + +BIF_RETTYPE binary_binary_part_3(BIF_ALIST_3) +{ + return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE binary_binary_part_2(BIF_ALIST_2) +{ + Eterm *tp; + if (is_not_tuple(BIF_ARG_2)) { + goto badarg; + } + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2) { + goto badarg; + } + return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +typedef struct { + int type; /* CL_TYPE_XXX */ + byte *temp_alloc; /* Used for erts_get/free_aligned, i.e. CL_TYPE_ALIGNED */ + unsigned char *buff; /* Used for all types, malloced if CL_TYPE_HEAP */ + Uint bufflen; /* The length (in bytes) of buffer */ +} CommonData; + +#define COMMON_LOOP_FACTOR 10 + +#define DIRECTION_PREFIX 0 +#define DIRECTION_SUFFIX 1 + +#define CL_OK 0 +#define CL_RESTART 1 + +/* The type field in the above structure */ +#define CL_TYPE_EMPTY 0 /* End of array */ +#define CL_TYPE_HEAP 1 +#define CL_TYPE_ALIGNED 2 +#define CL_TYPE_COMMON 3 /* emacsulated */ +#define CL_TYPE_HEAP_NOALLOC 4 /* Will need allocating when trapping */ + + +static int do_search_forward(CommonData *cd, Uint *posp, Uint *redsp) +{ + Uint pos = *posp; + Sint reds = (Sint) *redsp; + int i; + unsigned char current = 0; + + for(;;) { + for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) { + if (pos >= cd[i].bufflen) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + if (i == 0) { + current = cd[i].buff[pos]; + } else { + if (cd[i].buff[pos] != current) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + } + --reds; + } + ++pos; + if (reds <= 0) { + *posp = pos; + *redsp = 0; + return CL_RESTART; + } + } +} +static int do_search_backward(CommonData *cd, Uint *posp, Uint *redsp) +{ + Uint pos = *posp; + Sint reds = (Sint) *redsp; + int i; + unsigned char current = 0; + + for(;;) { + for(i = 0; cd[i].type != CL_TYPE_EMPTY; ++i) { + if (pos >= cd[i].bufflen) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + if (i == 0) { + current = cd[i].buff[cd[i].bufflen - 1 - pos]; + } else { + if (cd[i].buff[cd[i].bufflen - 1 - pos] != current) { + *posp = pos; + if (reds > 0) { + *redsp = (Uint) reds; + } else { + *redsp = 0; + } + return CL_OK; + } + } + --reds; + } + ++pos; + if (reds <= 0) { + *posp = pos; + *redsp = 0; + return CL_RESTART; + } + } +} + +static void cleanup_common_data(Binary *bp) +{ + int i; + CommonData *cd; + cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bp); + for (i=0;cd[i].type != CL_TYPE_EMPTY;++i) { + switch (cd[i].type) { + case CL_TYPE_HEAP: + erts_free(ERTS_ALC_T_BINARY_BUFFER,cd[i].buff); + break; + case CL_TYPE_ALIGNED: + erts_free_aligned_binary_bytes_extra(cd[i].temp_alloc, ERTS_ALC_T_BINARY_BUFFER); + break; + default: + break; + } + } + return; +} + +static BIF_RETTYPE do_longest_common(Process *p, Eterm list, int direction) +{ + Eterm l = list; + int n = 0; + Binary *mb; + CommonData *cd; + int i = 0; + Uint reds = get_reds(p, COMMON_LOOP_FACTOR); + Uint save_reds = reds; + int res; + Export *trapper; + Uint pos; + Eterm epos; + Eterm *hp; + Eterm bin_term; + Eterm b; + + /* First just count the number of binaries */ + while (is_list(l)) { + b = CAR(list_val(l)); + if (!is_binary(b)) { + goto badarg; + } + ++n; + l = CDR(list_val(l)); + } + if (l != NIL || n == 0) { + goto badarg; + } + + /* OK, now create a buffer of the right size, we can do a magic binary right away, + that's not too costly. */ + mb = erts_create_magic_binary((n+1)*sizeof(CommonData),cleanup_common_data); + cd = (CommonData *) ERTS_MAGIC_BIN_DATA(mb); + l = list; + while (is_list(l)) { + Uint bitoffs; + Uint bitsize; + Uint offset; + Eterm real_bin; + ProcBin* pb; + + cd[i].type = CL_TYPE_EMPTY; + b = CAR(list_val(l)); + ERTS_GET_REAL_BIN(b, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + erts_bin_free(mb); + goto badarg; + } + cd[i].bufflen = binary_size(b); + cd[i].temp_alloc = NULL; + if (*(binary_val(real_bin)) == HEADER_PROC_BIN) { + pb = (ProcBin *) binary_val(real_bin); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc), + ERTS_ALC_T_BINARY_BUFFER,0); + cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_COMMON; + } else { /* Heap binary */ + cd[i].buff = erts_get_aligned_binary_bytes_extra(b, &(cd[i].temp_alloc), + ERTS_ALC_T_BINARY_BUFFER,0); + /* CL_TYPE_HEAP_NOALLOC means you have to copy if trapping */ + cd[i].type = (cd[i].temp_alloc != NULL) ? CL_TYPE_ALIGNED : CL_TYPE_HEAP_NOALLOC; + } + ++i; + l = CDR(list_val(l)); + } + cd[i].type = CL_TYPE_EMPTY; +#if defined(DEBUG) || defined(VALGRIND) + cd[i].temp_alloc = NULL; + cd[i].buff = NULL; + cd[i].bufflen = 0; +#endif + + pos = 0; + if (direction == DIRECTION_PREFIX) { + trapper = &binary_longest_prefix_trap_export; + res = do_search_forward(cd,&pos,&reds); + } else { + ASSERT(direction == DIRECTION_SUFFIX); + trapper = &binary_longest_suffix_trap_export; + res = do_search_backward(cd,&pos,&reds); + } + epos = erts_make_integer(pos,p); + if (res == CL_OK) { + erts_bin_free(mb); + BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR); + BIF_RET(epos); + } else { + ASSERT(res == CL_RESTART); + /* Copy all heap binaries that are not already copied (aligned) */ + for(i = 0; i < n; ++i) { + if (cd[i].type == CL_TYPE_HEAP_NOALLOC) { + unsigned char *tmp = cd[i].buff; + cd[i].buff = erts_alloc(ERTS_ALC_T_BINARY_BUFFER, cd[i].bufflen); + memcpy(cd[i].buff,tmp,cd[i].bufflen); + cd[i].type = CL_TYPE_HEAP; + } + } + hp = HAlloc(p, PROC_BIN_SIZE); + bin_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb); + BUMP_ALL_REDS(p); + BIF_TRAP3(trapper, p, bin_term, epos,list); + } + badarg: + BIF_ERROR(p,BADARG); +} + +static BIF_RETTYPE do_longest_common_trap(Process *p, Eterm bin_term, Eterm current_pos, + Eterm orig_list, int direction) +{ + Uint reds = get_reds(p, COMMON_LOOP_FACTOR); + Uint save_reds = reds; + Uint pos; + Binary *bin; + CommonData *cd; + int res; + Eterm epos; + Export *trapper; + +#ifdef DEBUG + int r; + r = term_to_Uint(current_pos, &pos); + ASSERT(r != 0); +#else + term_to_Uint(current_pos, &pos); +#endif + ASSERT(ERTS_TERM_IS_MAGIC_BINARY(bin_term)); + bin = ((ProcBin *) binary_val(bin_term))->val; + cd = (CommonData *) ERTS_MAGIC_BIN_DATA(bin); + if (direction == DIRECTION_PREFIX) { + trapper = &binary_longest_prefix_trap_export; + res = do_search_forward(cd,&pos,&reds); + } else { + ASSERT(direction == DIRECTION_SUFFIX); + trapper = &binary_longest_suffix_trap_export; + res = do_search_backward(cd,&pos,&reds); + } + epos = erts_make_integer(pos,p); + if (res == CL_OK) { + BUMP_REDS(p, (save_reds - reds) / COMMON_LOOP_FACTOR); + BIF_RET(epos); + } else { + ASSERT(res == CL_RESTART); + /* Copy all heap binaries that are not already copied (aligned) */ + BUMP_ALL_REDS(p); + BIF_TRAP3(trapper, p, bin_term, epos, orig_list); + } +} + +static BIF_RETTYPE binary_longest_prefix_trap(BIF_ALIST_3) +{ + return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_PREFIX); +} + +static BIF_RETTYPE binary_longest_suffix_trap(BIF_ALIST_3) +{ + return do_longest_common_trap(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3,DIRECTION_SUFFIX); +} + +BIF_RETTYPE binary_longest_common_prefix_1(BIF_ALIST_1) +{ + return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_PREFIX); +} + +BIF_RETTYPE binary_longest_common_suffix_1(BIF_ALIST_1) +{ + return do_longest_common(BIF_P,BIF_ARG_1,DIRECTION_SUFFIX); +} + +BIF_RETTYPE binary_first_1(BIF_ALIST_1) +{ + byte* bytes; + Uint byte_size; + Uint bit_offs; + Uint bit_size; + Uint res; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + byte_size = binary_size(BIF_ARG_1); + if (!byte_size) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size) { + goto badarg; + } + if (bit_offs) { + res = ((((Uint) bytes[0]) << bit_offs) | (((Uint) bytes[1]) >> (8-bit_offs))) & 0xFF; + } else { + res = bytes[0]; + } + BIF_RET(make_small(res)); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_last_1(BIF_ALIST_1) +{ + byte* bytes; + Uint byte_size; + Uint bit_offs; + Uint bit_size; + Uint res; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + byte_size = binary_size(BIF_ARG_1); + if (!byte_size) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size) { + goto badarg; + } + if (bit_offs) { + res = ((((Uint) bytes[byte_size-1]) << bit_offs) | + (((Uint) bytes[byte_size]) >> (8-bit_offs))) & 0xFF; + } else { + res = bytes[byte_size-1]; + } + BIF_RET(make_small(res)); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_at_2(BIF_ALIST_2) +{ + byte* bytes; + Uint byte_size; + Uint bit_offs; + Uint bit_size; + Uint res; + Uint index; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + byte_size = binary_size(BIF_ARG_1); + if (!byte_size) { + goto badarg; + } + if (!term_to_Uint(BIF_ARG_2, &index)) { + goto badarg; + } + if (index >= byte_size) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size) { + goto badarg; + } + if (bit_offs) { + res = ((((Uint) bytes[index]) << bit_offs) | + (((Uint) bytes[index+1]) >> (8-bit_offs))) & 0xFF; + } else { + res = bytes[index]; + } + BIF_RET(make_small(res)); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +#define BIN_TO_LIST_OK 0 +#define BIN_TO_LIST_TRAP 1 +/* No badarg, checked before call */ + +#define BIN_TO_LIST_LOOP_FACTOR 10 + +static int do_bin_to_list(Process *p, byte *bytes, Uint bit_offs, + Uint start, Sint *lenp, Eterm *termp) +{ + Uint reds = get_reds(p, BIN_TO_LIST_LOOP_FACTOR); /* reds can never be 0 */ + Uint len = *lenp; + Uint loops; + Eterm *hp; + Eterm term = *termp; + Uint n; + + ASSERT(reds > 0); + + loops = MIN(reds,len); + + BUMP_REDS(p, loops / BIN_TO_LIST_LOOP_FACTOR); + + hp = HAlloc(p,2*loops); + while (loops--) { + --len; + if (bit_offs) { + n = ((((Uint) bytes[start+len]) << bit_offs) | + (((Uint) bytes[start+len+1]) >> (8-bit_offs))) & 0xFF; + } else { + n = bytes[start+len]; + } + + term = CONS(hp,make_small(n),term); + hp +=2; + } + *termp = term; + *lenp = len; + if (len) { + BUMP_ALL_REDS(p); + return BIN_TO_LIST_TRAP; + } + return BIN_TO_LIST_OK; +} + + +static BIF_RETTYPE do_trap_bin_to_list(Process *p, Eterm binary, + Uint start, Sint len, Eterm sofar) +{ + Eterm *hp; + Eterm blob; + + hp = HAlloc(p,3); + hp[0] = make_pos_bignum_header(2); + hp[1] = start; + hp[2] = (Uint) len; + blob = make_big(hp); + BIF_TRAP3(&binary_bin_to_list_trap_export, p, binary, blob, sofar); +} + +static BIF_RETTYPE binary_bin_to_list_trap(BIF_ALIST_3) +{ + Eterm *ptr; + Uint start; + Sint len; + byte *bytes; + Uint bit_offs; + Uint bit_size; + Eterm res = BIF_ARG_3; + + ptr = big_val(BIF_ARG_2); + start = ptr[1]; + len = (Sint) ptr[2]; + + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (do_bin_to_list(BIF_P, bytes, bit_offs, start, &len, &res) == + BIN_TO_LIST_OK) { + BIF_RET(res); + } + return do_trap_bin_to_list(BIF_P,BIF_ARG_1,start,len,res); +} + +static BIF_RETTYPE binary_bin_to_list_common(Process *p, + Eterm bin, + Eterm epos, + Eterm elen) +{ + Uint pos; + Sint len; + size_t sz; + byte *bytes; + Uint bit_offs; + Uint bit_size; + Eterm res = NIL; + + if (is_not_binary(bin)) { + goto badarg; + } + if (!term_to_Uint(epos, &pos)) { + goto badarg; + } + if (!term_to_Sint(elen, &len)) { + goto badarg; + } + if (len < 0) { + Sint lentmp = -len; + /* overflow */ + if (lentmp == len || lentmp < 0 || -lentmp != len) { + goto badarg; + } + len = lentmp; + if (len > pos) { + goto badarg; + } + pos -= len; + } + /* overflow */ + if ((pos + len) < pos || (len > 0 && (pos + len) == pos)) { + goto badarg; + } + sz = binary_size(bin); + + if (pos+len > sz) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); + if (bit_size != 0) { + goto badarg; + } + if(do_bin_to_list(p, bytes, bit_offs, pos, &len, &res) == + BIN_TO_LIST_OK) { + BIF_RET(res); + } + return do_trap_bin_to_list(p,bin,pos,len,res); + + badarg: + BIF_ERROR(p,BADARG); +} + +BIF_RETTYPE binary_bin_to_list_3(BIF_ALIST_3) +{ + return binary_bin_to_list_common(BIF_P,BIF_ARG_1,BIF_ARG_2,BIF_ARG_3); +} + +BIF_RETTYPE binary_bin_to_list_2(BIF_ALIST_2) +{ + Eterm *tp; + + if (is_not_tuple(BIF_ARG_2)) { + goto badarg; + } + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2) { + goto badarg; + } + return binary_bin_to_list_common(BIF_P,BIF_ARG_1,tp[1],tp[2]); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +BIF_RETTYPE binary_bin_to_list_1(BIF_ALIST_1) +{ + Uint pos = 0; + Sint len; + byte *bytes; + Uint bit_offs; + Uint bit_size; + Eterm res = NIL; + + if (is_not_binary(BIF_ARG_1)) { + goto badarg; + } + len = binary_size(BIF_ARG_1); + ERTS_GET_BINARY_BYTES(BIF_ARG_1,bytes,bit_offs,bit_size); + if (bit_size != 0) { + goto badarg; + } + if(do_bin_to_list(BIF_P, bytes, bit_offs, pos, &len, &res) == + BIN_TO_LIST_OK) { + BIF_RET(res); + } + return do_trap_bin_to_list(BIF_P,BIF_ARG_1,pos,len,res); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + +/* + * Ok, erlang:list_to_binary does not interrupt, and we really don't want + * an alternative implementation for the exact same thing, why we + * have descided to use the old non-restarting implementation for now. + * In reality, there are seldom many iterations involved in doing this, so the + * problem of long-running bifs is not really that big in this case. + * So, for now we use the old implementation also in the module binary. + */ + +BIF_RETTYPE binary_list_to_bin_1(BIF_ALIST_1) +{ + return erts_list_to_binary_bif(BIF_P, BIF_ARG_1); +} + +typedef struct { + Uint times_left; + Uint source_size; + int source_type; + byte *source; + byte *temp_alloc; + Uint result_pos; + Binary *result; +} CopyBinState; + +#define BC_TYPE_EMPTY 0 +#define BC_TYPE_HEAP 1 +#define BC_TYPE_ALIGNED 2 /* May or may not point to (emasculated) binary, temp_alloc field is set + so that erts_free_aligned_binary_bytes_extra can handle either */ + + +#define BINARY_COPY_LOOP_FACTOR 100 + +static void cleanup_copy_bin_state(Binary *bp) +{ + CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(bp); + if (cbs->result != NULL) { + erts_bin_free(cbs->result); + cbs->result = NULL; + } + switch (cbs->source_type) { + case BC_TYPE_HEAP: + erts_free(ERTS_ALC_T_BINARY_BUFFER,cbs->source); + break; + case BC_TYPE_ALIGNED: + erts_free_aligned_binary_bytes_extra(cbs->temp_alloc, + ERTS_ALC_T_BINARY_BUFFER); + break; + default: + /* otherwise do nothing */ + break; + } + cbs->source_type = BC_TYPE_EMPTY; +} + +/* + * Binary *erts_bin_nrml_alloc(Uint size); + * Binary *erts_bin_realloc(Binary *bp, Uint size); + * void erts_bin_free(Binary *bp); + */ +static BIF_RETTYPE do_binary_copy(Process *p, Eterm bin, Eterm en) +{ + Uint n; + byte *bytes; + Uint bit_offs; + Uint bit_size; + size_t size; + Uint reds = get_reds(p, BINARY_COPY_LOOP_FACTOR); + Uint target_size; + byte *t; + Uint pos; + + + if (is_not_binary(bin)) { + goto badarg; + } + if (!term_to_Uint(en, &n)) { + goto badarg; + } + if (!n) { + Eterm res_term = erts_new_heap_binary(p,NULL,0,&bytes); + BIF_RET(res_term); + } + ERTS_GET_BINARY_BYTES(bin,bytes,bit_offs,bit_size); + if (bit_size != 0) { + goto badarg; + } + + size = binary_size(bin); + target_size = size * n; + + if ((target_size - size) >= reds) { + Eterm orig; + Uint offset; + Uint bit_offset; + Uint bit_size; + CopyBinState *cbs; + Eterm *hp; + Eterm trap_term; + int i; + + /* We will trap, set up the structure for trapping right away */ + Binary *mb = erts_create_magic_binary(sizeof(CopyBinState), + cleanup_copy_bin_state); + cbs = ERTS_MAGIC_BIN_DATA(mb); + + cbs->temp_alloc = NULL; + cbs->source = NULL; + + ERTS_GET_REAL_BIN(bin, orig, offset, bit_offset, bit_size); + if (*(binary_val(orig)) == HEADER_PROC_BIN) { + ProcBin* pb = (ProcBin *) binary_val(orig); + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + cbs->source = + erts_get_aligned_binary_bytes_extra(bin, + &(cbs->temp_alloc), + ERTS_ALC_T_BINARY_BUFFER, + 0); + cbs->source_type = BC_TYPE_ALIGNED; + } else { /* Heap binary */ + cbs->source = + erts_get_aligned_binary_bytes_extra(bin, + &(cbs->temp_alloc), + ERTS_ALC_T_BINARY_BUFFER, + 0); + if (!(cbs->temp_alloc)) { /* alignment not needed, need to copy */ + byte *tmp = erts_alloc(ERTS_ALC_T_BINARY_BUFFER,size); + memcpy(tmp,cbs->source,size); + cbs->source = tmp; + cbs->source_type = BC_TYPE_HEAP; + } else { + cbs->source_type = BC_TYPE_ALIGNED; + } + } + cbs->result = erts_bin_nrml_alloc(target_size); /* Always offheap + if trapping */ + cbs->result->flags = 0; + cbs->result->orig_size = target_size; + erts_refc_init(&(cbs->result->refc), 1); + t = (byte *) cbs->result->orig_bytes; /* No offset or anything */ + pos = 0; + i = 0; + while (pos < reds) { + memcpy(t+pos,cbs->source, size); + pos += size; + ++i; + } + cbs->source_size = size; + cbs->result_pos = pos; + cbs->times_left = n-i; + hp = HAlloc(p,PROC_BIN_SIZE); + trap_term = erts_mk_magic_binary_term(&hp, &MSO(p), mb); + BUMP_ALL_REDS(p); + BIF_TRAP2(&binary_copy_trap_export, p, bin, trap_term); + } else { + Eterm res_term; + byte *temp_alloc = NULL; + byte *source = + erts_get_aligned_binary_bytes(bin, + &temp_alloc); + if (target_size <= ERL_ONHEAP_BIN_LIMIT) { + res_term = erts_new_heap_binary(p,NULL,target_size,&t); + } else { + res_term = erts_new_mso_binary(p,NULL,target_size); + t = ((ProcBin *) binary_val(res_term))->bytes; + } + pos = 0; + while (pos < target_size) { + memcpy(t+pos,source, size); + pos += size; + } + erts_free_aligned_binary_bytes(temp_alloc); + BUMP_REDS(p,pos / BINARY_COPY_LOOP_FACTOR); + BIF_RET(res_term); + } + badarg: + BIF_ERROR(p,BADARG); +} + +BIF_RETTYPE binary_copy_trap(BIF_ALIST_2) +{ + Uint n; + size_t size; + Uint reds = get_reds(BIF_P, BINARY_COPY_LOOP_FACTOR); + byte *t; + Uint pos; + Binary *mb = ((ProcBin *) binary_val(BIF_ARG_2))->val; + CopyBinState *cbs = (CopyBinState *) ERTS_MAGIC_BIN_DATA(mb); + Uint opos; + + /* swapout... */ + n = cbs->times_left; + size = cbs->source_size; + opos = pos = cbs->result_pos; + t = (byte *) cbs->result->orig_bytes; /* "well behaved" binary */ + if ((n-1) * size >= reds) { + Uint i = 0; + while ((pos - opos) < reds) { + memcpy(t+pos,cbs->source, size); + pos += size; + ++i; + } + cbs->result_pos = pos; + cbs->times_left -= i; + BUMP_ALL_REDS(BIF_P); + BIF_TRAP2(&binary_copy_trap_export, BIF_P, BIF_ARG_1, BIF_ARG_2); + } else { + Binary *save; + ProcBin* pb; + Uint target_size = cbs->result->orig_size; + while (pos < target_size) { + memcpy(t+pos,cbs->source, size); + pos += size; + } + save = cbs->result; + cbs->result = NULL; + cleanup_copy_bin_state(mb); /* now cbs is dead */ + pb = (ProcBin *) HAlloc(BIF_P, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = target_size; + pb->next = MSO(BIF_P).first; + MSO(BIF_P).first = (struct erl_off_heap_header*) pb; + pb->val = save; + pb->bytes = t; + pb->flags = 0; + + OH_OVERHEAD(&(MSO(BIF_P)), target_size / sizeof(Eterm)); + BUMP_REDS(BIF_P,(pos - opos) / BINARY_COPY_LOOP_FACTOR); + + BIF_RET(make_binary(pb)); + } +} + + +BIF_RETTYPE binary_copy_1(BIF_ALIST_1) +{ + return do_binary_copy(BIF_P,BIF_ARG_1,make_small(1)); +} + +BIF_RETTYPE binary_copy_2(BIF_ALIST_2) +{ + return do_binary_copy(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +BIF_RETTYPE binary_referenced_byte_size_1(BIF_ALIST_1) +{ + ErlSubBin *sb; + ProcBin *pb; + Eterm res; + Eterm bin = BIF_ARG_1; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + sb = (ErlSubBin *) binary_val(bin); + if (sb->thing_word == HEADER_SUB_BIN) { + bin = sb->orig; + } + pb = (ProcBin *) binary_val(bin); + if (pb->thing_word == HEADER_PROC_BIN) { + /* XXX:PaN - Halfword - orig_size is a long, we should handle that */ + res = erts_make_integer((Uint) pb->val->orig_size, BIF_P); + } else { /* heap binary */ + res = erts_make_integer((Uint) ((ErlHeapBin *) pb)->size, BIF_P); + } + BIF_RET(res); +} + +#define END_BIG 0 +#define END_SMALL 1 + +#ifdef WORDS_BIGENDIAN +#define END_NATIVE END_BIG +#else +#define END_NATIVE END_SMALL +#endif + +static int get_need(Uint u) { +#if defined(ARCH_64) && !HALFWORD_HEAP + if (u > 0xFFFFFFFFUL) { + if (u > 0xFFFFFFFFFFFFUL) { + if (u > 0xFFFFFFFFFFFFFFUL) { + return 8; + } + return 7; + } + if (u > 0xFFFFFFFFFFUL) { + return 6; + } + return 5; + } +#endif + if (u > 0xFFFFUL) { + if (u > 0xFFFFFFUL) { + return 4; + } + return 3; + } + if (u > 0xFFUL) { + return 2; + } + return 1; +} + +static BIF_RETTYPE do_encode_unsigned(Process *p, Eterm uns, Eterm endianess) +{ + Eterm res; + if ((is_not_small(uns) && is_not_big(uns)) || is_not_atom(endianess) || + (endianess != am_big && endianess != am_little)) { + goto badarg; + } + if (is_small(uns)) { + Sint x = signed_val(uns); + Uint u; + int n,i; + byte *b; + + if (x < 0) { + goto badarg; + } + + u = (Uint) x; + n = get_need(u); + ASSERT(n <= ERL_ONHEAP_BIN_LIMIT); + res = erts_new_heap_binary(p, NULL, n, &b); + if (endianess == am_big) { + for(i=n-1;i>=0;--i) { + b[i] = u & 0xFF; + u >>= 8; + } + } else { + for(i=0;i<n;++i) { + b[i] = u & 0xFF; + u >>= 8; + } + } + BIF_RET(res); + } else { + /* Big */ + Eterm *bigp = big_val(uns); + Uint n; + dsize_t num_parts = BIG_SIZE(bigp); + Eterm res; + byte *b; + ErtsDigit d; + + if(BIG_SIGN(bigp)) { + goto badarg; + } + n = (num_parts-1)*sizeof(ErtsDigit)+get_need(BIG_DIGIT(bigp,(num_parts-1))); + if (n <= ERL_ONHEAP_BIN_LIMIT) { + res = erts_new_heap_binary(p,NULL,n,&b); + } else { + res = erts_new_mso_binary(p,NULL,n); + b = ((ProcBin *) binary_val(res))->bytes; + } + + if (endianess == am_big) { + Sint i,j; + j = 0; + d = BIG_DIGIT(bigp,0); + for (i=n-1;i>=0;--i) { + b[i] = d & 0xFF; + if (!((++j) % sizeof(ErtsDigit))) { + d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit)); + } else { + d >>= 8; + } + } + } else { + Sint i,j; + j = 0; + d = BIG_DIGIT(bigp,0); + for (i=0;i<n;++i) { + b[i] = d & 0xFF; + if (!((++j) % sizeof(ErtsDigit))) { + d = BIG_DIGIT(bigp,j / sizeof(ErtsDigit)); + } else { + d >>= 8; + } + } + + } + BIF_RET(res); + } + badarg: + BIF_ERROR(p,BADARG); +} + +static BIF_RETTYPE do_decode_unsigned(Process *p, Eterm uns, Eterm endianess) +{ + byte *bytes; + Uint bitoffs, bitsize; + Uint size; + Eterm res; + + if (is_not_binary(uns) || is_not_atom(endianess) || + (endianess != am_big && endianess != am_little)) { + goto badarg; + } + ERTS_GET_BINARY_BYTES(uns, bytes, bitoffs, bitsize); + if (bitsize != 0) { + goto badarg; + } + /* align while rolling */ + size = binary_size(uns); + if (bitoffs) { + if (endianess == am_big) { + while (size && (((((Uint) bytes[0]) << bitoffs) | + (((Uint) bytes[1]) >> (8-bitoffs))) & 0xFF) == 0) { + ++bytes; + --size; + } + } else { + while(size && + (((((Uint) bytes[size-1]) << bitoffs) | + (((Uint) bytes[size]) >> (8-bitoffs))) & 0xFF) == 0) { + --size; + } + } + } else { + if (endianess == am_big) { + while (size && *bytes == 0) { + ++bytes; + --size; + } + } else { + while(size && bytes[size-1] == 0) { + --size; + } + } + } + if (!size) { + BIF_RET(make_small(0)); + } + + if (size <= sizeof(Uint)) { + Uint u = 0; + Sint i; + + if (endianess == am_big) { + if (bitoffs) { + for(i=0;i<size;++i) { + u <<=8; + u |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + } + } else { + for(i=0;i<size;++i) { + u <<=8; + u |= bytes[i]; + } + } + } else { + + if (bitoffs) { + for(i=size-1;i>=0;--i) { + u <<=8; + u |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + } + } else { + for(i=size-1;i>=0;--i) { + u <<=8; + u |= bytes[i]; + } + } + } + res = erts_make_integer(u,p); + BIF_RET(res); + } else { + /* Assume big, as we stripped away all zeroes from the MSB part of the binary */ + dsize_t num_parts = size / sizeof(ErtsDigit) + !!(size % sizeof(ErtsDigit)); + Eterm *bigp; + + bigp = HAlloc(p, BIG_NEED_SIZE(num_parts)); + *bigp = make_pos_bignum_header(num_parts); + res = make_big(bigp); + + if (endianess == am_big) { + Sint i,j; + ErtsDigit *d; + j = size; + d = &(BIG_DIGIT(bigp,num_parts - 1)); + *d = 0; + i = 0; + if(bitoffs) { + for (;;){ + (*d) <<= 8; + (*d) |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + if (++i >= size) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } else { + for (;;){ + (*d) <<= 8; + (*d) |= bytes[i]; + if (++i >= size) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } + } else { + Sint i,j; + ErtsDigit *d; + j = size; + d = &(BIG_DIGIT(bigp,num_parts - 1)); + *d = 0; + i = size-1; + if (bitoffs) { + for (;;){ + (*d) <<= 8; + (*d) |= (((((Uint) bytes[i]) << bitoffs) | + (((Uint) bytes[i+1]) >> (8-bitoffs))) & 0xFF); + if (--i < 0) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } else { + for (;;){ + (*d) <<= 8; + (*d) |= bytes[i]; + if (--i < 0) { + break; + } + if (!(--j % sizeof(ErtsDigit))) { + --d; + *d = 0; + } + } + } + } + BIF_RET(res); + } + badarg: + BIF_ERROR(p,BADARG); +} + +BIF_RETTYPE binary_encode_unsigned_1(BIF_ALIST_1) +{ + return do_encode_unsigned(BIF_P,BIF_ARG_1,am_big); +} + +BIF_RETTYPE binary_encode_unsigned_2(BIF_ALIST_2) +{ + return do_encode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +BIF_RETTYPE binary_decode_unsigned_1(BIF_ALIST_1) +{ + return do_decode_unsigned(BIF_P,BIF_ARG_1,am_big); +} + +BIF_RETTYPE binary_decode_unsigned_2(BIF_ALIST_2) +{ + return do_decode_unsigned(BIF_P,BIF_ARG_1,BIF_ARG_2); +} + +/* + * Hard debug functions (dump) for the search structures + */ + +#ifdef HARDDEBUG +static void dump_bm_data(BMData *bm) +{ + int i,j; + erts_printf("Dumping Boyer-Moore structure.\n"); + erts_printf("=============================\n"); + erts_printf("Searchstring [%ld]:\n", bm->len); + erts_printf("<<"); + for (i = 0; i < bm->len; ++i) { + if (i > 0) { + erts_printf(", "); + } + erts_printf("%d", (int) bm->x[i]); + if (bm->x[i] >= 'A') { + erts_printf(" ($%c)",(char) bm->x[i]); + } + } + erts_printf(">>\n"); + erts_printf("GoodShift array:\n"); + for (i = 0; i < bm->len; ++i) { + erts_printf("GoodShift[%d]: %ld\n", i, bm->goodshift[i]); + } + erts_printf("BadShift array:\n"); + j = 0; + for (i = 0; i < ALPHABET_SIZE; i += j) { + for (j = 0; i + j < ALPHABET_SIZE && j < 6; ++j) { + erts_printf("BS[%03d]:%02ld, ", i+j, bm->badshift[i+j]); + } + erts_printf("\n"); + } +} + +static void dump_ac_node(ACNode *node, int indent, int ch) { + int i; + char *spaces = erts_alloc(ERTS_ALC_T_TMP, 10 * indent + 1); + memset(spaces,' ',10*indent); + spaces[10*indent] = '\0'; + erts_printf("%s-> %c\n",spaces,ch); + erts_printf("%sId: %u\n",spaces,(unsigned) node->id); + erts_printf("%sD: %u\n",spaces,(unsigned)node->d); + erts_printf("%sFinal: %d\n",spaces,(int)node->final); + erts_printf("%sFail: %u\n",spaces,(unsigned)node->h->id); + erts_free(ERTS_ALC_T_TMP,spaces); + for(i=0;i<ALPHABET_SIZE;++i) { + if (node->g[i] != NULL && node->g[i] != node) { + dump_ac_node(node->g[i],indent+1,i); + } + } +} + + +static void dump_ac_trie(ACTrie *act) +{ + erts_printf("Aho Corasick Trie dump.\n"); + erts_printf("=======================\n"); + erts_printf("Node counter: %u\n", (unsigned) act->idc); + erts_printf("Searchstring counter: %u\n", (unsigned) act->counter); + erts_printf("Trie:\n"); + dump_ac_node(act->root, 0, '0'); + return; +} +#endif diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 9d5f0d9c02..c9cdcb87a6 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1193,7 +1193,7 @@ int erts_ddll_driver_ok(DE_Handle *dh) static void ddll_no_more_references(void *vdh) { DE_Handle *dh = (DE_Handle *) vdh; - int x; + erts_aint_t x; lock_drv_list(); @@ -1604,7 +1604,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) erts_sys_ddll_close(dh->handle); return ERL_DE_LOAD_ERROR_BAD_NAME; } - erts_smp_atomic_init(&(dh->refc), (long) 0); + erts_smp_atomic_init(&(dh->refc), (erts_aint_t) 0); dh->port_count = 0; dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); sys_strcpy(dh->full_path, path); @@ -1646,7 +1646,8 @@ static int do_unload_driver_entry(DE_Handle *dh, Eterm *save_name) if (save_name != NULL) { *save_name = mkatom(q->name); } - /* XXX:PaN Future locking problems? Don't dare to let go of the diver_list lock here!*/ + /* Future locking problems? Don't dare to let go of the + diver_list lock here!*/ if (q->finish) { int fpe_was_unmasked = erts_block_fpe(); (*(q->finish))(); @@ -1671,7 +1672,7 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name) dh->handle = NULL; dh->procs = NULL; dh->port_count = 0; - erts_refc_init(&(dh->refc), (long) 0); + erts_refc_init(&(dh->refc), (erts_aint_t) 0); dh->status = -1; dh->reload_full_path = NULL; dh->reload_driver_name = NULL; diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index 440b0b4f14..01e6977a2c 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -314,6 +314,30 @@ double_to_integer(Process* p, double x) return res; } +/******************************************************************************** + * binary_part guards. The actual implementation is in erl_bif_binary.c + ********************************************************************************/ +BIF_RETTYPE binary_part_3(BIF_ALIST_3) +{ + return erts_binary_part(BIF_P,BIF_ARG_1,BIF_ARG_2, BIF_ARG_3); +} + +BIF_RETTYPE binary_part_2(BIF_ALIST_2) +{ + Eterm *tp; + if (is_not_tuple(BIF_ARG_2)) { + goto badarg; + } + tp = tuple_val(BIF_ARG_2); + if (arityval(*tp) != 2) { + goto badarg; + } + return erts_binary_part(BIF_P,BIF_ARG_1,tp[1], tp[2]); + badarg: + BIF_ERROR(BIF_P,BADARG); +} + + /* * The following code is used when a guard that may build on the * heap is called directly. They must not use HAlloc(), but must @@ -630,3 +654,16 @@ gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live) } return res; } + +/******************************************************************************** + * binary_part guards. The actual implementation is in erl_bif_binary.c + ********************************************************************************/ +Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live) +{ + return erts_gc_binary_part(p,reg,live,0); +} + +Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live) +{ + return erts_gc_binary_part(p,reg,live,1); +} diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 18cb09d8cd..4a717d7271 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -38,9 +38,7 @@ #include "erl_instrument.h" #include "dist.h" #include "erl_gc.h" -#ifdef ELIB_ALLOC_IS_CLIB -#include "elib_stat.h" -#endif +#include "erl_cpu_topology.h" #ifdef HIPE #include "hipe_arch.h" #endif @@ -122,22 +120,26 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE #endif static Eterm -bld_bin_list(Uint **hpp, Uint *szp, ProcBin* pb) +bld_bin_list(Uint **hpp, Uint *szp, ErlOffHeap* oh) { + struct erl_off_heap_header* ohh; Eterm res = NIL; Eterm tuple; - for (; pb; pb = pb->next) { - Eterm val = erts_bld_uword(hpp, szp, (UWord) pb->val); - Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size); - - if (szp) - *szp += 4+2; - if (hpp) { - Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); - tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); - res = CONS(*hpp + 4, tuple, res); - *hpp += 4+2; + for (ohh = oh->first; ohh; ohh = ohh->next) { + if (ohh->thing_word == HEADER_PROC_BIN) { + ProcBin* pb = (ProcBin*) ohh; + Eterm val = erts_bld_uword(hpp, szp, (UWord) pb->val); + Eterm orig_size = erts_bld_uint(hpp, szp, pb->val->orig_size); + + if (szp) + *szp += 4+2; + if (hpp) { + Uint refc = (Uint) erts_smp_atomic_read(&pb->val->refc); + tuple = TUPLE3(*hpp, val, orig_size, make_small(refc)); + res = CONS(*hpp + 4, tuple, res); + *hpp += 4+2; + } } } return res; @@ -176,10 +178,10 @@ static void do_make_one_mon_element(ErtsMonitor *mon, void * vpmlc) Eterm tup; Eterm r = (IS_CONST(mon->ref) ? mon->ref - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->ref)); + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->ref)); Eterm p = (IS_CONST(mon->pid) ? mon->pid - : STORE_NC(&(pmlc->hp), &MSO(pmlc->p).externals, mon->pid)); + : STORE_NC(&(pmlc->hp), &MSO(pmlc->p), mon->pid)); tup = TUPLE5(pmlc->hp, pmlc->tag, make_small(mon->type), r, p, mon->name); pmlc->hp += 6; pmlc->res = CONS(pmlc->hp, tup, pmlc->res); @@ -240,7 +242,7 @@ static void do_make_one_lnk_element(ErtsLink *lnk, void * vpllc) Eterm old_res, targets = NIL; Eterm p = (IS_CONST(lnk->pid) ? lnk->pid - : STORE_NC(&(pllc->hp), &MSO(pllc->p).externals, lnk->pid)); + : STORE_NC(&(pllc->hp), &MSO(pllc->p), lnk->pid)); if (lnk->type == LINK_NODE) { targets = make_small(ERTS_LINK_REFC(lnk)); } else if (ERTS_LINK_ROOT(lnk) != NULL) { @@ -1140,9 +1142,9 @@ process_info_aux(Process *BIF_P, } else { /* Make our copy of the message */ - ASSERT(size_object(msg) == hfp->size); + ASSERT(size_object(msg) == hfp->used_size); msg = copy_struct(msg, - hfp->size, + hfp->used_size, &hp, &MSO(BIF_P)); } @@ -1225,7 +1227,7 @@ process_info_aux(Process *BIF_P, hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -1258,9 +1260,7 @@ process_info_aux(Process *BIF_P, else { /* Monitor by pid. Build {process, Pid} and cons it. */ Eterm t; - Eterm pid = STORE_NC(&hp, - &MSO(BIF_P).externals, - mic.mi[i].entity); + Eterm pid = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); t = TUPLE2(hp, am_process, pid); hp += 3; res = CONS(hp, t, res); @@ -1282,7 +1282,7 @@ process_info_aux(Process *BIF_P, res = NIL; for (i = 0; i < mic.mi_i; ++i) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -1491,7 +1491,7 @@ process_info_aux(Process *BIF_P, case am_group_leader: { int sz = NC_HEAP_SIZE(rp->group_leader); hp = HAlloc(BIF_P, 3 + sz); - res = STORE_NC(&hp, &MSO(BIF_P).externals, rp->group_leader); + res = STORE_NC(&hp, &MSO(BIF_P), rp->group_leader); break; } @@ -1516,9 +1516,9 @@ process_info_aux(Process *BIF_P, case am_binary: { Uint sz = 3; - (void) bld_bin_list(NULL, &sz, MSO(rp).mso); + (void) bld_bin_list(NULL, &sz, &MSO(rp)); hp = HAlloc(BIF_P, sz); - res = bld_bin_list(&hp, NULL, MSO(rp).mso); + res = bld_bin_list(&hp, NULL, &MSO(rp)); break; } @@ -1688,6 +1688,8 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return erts_get_cpu_topology_term(BIF_P, *tp); } else if (ERTS_IS_ATOM_STR("cpu_topology", sel) && arity == 2) { Eterm res = erts_get_cpu_topology_term(BIF_P, *tp); + if (res == THE_NON_VALUE) + goto badarg; ERTS_BIF_PREP_TRAP1(ret, erts_format_cpu_topology_trap, BIF_P, res); return ret; #if defined(PURIFY) || defined(VALGRIND) @@ -1892,6 +1894,37 @@ c_compiler_used(Eterm **hpp, Uint *szp) } +static int is_snif_term(Eterm module_atom) { + int i; + Atom *a = atom_tab(atom_val(module_atom)); + char *aname = (char *) a->name; + + /* if a->name has a '.' then the bif (snif) is bogus i.e a package */ + for (i = 0; i < a->len; i++) { + if (aname[i] == '.') + return 0; + } + + return 1; +} + +static Eterm build_snif_term(Eterm **hpp, Uint *szp, int ix, Eterm res) { + Eterm tup; + tup = erts_bld_tuple(hpp, szp, 3, bif_table[ix].module, bif_table[ix].name, make_small(bif_table[ix].arity)); + res = erts_bld_cons( hpp, szp, tup, res); + return res; +} + +static Eterm build_snifs_term(Eterm **hpp, Uint *szp, Eterm res) { + int i; + for (i = 0; i < BIF_SIZE; i++) { + if (is_snif_term(bif_table[i].module)) { + res = build_snif_term(hpp, szp, i, res); + } + } + return res; +} + BIF_RETTYPE system_info_1(BIF_ALIST_1) { Eterm res; @@ -1925,6 +1958,35 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) : am_enabled); } #endif + } else if (BIF_ARG_1 == am_build_type) { +#if defined(DEBUG) + ERTS_DECL_AM(debug); + BIF_RET(AM_debug); +#elif defined(PURIFY) + ERTS_DECL_AM(purify); + BIF_RET(AM_purify); +#elif defined(QUANTIFY) + ERTS_DECL_AM(quantify); + BIF_RET(AM_quantify); +#elif defined(PURECOV) + ERTS_DECL_AM(purecov); + BIF_RET(AM_purecov); +#elif defined(ERTS_GCOV) + ERTS_DECL_AM(gcov); + BIF_RET(AM_gcov); +#elif defined(VALGRIND) + ERTS_DECL_AM(valgrind); + BIF_RET(AM_valgrind); +#elif defined(GPROF) + ERTS_DECL_AM(gprof); + BIF_RET(AM_gprof); +#elif defined(ERTS_ENABLE_LOCK_COUNT) + ERTS_DECL_AM(lcnt); + BIF_RET(AM_lcnt); +#else + BIF_RET(am_opt); +#endif + BIF_RET(res); } else if (BIF_ARG_1 == am_allocated_areas) { res = erts_allocated_areas(NULL, NULL, BIF_P); BIF_RET(res); @@ -1940,6 +2002,17 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(db_get_trace_control_word_0(BIF_P)); } else if (ERTS_IS_ATOM_STR("ets_realloc_moves", BIF_ARG_1)) { BIF_RET((erts_ets_realloc_always_moves) ? am_true : am_false); + } else if (ERTS_IS_ATOM_STR("ets_always_compress", BIF_ARG_1)) { + BIF_RET((erts_ets_always_compress) ? am_true : am_false); + } else if (ERTS_IS_ATOM_STR("snifs", BIF_ARG_1)) { + Uint size = 0; + Uint *szp; + + szp = &size; + build_snifs_term(NULL, szp, NIL); + hp = HAlloc(BIF_P, size); + res = build_snifs_term(&hp, NULL, NIL); + BIF_RET(res); } else if (BIF_ARG_1 == am_sequential_tracer) { val = erts_get_system_seq_tracer(); ASSERT(is_internal_pid(val) || is_internal_port(val) || val==am_false) @@ -1947,7 +2020,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = TUPLE2(hp, am_sequential_tracer, val); BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection){ - Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); Eterm tup; hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2); @@ -1962,7 +2035,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); } else if (BIF_ARG_1 == am_fullsweep_after){ - Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_fullsweep_after, make_small(val)); BIF_RET(res); @@ -2084,86 +2157,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(erts_alloc_util_allocators((void *) BIF_P)); } else if (BIF_ARG_1 == am_elib_malloc) { -#ifdef ELIB_ALLOC_IS_CLIB - struct elib_stat stat; - DECL_AM(heap_size); - DECL_AM(max_alloced_size); - DECL_AM(alloced_size); - DECL_AM(free_size); - DECL_AM(no_alloced_blocks); - DECL_AM(no_free_blocks); - DECL_AM(smallest_alloced_block); - DECL_AM(largest_free_block); - Eterm atoms[8]; - Eterm ints[8]; - Uint **hpp; - Uint sz; - Uint *szp; - int length; -#ifdef DEBUG - Uint *endp; -#endif - - elib_stat(&stat); - - /* First find out the heap size needed ... */ - hpp = NULL; - szp = &sz; - sz = 0; - - build_elib_malloc_term: - length = 0; - atoms[length] = AM_heap_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_total*sizeof(Uint)); - atoms[length] = AM_max_alloced_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_max_alloc*sizeof(Uint)); - atoms[length] = AM_alloced_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_alloc*sizeof(Uint)); - atoms[length] = AM_free_size; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.mem_free*sizeof(Uint)); - atoms[length] = AM_no_alloced_blocks; - ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.mem_blocks); - atoms[length] = AM_no_free_blocks; - ints[length++] = erts_bld_uint(hpp, szp, (Uint) stat.free_blocks); - atoms[length] = AM_smallest_alloced_block; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.min_used*sizeof(Uint)); - atoms[length] = AM_largest_free_block; - ints[length++] = erts_bld_uint(hpp, szp, - (Uint) stat.max_free*sizeof(Uint)); - - - - ASSERT(length <= sizeof(atoms)/sizeof(Eterm)); - ASSERT(length <= sizeof(ints)/sizeof(Eterm)); - - res = erts_bld_2tup_list(hpp, szp, length, atoms, ints); - - if (szp) { - /* ... and then build the term */ - hp = HAlloc(BIF_P, sz); -#ifdef DEBUG - endp = hp + sz; -#endif - - szp = NULL; - hpp = &hp; - goto build_elib_malloc_term; - } - -#ifdef DEBUG - ASSERT(endp == hp); -#endif - -#else /* #ifdef ELIB_ALLOC_IS_CLIB */ - res = am_false; -#endif /* #ifdef ELIB_ALLOC_IS_CLIB */ - - BIF_RET(res); + /* To be removed in R15 */ + BIF_RET(am_false); } else if (BIF_ARG_1 == am_os_version) { int major, minor, build; @@ -2274,6 +2269,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("cpu_topology", BIF_ARG_1)) { res = erts_get_cpu_topology_term(BIF_P, am_used); BIF_TRAP1(erts_format_cpu_topology_trap, BIF_P, res); + } else if (ERTS_IS_ATOM_STR("update_cpu_info", BIF_ARG_1)) { + if (erts_update_cpu_info()) { + ERTS_DECL_AM(changed); + BIF_RET(AM_changed); + } + else { + ERTS_DECL_AM(unchanged); + BIF_RET(AM_unchanged); + } #if defined(__GNUC__) && defined(HAVE_SOLARIS_SPARC_PERFMON) } else if (ERTS_IS_ATOM_STR("ultrasparc_read_tick1", BIF_ARG_1)) { register unsigned high asm("%l0"); @@ -2345,7 +2349,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } /* Arguments that are unusual follow ... */ else if (ERTS_IS_ATOM_STR("logical_processors", BIF_ARG_1)) { - int no = erts_get_cpu_configured(erts_cpuinfo); + int no; + erts_get_logical_processors(&no, NULL, NULL); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2354,7 +2359,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } } else if (ERTS_IS_ATOM_STR("logical_processors_online", BIF_ARG_1)) { - int no = erts_get_cpu_online(erts_cpuinfo); + int no; + erts_get_logical_processors(NULL, &no, NULL); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2363,7 +2369,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } } else if (ERTS_IS_ATOM_STR("logical_processors_available", BIF_ARG_1)) { - int no = erts_get_cpu_available(erts_cpuinfo); + int no; + erts_get_logical_processors(NULL, NULL, &no); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2523,6 +2530,15 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(erts_sched_stat_term(BIF_P, 1)); } else if (ERTS_IS_ATOM_STR("taints", BIF_ARG_1)) { BIF_RET(erts_nif_taints(BIF_P)); + } else if (ERTS_IS_ATOM_STR("reader_groups_map", BIF_ARG_1)) { + BIF_RET(erts_get_reader_groups_map(BIF_P)); + } else if (ERTS_IS_ATOM_STR("dist_buf_busy_limit", BIF_ARG_1)) { + Uint hsz = 0; + + (void) erts_bld_uint(NULL, &hsz, erts_dist_buf_busy_limit); + hp = hsz ? HAlloc(BIF_P, hsz) : NULL; + res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); + BIF_RET(res); } BIF_ERROR(BIF_P, BADARG); @@ -2638,7 +2654,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) hp = HAlloc(BIF_P, 3 + mic.sz); res = NIL; for (i = 0; i < mic.mi_i; i++) { - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); res = CONS(hp, item, res); hp += 2; } @@ -2658,7 +2674,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) res = NIL; for (i = 0; i < mic.mi_i; i++) { Eterm t; - item = STORE_NC(&hp, &MSO(BIF_P).externals, mic.mi[i].entity); + item = STORE_NC(&hp, &MSO(BIF_P), mic.mi[i].entity); t = TUPLE2(hp, am_process, item); hp += 3; res = CONS(hp, t, res); @@ -2715,7 +2731,7 @@ BIF_RETTYPE port_info_2(BIF_ALIST_2) erts_doforall_links(prt->nlinks, &one_link_size, &size); for (bp = prt->bp; bp; bp = bp->next) - size += sizeof(ErlHeapFragment) + (bp->size - 1)*sizeof(Eterm); + size += sizeof(ErlHeapFragment) + (bp->alloc_size - 1)*sizeof(Eterm); if (prt->linebuf) size += sizeof(LineBuf) + prt->linebuf->ovsiz; @@ -3386,6 +3402,16 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("fake_scheduler_bindings", tp[1])) { return erts_fake_scheduler_bindings(BIF_P, tp[2]); } + else if (ERTS_IS_ATOM_STR("reader_groups_map", tp[1])) { + Sint groups; + if (is_not_small(tp[2])) + BIF_ERROR(BIF_P, BADARG); + groups = signed_val(tp[2]); + if (groups < (Sint) 1 || groups > (Sint) INT_MAX) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(erts_debug_reader_groups_map(BIF_P, (int) groups)); + } break; } default: @@ -3404,8 +3430,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) */ if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1) && (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) { - long on = (long) (BIF_ARG_2 == am_true); - long prev_on = erts_smp_atomic_xchg(&available_internal_state, on); + erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true); + erts_aint_t prev_on = erts_smp_atomic_xchg(&available_internal_state, on); if (on) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Process %T ", BIF_P->id); @@ -3567,6 +3593,17 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } } } + else if (ERTS_IS_ATOM_STR("binary_loop_limit", BIF_ARG_1)) { + /* Used by binary_module_SUITE (stdlib) */ + Uint max_loops; + if (is_atom(BIF_ARG_2) && ERTS_IS_ATOM_STR("default", BIF_ARG_2)) { + max_loops = erts_binary_set_loop_limit(-1); + BIF_RET(make_small(max_loops)); + } else if (term_to_Uint(BIF_ARG_2, &max_loops) != 0) { + max_loops = erts_binary_set_loop_limit(max_loops); + BIF_RET(make_small(max_loops)); + } + } else if (ERTS_IS_ATOM_STR("re_loop_limit", BIF_ARG_1)) { /* Used by re_SUITE (stdlib) */ Uint max_loops; @@ -3591,7 +3628,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) { /* Used by hipe test suites */ - long flag = erts_smp_atomic_read(&hipe_test_reschedule_flag); + erts_aint_t flag = erts_smp_atomic_read(&hipe_test_reschedule_flag); if (!flag && BIF_ARG_2 != am_false) { erts_smp_atomic_set(&hipe_test_reschedule_flag, 1); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); @@ -3666,7 +3703,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) #ifdef ERTS_ENABLE_LOCK_COUNT static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_stats_t *stats, Eterm res) { - unsigned long tries = 0, colls = 0; + Uint tries = 0, colls = 0; unsigned long timer_s = 0, timer_ns = 0, timer_n = 0; unsigned int line = 0; @@ -3679,8 +3716,8 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}] */ - ethr_atomic_read(&stats->tries, (long *)&tries); - ethr_atomic_read(&stats->colls, (long *)&colls); + tries = (Uint) ethr_atomic_read(&stats->tries); + colls = (Uint) ethr_atomic_read(&stats->colls); line = stats->line; timer_s = stats->timer.s; diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 9b56ddd4f8..fbc92b9730 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -579,8 +579,8 @@ BIF_RETTYPE port_get_data_1(BIF_ALIST_1) if (prt->bp == NULL) { /* MUST be CONST! */ res = prt->data; } else { - Eterm* hp = HAlloc(BIF_P, prt->bp->size); - res = copy_struct(prt->data, prt->bp->size, &hp, &MSO(BIF_P)); + Eterm* hp = HAlloc(BIF_P, prt->bp->used_size); + res = copy_struct(prt->data, prt->bp->used_size, &hp, &MSO(BIF_P)); } erts_smp_port_unlock(prt); BIF_RET(res); @@ -610,6 +610,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) int binary_io; int soft_eof; Sint linebuf; + Eterm edir = NIL; byte dir[MAXPATHLEN]; /* These are the defaults */ @@ -686,19 +687,10 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) } else if (option == am_arg0) { char *a0; - int n; - if (is_nil(*tp)) { - n = 0; - } else if( (n = is_string(*tp)) == 0) { + + if ((a0 = erts_convert_filename_to_native(*tp, ERTS_ALC_T_TMP, 1)) == NULL) { goto badarg; } - a0 = (char *) erts_alloc(ERTS_ALC_T_TMP, - (n + 1) * sizeof(byte)); - if (intlist_to_buf(*tp, a0, n) != n) { - erl_exit(1, "%s:%d: Internal error\n", - __FILE__, __LINE__); - } - a0[n] = '\0'; if (opts.argv == NULL) { opts.argv = erts_alloc(ERTS_ALC_T_TMP, 2 * sizeof(char **)); @@ -711,22 +703,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) opts.argv[0] = a0; } } else if (option == am_cd) { - Eterm iolist; - DeclareTmpHeap(heap,4,p); - int r; - - UseTmpHeap(4,p); - heap[0] = *tp; - heap[1] = make_list(heap+2); - heap[2] = make_small(0); - heap[3] = NIL; - iolist = make_list(heap); - r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN); - UnUseTmpHeap(4,p); - if (r < 0) { - goto badarg; - } - opts.wd = (char *) dir; + edir = *tp; } else { goto badarg; } @@ -838,19 +815,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) goto badarg; } name = tp[1]; - if (is_atom(name)) { - name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, - atom_tab(atom_val(name))->len+1); - sys_memcpy((void *) name_buf, - (void *) atom_tab(atom_val(name))->name, - atom_tab(atom_val(name))->len); - name_buf[atom_tab(atom_val(name))->len] = '\0'; - } else if ((i = is_string(name))) { - name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); - if (intlist_to_buf(name, name_buf, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - name_buf[i] = '\0'; - } else { + if ((name_buf = erts_convert_filename_to_native(name,ERTS_ALC_T_TMP,0)) == NULL) { goto badarg; } opts.spawn_type = ERTS_SPAWN_EXECUTABLE; @@ -892,7 +857,33 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) /* Argument vector only if explicit spawn_executable */ goto badarg; } - + + if (edir != NIL) { + /* A working directory is expressed differently if spawn_executable, i.e. Unicode is handles + for spawn_executable... */ + if (opts.spawn_type != ERTS_SPAWN_EXECUTABLE) { + Eterm iolist; + DeclareTmpHeap(heap,4,p); + int r; + + UseTmpHeap(4,p); + heap[0] = edir; + heap[1] = make_list(heap+2); + heap[2] = make_small(0); + heap[3] = NIL; + iolist = make_list(heap); + r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN); + UnUseTmpHeap(4,p); + if (r < 0) { + goto badarg; + } + opts.wd = (char *) dir; + } else { + if ((opts.wd = erts_convert_filename_to_native(edir,ERTS_ALC_T_TMP,0)) == NULL) { + goto badarg; + } + } + } if (driver != &spawn_driver && opts.exit_status) { goto badarg; @@ -941,6 +932,9 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) if (opts.argv) { free_args(opts.argv); } + if (opts.wd && opts.wd != ((char *)dir)) { + erts_free(ERTS_ALC_T_TMP, (void *) opts.wd); + } return port_num; badarg: @@ -950,6 +944,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) #undef OPEN_PORT_ERROR } +/* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */ static char **convert_args(Eterm l) { char **pp; @@ -966,22 +961,14 @@ static char **convert_args(Eterm l) pp[i++] = erts_default_arg0; while (is_list(l)) { str = CAR(list_val(l)); - - if (is_nil(str)) { - n = 0; - } else if( (n = is_string(str)) == 0) { - /* Not a string... */ + if ((b = erts_convert_filename_to_native(str,ERTS_ALC_T_TMP,1)) == NULL) { int j; for (j = 1; j < i; ++j) erts_free(ERTS_ALC_T_TMP, pp[j]); erts_free(ERTS_ALC_T_TMP, pp); return NULL; - } - b = (char *) erts_alloc(ERTS_ALC_T_TMP, (n + 1) * sizeof(byte)); - pp[i++] = (char *) b; - if (intlist_to_buf(str, b, n) != n) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - b[n] = '\0'; + } + pp[i++] = b; l = CDR(list_val(l)); } pp[i] = NULL; diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index eb40c75110..3508e8e0dc 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -357,7 +357,7 @@ bif_timer_timeout(ErtsBifTimer* btm) rp, &rp_locks); } else { - Eterm old_size = bp->size; + Eterm old_size = bp->used_size; bp = erts_resize_message_buffer(bp, old_size + wrap_size, &message, 1); hp = &bp->mem[0] + old_size; @@ -478,7 +478,7 @@ setup_bif_timer(Uint32 xflags, tab_insert(btm); ASSERT(btm == tab_find(ref)); btm->tm.active = 0; /* MUST be initalized */ - erl_set_timer(&btm->tm, + erts_set_timer(&btm->tm, (ErlTimeoutProc) bif_timer_timeout, (ErlCancelProc) bif_timer_cleanup, (void *) btm, @@ -550,7 +550,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) res = am_false; } else { - Uint left = time_left(&btm->tm); + Uint left = erts_time_left(&btm->tm); if (!(btm->flags & BTM_FLG_BYNAME)) { erts_smp_proc_lock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ); unlink_proc(btm); @@ -558,7 +558,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) } tab_remove(btm); ASSERT(!tab_find(BIF_ARG_1)); - erl_cancel_timer(&btm->tm); + erts_cancel_timer(&btm->tm); erts_smp_btm_rwunlock(); res = erts_make_integer(left, BIF_P); } @@ -587,7 +587,7 @@ BIF_RETTYPE read_timer_1(BIF_ALIST_1) res = am_false; } else { - Uint left = time_left(&btm->tm); + Uint left = erts_time_left(&btm->tm); res = erts_make_integer(left, BIF_P); } @@ -613,7 +613,7 @@ erts_print_bif_timer_info(int to, void *to_arg) : btm->receiver.proc.ess->id); erts_print(to, to_arg, "=timer:%T\n", receiver); erts_print(to, to_arg, "Message: %T\n", btm->message); - erts_print(to, to_arg, "Time left: %d ms\n", time_left(&btm->tm)); + erts_print(to, to_arg, "Time left: %d ms\n", erts_time_left(&btm->tm)); } } @@ -640,7 +640,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks) tab_remove(btm); tmp_btm = btm; btm = btm->receiver.proc.next; - erl_cancel_timer(&tmp_btm->tm); + erts_cancel_timer(&tmp_btm->tm); } p->bif_timers = NULL; diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 7fe4410e0d..0509e51a6f 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -40,8 +40,7 @@ #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) -static erts_smp_mtx_t trace_pattern_mutex; -const struct trace_pattern_flags erts_trace_pattern_flags_off = {0, 0, 0, 0}; +const struct trace_pattern_flags erts_trace_pattern_flags_off = {0, 0, 0, 0, 0}; static int erts_default_trace_pattern_is_on; static Binary *erts_default_match_spec; static Binary *erts_default_meta_match_spec; @@ -65,7 +64,6 @@ static void clear_trace_bif(int bif_index); void erts_bif_trace_init(void) { - erts_smp_mtx_init(&trace_pattern_mutex, "trace_pattern"); erts_default_trace_pattern_is_on = 0; erts_default_match_spec = NULL; erts_default_meta_match_spec = NULL; @@ -186,6 +184,14 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) flags.breakpoint = 1; flags.call_count = 1; break; + case am_call_time: + if (is_global) { + goto error; + } + flags.breakpoint = 1; + flags.call_time = 1; + break; + default: goto error; } @@ -195,8 +201,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) goto error; } - if (match_prog_set && !flags.local && !flags.meta && flags.call_count) { - /* A match prog is not allowed with just call_count */ + if (match_prog_set && !flags.local && !flags.meta && (flags.call_count || flags.call_time)) { + /* A match prog is not allowed with just call_count or call_time*/ goto error; } @@ -235,6 +241,8 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) |= flags.meta; erts_default_trace_pattern_flags.call_count |= (on == 1) ? flags.call_count : 0; + erts_default_trace_pattern_flags.call_time + |= (on == 1) ? flags.call_time : 0; } else { erts_default_trace_pattern_flags.local &= ~flags.local; @@ -242,10 +250,13 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) &= ~flags.meta; erts_default_trace_pattern_flags.call_count &= ~flags.call_count; + erts_default_trace_pattern_flags.call_time + &= ~flags.call_time; if (! (erts_default_trace_pattern_flags.breakpoint = erts_default_trace_pattern_flags.local | erts_default_trace_pattern_flags.meta | - erts_default_trace_pattern_flags.call_count)) { + erts_default_trace_pattern_flags.call_count | + erts_default_trace_pattern_flags.call_time)) { erts_default_trace_pattern_is_on = !!on; /* i.e off */ } } @@ -267,8 +278,9 @@ trace_pattern_3(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) if (on) { if (on != 1) { flags.call_count = 0; + flags.call_time = 0; } - flags.breakpoint = flags.local | flags.meta | flags.call_count; + flags.breakpoint = flags.local | flags.meta | flags.call_count | flags.call_time; erts_default_trace_pattern_flags = flags; /* Struct copy */ erts_default_trace_pattern_is_on = !!flags.breakpoint; } @@ -336,7 +348,6 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, struct trace_pattern_flags *trace_pattern_flags, Eterm *meta_tracer_pid) { - erts_smp_mtx_lock(&trace_pattern_mutex); if (trace_pattern_is_on) *trace_pattern_is_on = erts_default_trace_pattern_is_on; if (match_spec) @@ -347,12 +358,10 @@ erts_get_default_trace_pattern(int *trace_pattern_is_on, *trace_pattern_flags = erts_default_trace_pattern_flags; if (meta_tracer_pid) *meta_tracer_pid = erts_default_meta_tracer_pid; - erts_smp_mtx_unlock(&trace_pattern_mutex); } - Uint erts_trace_flag2bit(Eterm flag) { @@ -380,7 +389,7 @@ erts_trace_flag2bit(Eterm flag) default: return 0; } } - + /* Scan the argument list and sort out the trace flags. ** ** Returns !0 on success, 0 on failure. @@ -931,6 +940,7 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) #define FUNC_TRACE_LOCAL_TRACE (1<<2) #define FUNC_TRACE_META_TRACE (1<<3) #define FUNC_TRACE_COUNT_TRACE (1<<4) +#define FUNC_TRACE_TIME_TRACE (1<<5) /* * Returns either FUNC_TRACE_NOEXIST, FUNC_TRACE_UNTRACED, * FUNC_TRACE_GLOBAL_TRACE, or, @@ -945,11 +955,13 @@ trace_info_pid(Process* p, Eterm pid_spec, Eterm key) * * If the return value contains FUNC_TRACE_COUNT_TRACE, *count is set. */ -static int function_is_traced(Eterm mfa[3], - Binary **ms, /* out */ - Binary **ms_meta, /* out */ +static int function_is_traced(Process *p, + Eterm mfa[3], + Binary **ms, /* out */ + Binary **ms_meta, /* out */ Eterm *tracer_pid_meta, /* out */ - Sint *count) /* out */ + Sint *count, /* out */ + Eterm *call_time) /* out */ { Export e; Export* ep; @@ -980,10 +992,13 @@ static int function_is_traced(Eterm mfa[3], r |= FUNC_TRACE_LOCAL_TRACE; *ms = ep->match_prog_set; } - if (erts_is_mtrace_bif(ep->code+3, ms_meta, + if (erts_is_mtrace_break(ep->code+3, ms_meta, tracer_pid_meta)) { r |= FUNC_TRACE_META_TRACE; } + if (erts_is_time_break(p, ep->code+3, call_time)) { + r |= FUNC_TRACE_TIME_TRACE; + } } return r ? r : FUNC_TRACE_UNTRACED; } @@ -1001,7 +1016,9 @@ static int function_is_traced(Eterm mfa[3], | (erts_is_mtrace_break(code, ms_meta, tracer_pid_meta) ? FUNC_TRACE_META_TRACE : 0) | (erts_is_count_break(code, count) - ? FUNC_TRACE_COUNT_TRACE : 0); + ? FUNC_TRACE_COUNT_TRACE : 0) + | (erts_is_time_break(p, code, call_time) + ? FUNC_TRACE_TIME_TRACE : 0); return r ? r : FUNC_TRACE_UNTRACED; } @@ -1020,6 +1037,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) Eterm match_spec = am_false; Eterm retval = am_false; Eterm meta = am_false; + Eterm call_time = NIL; int r; @@ -1039,7 +1057,22 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) mfa[1] = tp[2]; mfa[2] = signed_val(tp[3]); - r = function_is_traced(mfa, &ms, &ms_meta, &meta, &count); +#ifdef ERTS_SMP + if ( (key == am_call_time) || (key == am_all)) { + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); + erts_smp_block_system(0); + } +#endif + + r = function_is_traced(p, mfa, &ms, &ms_meta, &meta, &count, &call_time); + +#ifdef ERTS_SMP + if ( (key == am_call_time) || (key == am_all)) { + erts_smp_release_system(); + erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); + } +#endif + switch (r) { case FUNC_TRACE_NOEXIST: UnUseTmpHeap(3,p); @@ -1092,8 +1125,13 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_make_integer(count, p); } break; + case am_call_time: + if (r & FUNC_TRACE_TIME_TRACE) { + retval = call_time; + } + break; case am_all: { - Eterm match_spec_meta = am_false, c = am_false, t; + Eterm match_spec_meta = am_false, c = am_false, t, ct = am_false; if (ms) { match_spec = MatchSetGetSource(ms); @@ -1111,10 +1149,15 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) erts_make_integer(-count-1, p) : erts_make_integer(count, p); } - hp = HAlloc(p, (3+2)*5); + if (r & FUNC_TRACE_TIME_TRACE) { + ct = call_time; + } + hp = HAlloc(p, (3+2)*6); retval = NIL; t = TUPLE2(hp, am_call_count, c); hp += 3; retval = CONS(hp, t, retval); hp += 2; + t = TUPLE2(hp, am_call_time, ct); hp += 3; + retval = CONS(hp, t, retval); hp += 2; t = TUPLE2(hp, am_meta_match_spec, match_spec_meta); hp += 3; retval = CONS(hp, t, retval); hp += 2; t = TUPLE2(hp, am_meta, meta); hp += 3; @@ -1210,6 +1253,13 @@ trace_info_on_load(Process* p, Eterm key) } else { return TUPLE2(hp, key, am_false); } + case am_call_time: + hp = HAlloc(p, 3); + if (erts_default_trace_pattern_flags.call_time) { + return TUPLE2(hp, key, am_true); + } else { + return TUPLE2(hp, key, am_false); + } case am_all: { Eterm match_spec = am_false, meta_match_spec = am_false, r = NIL, t; @@ -1284,6 +1334,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified, for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { /* Empty loop body */ } + if (j == specified) { if (on) { if (! flags.breakpoint) @@ -1356,6 +1407,12 @@ erts_set_trace_pattern(Eterm* mfa, int specified, erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_GLOBAL; m = 1; } + if (flags.call_time) { + erts_set_time_trace_bif(bif_export[i]->code + 3, on); + /* I don't want to remove any other tracers */ + erts_bif_trace_flags[i] |= BIF_TRACE_AS_CALL_TIME; + m = 1; + } if (erts_bif_trace_flags[i]) { setup_bif_trace(i); } @@ -1375,6 +1432,11 @@ erts_set_trace_pattern(Eterm* mfa, int specified, } m = 1; } + if (flags.call_time) { + erts_clear_time_trace_bif(bif_export[i]->code + 3); + erts_bif_trace_flags[i] &= ~BIF_TRACE_AS_CALL_TIME; + m = 1; + } if (! erts_bif_trace_flags[i]) { reset_bif_trace(i); } @@ -1392,6 +1454,7 @@ erts_set_trace_pattern(Eterm* mfa, int specified, erts_clear_trace_break(mfa, specified); erts_clear_mtrace_break(mfa, specified); erts_clear_count_break(mfa, specified); + erts_clear_time_break(mfa, specified); } else { int m = 0; if (flags.local) { @@ -1405,6 +1468,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (flags.call_count) { m = erts_set_count_break(mfa, specified, on); } + if (flags.call_time) { + m = erts_set_time_break(mfa, specified, on); + } /* All assignments to 'm' above should give the same value, * so just use the last */ matches += m; @@ -1420,6 +1486,9 @@ erts_set_trace_pattern(Eterm* mfa, int specified, if (flags.call_count) { m = erts_clear_count_break(mfa, specified); } + if (flags.call_time) { + m = erts_clear_time_break(mfa, specified); + } /* All assignments to 'm' above should give the same value, * so just use the last */ matches += m; @@ -1536,7 +1605,7 @@ static void reset_bif_trace(int bif_index) { ASSERT(ExportIsBuiltIn(ep)); ASSERT(ep->code[4]); ASSERT(! ep->match_prog_set); - ASSERT(! erts_is_mtrace_bif((BeamInstr *)ep->code+3, NULL, NULL)); + ASSERT(! erts_is_mtrace_break((BeamInstr *)ep->code+3, NULL, NULL)); ep->code[4] = (BeamInstr) bif_table[bif_index].f; } @@ -2092,7 +2161,7 @@ trace_delivered_1(BIF_ALIST_1) #ifdef ERTS_SMP bp = new_message_buffer(REF_THING_SIZE + 4); hp = &bp->mem[0]; - msg_ref = STORE_NC(&hp, &bp->off_heap.externals, ref); + msg_ref = STORE_NC(&hp, &bp->off_heap, ref); #else hp = HAlloc(BIF_P, 4); msg_ref = ref; diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index 5b0b3bcec2..bdf0fe23fc 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -21,6 +21,7 @@ #define __ERL_BINARY_H #include "erl_threads.h" +#include "bif.h" /* * Maximum number of bytes to place in a heap binary. @@ -150,7 +151,18 @@ do { \ void erts_init_binary(void); -byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra); +byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, ErtsAlcType_t, unsigned extra); +/* Used by unicode module */ +Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs); + +/* + * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1 + */ + +BIF_RETTYPE erts_list_to_binary_bif(Process *p, Eterm arg); +BIF_RETTYPE erts_gc_binary_part(Process *p, Eterm *reg, Eterm live, int range_is_tuple); +BIF_RETTYPE erts_binary_part(Process *p, Eterm binary, Eterm epos, Eterm elen); + #if defined(__i386__) || !defined(__GNUC__) /* @@ -168,6 +180,7 @@ byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, unsigned extra); ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr); ERTS_GLB_INLINE void erts_free_aligned_binary_bytes(byte* buf); +ERTS_GLB_INLINE void erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t); ERTS_GLB_INLINE Binary *erts_bin_drv_alloc_fnf(Uint size); ERTS_GLB_INLINE Binary *erts_bin_drv_alloc(Uint size); ERTS_GLB_INLINE Binary *erts_bin_nrml_alloc(Uint size); @@ -184,17 +197,23 @@ ERTS_GLB_INLINE Binary *erts_create_magic_binary(Uint size, ERTS_GLB_INLINE byte* erts_get_aligned_binary_bytes(Eterm bin, byte** base_ptr) { - return erts_get_aligned_binary_bytes_extra(bin, base_ptr, 0); + return erts_get_aligned_binary_bytes_extra(bin, base_ptr, ERTS_ALC_T_TMP, 0); } ERTS_GLB_INLINE void -erts_free_aligned_binary_bytes(byte* buf) +erts_free_aligned_binary_bytes_extra(byte* buf, ErtsAlcType_t allocator) { if (buf) { - erts_free(ERTS_ALC_T_TMP, (void *) buf); + erts_free(allocator, (void *) buf); } } +ERTS_GLB_INLINE void +erts_free_aligned_binary_bytes(byte* buf) +{ + erts_free_aligned_binary_bytes_extra(buf,ERTS_ALC_T_TMP); +} + /* Explicit extra bytes allocated to counter buggy drivers. ** These extra bytes where earlier (< R13B04) added by an alignment-bug ** in this code. Do we dare remove this in some major release (R14?) maybe? diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index defe18c92b..6f8a7436d5 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -555,10 +555,11 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags) { unsigned long offs; - ASSERT(size != 0); offs = BIT_OFFSET(size); if (is_small(val)) { Sint v = signed_val(val); + + ASSERT(size != 0); /* Tested by caller */ if (flags & BSF_LITTLE) { /* Little endian */ sz--; COPY_VAL(buf,1,v,sz); @@ -578,6 +579,9 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags) ErtsDigit* dp = big_v(val); int n = MIN(sz,ds); + if (size == 0) { + return 0; + } if (flags & BSF_LITTLE) { sz -= n; /* pad with this amount */ if (sign) { @@ -729,15 +733,13 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag Uint b; byte *iptr; - if (num_bits == 0) { - return 1; - } - bit_offset = BIT_OFFSET(bin_offset); if (is_small(arg)) { Uint rbits = 8 - bit_offset; - if (bit_offset + num_bits <= 8) { + if (num_bits == 0) { + return 1; + } else if (bit_offset + num_bits <= 8) { /* * All bits are in the same byte. */ @@ -1335,12 +1337,12 @@ erts_bs_append(Process* c_p, Eterm* reg, Uint live, Eterm build_size_term, hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = used_size_in_bytes; - pb->next = MSO(c_p).mso; - MSO(c_p).mso = pb; + pb->next = MSO(c_p).first; + MSO(c_p).first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER; - MSO(c_p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(c_p)), pb->size / sizeof(Eterm)); /* * Now allocate the sub binary and set its size to include the @@ -1506,12 +1508,12 @@ erts_bs_init_writable(Process* p, Eterm sz) hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = 0; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = PB_IS_WRITABLE | PB_ACTIVE_WRITER; - MSO(p).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); /* * Now allocate the sub binary. diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c new file mode 100644 index 0000000000..8a6b4d8d6c --- /dev/null +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -0,0 +1,2359 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: CPU topology and related functionality + * + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <ctype.h> + +#include "global.h" +#include "error.h" +#include "bif.h" +#include "erl_cpu_topology.h" + +#define ERTS_MAX_READER_GROUPS 8 + +/* + * Cpu topology hierarchy. + */ +#define ERTS_TOPOLOGY_NODE 0 +#define ERTS_TOPOLOGY_PROCESSOR 1 +#define ERTS_TOPOLOGY_PROCESSOR_NODE 2 +#define ERTS_TOPOLOGY_CORE 3 +#define ERTS_TOPOLOGY_THREAD 4 +#define ERTS_TOPOLOGY_LOGICAL 5 + +#define ERTS_TOPOLOGY_MAX_DEPTH 6 + +typedef struct { + int bind_id; + int bound_id; +} ErtsCpuBindData; + +static erts_cpu_info_t *cpuinfo; + +static int max_main_threads; +static int reader_groups; + +static ErtsCpuBindData *scheduler2cpu_map; +static erts_smp_rwmtx_t cpuinfo_rwmtx; + +typedef enum { + ERTS_CPU_BIND_UNDEFINED, + ERTS_CPU_BIND_SPREAD, + ERTS_CPU_BIND_PROCESSOR_SPREAD, + ERTS_CPU_BIND_THREAD_SPREAD, + ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD, + ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD, + ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD, + ERTS_CPU_BIND_NO_SPREAD, + ERTS_CPU_BIND_NONE +} ErtsCpuBindOrder; + +#define ERTS_CPU_BIND_DEFAULT_BIND \ + ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD + +static int no_cpu_groups_callbacks; +static ErtsCpuBindOrder cpu_bind_order; + +static erts_cpu_topology_t *user_cpudata; +static int user_cpudata_size; +static erts_cpu_topology_t *system_cpudata; +static int system_cpudata_size; + +typedef struct { + int level[ERTS_TOPOLOGY_MAX_DEPTH+1]; +} erts_avail_cput; + +typedef struct { + int id; + int sub_levels; + int cpu_groups; +} erts_cpu_groups_count_t; + +typedef struct { + int logical; + int cpu_group; +} erts_cpu_groups_map_array_t; + +typedef struct erts_cpu_groups_callback_list_t_ erts_cpu_groups_callback_list_t; +struct erts_cpu_groups_callback_list_t_ { + erts_cpu_groups_callback_list_t *next; + erts_cpu_groups_callback_t callback; + void *arg; +}; + +typedef struct erts_cpu_groups_map_t_ erts_cpu_groups_map_t; +struct erts_cpu_groups_map_t_ { + erts_cpu_groups_map_t *next; + int groups; + erts_cpu_groups_map_array_t *array; + int size; + int logical_processors; + erts_cpu_groups_callback_list_t *callback_list; +}; + +typedef struct { + erts_cpu_groups_callback_t callback; + int ix; + void *arg; +} erts_cpu_groups_callback_call_t; + +static erts_cpu_groups_map_t *cpu_groups_maps; + +static erts_cpu_groups_map_t *reader_groups_map; + +#define ERTS_TOPOLOGY_CG ERTS_TOPOLOGY_MAX_DEPTH + +#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff) + +#ifdef ERTS_SMP +static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata, + int size, + ErtsCpuBindOrder bind_order, + int mk_seq); +static void write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size); +#endif + +static void reader_groups_callback(int, ErtsSchedulerData *, int, void *); +static erts_cpu_groups_map_t *add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg); +static void update_cpu_groups_maps(void); +static void make_cpu_groups_map(erts_cpu_groups_map_t *map, int test); +static int cpu_groups_lookup(erts_cpu_groups_map_t *map, + ErtsSchedulerData *esdp); + +static void create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, + int *cpudata_size); +static void destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata); + +static int +int_cmp(const void *vx, const void *vy) +{ + return *((int *) vx) - *((int *) vy); +} + +static int +cpu_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->core != y->core) + return x->core - y->core; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->node != y->node) + return x->node - y->node; + return 0; +} + +static int +cpu_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_thread_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + return 0; +} + +static int +cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->node != y->node) + return x->node - y->node; + if (x->core != y->core) + return x->core - y->core; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->core != y->core) + return x->core - y->core; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->core != y->core) + return x->core - y->core; + return 0; +} + +static int +cpu_no_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->thread != y->thread) + return x->thread - y->thread; + return 0; +} + +static ERTS_INLINE void +make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node) +{ + int ix; + int node = -1; + int processor = -1; + int processor_node = -1; + int processor_node_node = -1; + int core = -1; + int thread = -1; + int old_node = -1; + int old_processor = -1; + int old_processor_node = -1; + int old_core = -1; + int old_thread = -1; + + for (ix = 0; ix < size; ix++) { + if (!no_node || cpudata[ix].node >= 0) { + if (old_node == cpudata[ix].node) + cpudata[ix].node = node; + else { + old_node = cpudata[ix].node; + old_processor = processor = -1; + if (!no_node) + old_processor_node = processor_node = -1; + old_core = core = -1; + old_thread = thread = -1; + if (no_node || cpudata[ix].node >= 0) + cpudata[ix].node = ++node; + } + } + if (old_processor == cpudata[ix].processor) + cpudata[ix].processor = processor; + else { + old_processor = cpudata[ix].processor; + if (!no_node) + processor_node_node = old_processor_node = processor_node = -1; + old_core = core = -1; + old_thread = thread = -1; + cpudata[ix].processor = ++processor; + } + if (no_node && cpudata[ix].processor_node < 0) + old_processor_node = -1; + else { + if (old_processor_node == cpudata[ix].processor_node) { + if (no_node) + cpudata[ix].node = cpudata[ix].processor_node = node; + else { + if (processor_node_node >= 0) + cpudata[ix].node = processor_node_node; + cpudata[ix].processor_node = processor_node; + } + } + else { + old_processor_node = cpudata[ix].processor_node; + old_core = core = -1; + old_thread = thread = -1; + if (no_node) + cpudata[ix].node = cpudata[ix].processor_node = ++node; + else { + cpudata[ix].node = processor_node_node = ++node; + cpudata[ix].processor_node = ++processor_node; + } + } + } + if (!no_node && cpudata[ix].processor_node < 0) + cpudata[ix].processor_node = 0; + if (old_core == cpudata[ix].core) + cpudata[ix].core = core; + else { + old_core = cpudata[ix].core; + old_thread = thread = -1; + cpudata[ix].core = ++core; + } + if (old_thread == cpudata[ix].thread) + cpudata[ix].thread = thread; + else + old_thread = cpudata[ix].thread = ++thread; + } +} + +static void +cpu_bind_order_sort(erts_cpu_topology_t *cpudata, + int size, + ErtsCpuBindOrder bind_order, + int mk_seq) +{ + if (size > 1) { + int no_node = 0; + int (*cmp_func)(const void *, const void *); + switch (bind_order) { + case ERTS_CPU_BIND_SPREAD: + cmp_func = cpu_spread_order_cmp; + break; + case ERTS_CPU_BIND_PROCESSOR_SPREAD: + cmp_func = cpu_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_THREAD_SPREAD: + cmp_func = cpu_thread_spread_order_cmp; + break; + case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: + no_node = 1; + cmp_func = cpu_thread_no_node_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: + no_node = 1; + cmp_func = cpu_no_node_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: + no_node = 1; + cmp_func = cpu_no_node_thread_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_SPREAD: + cmp_func = cpu_no_spread_order_cmp; + break; + default: + cmp_func = NULL; + erl_exit(ERTS_ABORT_EXIT, + "Bad cpu bind type: %d\n", + (int) cpu_bind_order); + break; + } + + if (mk_seq) + make_cpudata_id_seq(cpudata, size, no_node); + + qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func); + } +} + +static int +processor_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->node != y->node) + return x->node - y->node; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->thread != y->thread) + return x->thread - y->thread; + return 0; +} + +#ifdef ERTS_SMP +void +erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp) +{ + erts_cpu_groups_map_t *cgm; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_callback_call_t *cgcc; + int cgcc_ix; + + /* Unbind from cpu */ + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + if (scheduler2cpu_map[esdp->no].bound_id >= 0 + && erts_unbind_from_cpu(cpuinfo) == 0) { + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + } + + cgcc = erts_alloc(ERTS_ALC_T_TMP, + (no_cpu_groups_callbacks + * sizeof(erts_cpu_groups_callback_call_t))); + cgcc_ix = 0; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + cgcc[cgcc_ix].callback = cgcl->callback; + cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp); + cgcc[cgcc_ix].arg = cgcl->arg; + cgcc_ix++; + } + } + ASSERT(no_cpu_groups_callbacks == cgcc_ix); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + + for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) + cgcc[cgcc_ix].callback(1, + esdp, + cgcc[cgcc_ix].ix, + cgcc[cgcc_ix].arg); + + erts_free(ERTS_ALC_T_TMP, cgcc); + + if (esdp->no <= max_main_threads) + erts_thr_set_main_status(0, 0); + +} + +void +erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue)); + + if (esdp->no <= max_main_threads) + erts_thr_set_main_status(1, (int) esdp->no); + + /* Make sure we check if we should bind to a cpu or not... */ + if (esdp->run_queue->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) + erts_smp_atomic32_set(&esdp->chk_cpu_bind, 1); + else + esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; +} + +#endif + +void +erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) +{ + int res, cpu_id, cgcc_ix; + erts_cpu_groups_map_t *cgm; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_callback_call_t *cgcc; +#ifdef ERTS_SMP + if (erts_common_run_queue) + erts_smp_atomic32_set(&esdp->chk_cpu_bind, 0); + else { + esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; + } +#endif + erts_smp_runq_unlock(esdp->run_queue); + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + cpu_id = scheduler2cpu_map[esdp->no].bind_id; + if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) { + res = erts_bind_to_cpu(cpuinfo, cpu_id); + if (res == 0) + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id; + else { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n", + (int) esdp->no, cpu_id, erl_errno_id(-res)); + erts_send_error_to_logger_nogl(dsbufp); + if (scheduler2cpu_map[esdp->no].bound_id >= 0) + goto unbind; + } + } + else if (cpu_id < 0) { + unbind: + /* Get rid of old binding */ + res = erts_unbind_from_cpu(cpuinfo); + if (res == 0) + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + else if (res != -ENOTSUP) { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n", + (int) esdp->no, cpu_id, erl_errno_id(-res)); + erts_send_error_to_logger_nogl(dsbufp); + } + } + + cgcc = erts_alloc(ERTS_ALC_T_TMP, + (no_cpu_groups_callbacks + * sizeof(erts_cpu_groups_callback_call_t))); + cgcc_ix = 0; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + cgcc[cgcc_ix].callback = cgcl->callback; + cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp); + cgcc[cgcc_ix].arg = cgcl->arg; + cgcc_ix++; + } + } + + ASSERT(no_cpu_groups_callbacks == cgcc_ix); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + + for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) + cgcc[cgcc_ix].callback(0, + esdp, + cgcc[cgcc_ix].ix, + cgcc[cgcc_ix].arg); + + erts_free(ERTS_ALC_T_TMP, cgcc); + + erts_smp_runq_lock(esdp->run_queue); +} + +#ifdef ERTS_SMP +void +erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp) +{ + int cgcc_ix; + erts_cpu_groups_map_t *cgm; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_callback_call_t *cgcc; + + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + + cgcc = erts_alloc(ERTS_ALC_T_TMP, + (no_cpu_groups_callbacks + * sizeof(erts_cpu_groups_callback_call_t))); + cgcc_ix = 0; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + cgcc[cgcc_ix].callback = cgcl->callback; + cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp); + cgcc[cgcc_ix].arg = cgcl->arg; + cgcc_ix++; + } + } + + ASSERT(no_cpu_groups_callbacks == cgcc_ix); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + + for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) + cgcc[cgcc_ix].callback(0, + esdp, + cgcc[cgcc_ix].ix, + cgcc[cgcc_ix].arg); + + erts_free(ERTS_ALC_T_TMP, cgcc); + + if (esdp->no <= max_main_threads) + erts_thr_set_main_status(1, (int) esdp->no); +} +#endif + +static void +write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) +{ + int s_ix = 1; + int cpu_ix; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + if (cpu_bind_order != ERTS_CPU_BIND_NONE && size) { + + cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1); + + for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++) + if (erts_is_cpu_available(cpuinfo, cpudata[cpu_ix].logical)) + scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical; + } + + if (s_ix <= erts_no_schedulers) + for (; s_ix <= erts_no_schedulers; s_ix++) + scheduler2cpu_map[s_ix].bind_id = -1; +} + +int +erts_init_scheduler_bind_type_string(char *how) +{ + if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) + return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED; + + if (!system_cpudata && !user_cpudata) + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; + + if (sys_strcmp(how, "db") == 0) + cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (sys_strcmp(how, "s") == 0) + cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (sys_strcmp(how, "ps") == 0) + cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "ts") == 0) + cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (sys_strcmp(how, "tnnps") == 0) + cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "nnps") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "nnts") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (sys_strcmp(how, "ns") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (sys_strcmp(how, "u") == 0) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE; + + return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; +} + +static Eterm +bound_schedulers_term(ErtsCpuBindOrder order) +{ + switch (order) { + case ERTS_CPU_BIND_SPREAD: { + ERTS_DECL_AM(spread); + return AM_spread; + } + case ERTS_CPU_BIND_PROCESSOR_SPREAD: { + ERTS_DECL_AM(processor_spread); + return AM_processor_spread; + } + case ERTS_CPU_BIND_THREAD_SPREAD: { + ERTS_DECL_AM(thread_spread); + return AM_thread_spread; + } + case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: { + ERTS_DECL_AM(thread_no_node_processor_spread); + return AM_thread_no_node_processor_spread; + } + case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: { + ERTS_DECL_AM(no_node_processor_spread); + return AM_no_node_processor_spread; + } + case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: { + ERTS_DECL_AM(no_node_thread_spread); + return AM_no_node_thread_spread; + } + case ERTS_CPU_BIND_NO_SPREAD: { + ERTS_DECL_AM(no_spread); + return AM_no_spread; + } + case ERTS_CPU_BIND_NONE: { + ERTS_DECL_AM(unbound); + return AM_unbound; + } + default: + ASSERT(0); + return THE_NON_VALUE; + } +} + +Eterm +erts_bound_schedulers_term(Process *c_p) +{ + ErtsCpuBindOrder order; + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + order = cpu_bind_order; + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return bound_schedulers_term(order); +} + +Eterm +erts_bind_schedulers(Process *c_p, Eterm how) +{ + int notify = 0; + Eterm res; + erts_cpu_topology_t *cpudata; + int cpudata_size; + ErtsCpuBindOrder old_cpu_bind_order; + + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + + if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) { + ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP); + } + else { + + old_cpu_bind_order = cpu_bind_order; + + if (ERTS_IS_ATOM_STR("default_bind", how)) + cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (ERTS_IS_ATOM_STR("spread", how)) + cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (ERTS_IS_ATOM_STR("processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("no_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (ERTS_IS_ATOM_STR("unbound", how)) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else { + cpu_bind_order = old_cpu_bind_order; + ERTS_BIF_PREP_ERROR(res, c_p, BADARG); + goto done; + } + + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + + if (!cpudata) { + cpu_bind_order = old_cpu_bind_order; + ERTS_BIF_PREP_ERROR(res, c_p, BADARG); + goto done; + } + + write_schedulers_bind_change(cpudata, cpudata_size); + notify = 1; + + destroy_tmp_cpu_topology_copy(cpudata); + + res = bound_schedulers_term(old_cpu_bind_order); + } + + done: + + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + + if (notify) + erts_sched_notify_check_cpu_bind(); + + return res; +} + +int +erts_sched_bind_atthrcreate_prepare(void) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + return esdp != NULL && erts_is_scheduler_bound(esdp); +} + +int +erts_sched_bind_atthrcreate_child(int unbind) +{ + int res = 0; + if (unbind) { + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + res = erts_unbind_from_cpu(cpuinfo); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + } + return res; +} + +void +erts_sched_bind_atthrcreate_parent(int unbind) +{ + +} + +int +erts_sched_bind_atfork_prepare(void) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int unbind = esdp != NULL && erts_is_scheduler_bound(esdp); + if (unbind) + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + return unbind; +} + +int +erts_sched_bind_atfork_child(int unbind) +{ + if (unbind) { + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + return erts_unbind_from_cpu(cpuinfo); + } + return 0; +} + +char * +erts_sched_bind_atvfork_child(int unbind) +{ + if (unbind) { + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + return erts_get_unbind_from_cpu_str(cpuinfo); + } + return "false"; +} + +void +erts_sched_bind_atfork_parent(int unbind) +{ + if (unbind) + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); +} + +Eterm +erts_fake_scheduler_bindings(Process *p, Eterm how) +{ + ErtsCpuBindOrder fake_cpu_bind_order; + erts_cpu_topology_t *cpudata; + int cpudata_size; + Eterm res; + + if (ERTS_IS_ATOM_STR("default_bind", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (ERTS_IS_ATOM_STR("spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (ERTS_IS_ATOM_STR("processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("no_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (ERTS_IS_ATOM_STR("unbound", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NONE; + else { + ERTS_BIF_PREP_ERROR(res, p, BADARG); + return res; + } + + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + + if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE) + ERTS_BIF_PREP_RET(res, am_false); + else { + int i; + Eterm *hp; + + cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1); + +#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA + + erts_fprintf(stderr, "node: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].node); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "processor: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].processor); + erts_fprintf(stderr, "\n"); + if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD + && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD + && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) { + erts_fprintf(stderr, "processor_node:"); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].processor_node); + erts_fprintf(stderr, "\n"); + } + erts_fprintf(stderr, "core: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].core); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "thread: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].thread); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "logical: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].logical); + erts_fprintf(stderr, "\n"); +#endif + + hp = HAlloc(p, cpudata_size+1); + ERTS_BIF_PREP_RET(res, make_tuple(hp)); + *hp++ = make_arityval((Uint) cpudata_size); + for (i = 0; i < cpudata_size; i++) + *hp++ = make_small((Uint) cpudata[i].logical); + } + + destroy_tmp_cpu_topology_copy(cpudata); + + return res; +} + +Eterm +erts_get_schedulers_binds(Process *c_p) +{ + int ix; + ERTS_DECL_AM(unbound); + Eterm *hp = HAlloc(c_p, erts_no_schedulers+1); + Eterm res = make_tuple(hp); + + *(hp++) = make_arityval(erts_no_schedulers); + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + for (ix = 1; ix <= erts_no_schedulers; ix++) + *(hp++) = (scheduler2cpu_map[ix].bound_id >= 0 + ? make_small(scheduler2cpu_map[ix].bound_id) + : AM_unbound); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return res; +} + +/* + * CPU topology + */ + +typedef struct { + int *id; + int used; + int size; +} ErtsCpuTopIdSeq; + +typedef struct { + ErtsCpuTopIdSeq logical; + ErtsCpuTopIdSeq thread; + ErtsCpuTopIdSeq core; + ErtsCpuTopIdSeq processor_node; + ErtsCpuTopIdSeq processor; + ErtsCpuTopIdSeq node; +} ErtsCpuTopEntry; + +static void +init_cpu_top_entry(ErtsCpuTopEntry *cte) +{ + int size = 10; + cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->logical.size = size; + cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->thread.size = size; + cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->core.size = size; + cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->processor_node.size = size; + cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->processor.size = size; + cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->node.size = size; +} + +static void +destroy_cpu_top_entry(ErtsCpuTopEntry *cte) +{ + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id); +} + +static int +get_cput_value_or_range(int *v, int *vr, char **str) +{ + long l; + char *c = *str; + errno = 0; + if (!isdigit((unsigned char)*c)) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; + l = strtol(c, &c, 10); + if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; + *v = (int) l; + if (*c == '-') { + c++; + if (!isdigit((unsigned char)*c)) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + l = strtol(c, &c, 10); + if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + *vr = (int) l; + } + *str = c; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str) +{ + int ix = 0; + int need_size = 0; + char *c = *str; + + while (1) { + int res; + int val; + int nids; + int val_range = -1; + res = get_cput_value_or_range(&val, &val_range, &c); + if (res != ERTS_INIT_CPU_TOPOLOGY_OK) + return res; + if (val_range < 0 || val_range == val) + nids = 1; + else { + if (val_range > val) + nids = val_range - val + 1; + else + nids = val - val_range + 1; + } + need_size += nids; + if (need_size > idseq->size) { + idseq->size = need_size + 10; + idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS, + idseq->id, + sizeof(int)*idseq->size); + } + if (nids == 1) + idseq->id[ix++] = val; + else if (val_range > val) { + for (; val <= val_range; val++) + idseq->id[ix++] = val; + } + else { + for (; val >= val_range; val--) + idseq->id[ix++] = val; + } + if (*c != ',') + break; + c++; + } + *str = c; + idseq->used = ix; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +get_cput_entry(ErtsCpuTopEntry *cput, char **str) +{ + int h; + char *c = *str; + + cput->logical.used = 0; + cput->thread.id[0] = 0; + cput->thread.used = 1; + cput->core.id[0] = 0; + cput->core.used = 1; + cput->processor_node.id[0] = -1; + cput->processor_node.used = 1; + cput->processor.id[0] = 0; + cput->processor.used = 1; + cput->node.id[0] = -1; + cput->node.used = 1; + + h = ERTS_TOPOLOGY_MAX_DEPTH; + while (*c != ':' && *c != '\0') { + int res; + ErtsCpuTopIdSeq *idseqp; + switch (*c++) { + case 'L': + if (h <= ERTS_TOPOLOGY_LOGICAL) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->logical; + h = ERTS_TOPOLOGY_LOGICAL; + break; + case 't': + case 'T': + if (h <= ERTS_TOPOLOGY_THREAD) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->thread; + h = ERTS_TOPOLOGY_THREAD; + break; + case 'c': + case 'C': + if (h <= ERTS_TOPOLOGY_CORE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->core; + h = ERTS_TOPOLOGY_CORE; + break; + case 'p': + case 'P': + if (h <= ERTS_TOPOLOGY_PROCESSOR) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->processor; + h = ERTS_TOPOLOGY_PROCESSOR; + break; + case 'n': + case 'N': + if (h <= ERTS_TOPOLOGY_PROCESSOR) { + do_node: + if (h <= ERTS_TOPOLOGY_NODE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->node; + h = ERTS_TOPOLOGY_NODE; + } + else { + int p_node = 0; + char *p_chk = c; + while (*p_chk != '\0' && *p_chk != ':') { + if (*p_chk == 'p' || *p_chk == 'P') { + p_node = 1; + break; + } + p_chk++; + } + if (!p_node) + goto do_node; + if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->processor_node; + h = ERTS_TOPOLOGY_PROCESSOR_NODE; + } + break; + default: + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE; + } + res = get_cput_id_seq(idseqp, &c); + if (res != ERTS_INIT_CPU_TOPOLOGY_OK) + return res; + } + + if (cput->logical.used < 1) + return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID; + + if (*c == ':') { + c++; + } + + if (cput->thread.used != 1 + && cput->thread.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->core.used != 1 + && cput->core.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->processor_node.used != 1 + && cput->processor_node.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->processor.used != 1 + && cput->processor.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->node.used != 1 + && cput->node.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + + *str = c; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +verify_topology(erts_cpu_topology_t *cpudata, int size) +{ + if (size > 0) { + int *logical; + int node, processor, no_nodes, i; + + /* Verify logical ids */ + logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size); + + for (i = 0; i < size; i++) + logical[i] = cpudata[i].logical; + + qsort(logical, size, sizeof(int), int_cmp); + for (i = 0; i < size-1; i++) { + if (logical[i] == logical[i+1]) { + erts_free(ERTS_ALC_T_TMP, logical); + return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS; + } + } + + erts_free(ERTS_ALC_T_TMP, logical); + + qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp); + + /* Verify unique entities */ + + for (i = 1; i < size; i++) { + if (cpudata[i-1].processor == cpudata[i].processor + && cpudata[i-1].node == cpudata[i].node + && (cpudata[i-1].processor_node + == cpudata[i].processor_node) + && cpudata[i-1].core == cpudata[i].core + && cpudata[i-1].thread == cpudata[i].thread) { + return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES; + } + } + + /* Verify numa nodes */ + node = cpudata[0].node; + processor = cpudata[0].processor; + no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0; + for (i = 1; i < size; i++) { + if (no_nodes) { + if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + } + else { + if (cpudata[i].processor == processor && cpudata[i].node != node) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + node = cpudata[i].node; + processor = cpudata[i].processor; + if (node >= 0 && cpudata[i].processor_node >= 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + if (node < 0 && cpudata[i].processor_node < 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + } + } + } + + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +int +erts_init_cpu_topology_string(char *topology_str) +{ + ErtsCpuTopEntry cput; + int need_size; + char *c; + int ix; + int error = ERTS_INIT_CPU_TOPOLOGY_OK; + + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata_size = 10; + + user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + + init_cpu_top_entry(&cput); + + ix = 0; + need_size = 0; + + c = topology_str; + if (*c == '\0') { + error = ERTS_INIT_CPU_TOPOLOGY_MISSING; + goto fail; + } + do { + int r; + error = get_cput_entry(&cput, &c); + if (error != ERTS_INIT_CPU_TOPOLOGY_OK) + goto fail; + need_size += cput.logical.used; + if (user_cpudata_size < need_size) { + user_cpudata_size = need_size + 10; + user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, + user_cpudata, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + } + + ASSERT(cput.thread.used == 1 + || cput.thread.used == cput.logical.used); + ASSERT(cput.core.used == 1 + || cput.core.used == cput.logical.used); + ASSERT(cput.processor_node.used == 1 + || cput.processor_node.used == cput.logical.used); + ASSERT(cput.processor.used == 1 + || cput.processor.used == cput.logical.used); + ASSERT(cput.node.used == 1 + || cput.node.used == cput.logical.used); + + for (r = 0; r < cput.logical.used; r++) { + user_cpudata[ix].logical = cput.logical.id[r]; + user_cpudata[ix].thread = + cput.thread.id[cput.thread.used == 1 ? 0 : r]; + user_cpudata[ix].core = + cput.core.id[cput.core.used == 1 ? 0 : r]; + user_cpudata[ix].processor_node = + cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r]; + user_cpudata[ix].processor = + cput.processor.id[cput.processor.used == 1 ? 0 : r]; + user_cpudata[ix].node = + cput.node.id[cput.node.used == 1 ? 0 : r]; + ix++; + } + } while (*c != '\0'); + + if (user_cpudata_size != ix) { + user_cpudata_size = ix; + user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, + user_cpudata, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + } + + error = verify_topology(user_cpudata, user_cpudata_size); + if (error == ERTS_INIT_CPU_TOPOLOGY_OK) { + destroy_cpu_top_entry(&cput); + return ERTS_INIT_CPU_TOPOLOGY_OK; + } + + fail: + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata_size = 0; + destroy_cpu_top_entry(&cput); + return error; +} + +#define ERTS_GET_CPU_TOPOLOGY_ERROR -1 +#define ERTS_GET_USED_CPU_TOPOLOGY 0 +#define ERTS_GET_DETECTED_CPU_TOPOLOGY 1 +#define ERTS_GET_DEFINED_CPU_TOPOLOGY 2 + +static Eterm get_cpu_topology_term(Process *c_p, int type); + +Eterm +erts_set_cpu_topology(Process *c_p, Eterm term) +{ + erts_cpu_topology_t *cpudata = NULL; + int cpudata_size = 0; + Eterm res; + + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY); + if (term == am_undefined) { + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata = NULL; + user_cpudata_size = 0; + + if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) { + cpudata_size = system_cpudata_size; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + + sys_memcpy((void *) cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*cpudata_size); + } + } + else if (is_not_list(term)) { + error: + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + res = THE_NON_VALUE; + goto done; + } + else { + Eterm list = term; + int ix = 0; + + cpudata_size = 100; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + + while (is_list(list)) { + Eterm *lp = list_val(list); + Eterm cpu = CAR(lp); + Eterm* tp; + Sint id; + + if (is_not_tuple(cpu)) + goto error; + + tp = tuple_val(cpu); + + if (arityval(tp[0]) != 7 || tp[1] != am_cpu) + goto error; + + if (ix >= cpudata_size) { + cpudata_size += 100; + cpudata = erts_realloc(ERTS_ALC_T_TMP, + cpudata, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + } + + id = signed_val(tp[2]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].node = (int) id; + + id = signed_val(tp[3]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].processor = (int) id; + + id = signed_val(tp[4]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].processor_node = (int) id; + + id = signed_val(tp[5]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].core = (int) id; + + id = signed_val(tp[6]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].thread = (int) id; + + id = signed_val(tp[7]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].logical = (int) id; + + list = CDR(lp); + ix++; + } + + if (is_not_nil(list)) + goto error; + + cpudata_size = ix; + + if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size)) + goto error; + + if (user_cpudata_size != cpudata_size) { + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + sizeof(erts_cpu_topology_t)*cpudata_size); + user_cpudata_size = cpudata_size; + } + + sys_memcpy((void *) user_cpudata, + (void *) cpudata, + sizeof(erts_cpu_topology_t)*cpudata_size); + } + + update_cpu_groups_maps(); + + write_schedulers_bind_change(cpudata, cpudata_size); + + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_sched_notify_check_cpu_bind(); + + done: + + if (cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); + + return res; +} + +static void +create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size) +{ + if (user_cpudata) { + *cpudata_size = user_cpudata_size; + *cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * (*cpudata_size))); + sys_memcpy((void *) *cpudata, + (void *) user_cpudata, + sizeof(erts_cpu_topology_t)*(*cpudata_size)); + } + else if (system_cpudata) { + *cpudata_size = system_cpudata_size; + *cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * (*cpudata_size))); + sys_memcpy((void *) *cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*(*cpudata_size)); + } + else { + *cpudata = NULL; + *cpudata_size = 0; + } +} + +static void +destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata) +{ + if (cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); +} + + +static Eterm +bld_topology_term(Eterm **hpp, + Uint *hszp, + erts_cpu_topology_t *cpudata, + int size) +{ + Eterm res = NIL; + int i; + + if (size == 0) + return am_undefined; + + for (i = size-1; i >= 0; i--) { + res = erts_bld_cons(hpp, + hszp, + erts_bld_tuple(hpp, + hszp, + 7, + am_cpu, + make_small(cpudata[i].node), + make_small(cpudata[i].processor), + make_small(cpudata[i].processor_node), + make_small(cpudata[i].core), + make_small(cpudata[i].thread), + make_small(cpudata[i].logical)), + res); + } + return res; +} + +static Eterm +get_cpu_topology_term(Process *c_p, int type) +{ +#ifdef DEBUG + Eterm *hp_end; +#endif + Eterm *hp; + Uint hsz; + Eterm res = THE_NON_VALUE; + erts_cpu_topology_t *cpudata = NULL; + int size = 0; + + switch (type) { + case ERTS_GET_USED_CPU_TOPOLOGY: + if (user_cpudata) + goto defined; + else + goto detected; + case ERTS_GET_DETECTED_CPU_TOPOLOGY: + detected: + if (!system_cpudata) + res = am_undefined; + else { + size = system_cpudata_size; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * size)); + sys_memcpy((void *) cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*size); + } + break; + case ERTS_GET_DEFINED_CPU_TOPOLOGY: + defined: + if (!user_cpudata) + res = am_undefined; + else { + size = user_cpudata_size; + cpudata = user_cpudata; + } + break; + default: + erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); + break; + } + + if (res == am_undefined) { + ASSERT(!cpudata); + return res; + } + + hsz = 0; + + bld_topology_term(NULL, &hsz, + cpudata, size); + + hp = HAlloc(c_p, hsz); + +#ifdef DEBUG + hp_end = hp + hsz; +#endif + + res = bld_topology_term(&hp, NULL, + cpudata, size); + + ASSERT(hp_end == hp); + + if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); + + return res; +} + +Eterm +erts_get_cpu_topology_term(Process *c_p, Eterm which) +{ + Eterm res; + int type; + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + if (ERTS_IS_ATOM_STR("used", which)) + type = ERTS_GET_USED_CPU_TOPOLOGY; + else if (ERTS_IS_ATOM_STR("detected", which)) + type = ERTS_GET_DETECTED_CPU_TOPOLOGY; + else if (ERTS_IS_ATOM_STR("defined", which)) + type = ERTS_GET_DEFINED_CPU_TOPOLOGY; + else + type = ERTS_GET_CPU_TOPOLOGY_ERROR; + if (type == ERTS_GET_CPU_TOPOLOGY_ERROR) + res = THE_NON_VALUE; + else + res = get_cpu_topology_term(c_p, type); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return res; +} + +static void +get_logical_processors(int *conf, int *onln, int *avail) +{ + if (conf) + *conf = erts_get_cpu_configured(cpuinfo); + if (onln) + *onln = erts_get_cpu_online(cpuinfo); + if (avail) + *avail = erts_get_cpu_available(cpuinfo); +} + +void +erts_get_logical_processors(int *conf, int *onln, int *avail) +{ + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + get_logical_processors(conf, onln, avail); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); +} + +void +erts_pre_early_init_cpu_topology(int *max_rg_p, + int *conf_p, + int *onln_p, + int *avail_p) +{ + cpu_groups_maps = NULL; + no_cpu_groups_callbacks = 0; + *max_rg_p = ERTS_MAX_READER_GROUPS; + cpuinfo = erts_cpu_info_create(); + get_logical_processors(conf_p, onln_p, avail_p); +} + +void +erts_early_init_cpu_topology(int no_schedulers, + int *max_main_threads_p, + int max_reader_groups, + int *reader_groups_p) +{ + user_cpudata = NULL; + user_cpudata_size = 0; + + system_cpudata_size = erts_get_cpu_topology_size(cpuinfo); + system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * system_cpudata_size)); + + cpu_bind_order = ERTS_CPU_BIND_UNDEFINED; + + if (!erts_get_cpu_topology(cpuinfo, system_cpudata) + || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, + system_cpudata_size)) { + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + system_cpudata = NULL; + system_cpudata_size = 0; + } + + max_main_threads = erts_get_cpu_configured(cpuinfo); + if (max_main_threads > no_schedulers) + max_main_threads = no_schedulers; + *max_main_threads_p = max_main_threads; + + reader_groups = max_main_threads; + if (reader_groups <= 1 || max_reader_groups <= 1) + reader_groups = 0; + if (reader_groups > max_reader_groups) + reader_groups = max_reader_groups; + *reader_groups_p = reader_groups; +} + +void +erts_init_cpu_topology(void) +{ + int ix; + + erts_smp_rwmtx_init(&cpuinfo_rwmtx, "cpu_info"); + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + + scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(ErtsCpuBindData) + * (erts_no_schedulers+1))); + for (ix = 1; ix <= erts_no_schedulers; ix++) { + scheduler2cpu_map[ix].bind_id = -1; + scheduler2cpu_map[ix].bound_id = -1; + } + + if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED) { + int ncpus = erts_get_cpu_configured(cpuinfo); + if (ncpus < 1 || erts_no_schedulers < ncpus) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else + cpu_bind_order = ((system_cpudata || user_cpudata) + && (erts_bind_to_cpu(cpuinfo, -1) != -ENOTSUP) + ? ERTS_CPU_BIND_DEFAULT_BIND + : ERTS_CPU_BIND_NONE); + } + + reader_groups_map = add_cpu_groups(reader_groups, + reader_groups_callback, + NULL); + + if (cpu_bind_order == ERTS_CPU_BIND_NONE) + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + else { + erts_cpu_topology_t *cpudata; + int cpudata_size; + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + write_schedulers_bind_change(cpudata, cpudata_size); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_sched_notify_check_cpu_bind(); + destroy_tmp_cpu_topology_copy(cpudata); + } +} + +int +erts_update_cpu_info(void) +{ + int changed; + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + changed = erts_cpu_info_update(cpuinfo); + if (changed) { + erts_cpu_topology_t *cpudata; + int cpudata_size; + + if (system_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + + system_cpudata_size = erts_get_cpu_topology_size(cpuinfo); + if (!system_cpudata_size) + system_cpudata = NULL; + else { + system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * system_cpudata_size)); + + if (!erts_get_cpu_topology(cpuinfo, system_cpudata) + || (ERTS_INIT_CPU_TOPOLOGY_OK + != verify_topology(system_cpudata, + system_cpudata_size))) { + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + system_cpudata = NULL; + system_cpudata_size = 0; + } + } + + update_cpu_groups_maps(); + + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + write_schedulers_bind_change(cpudata, cpudata_size); + destroy_tmp_cpu_topology_copy(cpudata); + } + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + if (changed) + erts_sched_notify_check_cpu_bind(); + return changed; +} + +/* + * reader groups map + */ + +void +reader_groups_callback(int suspending, + ErtsSchedulerData *esdp, + int group, + void *unused) +{ + if (reader_groups && esdp->no <= max_main_threads) + erts_smp_rwmtx_set_reader_group(suspending ? 0 : group+1); +} + +static Eterm get_cpu_groups_map(Process *c_p, + erts_cpu_groups_map_t *map, + int offset); +Eterm +erts_debug_reader_groups_map(Process *c_p, int groups) +{ + Eterm res; + erts_cpu_groups_map_t test; + + test.array = NULL; + test.groups = groups; + make_cpu_groups_map(&test, 1); + if (!test.array) + res = NIL; + else { + res = get_cpu_groups_map(c_p, &test, 1); + erts_free(ERTS_ALC_T_TMP, test.array); + } + return res; +} + + +Eterm +erts_get_reader_groups_map(Process *c_p) +{ + Eterm res; + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + res = get_cpu_groups_map(c_p, reader_groups_map, 1); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return res; +} + +/* + * CPU groups + */ + +static Eterm +get_cpu_groups_map(Process *c_p, + erts_cpu_groups_map_t *map, + int offset) +{ +#ifdef DEBUG + Eterm *endp; +#endif + Eterm res = NIL, tuple; + Eterm *hp; + int i; + + hp = HAlloc(c_p, map->logical_processors*(2+3)); +#ifdef DEBUG + endp = hp + map->logical_processors*(2+3); +#endif + for (i = map->size - 1; i >= 0; i--) { + if (map->array[i].logical >= 0) { + tuple = TUPLE2(hp, + make_small(map->array[i].logical), + make_small(map->array[i].cpu_group + offset)); + hp += 3; + res = CONS(hp, tuple, res); + hp += 2; + } + } + ASSERT(hp == endp); + return res; +} + +static void +make_available_cpu_topology(erts_avail_cput *no, + erts_avail_cput *avail, + erts_cpu_topology_t *cpudata, + int *size, + int test) +{ + int len = *size; + erts_cpu_topology_t last; + int a, i, j; + + no->level[ERTS_TOPOLOGY_NODE] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE] = -1; + no->level[ERTS_TOPOLOGY_CORE] = -1; + no->level[ERTS_TOPOLOGY_THREAD] = -1; + no->level[ERTS_TOPOLOGY_LOGICAL] = -1; + + last.node = INT_MIN; + last.processor = INT_MIN; + last.processor_node = INT_MIN; + last.core = INT_MIN; + last.thread = INT_MIN; + last.logical = INT_MIN; + + a = 0; + + for (i = 0; i < len; i++) { + + if (!test && !erts_is_cpu_available(cpuinfo, cpudata[i].logical)) + continue; + + if (last.node != cpudata[i].node) + goto node; + if (last.processor != cpudata[i].processor) + goto processor; + if (last.processor_node != cpudata[i].processor_node) + goto processor_node; + if (last.core != cpudata[i].core) + goto core; + ASSERT(last.thread != cpudata[i].thread); + goto thread; + + node: + no->level[ERTS_TOPOLOGY_NODE]++; + processor: + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + processor_node: + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + core: + no->level[ERTS_TOPOLOGY_CORE]++; + thread: + no->level[ERTS_TOPOLOGY_THREAD]++; + + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + for (j = 0; j < ERTS_TOPOLOGY_LOGICAL; j++) + avail[a].level[j] = no->level[j]; + + avail[a].level[ERTS_TOPOLOGY_LOGICAL] = cpudata[i].logical; + avail[a].level[ERTS_TOPOLOGY_CG] = 0; + + ASSERT(last.logical != cpudata[i].logical); + + last = cpudata[i]; + a++; + } + + no->level[ERTS_TOPOLOGY_NODE]++; + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + no->level[ERTS_TOPOLOGY_CORE]++; + no->level[ERTS_TOPOLOGY_THREAD]++; + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + *size = a; +} + +static void +cpu_group_insert(erts_cpu_groups_map_t *map, + int logical, int cpu_group) +{ + int start = logical % map->size; + int ix = start; + + do { + if (map->array[ix].logical < 0) { + map->array[ix].logical = logical; + map->array[ix].cpu_group = cpu_group; + return; + } + ix++; + if (ix == map->size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); +} + + +static int +sub_levels(erts_cpu_groups_count_t *cgc, int level, int aix, + int avail_sz, erts_avail_cput *avail) +{ + int sub_level = level+1; + int last = -1; + cgc->sub_levels = 0; + + do { + if (last != avail[aix].level[sub_level]) { + cgc->sub_levels++; + last = avail[aix].level[sub_level]; + } + aix++; + } + while (aix < avail_sz && cgc->id == avail[aix].level[level]); + cgc->cpu_groups = 0; + return aix; +} + +static int +write_cpu_groups(int *cgp, erts_cpu_groups_count_t *cgcp, + int level, int a, + int avail_sz, erts_avail_cput *avail) +{ + int cg = *cgp; + int sub_level = level+1; + int sl_per_gr = cgcp->sub_levels / cgcp->cpu_groups; + int xsl = cgcp->sub_levels % cgcp->cpu_groups; + int sls = 0; + int last = -1; + int xsl_cg_lim = (cgcp->cpu_groups - xsl) + cg + 1; + + ASSERT(level < 0 || avail[a].level[level] == cgcp->id); + + do { + if (last != avail[a].level[sub_level]) { + if (!sls) { + sls = sl_per_gr; + cg++; + if (cg >= xsl_cg_lim) + sls++; + } + last = avail[a].level[sub_level]; + sls--; + } + avail[a].level[ERTS_TOPOLOGY_CG] = cg; + a++; + } while (a < avail_sz && (level < 0 + || avail[a].level[level] == cgcp->id)); + + ASSERT(cgcp->cpu_groups == cg - *cgp); + + *cgp = cg; + + return a; +} + +static int +cg_count_sub_levels_compare(const void *vx, const void *vy) +{ + erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx; + erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy; + if (x->sub_levels != y->sub_levels) + return y->sub_levels - x->sub_levels; + return x->id - y->id; +} + +static int +cg_count_id_compare(const void *vx, const void *vy) +{ + erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx; + erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy; + return x->id - y->id; +} + +static void +make_cpu_groups_map(erts_cpu_groups_map_t *map, int test) +{ + int i, spread_level, avail_sz; + erts_avail_cput no, *avail; + erts_cpu_topology_t *cpudata; + ErtsAlcType_t alc_type = (test + ? ERTS_ALC_T_TMP + : ERTS_ALC_T_CPU_GRPS_MAP); + + if (map->array) + erts_free(alc_type, map->array); + + map->array = NULL; + map->logical_processors = 0; + map->size = 0; + + if (!map->groups) + return; + + create_tmp_cpu_topology_copy(&cpudata, &avail_sz); + + if (!cpudata) + return; + + cpu_bind_order_sort(cpudata, + avail_sz, + ERTS_CPU_BIND_NO_SPREAD, + 1); + + avail = erts_alloc(ERTS_ALC_T_TMP, + sizeof(erts_avail_cput)*avail_sz); + + make_available_cpu_topology(&no, avail, cpudata, + &avail_sz, test); + + destroy_tmp_cpu_topology_copy(cpudata); + + map->size = avail_sz*2+1; + + map->array = erts_alloc(alc_type, + (sizeof(erts_cpu_groups_map_array_t) + * map->size));; + map->logical_processors = avail_sz; + + for (i = 0; i < map->size; i++) { + map->array[i].logical = -1; + map->array[i].cpu_group = -1; + } + + spread_level = ERTS_TOPOLOGY_CORE; + for (i = ERTS_TOPOLOGY_NODE; i < ERTS_TOPOLOGY_THREAD; i++) { + if (no.level[i] > map->groups) { + spread_level = i; + break; + } + } + + if (no.level[spread_level] <= map->groups) { + int a, cg, last = -1; + cg = -1; + ASSERT(spread_level == ERTS_TOPOLOGY_CORE); + for (a = 0; a < avail_sz; a++) { + if (last != avail[a].level[spread_level]) { + cg++; + last = avail[a].level[spread_level]; + } + cpu_group_insert(map, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + cg); + } + } + else { /* map->groups < no.level[spread_level] */ + erts_cpu_groups_count_t *cg_count; + int a, cg, tl, toplevels; + + tl = spread_level-1; + + if (spread_level == ERTS_TOPOLOGY_NODE) + toplevels = 1; + else + toplevels = no.level[tl]; + + cg_count = erts_alloc(ERTS_ALC_T_TMP, + toplevels*sizeof(erts_cpu_groups_count_t)); + + if (toplevels == 1) { + cg_count[0].id = 0; + cg_count[0].sub_levels = no.level[spread_level]; + cg_count[0].cpu_groups = map->groups; + } + else { + int cgs_per_tl, cgs; + cgs = map->groups; + cgs_per_tl = cgs / toplevels; + + a = 0; + for (i = 0; i < toplevels; i++) { + cg_count[i].id = avail[a].level[tl]; + a = sub_levels(&cg_count[i], tl, a, avail_sz, avail); + } + + qsort(cg_count, + toplevels, + sizeof(erts_cpu_groups_count_t), + cg_count_sub_levels_compare); + + for (i = 0; i < toplevels; i++) { + if (cg_count[i].sub_levels < cgs_per_tl) { + cg_count[i].cpu_groups = cg_count[i].sub_levels; + cgs -= cg_count[i].sub_levels; + } + else { + cg_count[i].cpu_groups = cgs_per_tl; + cgs -= cgs_per_tl; + } + } + + while (cgs > 0) { + for (i = 0; i < toplevels; i++) { + if (cg_count[i].sub_levels == cg_count[i].cpu_groups) + break; + else { + cg_count[i].cpu_groups++; + if (--cgs == 0) + break; + } + } + } + + qsort(cg_count, + toplevels, + sizeof(erts_cpu_groups_count_t), + cg_count_id_compare); + } + + a = i = 0; + cg = -1; + while (a < avail_sz) { + a = write_cpu_groups(&cg, &cg_count[i], tl, + a, avail_sz, avail); + i++; + } + + ASSERT(map->groups == cg + 1); + + for (a = 0; a < avail_sz; a++) + cpu_group_insert(map, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + avail[a].level[ERTS_TOPOLOGY_CG]); + + erts_free(ERTS_ALC_T_TMP, cg_count); + } + + erts_free(ERTS_ALC_T_TMP, avail); +} + +static erts_cpu_groups_map_t * +add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg) +{ + int use_groups = groups; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_map_t *cgm; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + if (use_groups > max_main_threads) + use_groups = max_main_threads; + + if (!use_groups) + return NULL; + + no_cpu_groups_callbacks++; + cgcl = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP, + sizeof(erts_cpu_groups_callback_list_t)); + cgcl->callback = callback; + cgcl->arg = arg; + + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + if (cgm->groups == use_groups) { + cgcl->next = cgm->callback_list; + cgm->callback_list = cgcl; + return cgm; + } + } + + + cgm = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP, + sizeof(erts_cpu_groups_map_t)); + cgm->next = cpu_groups_maps; + cgm->groups = use_groups; + cgm->array = NULL; + cgm->size = 0; + cgm->logical_processors = 0; + cgm->callback_list = cgcl; + + cgcl->next = NULL; + + make_cpu_groups_map(cgm, 0); + + cpu_groups_maps = cgm; + + return cgm; +} + +static void +remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg) +{ + erts_cpu_groups_map_t *prev_cgm, *cgm; + erts_cpu_groups_callback_list_t *prev_cgcl, *cgcl; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + no_cpu_groups_callbacks--; + + prev_cgm = NULL; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + prev_cgcl = NULL; + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + if (cgcl->callback == callback && cgcl->arg == arg) { + if (prev_cgcl) + prev_cgcl->next = cgcl->next; + else + cgm->callback_list = cgcl->next; + erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgcl); + if (!cgm->callback_list) { + if (prev_cgm) + prev_cgm->next = cgm->next; + else + cpu_groups_maps = cgm->next; + if (cgm->array) + erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm->array); + erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm); + } + return; + } + prev_cgcl = cgcl; + } + prev_cgm = cgm; + } + + erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); +} + +static int +cpu_groups_lookup(erts_cpu_groups_map_t *map, + ErtsSchedulerData *esdp) +{ + int start, logical, ix; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + if (esdp->cpu_id < 0) + return (((int) esdp->no) - 1) % map->groups; + + logical = esdp->cpu_id; + start = logical % map->size; + ix = start; + + do { + if (map->array[ix].logical == logical) { + int group = map->array[ix].cpu_group; + ASSERT(0 <= group && group < map->groups); + return group; + } + ix++; + if (ix == map->size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); +} + +static void +update_cpu_groups_maps(void) +{ + erts_cpu_groups_map_t *cgm; + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) + make_cpu_groups_map(cgm, 0); +} + +void +erts_add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg) +{ + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + add_cpu_groups(groups, callback, arg); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); +} + +void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback, + void *arg) +{ + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + remove_cpu_groups(callback, arg); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); +} diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h new file mode 100644 index 0000000000..c5a9520b61 --- /dev/null +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -0,0 +1,105 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: CPU topology and related functionality + * + * Author: Rickard Green + */ + +#ifndef ERL_CPU_TOPOLOGY_H__ +#define ERL_CPU_TOPOLOGY_H__ + +void erts_pre_early_init_cpu_topology(int *max_rg_p, + int *conf_p, + int *onln_p, + int *avail_p); +void erts_early_init_cpu_topology(int no_schedulers, + int *max_main_threads_p, + int max_reader_groups, + int *reader_groups_p); +void erts_init_cpu_topology(void); + + +#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS 0 +#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED 1 +#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY 2 +#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE 3 + +int erts_init_scheduler_bind_type_string(char *how); + + +#define ERTS_INIT_CPU_TOPOLOGY_OK 0 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID 1 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE 2 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY 3 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE 4 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES 5 +#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID 6 +#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS 7 +#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES 8 +#define ERTS_INIT_CPU_TOPOLOGY_MISSING 9 + +int erts_init_cpu_topology_string(char *topology_str); + +void erts_sched_check_cpu_bind(ErtsSchedulerData *esdp); +#ifdef ERTS_SMP +void erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp); +void erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp); +void erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp); +#endif + +int erts_update_cpu_info(void); + +Eterm erts_bind_schedulers(Process *c_p, Eterm how); +Eterm erts_get_schedulers_binds(Process *c_p); + +Eterm erts_get_reader_groups_map(Process *c_p); + +Eterm erts_set_cpu_topology(Process *c_p, Eterm term); +Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which); + +int erts_update_cpu_info(void); +void erts_get_logical_processors(int *conf, int *onln, int *avail); + +int erts_sched_bind_atthrcreate_prepare(void); +int erts_sched_bind_atthrcreate_child(int unbind); +void erts_sched_bind_atthrcreate_parent(int unbind); + +int erts_sched_bind_atfork_prepare(void); +int erts_sched_bind_atfork_child(int unbind); +char *erts_sched_bind_atvfork_child(int unbind); +void erts_sched_bind_atfork_parent(int unbind); + +Eterm erts_fake_scheduler_bindings(Process *p, Eterm how); +Eterm erts_debug_cpu_groups_map(Process *c_p, int groups); + + +typedef void (*erts_cpu_groups_callback_t)(int, + ErtsSchedulerData *, + int, + void *); + +void erts_add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg); +void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback, + void *arg); + +#endif diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index cbdaa459de..3173d3510e 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -78,11 +78,19 @@ enum DbIterSafety { ** The main meta table, containing all ets tables. */ #ifdef ERTS_SMP -# define META_MAIN_TAB_LOCK_CNT 16 -static union { - erts_smp_spinlock_t lck; - byte _cache_line_alignment[64]; -}meta_main_tab_locks[META_MAIN_TAB_LOCK_CNT]; + +#define ERTS_META_MAIN_TAB_LOCK_TAB_BITS 8 +#define ERTS_META_MAIN_TAB_LOCK_TAB_SIZE (1 << ERTS_META_MAIN_TAB_LOCK_TAB_BITS) +#define ERTS_META_MAIN_TAB_LOCK_TAB_MASK (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE - 1) + +typedef union { + erts_smp_rwmtx_t rwmtx; + byte cache_line_align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE( + sizeof(erts_smp_rwmtx_t))]; +} erts_meta_main_tab_lock_t; + +static erts_meta_main_tab_lock_t *meta_main_tab_locks; + #endif static struct { union { @@ -104,17 +112,13 @@ static struct { #define MARK_SLOT_DEAD(i) (meta_main_tab[(i)].u.next_free |= 2) #define GET_ANY_SLOT_TAB(i) ((DbTable*)(meta_main_tab[(i)].u.next_free & ~(1|2))) /* dead or alive */ -static ERTS_INLINE void meta_main_tab_lock(unsigned slot) -{ -#ifdef ERTS_SMP - erts_smp_spin_lock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); -#endif -} - -static ERTS_INLINE void meta_main_tab_unlock(unsigned slot) +static ERTS_INLINE erts_smp_rwmtx_t * +get_meta_main_tab_lock(unsigned slot) { #ifdef ERTS_SMP - erts_smp_spin_unlock(&meta_main_tab_locks[slot % META_MAIN_TAB_LOCK_CNT].lck); + return &meta_main_tab_locks[slot & ERTS_META_MAIN_TAB_LOCK_TAB_MASK].rwmtx; +#else + return NULL; #endif } @@ -166,7 +170,8 @@ struct meta_name_tab_entry* meta_name_tab_bucket(Eterm name, typedef enum { LCK_READ=1, /* read only access */ LCK_WRITE=2, /* exclusive table write access */ - LCK_WRITE_REC=3 /* record write access */ + LCK_WRITE_REC=3, /* record write access */ + LCK_NONE=4 } db_lock_kind_t; extern DbTableMethod db_hash; @@ -174,6 +179,7 @@ extern DbTableMethod db_tree; int user_requested_db_max_tabs; int erts_ets_realloc_always_moves; +int erts_ets_always_compress; static int db_max_tabs; static DbTable *meta_pid_to_tab; /* Pid mapped to owned tables */ static DbTable *meta_pid_to_fixed_tab; /* Pid mapped to fixed tables */ @@ -213,61 +219,87 @@ Export ets_select_continue_exp; * Static traps */ static Export ets_delete_continue_exp; - -static ERTS_INLINE DbTable* db_ref(DbTable* tb) -{ - if (tb != NULL) { - erts_refc_inc(&tb->common.ref, 2); - } - return tb; -} - -static ERTS_INLINE DbTable* db_unref(DbTable* tb) + +static void +free_dbtable(DbTable* tb) { - if (!erts_refc_dectest(&tb->common.ref, 0)) { #ifdef HARDDEBUG if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) { - erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n", - erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), + erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n", + erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), tb->common.fixations); } - erts_fprintf(stderr, "ets: db_unref(%T) deleted!!!\r\n", + erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n", tb->common.id); - erts_fprintf(stderr, "ets: db_unref: meta_pid_to_tab common.memory_size = %ld\n", + erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n", erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab); - erts_fprintf(stderr, "ets: db_unref: meta_pid_to_fixed_tab common.memory_size = %ld\n", + erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n", erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab); - #endif #ifdef ERTS_SMP erts_smp_rwmtx_destroy(&tb->common.rwlock); erts_smp_mtx_destroy(&tb->common.fixlock); #endif ASSERT(is_immed(tb->common.heir_data)); - erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); + erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); - return NULL; - } - return tb; } -static ERTS_INLINE void db_init_lock(DbTable* tb, char *rwname, char* fixname) +#ifdef ERTS_SMP +static void +chk_free_dbtable(void *vtb) +{ + DbTable * tb = (DbTable *) vtb; + ERTS_THR_MEMORY_BARRIER; + if (erts_refc_dectest(&tb->common.ref, 0) == 0) + free_dbtable(tb); +} +#endif + +static void schedule_free_dbtable(DbTable* tb) +{ + /* + * NON-SMP case: Caller is *not* allowed to access the *tb + * structure after this function has returned! + * SMP case: Caller is allowed to access the *tb structure + * until the bif has returned (we typically + * need to unlock the table lock after this + * function has returned). + */ +#ifdef ERTS_SMP + int scheds = erts_get_max_no_executing_schedulers(); + ASSERT(scheds >= 1); + ASSERT(erts_refc_read(&tb->common.ref, 0) == 0); + erts_refc_init(&tb->common.ref, scheds); + ERTS_THR_MEMORY_BARRIER; + erts_smp_schedule_misc_aux_work(0, scheds, chk_free_dbtable, tb); +#else + free_dbtable(tb); +#endif +} + +static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, + char *rwname, char* fixname) { - erts_refc_init(&tb->common.ref, 1); - erts_refc_init(&tb->common.fixref, 0); #ifdef ERTS_SMP - erts_smp_rwmtx_init_x(&tb->common.rwlock, rwname, tb->common.the_name); + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + if (use_frequent_read_lock) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; +#endif +#ifdef ERTS_SMP + erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, + rwname, tb->common.the_name); erts_smp_mtx_init_x(&tb->common.fixlock, fixname, tb->common.the_name); tb->common.is_thread_safe = !(tb->common.status & DB_FINE_LOCKED); #endif } -static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind) +static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) { #ifdef ERTS_SMP ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); @@ -295,16 +327,13 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind) #endif } -static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) -{ - (void) db_ref(tb); -#ifdef ERTS_SMP - db_lock_take_over_ref(tb, kind); -#endif -} - static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) { + /* + * In NON-SMP case tb may refer to an already deallocated + * DbTable structure. That is, ONLY the SMP case is allowed + * to follow the tb pointer! + */ #ifdef ERTS_SMP ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); @@ -331,7 +360,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) } } #endif - (void) db_unref(tb); /* May delete table... */ } @@ -349,56 +377,85 @@ static ERTS_INLINE void db_meta_unlock(DbTable* tb, db_lock_kind_t kind) } static ERTS_INLINE -DbTable* db_get_table(Process *p, - Eterm id, - int what, - db_lock_kind_t kind) +DbTable* db_get_table_aux(Process *p, + Eterm id, + int what, + db_lock_kind_t kind, + int meta_already_locked) { DbTable *tb = NULL; + erts_smp_rwmtx_t *mtl = NULL; + + /* + * IMPORTANT: Only scheduler threads are allowed + * to access tables. Memory management + * depend on it. + */ + ASSERT(erts_get_scheduler_data()); if (is_small(id)) { Uint slot = unsigned_val(id) & meta_main_tab_slot_mask; - meta_main_tab_lock(slot); - if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) { - /* SMP: inc to prevent race, between unlock of meta_main_tab_lock - * and the table locking outside the meta_main_tab_lock - */ - tb = db_ref(meta_main_tab[slot].u.tb); + if (!meta_already_locked) { + mtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rlock(mtl); + } +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) + else { + erts_smp_rwmtx_t *test_mtl = get_meta_main_tab_lock(slot); + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(test_mtl) + || erts_lc_rwmtx_is_rwlocked(test_mtl)); } - meta_main_tab_unlock(slot); +#endif + if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) + tb = meta_main_tab[slot].u.tb; } else if (is_atom(id)) { - erts_smp_rwmtx_t* rwlock; - struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&rwlock); - erts_smp_rwmtx_rlock(rwlock); + struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl); + if (!meta_already_locked) + erts_smp_rwmtx_rlock(mtl); + else{ + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(mtl) + || erts_lc_rwmtx_is_rwlocked(mtl)); + mtl = NULL; + } + if (bucket->pu.tb != NULL) { if (is_atom(bucket->u.name_atom)) { /* single */ - if (bucket->u.name_atom == id) { - tb = db_ref(bucket->pu.tb); - } + if (bucket->u.name_atom == id) + tb = bucket->pu.tb; } else { /* multi */ Uint cnt = unsigned_val(bucket->u.mcnt); Uint i; for (i=0; i<cnt; i++) { if (bucket->pu.mvec[i].u.name_atom == id) { - tb = db_ref(bucket->pu.mvec[i].pu.tb); + tb = bucket->pu.mvec[i].pu.tb; break; } } } } - erts_smp_rwmtx_runlock(rwlock); } if (tb) { - db_lock_take_over_ref(tb, kind); - if (tb->common.id == id && ((tb->common.status & what) != 0 || - p->id == tb->common.owner)) { - return tb; + db_lock(tb, kind); + if (tb->common.id != id + || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) { + db_unlock(tb, kind); + tb = NULL; } - db_unlock(tb, kind); } - return NULL; + if (mtl) + erts_smp_rwmtx_runlock(mtl); + return tb; +} + +static ERTS_INLINE +DbTable* db_get_table(Process *p, + Eterm id, + int what, + db_lock_kind_t kind) +{ + return db_get_table_aux(p, id, what, kind, 0); } /* Requires meta_main_tab_locks[slot] locked. @@ -413,15 +470,15 @@ static ERTS_INLINE void free_slot(int slot) erts_smp_spin_unlock(&meta_main_tab_main_lock); } -static int insert_named_tab(Eterm name_atom, DbTable* tb) +static int insert_named_tab(Eterm name_atom, DbTable* tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; struct meta_name_tab_entry* new_entry; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - - erts_smp_rwmtx_rwlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwlock(rwlock); if (bucket->pu.tb == NULL) { /* empty */ new_entry = bucket; @@ -468,17 +525,32 @@ static int insert_named_tab(Eterm name_atom, DbTable* tb) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } -static int remove_named_tab(Eterm name_atom) +static int remove_named_tab(DbTable *tb, int have_lock) { int ret = 0; erts_smp_rwmtx_t* rwlock; + Eterm name_atom = tb->common.id; struct meta_name_tab_entry* bucket = meta_name_tab_bucket(name_atom, &rwlock); - erts_smp_rwmtx_rwlock(rwlock); +#ifdef ERTS_SMP + if (!have_lock && erts_smp_rwmtx_tryrwlock(rwlock) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(rwlock); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(rwlock)); + if (bucket->pu.tb == NULL) { goto done; } @@ -529,7 +601,8 @@ static int remove_named_tab(Eterm name_atom) ret = 1; /* Ok */ done: - erts_smp_rwmtx_rwunlock(rwlock); + if (!have_lock) + erts_smp_rwmtx_rwunlock(rwlock); return ret; } @@ -538,11 +611,11 @@ done: */ static ERTS_INLINE void local_fix_table(DbTable* tb) { - erts_refc_inc(&tb->common.fixref, 1); + erts_refc_inc(&tb->common.ref, 1); } static ERTS_INLINE void local_unfix_table(DbTable* tb) { - if (erts_refc_dectest(&tb->common.fixref, 0) == 0) { + if (erts_refc_dectest(&tb->common.ref, 0) == 0) { ASSERT(IS_HASH_TABLE(tb->common.status)); db_unfix_table_hash(&(tb->hash)); } @@ -876,7 +949,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) position > arityval(handle.dbterm->tpl[0])) { goto finalize; } - oldcnt = handle.dbterm->tpl[position]; + oldcnt = db_do_read_element(&handle, position); if (is_big(oldcnt)) { halloc_size += BIG_NEED_SIZE(big_arity(oldcnt)); } @@ -1133,6 +1206,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) { DbTable* tb; Eterm ret; + erts_smp_rwmtx_t *lck1, *lck2; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1141,34 +1215,65 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) BIF_P->initial[0], BIF_P->initial[1], BIF_P->initial[2]); #endif - if ((tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE)) == NULL) { + + if (is_not_atom(BIF_ARG_2)) { BIF_ERROR(BIF_P, BADARG); } - if (is_not_atom(BIF_ARG_2)) { - goto badarg; + (void) meta_name_tab_bucket(BIF_ARG_2, &lck1); + + if (is_small(BIF_ARG_1)) { + Uint slot = unsigned_val(BIF_ARG_1) & meta_main_tab_slot_mask; + lck2 = get_meta_main_tab_lock(slot); } + else if (is_atom(BIF_ARG_1)) { + (void) meta_name_tab_bucket(BIF_ARG_1, &lck2); + if (lck1 == lck2) + lck2 = NULL; + else if (lck1 > lck2) { + erts_smp_rwmtx_t *tmp = lck1; + lck1 = lck2; + lck2 = tmp; + } + } + else { + BIF_ERROR(BIF_P, BADARG); + } + + erts_smp_rwmtx_rwlock(lck1); + if (lck2) + erts_smp_rwmtx_rwlock(lck2); + + tb = db_get_table_aux(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE, 1); + if (!tb) + goto badarg; if (is_not_atom(tb->common.id)) { /* Not a named table */ tb->common.the_name = BIF_ARG_2; goto done; } - if (!insert_named_tab(BIF_ARG_2,tb)) { + if (!insert_named_tab(BIF_ARG_2, tb, 1)) goto badarg; - } - if (!remove_named_tab(tb->common.id)) { + + if (!remove_named_tab(tb, 1)) erl_exit(1,"Could not find named tab %s", tb->common.id); - } tb->common.id = tb->common.the_name = BIF_ARG_2; done: ret = tb->common.id; db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_RET(ret); badarg: - db_unlock(tb, LCK_WRITE); + if (tb) + db_unlock(tb, LCK_WRITE); + erts_smp_rwmtx_rwunlock(lck1); + if (lck2) + erts_smp_rwmtx_rwunlock(lck2); BIF_ERROR(BIF_P, BADARG); } @@ -1189,10 +1294,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) UWord heir_data; Uint32 status; Sint keypos; - int is_named, is_fine_locked; + int is_named, is_fine_locked, frequent_read, is_compressed; int cret; DeclareTmpHeap(meta_tuple,3,BIF_P); DbTableMethod* meth; + erts_smp_rwmtx_t *mmtl; if (is_not_atom(BIF_ARG_1)) { BIF_ERROR(BIF_P, BADARG); @@ -1205,8 +1311,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) keypos = 1; is_named = 0; is_fine_locked = 0; + frequent_read = 0; heir = am_none; heir_data = (UWord) am_undefined; + is_compressed = erts_ets_always_compress; list = BIF_ARG_2; while(is_list(list)) { @@ -1238,6 +1346,13 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) is_fine_locked = 0; } else break; } + else if (tp[1] == am_read_concurrency) { + if (tp[2] == am_true) { + frequent_read = 1; + } else if (tp[2] == am_false) { + frequent_read = 0; + } else break; + } else if (tp[1] == am_heir && tp[2] == am_none) { heir = am_none; heir_data = am_undefined; @@ -1262,6 +1377,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) else if (val == am_named_table) { is_named = 1; } + else if (val == am_compressed) { + is_compressed = 1; + } else if (val == am_set || val == am_protected) ; else break; @@ -1286,6 +1404,11 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } +#ifdef ERTS_SMP + if (frequent_read && !(status & DB_PRIVATE)) + status |= DB_FREQ_READ; +#endif + /* we create table outside any table lock * and take the unusal cost of destroy table if it * fails to find a slot @@ -1308,7 +1431,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.type = status & ERTS_ETS_TABLE_TYPES; /* Note, 'type' is *read only* from now on... */ #endif - db_init_lock(tb, "db_tab", "db_tab_fix"); + erts_refc_init(&tb->common.ref, 0); + db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), + "db_tab", "db_tab_fix"); tb->common.keypos = keypos; tb->common.owner = BIF_P->id; set_heir(BIF_P, tb, heir, heir_data); @@ -1316,6 +1441,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) erts_smp_atomic_init(&tb->common.nitems, 0); tb->common.fixations = NULL; + tb->common.compress = is_compressed; cret = meth->db_create(BIF_P, tb); ASSERT(cret == DB_ERROR_NONE); @@ -1328,8 +1454,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) "** Too many db tables **\n"); free_heir_data(tb); tb->common.meth->db_free_table(tb); - erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); - ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); + free_dbtable(tb); BIF_ERROR(BIF_P, SYSTEM_LIMIT); } @@ -1351,19 +1476,22 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.id = ret; tb->common.slot = slot; /* store slot for erase */ - meta_main_tab_lock(slot); + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); meta_main_tab[slot].u.tb = tb; ASSERT(IS_SLOT_ALIVE(slot)); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); - if (is_named && !insert_named_tab(BIF_ARG_1, tb)) { - meta_main_tab_lock(slot); + if (is_named && !insert_named_tab(BIF_ARG_1, tb, 0)) { + mmtl = get_meta_main_tab_lock(slot); + erts_smp_rwmtx_rwlock(mmtl); free_slot(slot); - meta_main_tab_unlock(slot); + erts_smp_rwmtx_rwunlock(mmtl); - db_lock_take_over_ref(tb,LCK_WRITE); + db_lock(tb,LCK_WRITE); free_heir_data(tb); tb->common.meth->db_free_table(tb); + schedule_free_dbtable(tb); db_unlock(tb,LCK_WRITE); BIF_ERROR(BIF_P, BADARG); } @@ -1499,6 +1627,7 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) { int trap; DbTable* tb; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG erts_fprintf(stderr, @@ -1520,13 +1649,23 @@ BIF_RETTYPE ets_delete_1(BIF_ALIST_1) tb->common.status &= ~(DB_PROTECTED|DB_PUBLIC|DB_PRIVATE); tb->common.status |= DB_DELETE; - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + /* + * We keep our increased refc over this op in order to + * prevent the table from disapearing. + */ + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif /* We must keep the slot, to be found by db_proc_dead() if process dies */ MARK_SLOT_DEAD(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); - if (is_atom(tb->common.id)) { - remove_named_tab(tb->common.id); - } + erts_smp_rwmtx_rwunlock(mmtl); + if (is_atom(tb->common.id)) + remove_named_tab(tb, 0); if (tb->common.owner != BIF_P->id) { DeclareTmpHeap(meta_tuple,3,BIF_P); @@ -1919,14 +2058,15 @@ BIF_RETTYPE ets_all_0(BIF_ALIST_0) previous = NIL; j = 0; for(i = 0; (i < t_max_tabs && j < t_tabs_cnt); i++) { - meta_main_tab_lock(i); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(i); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(i)) { j++; tb = meta_main_tab[i].u.tb; previous = CONS(hp, tb->common.id, previous); hp += 2; } - meta_main_tab_unlock(i); + erts_smp_rwmtx_runlock(mmtl); } HRelease(BIF_P, hendp, hp); BIF_RET(previous); @@ -2456,7 +2596,7 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3) BIF_RETTYPE ets_info_1(BIF_ALIST_1) { static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, - am_node, am_size, am_name, am_heir, am_owner, am_memory}; + am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -2630,12 +2770,30 @@ void init_db(void) size_t size; #ifdef ERTS_SMP - for (i=0; i<META_MAIN_TAB_LOCK_CNT; i++) { - erts_smp_spinlock_init_x(&meta_main_tab_locks[i].lck, "meta_main_tab_slot", make_small(i)); + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + meta_main_tab_locks = erts_alloc(ERTS_ALC_T_DB_TABLES, + (sizeof(erts_meta_main_tab_lock_t) + * (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE+1))); + + if ((((UWord) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) != 0) + meta_main_tab_locks = ((erts_meta_main_tab_lock_t *) + ((((UWord) meta_main_tab_locks) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + + ASSERT((((UWord) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) == 0); + + for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) { + erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt, + "meta_main_tab_slot", make_small(i)); } erts_smp_spinlock_init(&meta_main_tab_main_lock, "meta_main_tab_main"); for (i=0; i<META_NAME_TAB_LOCK_CNT; i++) { - erts_smp_rwmtx_init_x(&meta_name_tab_rwlocks[i].lck, "meta_name_tab", make_small(i)); + erts_smp_rwmtx_init_opt_x(&meta_name_tab_rwlocks[i].lck, &rwmtx_opt, + "meta_name_tab", make_small(i)); } #endif @@ -2703,9 +2861,9 @@ void init_db(void) erts_smp_atomic_init(&meta_pid_to_tab->common.nitems, 0); meta_pid_to_tab->common.slot = -1; meta_pid_to_tab->common.meth = &db_hash; + meta_pid_to_tab->common.compress = 0; - erts_refc_init(&meta_pid_to_tab->common.ref, 1); - erts_refc_init(&meta_pid_to_tab->common.fixref, 0); + erts_refc_init(&meta_pid_to_tab->common.ref, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ @@ -2735,9 +2893,9 @@ void init_db(void) erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.nitems, 0); meta_pid_to_fixed_tab->common.slot = -1; meta_pid_to_fixed_tab->common.meth = &db_hash; + meta_pid_to_fixed_tab->common.compress = 0; - erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 1); - erts_refc_init(&meta_pid_to_fixed_tab->common.fixref, 0); + erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ @@ -2895,12 +3053,10 @@ retry: to_pid, to_locks, ERTS_P2P_FLG_TRY_LOCK); if (to_proc == ERTS_PROC_LOCK_BUSY) { - db_ref(tb); /* while unlocked */ db_unlock(tb,LCK_WRITE); to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to_pid, to_locks); db_lock(tb,LCK_WRITE); - tb = db_unref(tb); ASSERT(tb != NULL); if (tb->common.owner != p->id) { @@ -2943,7 +3099,7 @@ retry: db_unlock(tb,LCK_WRITE); heir_data = tb->common.heir_data; if (!is_immed(heir_data)) { - Eterm* tpv = DBTERM_BUF((DbTerm*)heir_data); /* tuple_val */ + Eterm* tpv = ((DbTerm*)heir_data)->tpl; /* tuple_val */ ASSERT(arityval(*tpv) == 1); heir_data = tpv[1]; } @@ -3008,15 +3164,16 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (!IS_SLOT_FREE(ix)) { - tb = db_ref(GET_ANY_SLOT_TAB(ix)); + tb = GET_ANY_SLOT_TAB(ix); ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int do_yield; - db_lock_take_over_ref(tb, LCK_WRITE); + db_lock(tb, LCK_WRITE); /* Ownership may have changed since we looked up the table. */ if (tb->common.owner != pid) { @@ -3045,7 +3202,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) tb->common.status |= DB_DELETE; if (is_atom(tb->common.id)) - remove_named_tab(tb->common.id); + remove_named_tab(tb, 0); free_heir_data(tb); free_fixations_locked(tb); @@ -3095,17 +3252,18 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) while (state->slots.ix < state->slots.size) { DbTable *tb = NULL; Sint ix = unsigned_val(state->slots.arr[state->slots.ix]); - meta_main_tab_lock(ix); + erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); + erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(ix)) { - tb = db_ref(meta_main_tab[ix].u.tb); + tb = meta_main_tab[ix].u.tb; ASSERT(tb); } - meta_main_tab_unlock(ix); + erts_smp_rwmtx_runlock(mmtl); if (tb) { int reds; DbFixation** pp; - db_lock_take_over_ref(tb, LCK_WRITE_REC); + db_lock(tb, LCK_WRITE_REC); #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif @@ -3115,7 +3273,8 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) pp = &(*pp)->next) { if ((*pp)->pid == pid) { DbFixation* fix = *pp; - erts_refc_add(&tb->common.fixref,-fix->counter,0); + erts_aint_t diff = -((erts_aint_t) fix->counter); + erts_refc_add(&tb->common.ref,diff,0); *pp = fix->next; erts_db_free(ERTS_ALC_T_DB_FIXATION, tb, fix, sizeof(DbFixation)); @@ -3190,7 +3349,7 @@ static void fix_table_locked(Process* p, DbTable* tb) #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif - erts_refc_inc(&tb->common.fixref,1); + erts_refc_inc(&tb->common.ref,1); fix = tb->common.fixations; if (fix == NULL) { get_now(&(tb->common.megasec), @@ -3244,7 +3403,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) { if ((*pp)->pid == p->id) { DbFixation* fix = *pp; - erts_refc_dec(&tb->common.fixref,0); + erts_refc_dec(&tb->common.ref,0); --(fix->counter); ASSERT(fix->counter >= 0); if (fix->counter > 0) { @@ -3270,7 +3429,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, unlocked: if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status) - && erts_smp_atomic_read(&tb->hash.fixdel) != (long)NULL) { + && erts_smp_atomic_read(&tb->hash.fixdel) != (erts_aint_t)NULL) { #ifdef ERTS_SMP if (*kind_p == LCK_READ && tb->common.is_thread_safe) { /* Must have write lock while purging pseudo-deleted (OTP-8166) */ @@ -3292,6 +3451,8 @@ static void free_fixations_locked(DbTable *tb) fix = tb->common.fixations; while (fix != NULL) { + erts_aint_t diff = -((erts_aint_t) fix->counter); + erts_refc_add(&tb->common.ref,diff,0); next_fix = fix->next; db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_fixed_tab, @@ -3332,8 +3493,8 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data) UseTmpHeap(2,me); /* Make a dummy 1-tuple around data to use db_get_term() */ - heir_data = (UWord) db_get_term(&tb->common, NULL, 0, - TUPLE1(tmp,heir_data)); + heir_data = (UWord) db_store_term(&tb->common, NULL, 0, + TUPLE1(tmp,heir_data)); UnUseTmpHeap(2,me); ASSERT(!is_immed(heir_data)); } @@ -3344,7 +3505,7 @@ static void free_heir_data(DbTable* tb) { if (tb->common.heir != am_none && !is_immed(tb->common.heir_data)) { DbTerm* p = (DbTerm*) tb->common.heir_data; - db_free_term_data(p); + db_cleanup_offheap_comp(p); erts_db_free(ERTS_ALC_T_DB_TERM, tb, (void *)p, sizeof(DbTerm) + (p->size-1)*sizeof(Eterm)); } @@ -3386,6 +3547,7 @@ static int free_table_cont(Process *p, int clean_meta_tab) { Eterm result; + erts_smp_rwmtx_t *mmtl; #ifdef HARDDEBUG if (!first) { @@ -3411,9 +3573,16 @@ static int free_table_cont(Process *p, tb->common.id); #endif /* Completely done - we will not get called again. */ - meta_main_tab_lock(tb->common.slot); + mmtl = get_meta_main_tab_lock(tb->common.slot); +#ifdef ERTS_SMP + if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { + erts_smp_rwmtx_rwunlock(&tb->common.rwlock); + erts_smp_rwmtx_rwlock(mmtl); + erts_smp_rwmtx_rwlock(&tb->common.rwlock); + } +#endif free_slot(tb->common.slot); - meta_main_tab_unlock(tb->common.slot); + erts_smp_rwmtx_rwunlock(mmtl); if (clean_meta_tab) { db_meta_lock(meta_pid_to_tab, LCK_WRITE_REC); @@ -3421,7 +3590,7 @@ static int free_table_cont(Process *p, make_small(tb->common.slot)); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); } - db_unref(tb); + schedule_free_dbtable(tb); BUMP_REDS(p, 100); return 0; } @@ -3469,10 +3638,13 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = erts_this_dist_entry->sysname; } else if (What == am_named_table) { ret = is_atom(tb->common.id) ? am_true : am_false; + } else if (What == am_compressed) { + ret = tb->common.compress ? am_true : am_false; + } /* * For debugging purposes */ - } else if (What == am_data) { + else if (What == am_data) { print_table(ERTS_PRINT_STDOUT, NULL, 1, tb); ret = am_true; } else if (What == am_atom_put("fixed",5)) { diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 7da28fad29..e0bdebcb01 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -61,6 +61,7 @@ void erts_db_foreach_offheap(DbTable *, extern int user_requested_db_max_tabs; /* set in erl_init */ extern int erts_ets_realloc_always_moves; /* set in erl_init */ +extern int erts_ets_always_compress; /* set in erl_init */ extern Export ets_select_delete_continue_exp; extern Export ets_select_count_continue_exp; extern Export ets_select_continue_exp; @@ -82,7 +83,8 @@ Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); #define ERTS_DB_ALC_MEM_UPDATE_(TAB, FREE_SZ, ALLOC_SZ) \ do { \ - long sz__ = ((long) (ALLOC_SZ)) - ((long) (FREE_SZ)); \ + erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ)) \ + - ((erts_aint_t) (FREE_SZ))); \ ASSERT((TAB)); \ erts_smp_atomic_add(&(TAB)->common.memory_size, sz__); \ } while (0) diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 124129a371..1e50fee554 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -135,8 +135,8 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) */ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) { - long was_next; - long exp_next; + erts_aint_t was_next; + erts_aint_t exp_next; FixedDeletion* fixd = (FixedDeletion*) erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL, (DbTable *) tb, sizeof(FixedDeletion)); @@ -146,7 +146,9 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) do { /* Lockless atomic insertion in linked list: */ exp_next = was_next; fixd->next = (FixedDeletion*) exp_next; - was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixd, exp_next); + was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, + (erts_aint_t) fixd, + exp_next); }while (was_next != exp_next); } @@ -267,11 +269,11 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix, */ #define BIN_FLAG_ALL_OBJECTS BIN_FLAG_USR1 -/* - * Size calculations - */ -#define SIZ_OVERHEAD ((sizeof(HashDbTerm)/sizeof(Eterm)) - 1) -#define SIZ_DBTERM(HDT) (SIZ_OVERHEAD + (HDT)->dbterm.size) + +static ERTS_INLINE void free_term(DbTableHash *tb, HashDbTerm* p) +{ + db_free_term((DbTable*)tb, p, offsetof(HashDbTerm, dbterm)); +} /* * Local types @@ -358,10 +360,8 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, HashValue hval, HashDbTerm *list); static void shrink(DbTableHash* tb, int nactive); static void grow(DbTableHash* tb, int nactive); -static void free_term(DbTableHash *tb, HashDbTerm* p); -static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2); -static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, - Eterm obj, HashValue hval); +static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, + DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, struct mp_info *mpi); @@ -442,6 +442,7 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, if (b->hvalue != hval) return 0; else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); + ASSERT(!is_header(itemKey)); return EQ(key,itemKey); } } @@ -454,10 +455,38 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b, if (b->hvalue != hval && b->hvalue != INVALID_HASH) return 0; else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); + ASSERT(!is_header(itemKey)); return EQ(key,itemKey); } } +static ERTS_INLINE HashDbTerm* new_dbterm(DbTableHash* tb, Eterm obj) +{ + HashDbTerm* p; + if (tb->common.compress) { + p = db_store_term_comp(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj); + } + else { + p = db_store_term(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj); + } + return p; +} + +static ERTS_INLINE HashDbTerm* replace_dbterm(DbTableHash* tb, HashDbTerm* old, + Eterm obj) +{ + HashDbTerm* ret; + ASSERT(old != NULL); + if (tb->common.compress) { + ret = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj); + } + else { + ret = db_store_term(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj); + } + return ret; +} + + /* ** External interface @@ -514,12 +543,12 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) { /*int tries = 0;*/ DEBUG_WAIT(); - if (erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel, - (long)NULL) != (long)NULL) { + if (erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, + (erts_aint_t)NULL) != (erts_aint_t)NULL) { /* Oboy, must join lists */ FixedDeletion* last = fixdel; - long was_tail; - long exp_tail; + erts_aint_t was_tail; + erts_aint_t exp_tail; while (last->next != NULL) last = last->next; was_tail = erts_smp_atomic_read(&tb->fixdel); @@ -528,7 +557,7 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) last->next = (FixedDeletion*) exp_tail; /*++tries;*/ DEBUG_WAIT(); - was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel, + was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, exp_tail); }while (was_tail != exp_tail); } @@ -546,7 +575,7 @@ void db_unfix_table_hash(DbTableHash *tb) || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock) && !tb->common.is_thread_safe)); restart: - fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (long)NULL); + fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (erts_aint_t)NULL); while (fixdel != NULL) { FixedDeletion *fx = fixdel; int ix = fx->slot; @@ -615,20 +644,24 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_atomic_init(&tb->szm, SEGSZ_MASK); erts_smp_atomic_init(&tb->nactive, SEGSZ); - erts_smp_atomic_init(&tb->fixdel, (long)NULL); - erts_smp_atomic_init(&tb->segtab, (long) alloc_ext_seg(tb,0,NULL)->segtab); + erts_smp_atomic_init(&tb->fixdel, (erts_aint_t)NULL); + erts_smp_atomic_init(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab); tb->nsegs = NSEG_1; tb->nslots = SEGSZ; erts_smp_atomic_init(&tb->is_resizing, 0); #ifdef ERTS_SMP if (tb->common.type & DB_FINE_LOCKED) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; int i; + if (tb->common.type & DB_FREQ_READ) + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */ (DbTable *) tb, sizeof(DbTableHashFineLocks)); for (i=0; i<DB_HASH_LOCK_CNT; ++i) { - erts_rwmtx_init_x(&tb->locks->lck_vec[i].lck, "db_hash_slot", make_small(i)); + erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, + "db_hash_slot", make_small(i)); } /* This important property is needed to guarantee that the buckets * involved in a grow/shrink operation it protected by the same lock: @@ -760,7 +793,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) ret = DB_ERROR_BADKEY; goto Ldone; } - q = get_term(tb, b, obj, hval); + q = replace_dbterm(tb, b, obj); q->next = bnext; q->hvalue = hval; /* In case of INVALID_HASH */ *bp = q; @@ -780,7 +813,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) HashDbTerm** qp = bp; q = b; do { - if (eq(make_tuple(q->dbterm.tpl), obj)) { + if (db_eq(&tb->common,obj,&q->dbterm)) { if (q->hvalue == INVALID_HASH) { erts_smp_atomic_inc(&tb->common.nitems); q->hvalue = hval; @@ -799,7 +832,8 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) /*else DB_DUPLICATE_BAG */ Lnew: - q = get_term(tb, NULL, obj, hval); + q = new_dbterm(tb, obj); + q->hvalue = hval; q->next = b; *bp = q; nitems = erts_smp_atomic_inctest(&tb->common.nitems); @@ -840,7 +874,7 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) while(b2 != NULL && has_key(tb,b2,key,hval)) b2 = b2->next; } - copy = put_term_list(p, b1, b2); + copy = build_term_list(p, b1, b2, tb); CHECK_TABLES(); *ret = copy; goto done; @@ -963,13 +997,10 @@ static int db_get_element_hash(Process *p, DbTable *tbl, while(b1 != 0) { if (has_live_key(tb,b1,key,hval)) { - Eterm copy; - if (ndex > arityval(b1->dbterm.tpl[0])) { retval = DB_ERROR_BADITEM; goto done; } - if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { HashDbTerm* b; HashDbTerm* b2 = b1->next; @@ -983,15 +1014,12 @@ static int db_get_element_hash(Process *p, DbTable *tbl, } b2 = b2->next; } - b = b1; while(b != b2) { if (b->hvalue != INVALID_HASH) { Eterm *hp; - Uint sz = size_object(b->dbterm.tpl[ndex])+2; - - hp = HAlloc(p, sz); - copy = copy_struct(b->dbterm.tpl[ndex], sz-2, &hp, &MSO(p)); + Eterm copy = db_copy_element_from_ets(&tb->common, p, + &b->dbterm, ndex, &hp, 2); elem_list = CONS(hp, copy, elem_list); hp += 2; } @@ -1000,8 +1028,8 @@ static int db_get_element_hash(Process *p, DbTable *tbl, *ret = elem_list; } else { - COPY_OBJECT(b1->dbterm.tpl[ndex], p, ©); - *ret = copy; + Eterm* hp; + *ret = db_copy_element_from_ets(&tb->common, p, &b1->dbterm, ndex, &hp, 0); } retval = DB_ERROR_NONE; goto done; @@ -1036,6 +1064,7 @@ int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value) ASSERT(!IS_FIXED(tb)); ASSERT((tb->common.status & DB_BAG)); + ASSERT(!tb->common.compress); while(b != 0) { if (has_live_key(tb,b,key,hval)) { @@ -1135,7 +1164,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) while(b != 0) { if (has_live_key(tb,b,key,hval)) { ++nkeys; - if (eq(object, make_tuple(b->dbterm.tpl))) { + if (db_eq(&tb->common,object, &b->dbterm)) { --nitems_diff; if (nkeys==1 && IS_FIXED(tb)) { /* Pseudo remove */ add_fixed_deletion(tb,ix); @@ -1184,7 +1213,7 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret) lck = RLOCK_HASH(tb, slot); nactive = NACTIVE(tb); if (slot < nactive) { - *ret = put_term_list(p, BUCKET(tb, slot), 0); + *ret = build_term_list(p, BUCKET(tb, slot), 0, tb); retval = DB_ERROR_NONE; } else if (slot == nactive) { @@ -1228,8 +1257,6 @@ static int db_select_continue_hash(Process *p, int num_left = 1000; HashDbTerm *current = 0; Eterm match_list; - Uint32 dummy; - unsigned sz; Eterm *hp; Eterm match_res; Sint got; @@ -1281,26 +1308,14 @@ static int db_select_continue_hash(Process *p, } for(;;) { if (current->hvalue != INVALID_HASH && - (match_res = - db_prog_match(p,mp, - make_tuple(current->dbterm.tpl), - NULL,0,&dummy), + (match_res = db_prog_match_and_copy(&tb->common, p, mp, all_objects, + ¤t->dbterm, &hp, 2), is_value(match_res))) { - if (all_objects) { - hp = HAlloc(p, current->dbterm.size + 2); - match_res = copy_shallow(DBTERM_BUF(¤t->dbterm), - current->dbterm.size, - &hp, - &MSO(p)); - } else { - sz = size_object(match_res); - - hp = HAlloc(p, sz + 2); - match_res = copy_struct(match_res, sz, &hp, &MSO(p)); - } - match_list = CONS(hp, match_res, match_list); + + match_list = CONS(hp, match_res, match_list); ++got; } + --num_left; save_slot_ix = slot_ix; if ((current = next(tb, (Uint*)&slot_ix, &lck, current)) == NULL) { @@ -1391,9 +1406,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, HashDbTerm *current = 0; unsigned current_list_pos = 0; Eterm match_list; - Uint32 dummy; Eterm match_res; - unsigned sz; Eterm *hp; int num_left = 1000; Uint got = 0; @@ -1460,22 +1473,9 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, for(;;) { if (current != NULL) { if (current->hvalue != INVALID_HASH) { - match_res = db_prog_match(p,mpi.mp, - make_tuple(current->dbterm.tpl), - NULL,0,&dummy); + match_res = db_prog_match_and_copy(&tb->common, p, mpi.mp, 0, + ¤t->dbterm, &hp, 2); if (is_value(match_res)) { - if (mpi.all_objects) { - hp = HAlloc(p, current->dbterm.size + 2); - match_res = copy_shallow(DBTERM_BUF(¤t->dbterm), - current->dbterm.size, - &hp, - &MSO(p)); - } else { - sz = size_object(match_res); - - hp = HAlloc(p, sz + 2); - match_res = copy_struct(match_res, sz, &hp, &MSO(p)); - } match_list = CONS(hp, match_res, match_list); ++got; } @@ -1590,7 +1590,6 @@ static int db_select_count_hash(Process *p, Uint slot_ix = 0; HashDbTerm* current = NULL; unsigned current_list_pos = 0; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got = 0; @@ -1640,8 +1639,8 @@ static int db_select_count_hash(Process *p, for(;;) { if (current != NULL) { if (current->hvalue != INVALID_HASH) { - if (db_prog_match(p, mpi.mp, make_tuple(current->dbterm.tpl), - NULL,0, &dummy) == am_true) { + if (db_prog_match_and_copy(&tb->common, p, mpi.mp, 0, + ¤t->dbterm, NULL,0) == am_true) { ++got; } --num_left; @@ -1709,7 +1708,6 @@ static int db_select_delete_hash(Process *p, Uint slot_ix = 0; HashDbTerm **current = NULL; unsigned current_list_pos = 0; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got = 0; @@ -1719,9 +1717,9 @@ static int db_select_delete_hash(Process *p, Eterm mpb; Eterm egot; #ifdef ERTS_SMP - int fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */ + erts_aint_t fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */ #else - int fixated_by_me = 0; + erts_aint_t fixated_by_me = 0; #endif erts_smp_rwmtx_t* lck; @@ -1790,9 +1788,8 @@ static int db_select_delete_hash(Process *p, } else { int did_erase = 0; - if ((db_prog_match(p,mpi.mp, - make_tuple((*current)->dbterm.tpl), - NULL,0,&dummy)) == am_true) { + if (db_prog_match_and_copy(&tb->common, p, mpi.mp, 0, + &(*current)->dbterm, NULL, 0) == am_true) { if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { add_fixed_deletion(tb, slot_ix); @@ -1855,7 +1852,6 @@ static int db_select_delete_continue_hash(Process *p, Uint slot_ix; Uint last_pseudo_delete = (Uint)-1; HashDbTerm **current = NULL; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got; @@ -1903,8 +1899,8 @@ static int db_select_delete_continue_hash(Process *p, } else { int did_erase = 0; - if ((db_prog_match(p,mp,make_tuple((*current)->dbterm.tpl), - NULL,0,&dummy)) == am_true) { + if (db_prog_match_and_copy(&tb->common, p, mp, 0, + &(*current)->dbterm, NULL, 0) == am_true) { if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { add_fixed_deletion(tb, slot_ix); @@ -1966,7 +1962,6 @@ static int db_select_count_continue_hash(Process *p, DbTableHash *tb = &tbl->hash; Uint slot_ix; HashDbTerm* current; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got; @@ -2004,8 +1999,8 @@ static int db_select_count_continue_hash(Process *p, current = current->next; continue; } - if (db_prog_match(p, mp, make_tuple(current->dbterm.tpl), - NULL,0,&dummy) == am_true) { + if (db_prog_match_and_copy(&tb->common, p, mp, 0, ¤t->dbterm, + NULL, 0) == am_true) { ++got; } --num_left; @@ -2131,11 +2126,11 @@ static int db_free_table_continue_hash(DbTable *tbl) sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion)); if (++done >= 2*DELETE_RECORD_LIMIT) { - erts_smp_atomic_set(&tb->fixdel, (long)fixdel); + erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)fixdel); return 0; /* Not done */ } } - erts_smp_atomic_set(&tb->fixdel, (long)NULL); + erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)NULL); done /= 2; while(tb->nslots != 0) { @@ -2352,7 +2347,7 @@ static int alloc_seg(DbTableHash *tb) struct ext_segment* eseg; eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - erts_smp_atomic_set(&tb->segtab, (long) eseg->segtab); + erts_smp_atomic_set(&tb->segtab, (erts_aint_t) eseg->segtab); tb->nsegs = eseg->nsegs; } ASSERT(seg_ix < tb->nsegs); @@ -2424,7 +2419,7 @@ static int free_seg(DbTableHash *tb, int free_records) MY_ASSERT(newtop->s.is_ext_segment); if (newtop->prev_segtab != NULL) { /* Time to use a smaller segtab */ - erts_smp_atomic_set(&tb->segtab, (long)newtop->prev_segtab); + erts_smp_atomic_set(&tb->segtab, (erts_aint_t)newtop->prev_segtab); tb->nsegs = seg_ix; ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); } @@ -2441,7 +2436,7 @@ static int free_seg(DbTableHash *tb, int free_records) if (seg_ix > 0) { if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; } else { - erts_smp_atomic_set(&tb->segtab, (long)NULL); + erts_smp_atomic_set(&tb->segtab, (erts_aint_t)NULL); } #endif tb->nslots -= SEGSZ; @@ -2450,31 +2445,19 @@ static int free_seg(DbTableHash *tb, int free_records) } -static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, - Eterm obj, HashValue hval) -{ - HashDbTerm* p = db_get_term((DbTableCommon *) tb, - (old != NULL) ? &(old->dbterm) : NULL, - ((char *) &(old->dbterm)) - ((char *) old), - obj); - p->hvalue = hval; - /*p->next = NULL;*/ /*No Need */ - return p; -} - - /* ** Copy terms from ptr1 until ptr2 ** works for ptr1 == ptr2 == 0 => [] ** or ptr2 == 0 */ -static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2) +static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, + DbTableHash* tb) { int sz = 0; HashDbTerm* ptr; Eterm list = NIL; Eterm copy; - Eterm *hp; + Eterm *hp, *hend; ptr = ptr1; while(ptr != ptr2) { @@ -2486,26 +2469,20 @@ static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2) } hp = HAlloc(p, sz); + hend = hp + sz; ptr = ptr1; while(ptr != ptr2) { if (ptr->hvalue != INVALID_HASH) { - copy = copy_shallow(DBTERM_BUF(&ptr->dbterm), ptr->dbterm.size, &hp, &MSO(p)); + copy = db_copy_object_from_ets(&tb->common, &ptr->dbterm, &hp, &MSO(p)); list = CONS(hp, copy, list); hp += 2; } ptr = ptr->next; } - return list; -} + HRelease(p,hend,hp); -static void free_term(DbTableHash *tb, HashDbTerm* p) -{ - db_free_term_data(&(p->dbterm)); - erts_db_free(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - (void *) p, - SIZ_DBTERM(p)*sizeof(Eterm)); + return list; } /* Grow table with one new bucket. @@ -2716,8 +2693,8 @@ static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle handle->tb = tbl; handle->bp = (void**) prevp; handle->dbterm = &b->dbterm; - handle->new_size = b->dbterm.size; handle->mustResize = 0; + handle->new_size = b->dbterm.size; handle->lck = lck; /* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */ return 1; @@ -2738,39 +2715,14 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle) erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck; ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(&tbl->hash,lck)); /* locked by db_lookup_dbterm_hash */ - ASSERT(&oldp->dbterm == handle->dbterm); - if (handle->mustResize) { - Eterm* top; - Eterm copy; - DbTerm* newDbTerm; - HashDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl, - sizeof(HashDbTerm)+sizeof(Eterm)*(handle->new_size-1)); - sys_memcpy(newp, oldp, sizeof(HashDbTerm)-sizeof(DbTerm)); /* copy only hashtab header */ - *(handle->bp) = newp; - newDbTerm = &newp->dbterm; - - newDbTerm->size = handle->new_size; - newDbTerm->off_heap.mso = NULL; - newDbTerm->off_heap.externals = NULL; - #ifndef HYBRID /* FIND ME! */ - newDbTerm->off_heap.funs = NULL; - #endif - newDbTerm->off_heap.overhead = 0; - - /* make a flat copy */ - top = DBTERM_BUF(newDbTerm); - copy = copy_struct(make_tuple(handle->dbterm->tpl), - handle->new_size, - &top, &newDbTerm->off_heap); - DBTERM_SET_TPL(newDbTerm,tuple_val(copy)); + ASSERT((&oldp->dbterm == handle->dbterm) == !(tbl->common.compress && handle->mustResize)); + if (handle->mustResize) { + db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); WUNLOCK_HASH(lck); - - db_free_term_data(handle->dbterm); - erts_db_free(ERTS_ALC_T_DB_TERM, tbl, - (void *) (((char *) handle->dbterm) - (sizeof(HashDbTerm) - sizeof(DbTerm))), - sizeof(HashDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1)); + + free_term(&tbl->hash, oldp); } else { WUNLOCK_HASH(lck); @@ -2779,7 +2731,7 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle) handle->dbterm = 0; #endif return; -} +} static int db_delete_all_objects_hash(Process* p, DbTable* tbl) { @@ -2805,7 +2757,11 @@ void db_foreach_offheap_hash(DbTable *tbl, for (i = 0; i < nactive; i++) { list = BUCKET(tb,i); while(list != 0) { - (*func)(&(list->dbterm.off_heap), arg); + ErlOffHeap tmp_offheap; + tmp_offheap.first = list->dbterm.first_oh; + tmp_offheap.overhead = 0; + (*func)(&tmp_offheap, arg); + list->dbterm.first_oh = tmp_offheap.first; list = list->next; } } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index b6b3cabafe..8108494fc5 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -122,12 +122,41 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack) } } -static void reset_static_stack(DbTableTree* tb) +static ERTS_INLINE void reset_static_stack(DbTableTree* tb) { tb->static_stack.pos = 0; tb->static_stack.slot = 0; } +static ERTS_INLINE void free_term(DbTableTree *tb, TreeDbTerm* p) +{ + db_free_term((DbTable*)tb, p, offsetof(TreeDbTerm, dbterm)); +} + +static ERTS_INLINE TreeDbTerm* new_dbterm(DbTableTree *tb, Eterm obj) +{ + TreeDbTerm* p; + if (tb->common.compress) { + p = db_store_term_comp(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj); + } + else { + p = db_store_term(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj); + } + return p; +} +static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old, + Eterm obj) +{ + TreeDbTerm* p; + ASSERT(old != NULL); + if (tb->common.compress) { + p = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj); + } + else { + p = db_store_term(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj); + } + return p; +} /* ** Some macros for "direction stacks" @@ -178,12 +207,6 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, #endif /* - * Size calculations - */ -#define SIZ_OVERHEAD ((sizeof(TreeDbTerm)/sizeof(Eterm)) - 1) -#define SIZ_DBTERM(TDT) (SIZ_OVERHEAD + (TDT)->dbterm.size) - -/* ** Datatypes */ @@ -263,9 +286,6 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key); static TreeDbTerm *linkout_object_tree(DbTableTree *tb, Eterm object); static int do_free_tree_cont(DbTableTree *tb, int num_left); -static TreeDbTerm* get_term(DbTableTree *tb, - TreeDbTerm* old, - Eterm obj); static void free_term(DbTableTree *tb, TreeDbTerm* p); static int balance_left(TreeDbTerm **this); static int balance_right(TreeDbTerm **this); @@ -622,7 +642,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) erts_smp_atomic_dec(&tb->common.nitems); return DB_ERROR_SYSRES; } - *this = get_term(tb, NULL, obj); + *this = new_dbterm(tb, obj); (*this)->balance = 0; (*this)->left = (*this)->right = NULL; break; @@ -636,7 +656,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) tstack[tpos++] = this; this = &((*this)->right); } else if (!key_clash_fail) { /* Equal key and this is a set, replace. */ - *this = get_term(tb, *this, obj); + *this = replace_dbterm(tb, *this, obj); break; } else { return DB_ERROR_BADKEY; /* key already exists */ @@ -714,7 +734,7 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) { DbTableTree *tb = &tbl->tree; Eterm copy; - Eterm *hp; + Eterm *hp, *hend; TreeDbTerm *this; /* @@ -728,11 +748,11 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) *ret = NIL; } else { hp = HAlloc(p, this->dbterm.size + 2); - copy = copy_shallow(DBTERM_BUF(&this->dbterm), - this->dbterm.size, - &hp, - &MSO(p)); + hend = hp + this->dbterm.size + 2; + copy = db_copy_object_from_ets(&tb->common, &this->dbterm, &hp, &MSO(p)); *ret = CONS(hp, copy, NIL); + hp += 2; + HRelease(p,hend,hp); } return DB_ERROR_NONE; } @@ -766,18 +786,10 @@ static int db_get_element_tree(Process *p, DbTable *tbl, if (this == NULL) { return DB_ERROR_BADKEY; } else { - Eterm element; - Uint sz; if (ndex > arityval(this->dbterm.tpl[0])) { return DB_ERROR_BADPARAM; } - element = this->dbterm.tpl[ndex]; - sz = size_object(element); - hp = HAlloc(p, sz); - *ret = copy_struct(element, - sz, - &hp, - &MSO(p)); + *ret = db_copy_element_from_ets(&tb->common, p, &this->dbterm, ndex, &hp, 0); } return DB_ERROR_NONE; } @@ -815,7 +827,7 @@ static int db_slot_tree(Process *p, DbTable *tbl, DbTableTree *tb = &tbl->tree; Sint slot; TreeDbTerm *st; - Eterm *hp; + Eterm *hp, *hend; Eterm copy; /* @@ -847,11 +859,11 @@ static int db_slot_tree(Process *p, DbTable *tbl, return DB_ERROR_UNSPEC; } hp = HAlloc(p, st->dbterm.size + 2); - copy = copy_shallow(DBTERM_BUF(&st->dbterm), - st->dbterm.size, - &hp, - &MSO(p)); + hend = hp + st->dbterm.size + 2; + copy = db_copy_object_from_ets(&tb->common, &st->dbterm, &hp, &MSO(p)); *ret = CONS(hp, copy, NIL); + hp += 2; + HRelease(p,hend,hp); return DB_ERROR_NONE; } @@ -1081,11 +1093,12 @@ static int db_select_continue_tree(Process *p, static int db_select_tree(Process *p, DbTable *tbl, Eterm pattern, int reverse, Eterm *ret) { + /* Strategy: Traverse backwards to build resulting list from tail to head */ DbTableTree *tb = &tbl->tree; DbTreeStack* stack; struct select_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; Eterm key; Eterm continuation; unsigned sz; @@ -1293,7 +1306,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, DbTreeStack* stack; struct select_count_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; Eterm key; Eterm continuation; unsigned sz; @@ -1395,7 +1408,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, DbTreeStack* stack; struct select_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; Eterm key; Eterm continuation; unsigned sz; @@ -1636,7 +1649,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, DbTableTree *tb = &tbl->tree; struct select_delete_context sc; struct mp_info mpi; - Eterm lastkey = NIL; + Eterm lastkey = THE_NON_VALUE; Eterm key; Eterm continuation; unsigned sz; @@ -1737,7 +1750,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, ** Other interface routines (not directly coupled to one bif) */ -/* Display hash table contents (for dump) */ +/* Display tree contents (for dump) */ static void db_print_tree(int to, void *to_arg, int show, DbTable *tbl) @@ -1817,10 +1830,14 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt, void (*func)(ErlOffHeap *, void *), void * arg) { + ErlOffHeap tmp_offheap; if(!tdbt) return; do_db_tree_foreach_offheap(tdbt->left, func, arg); - (*func)(&(tdbt->dbterm.off_heap), arg); + tmp_offheap.first = tdbt->dbterm.first_oh; + tmp_offheap.overhead = 0; + (*func)(&tmp_offheap, arg); + tdbt->dbterm.first_oh = tmp_offheap.first; do_db_tree_foreach_offheap(tdbt->right, func, arg); } @@ -1921,7 +1938,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, tstack[tpos++] = this; this = &((*this)->right); } else { /* Equal key, found the only possible matching object*/ - if (!eq(object,make_tuple((*this)->dbterm.tpl))) { + if (!db_eq(&tb->common,object,&(*this)->dbterm)) { return NULL; } q = (*this); @@ -2074,15 +2091,6 @@ static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t) } } -static void free_term(DbTableTree *tb, TreeDbTerm* p) -{ - db_free_term_data(&(p->dbterm)); - erts_db_free(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - (void *) p, - SIZ_DBTERM(p)*sizeof(Uint)); -} - static int do_free_tree_cont(DbTableTree *tb, int num_left) { TreeDbTerm *root; @@ -2113,17 +2121,6 @@ static int do_free_tree_cont(DbTableTree *tb, int num_left) return 1; } -static TreeDbTerm* get_term(DbTableTree *tb, - TreeDbTerm* old, - Eterm obj) -{ - TreeDbTerm* p = db_get_term((DbTableCommon *) tb, - (old != NULL) ? &(old->dbterm) : NULL, - ((char *) &(old->dbterm)) - ((char *) old), - obj); - return p; -} - /* * Deletion helpers */ @@ -2565,48 +2562,21 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle handle->tb = tbl; handle->dbterm = &(*pp)->dbterm; + handle->mustResize = 0; handle->bp = (void**) pp; handle->new_size = (*pp)->dbterm.size; - handle->mustResize = 0; return 1; } static void db_finalize_dbterm_tree(DbUpdateHandle* handle) { if (handle->mustResize) { - Eterm* top; - Eterm copy; - DbTerm* newDbTerm; - DbTableTree *tb = &handle->tb->tree; TreeDbTerm* oldp = (TreeDbTerm*) *handle->bp; - TreeDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, - handle->tb, - sizeof(TreeDbTerm)+sizeof(Eterm)*(handle->new_size-1)); - memcpy(newp, oldp, sizeof(TreeDbTerm)-sizeof(DbTerm)); /* copy only tree header */ - *(handle->bp) = newp; - reset_static_stack(tb); - newDbTerm = &newp->dbterm; - - newDbTerm->size = handle->new_size; - newDbTerm->off_heap.mso = NULL; - newDbTerm->off_heap.externals = NULL; - #ifndef HYBRID /* FIND ME! */ - newDbTerm->off_heap.funs = NULL; - #endif - newDbTerm->off_heap.overhead = 0; - - /* make a flat copy */ - top = DBTERM_BUF(newDbTerm); - copy = copy_struct(make_tuple(handle->dbterm->tpl), - handle->new_size, - &top, &newDbTerm->off_heap); - DBTERM_SET_TPL(newDbTerm,tuple_val(copy)); - - db_free_term_data(handle->dbterm); - erts_db_free(ERTS_ALC_T_DB_TERM, - handle->tb, - (void *) (((char *) handle->dbterm) - (sizeof(TreeDbTerm) - sizeof(DbTerm))), - sizeof(TreeDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1)); + + db_finalize_resize(handle, offsetof(TreeDbTerm,dbterm)); + reset_static_stack(&handle->tb->tree); + + free_term(&handle->tb->tree, oldp); } #ifdef DEBUG handle->dbterm = 0; @@ -2628,7 +2598,7 @@ static void traverse_backwards(DbTableTree *tb, { TreeDbTerm *this, *next; - if (lastkey == NIL) { + if (lastkey == THE_NON_VALUE) { stack->pos = stack->slot = 0; if (( this = tb->root ) == NULL) { return; @@ -2666,7 +2636,7 @@ static void traverse_forward(DbTableTree *tb, { TreeDbTerm *this, *next; - if (lastkey == NIL) { + if (lastkey == THE_NON_VALUE) { stack->pos = stack->slot = 0; if (( this = tb->root ) == NULL) { return; @@ -3006,7 +2976,7 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_context *sc = (struct select_context *) ptr; Eterm ret; - Uint32 dummy; + Eterm* hp; sc->lastobj = this->dbterm.tpl; @@ -3021,24 +2991,9 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, this->dbterm.tpl)) > 0))) { return 0; } - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - NULL,0, &dummy); + ret = db_prog_match_and_copy(&tb->common,sc->p,sc->mp,sc->all_objects, + &this->dbterm, &hp, 2); if (is_value(ret)) { - Uint sz; - Eterm *hp; - if (sc->all_objects) { - hp = HAlloc(sc->p, this->dbterm.size + 2); - ret = copy_shallow(DBTERM_BUF(&this->dbterm), - this->dbterm.size, - &hp, - &MSO(sc->p)); - } else { - sz = size_object(ret); - hp = HAlloc(sc->p, sz + 2); - ret = copy_struct(ret, sz, - &hp, &MSO(sc->p)); - } sc->accum = CONS(hp, ret, sc->accum); } if (MBUF(sc->p)) { @@ -3059,7 +3014,6 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_count_context *sc = (struct select_count_context *) ptr; Eterm ret; - Uint32 dummy; sc->lastobj = this->dbterm.tpl; @@ -3070,9 +3024,8 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr, this->dbterm.tpl)) > 0)) { return 0; } - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - NULL,0, &dummy); + ret = db_prog_match_and_copy(&tb->common, sc->p, sc->mp, 0, + &this->dbterm, NULL, 0); if (ret == am_true) { ++(sc->got); } @@ -3087,7 +3040,7 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_context *sc = (struct select_context *) ptr; Eterm ret; - Uint32 dummy; + Eterm* hp; sc->lastobj = this->dbterm.tpl; @@ -3103,25 +3056,10 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr, return 0; } - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - NULL,0, &dummy); + ret = db_prog_match_and_copy(&tb->common, sc->p, sc->mp, sc->all_objects, + &this->dbterm, &hp, 2); if (is_value(ret)) { - Uint sz; - Eterm *hp; - ++(sc->got); - if (sc->all_objects) { - hp = HAlloc(sc->p, this->dbterm.size + 2); - ret = copy_shallow(DBTERM_BUF(&this->dbterm), - this->dbterm.size, - &hp, - &MSO(sc->p)); - } else { - sz = size_object(ret); - hp = HAlloc(sc->p, sz + 2); - ret = copy_struct(ret, sz, &hp, &MSO(sc->p)); - } sc->accum = CONS(hp, ret, sc->accum); } if (MBUF(sc->p)) { @@ -3143,7 +3081,6 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_delete_context *sc = (struct select_delete_context *) ptr; Eterm ret; - Uint32 dummy; Eterm key; if (sc->erase_lastterm) @@ -3156,9 +3093,8 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl)) > 0) return 0; - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - NULL,0, &dummy); + ret = db_prog_match_and_copy(&tb->common, sc->p, sc->mp, 0, + &this->dbterm, NULL, 0); if (ret == am_true) { key = GETKEY(sc->tb, this->dbterm.tpl); linkout_tree(sc->tb, key); diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index fd7de98ac9..2852fb93fe 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -25,7 +25,6 @@ #ifdef HAVE_CONFIG_H # include "config.h" #endif - #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -363,12 +362,7 @@ static ErtsMatchPseudoProcess *match_pseudo_process; static ERTS_INLINE void cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) { - if (mpsp->process.mbuf - || mpsp->process.off_heap.mso -#ifndef HYBRID /* FIND ME! */ - || mpsp->process.off_heap.funs -#endif - || mpsp->process.off_heap.externals) { + if (mpsp->process.mbuf || mpsp->process.off_heap.first) { erts_cleanup_empty_process(&mpsp->process); } #ifdef DEBUG @@ -895,6 +889,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace); static Eterm seq_trace_fake(Process *p, Eterm arg1); +static void db_free_tmp_uncompressed(DbTerm* obj); + /* ** Interface routines. @@ -919,7 +915,7 @@ BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new) if (val != ((Uint32)val)) BIF_ERROR(p, BADARG); - old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (long) val); + old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (erts_aint_t) val); BIF_RET(erts_make_integer((Uint) old_tcw, p)); } @@ -1547,10 +1543,9 @@ restart: */ context.save = NULL; error: /* Here is were we land when compilation failed. */ - while (context.save != NULL) { - ErlHeapFragment *ll = context.save->next; + if (context.save != NULL) { free_message_buffer(context.save); - context.save = ll; + context.save = NULL; } DMC_FREE(stack); DMC_FREE(text); @@ -1567,15 +1562,11 @@ error: /* Here is were we land when compilation failed. */ void erts_db_match_prog_destructor(Binary *bprog) { MatchProg *prog; - ErlHeapFragment *tmp, *ll; if (bprog == NULL) return; prog = Binary2MatchProg(bprog); - tmp = prog->term_save; - while (tmp != NULL) { - ll = tmp->next; - free_message_buffer(tmp); - tmp = ll; + if (prog->term_save != NULL) { + free_message_buffer(prog->term_save); } if (prog->saved_program_buf != NULL) free_message_buffer(prog->saved_program_buf); @@ -1614,6 +1605,7 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) } return ret; } + /* ** Execution of the match program, this is Pam. ** May return THE_NON_VALUE, which is a bailout. @@ -1743,8 +1735,7 @@ restart: FAIL(); ep = termp; break; - case matchArrayBind: /* When the array size is unknown. */ /* XXX:PaN - where does - this array come from? */ + case matchArrayBind: /* When the array size is unknown. */ n = *pc++; hp[n] = dpm_array_to_list(psp, termp, arity); break; @@ -2402,33 +2393,46 @@ void db_do_update_element(DbUpdateHandle* handle, if (is_both_immed(newval,oldval)) { handle->dbterm->tpl[position] = newval; +#ifdef DEBUG_CLONE + if (handle->dbterm->debug_clone) { + handle->dbterm->debug_clone[position] = newval; + } +#endif return; } - else if (!handle->mustResize && is_boxed(newval)) { - newp = boxed_val(newval); - switch (*newp & _TAG_HEADER_MASK) { - case _TAG_HEADER_POS_BIG: - case _TAG_HEADER_NEG_BIG: - case _TAG_HEADER_FLOAT: - case _TAG_HEADER_HEAP_BIN: - newval_sz = header_arity(*newp) + 1; - if (is_boxed(oldval)) { - oldp = boxed_val(oldval); - switch (*oldp & _TAG_HEADER_MASK) { - case _TAG_HEADER_POS_BIG: - case _TAG_HEADER_NEG_BIG: - case _TAG_HEADER_FLOAT: - case _TAG_HEADER_HEAP_BIN: - oldval_sz = header_arity(*oldp) + 1; - if (oldval_sz == newval_sz) { - /* "self contained" terms of same size, do memcpy */ - sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm)); - return; + if (!handle->mustResize) { + if (handle->tb->common.compress) { + handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, + handle->dbterm); + handle->mustResize = 1; + oldval = handle->dbterm->tpl[position]; + } + else if (is_boxed(newval)) { + newp = boxed_val(newval); + switch (*newp & _TAG_HEADER_MASK) { + case _TAG_HEADER_POS_BIG: + case _TAG_HEADER_NEG_BIG: + case _TAG_HEADER_FLOAT: + case _TAG_HEADER_HEAP_BIN: + newval_sz = header_arity(*newp) + 1; + if (is_boxed(oldval)) { + oldp = boxed_val(oldval); + switch (*oldp & _TAG_HEADER_MASK) { + case _TAG_HEADER_POS_BIG: + case _TAG_HEADER_NEG_BIG: + case _TAG_HEADER_FLOAT: + case _TAG_HEADER_HEAP_BIN: + oldval_sz = header_arity(*oldp) + 1; + if (oldval_sz == newval_sz) { + /* "self contained" terms of same size, do memcpy */ + sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm)); + return; + } + goto both_size_set; } - goto both_size_set; } + goto new_size_set; } - goto new_size_set; } } /* Not possible for simple memcpy or dbterm is already non-contiguous, */ @@ -2447,79 +2451,371 @@ both_size_set: handle->mustResize = 1; } +static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old, + Uint old_sz, Uint new_sz, Uint offset) +{ + byte* ret; + if (erts_ets_realloc_always_moves) { + ret = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz); + sys_memcpy(ret, old, offset); + erts_db_free(ERTS_ALC_T_DB_TERM, (DbTable*)tb, old, old_sz); + } else { + ret = erts_db_realloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, + old, old_sz, new_sz); + } + return ret; +} + +/* Allocated size of a compressed dbterm +*/ +static ERTS_INLINE Uint db_alloced_size_comp(DbTerm* obj) +{ + return obj->tpl[arityval(*obj->tpl) + 1]; +} + +void db_free_term(DbTable *tb, void* basep, Uint offset) +{ + DbTerm* db = (DbTerm*) ((byte*)basep + offset); + Uint size; + if (tb->common.compress) { + db_cleanup_offheap_comp(db); + size = db_alloced_size_comp(db); + } + else { + ErlOffHeap tmp_oh; + tmp_oh.first = db->first_oh; + erts_cleanup_offheap(&tmp_oh); + size = offset + offsetof(DbTerm,tpl) + db->size*sizeof(Eterm); + } + erts_db_free(ERTS_ALC_T_DB_TERM, tb, basep, size); +} + +static ERTS_INLINE Uint align_up(Uint value, Uint pow2) +{ + ASSERT((pow2 & (pow2-1)) == 0); + return (value + (pow2-1)) & ~(pow2-1); +} + +/* Compressed size of an uncompressed term +*/ +static Uint db_size_dbterm_comp(DbTableCommon* tb, Eterm obj) +{ + Eterm* tpl = tuple_val(obj); + int i; + Uint size = sizeof(DbTerm) + + arityval(*tpl) * sizeof(Eterm) + + sizeof(Uint); /* "alloc_size" */ + + for (i = arityval(*tpl); i>0; i--) { + if (i != tb->keypos && is_not_immed(tpl[i])) { + size += erts_encode_ext_size_ets(tpl[i]); + } + } + size += size_object(tpl[tb->keypos]) * sizeof(Eterm); + return align_up(size, sizeof(Uint)); +} + +/* Conversion between top tuple element and pointer to compressed data +*/ +static ERTS_INLINE Eterm ext2elem(Eterm* tpl, byte* ext) +{ + return (((Uint)(ext - (byte*)tpl)) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; +} +static ERTS_INLINE byte* elem2ext(Eterm* tpl, Uint ix) +{ + ASSERT(is_header(tpl[ix])); + return (byte*)tpl + (tpl[ix] >> _TAG_PRIMARY_SIZE); +} + +static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, + Uint alloc_size) +{ + ErlOffHeap tmp_offheap; + Eterm* src = tuple_val(obj); + Eterm* tpl = dest->tpl; + Eterm key = src[tb->keypos]; + int arity = arityval(src[0]); + union { + Eterm* ep; + byte* cp; + UWord ui; + }top; + int i; + + top.ep = tpl+ 1 + arity + 1; + tpl[0] = src[0]; + tpl[arity + 1] = alloc_size; + + tmp_offheap.first = NULL; + tpl[tb->keypos] = copy_struct(key, size_object(key), &top.ep, &tmp_offheap); + dest->first_oh = tmp_offheap.first; + for (i=1; i<=arity; i++) { + if (i != tb->keypos) { + if (is_immed(src[i])) { + tpl[i] = src[i]; + } + else { + tpl[i] = ext2elem(tpl, top.cp); + top.cp = erts_encode_ext_ets(src[i], top.cp, &dest->first_oh); + } + } + } + +#ifdef DEBUG_CLONE + { + Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm)); + dest->debug_clone = dbg_top; + tmp_offheap.first = dest->first_oh; + copy_struct(obj, dest->size, &dbg_top, &tmp_offheap); + dest->first_oh = tmp_offheap.first; + ASSERT(dbg_top == dest->debug_clone + dest->size); + } +#endif + return top.cp; +} /* ** Copy the object into a possibly new DbTerm, ** offset is the offset of the DbTerm from the start -** of the sysAllocaed structure, The possibly realloced and copied +** of the allocated structure, The possibly realloced and copied ** structure is returned. Make sure (((char *) old) - offset) is a ** pointer to a ERTS_ALC_T_DB_TERM allocated data area. */ -void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) +void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) { + byte* basep; + DbTerm* newp; + Eterm* top; int size = size_object(obj); - void *structp = ((char*) old) - offset; - DbTerm* p; - Eterm copy; - Eterm *top; + ErlOffHeap tmp_offheap; if (old != 0) { - erts_cleanup_offheap(&old->off_heap); + basep = ((byte*) old) - offset; + tmp_offheap.first = old->first_oh; + erts_cleanup_offheap(&tmp_offheap); + old->first_oh = tmp_offheap.first; if (size == old->size) { - p = old; - } else { + newp = old; + } + else { Uint new_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1); Uint old_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(old->size-1); - if (erts_ets_realloc_always_moves) { - void *nstructp = erts_db_alloc(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - new_sz); - memcpy(nstructp,structp,offset); - erts_db_free(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - structp, - old_sz); - structp = nstructp; - } else { - structp = erts_db_realloc(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - structp, - old_sz, - new_sz); - } - p = (DbTerm*) ((void *)(((char *) structp) + offset)); + basep = db_realloc_term(tb, basep, old_sz, new_sz, offset); + newp = (DbTerm*) (basep + offset); + } + } + else { + basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable *)tb, + (offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1))); + newp = (DbTerm*) (basep + offset); + } + newp->size = size; + top = newp->tpl; + tmp_offheap.first = NULL; + copy_struct(obj, size, &top, &tmp_offheap); + newp->first_oh = tmp_offheap.first; +#ifdef DEBUG_CLONE + newp->debug_clone = NULL; +#endif + return basep; +} + + +void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) +{ + Uint new_sz = offset + db_size_dbterm_comp(tb, obj); + byte* basep; + DbTerm* newp; + byte* top; + + ASSERT(tb->compress); + if (old != 0) { + Uint old_sz = db_alloced_size_comp(old); + db_cleanup_offheap_comp(old); + + basep = ((byte*) old) - offset; + if (new_sz == old_sz) { + newp = old; + } + else { + basep = db_realloc_term(tb, basep, old_sz, new_sz, offset); + newp = (DbTerm*) (basep + offset); } } else { - structp = erts_db_alloc(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - (offset - + sizeof(DbTerm) - + sizeof(Eterm)*(size-1))); - p = (DbTerm*) ((void *)(((char *) structp) + offset)); - } - p->size = size; - p->off_heap.mso = NULL; - p->off_heap.externals = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; + basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz); + newp = (DbTerm*) (basep + offset); + } + + newp->size = size_object(obj); + top = copy_to_comp(tb, obj, newp, new_sz); + ASSERT(top <= basep + new_sz); + + // SVERK: realloc? + + return basep; +} + + +void db_finalize_resize(DbUpdateHandle* handle, Uint offset) +{ + DbTable* tbl = handle->tb; + DbTerm* newDbTerm; + Uint alloc_sz = offset + + (tbl->common.compress ? + db_size_dbterm_comp(&tbl->common, make_tuple(handle->dbterm->tpl)) : + sizeof(DbTerm)+sizeof(Eterm)*(handle->new_size-1)); + byte* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl, alloc_sz); + byte* oldp = *(handle->bp); + + sys_memcpy(newp, oldp, offset); /* copy only hash/tree header */ + *(handle->bp) = newp; + newDbTerm = (DbTerm*) (newp + offset); + newDbTerm->size = handle->new_size; + + /* make a flat copy */ + + if (tbl->common.compress) { + copy_to_comp(&tbl->common, make_tuple(handle->dbterm->tpl), + newDbTerm, alloc_sz); + db_free_tmp_uncompressed(handle->dbterm); + } + else { + Eterm* top; + ErlOffHeap tmp_offheap; + tmp_offheap.first = NULL; + top = newDbTerm->tpl; + copy_struct(make_tuple(handle->dbterm->tpl), handle->new_size, + &top, &tmp_offheap); + newDbTerm->first_oh = tmp_offheap.first; +#ifdef DEBUG_CLONE + newDbTerm->debug_clone = NULL; #endif - p->off_heap.overhead = 0; + ASSERT((byte*)top <= (newp + alloc_sz)); + } +} - top = DBTERM_BUF(p); - copy = copy_struct(obj, size, &top, &p->off_heap); - DBTERM_SET_TPL(p,tuple_val(copy)); +Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, + ErlOffHeap* off_heap) +{ + Eterm* hp = *hpp; + int i, arity = arityval(bp->tpl[0]); + + hp[0] = bp->tpl[0]; + *hpp += arity + 1; + + hp[tb->keypos] = copy_struct(bp->tpl[tb->keypos], + size_object(bp->tpl[tb->keypos]), + hpp, off_heap); + for (i=arity; i>0; i--) { + if (i != tb->keypos) { + if (is_immed(bp->tpl[i])) { + hp[i] = bp->tpl[i]; + } + else { + hp[i] = erts_decode_ext_ets(hpp, off_heap, + elem2ext(bp->tpl, i)); + } + } + } + ASSERT((*hpp - hp) <= bp->size); +#ifdef DEBUG_CLONE + ASSERT(eq(make_tuple(hp),make_tuple(bp->debug_clone))); +#endif + return make_tuple(hp); +} - return structp; +Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, + DbTerm* obj, Uint pos, + Eterm** hpp, Uint extra) +{ + if (is_immed(obj->tpl[pos])) { + *hpp = HAlloc(p, extra); + return obj->tpl[pos]; + } + if (tb->compress && pos != tb->keypos) { + byte* ext = elem2ext(obj->tpl, pos); + Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra; + Eterm* hp = HAlloc(p, sz); + Eterm* endp = hp + sz; + Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext); + *hpp = hp; + hp += extra; + HRelease(p, endp, hp); +#ifdef DEBUG_CLONE + ASSERT(eq(copy,obj->debug_clone[pos])); +#endif + return copy; + } + else { + Uint sz = size_object(obj->tpl[pos]); + *hpp = HAlloc(p, sz + extra); + return copy_struct(obj->tpl[pos], sz, hpp, &MSO(p)); + } } -void db_free_term_data(DbTerm* p) +/* Our own "cleanup_offheap" + * as refc-binaries may be unaligned in compressed terms +*/ +void db_cleanup_offheap_comp(DbTerm* obj) { - erts_cleanup_offheap(&p->off_heap); + union erl_off_heap_ptr u; + ProcBin tmp; + + for (u.hdr = obj->first_oh; u.hdr; u.hdr = u.hdr->next) { + if ((UWord)u.voidp % sizeof(UWord) != 0) { /* unaligned ptr */ + sys_memcpy(&tmp, u.voidp, sizeof(tmp)); + /* Warning, must pass (void*)-variable to memcpy. Otherwise it will + cause Bus error on Sparc due to false compile time assumptions + about word aligned memory (type cast is not enough) */ + u.pb = &tmp; + } + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { + erts_bin_free(u.pb->val); + } + break; + case FUN_SUBTAG: + ASSERT(u.pb != &tmp); + if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { + erts_erase_fun_entry(u.fun->fe); + } + break; + default: + ASSERT(is_external_header(u.hdr->thing_word)); + ASSERT(u.pb != &tmp); + erts_deref_node_entry(u.ext->node); + break; + } + } +#ifdef DEBUG_CLONE + if (obj->debug_clone != NULL) { + erts_free(ERTS_ALC_T_DB_TERM, obj->debug_clone); + obj->debug_clone = NULL; + } +#endif } +int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b) +{ + ErlOffHeap tmp_offheap; + Eterm* allocp; + Eterm* hp; + Eterm tmp_b; + int is_eq; + + ASSERT(tb->compress); + hp = allocp = erts_alloc(ERTS_ALC_T_TMP, b->size*sizeof(Eterm)); + tmp_offheap.first = NULL; + tmp_b = db_copy_from_comp(tb, b, &hp, &tmp_offheap); + is_eq = eq(a,tmp_b); + erts_cleanup_offheap(&tmp_offheap); + erts_free(ERTS_ALC_T_TMP, allocp); + return is_eq; +} /* ** Check if object represents a "match" variable @@ -4125,7 +4421,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info) DMC_INIT_STACK(heap); p = expr->mem; - i = expr->size; + i = expr->used_size; while (i--) { if (is_thing(*p)) { a = thing_arityval(*p); @@ -4154,7 +4450,7 @@ static int match_compact(ErlHeapFragment *expr, DMCErrInfo *err_info) } p = expr->mem; - i = expr->size; + i = expr->used_size; while (i--) { if (is_thing(*p)) { a = thing_arityval(*p); @@ -4411,7 +4707,65 @@ static Eterm seq_trace_fake(Process *p, Eterm arg1) } return result; } - + +DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org) +{ + ErlOffHeap tmp_offheap; + DbTerm* res = erts_alloc(ERTS_ALC_T_TMP, + sizeof(DbTerm) + org->size*sizeof(Eterm)); + Eterm* hp = res->tpl; + tmp_offheap.first = NULL; + db_copy_from_comp(tb, org, &hp, &tmp_offheap); + res->first_oh = tmp_offheap.first; + res->size = org->size; +#ifdef DEBUG_CLONE + res->debug_clone = NULL; +#endif + return res; +} + +void db_free_tmp_uncompressed(DbTerm* obj) +{ + ErlOffHeap off_heap; + off_heap.first = obj->first_oh; + erts_cleanup_offheap(&off_heap); +#ifdef DEBUG_CLONE + ASSERT(obj->debug_clone == NULL); +#endif + erts_free(ERTS_ALC_T_TMP, obj); +} + +Eterm db_prog_match_and_copy(DbTableCommon* tb, Process* c_p, Binary* bprog, + int all, DbTerm* obj, Eterm** hpp, Uint extra) +{ + Uint32 dummy; + Eterm res; + + if (tb->compress) { + obj = db_alloc_tmp_uncompressed(tb, obj); + } + + res = db_prog_match(c_p, bprog, make_tuple(obj->tpl), NULL, 0, &dummy); + + if (is_value(res) && hpp!=NULL) { + if (all) { + *hpp = HAlloc(c_p, obj->size + extra); + res = copy_shallow(obj->tpl, obj->size, hpp, &MSO(c_p)); + } + else { + Uint sz = size_object(res); + *hpp = HAlloc(c_p, sz + extra); + res = copy_struct(res, sz, hpp, &MSO(c_p)); + } + } + + if (tb->compress) { + db_free_tmp_uncompressed(obj); + } + return res; +} + + #ifdef DMC_DEBUG /* ** Disassemble match program diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 382e5dceb5..58ad39d772 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -52,22 +52,27 @@ is broken.*/ #define DB_ERROR_UNSPEC -10 /* Unspecified error */ +/*#define DEBUG_CLONE*/ /* * A datatype for a database entry stored out of a process heap */ typedef struct db_term { - ErlOffHeap off_heap; /* Off heap data for term. */ - Uint size; /* Size of term in "words" */ - Eterm tpl[1]; /* Untagged "constant pointer" to top tuple */ - /* (assumed to be first in buffer) */ + struct erl_off_heap_header* first_oh; /* Off heap data for term. */ + Uint size; /* Heap size of term in "words" */ +#ifdef DEBUG_CLONE + Eterm* debug_clone; /* An uncompressed copy */ +#endif + Eterm tpl[1]; /* Term data. Top tuple always first */ + + /* Compression: is_immed and key element are uncompressed. + Compressed elements are stored in external format after each other + last in dbterm. The top tuple elements contains byte offsets, to + the start of the data, tagged as headers. + The allocated size of the dbterm in bytes is stored at tpl[arity+1]. + */ } DbTerm; -/* "Assign" a value to DbTerm.tpl */ -#define DBTERM_SET_TPL(dbtermPtr,tplPtr) ASSERT((tplPtr)==(dbtermPtr->tpl)) -/* Get start of term buffer */ -#define DBTERM_BUF(dbtermPtr) ((dbtermPtr)->tpl) - union db_table; typedef union db_table DbTable; @@ -186,6 +191,12 @@ typedef struct db_table_method } DbTableMethod; +typedef struct db_fixation { + Eterm pid; + Uint counter; + struct db_fixation *next; +} DbFixation; + /* * This structure contains data for all different types of database * tables. Note that these fields must match the same fields @@ -194,16 +205,8 @@ typedef struct db_table_method * operations may be the same on different types of tables. */ -typedef struct db_fixation { - Eterm pid; - Uint counter; - struct db_fixation *next; -} DbFixation; - - typedef struct db_table_common { - erts_refc_t ref; - erts_refc_t fixref; /* fixation counter */ + erts_refc_t ref; /* fixation counter and delete counter */ #ifdef ERTS_SMP erts_smp_rwmtx_t rwlock; /* rw lock on table */ erts_smp_mtx_t fixlock; /* Protects fixations,megasec,sec,microsec */ @@ -226,6 +229,7 @@ typedef struct db_table_common { Uint32 status; /* bit masks defined below */ int slot; /* slot index in meta_main_tab */ int keypos; /* defaults to 1 */ + int compress; } DbTableCommon; /* These are status bit patterns */ @@ -240,17 +244,66 @@ typedef struct db_table_common { #define DB_DUPLICATE_BAG (1 << 8) #define DB_ORDERED_SET (1 << 9) #define DB_DELETE (1 << 10) /* table is being deleted */ +#define DB_FREQ_READ (1 << 11) -#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED) +#define ERTS_ETS_TABLE_TYPES (DB_BAG|DB_SET|DB_DUPLICATE_BAG|DB_ORDERED_SET|DB_FINE_LOCKED|DB_FREQ_READ) #define IS_HASH_TABLE(Status) (!!((Status) & \ (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) #define IS_TREE_TABLE(Status) (!!((Status) & \ DB_ORDERED_SET)) -#define NFIXED(T) (erts_refc_read(&(T)->common.fixref,0)) +#define NFIXED(T) (erts_refc_read(&(T)->common.ref,0)) #define IS_FIXED(T) (NFIXED(T) != 0) Eterm erts_ets_copy_object(Eterm, Process*); +Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, + ErlOffHeap* off_heap); +int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b); +DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org); + +ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp, + Eterm** hpp, ErlOffHeap* off_heap); +ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b); +ERTS_GLB_INLINE Eterm db_do_read_element(DbUpdateHandle* handle, Sint position); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp, + Eterm** hpp, ErlOffHeap* off_heap) +{ + if (tb->compress) { + return db_copy_from_comp(tb, bp, hpp, off_heap); + } + else { + return copy_shallow(bp->tpl, bp->size, hpp, off_heap); + } +} + +ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) +{ + if (!tb->compress) { + return eq(a, make_tuple(b->tpl)); + } + else { + return db_eq_comp(tb, a, b); + } +} + +/* Must be called to read elements after db_lookup_dbterm. +** Will decompress if needed. */ +ERTS_GLB_INLINE Eterm db_do_read_element(DbUpdateHandle* handle, Sint position) +{ + Eterm elem = handle->dbterm->tpl[position]; + if (!is_header(elem)) { + return elem; + } + ASSERT(((DbTableCommon*)handle->tb)->compress); + ASSERT(!handle->mustResize); + handle->dbterm = db_alloc_tmp_uncompressed((DbTableCommon*)handle->tb, handle->dbterm); + handle->mustResize = 1; + return handle->dbterm->tpl[position]; +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ /* optimised version of copy_object (normal case? atomic object) */ #define COPY_OBJECT(obj, p, objp) \ @@ -276,14 +329,19 @@ Eterm db_set_trace_control_word_1(Process *p, Eterm val); void db_initialize_util(void); Eterm db_getkey(int keypos, Eterm obj); -void db_free_term_data(DbTerm* p); -void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +void db_cleanup_offheap_comp(DbTerm* p); +void db_free_term(DbTable *tb, void* basep, Uint offset); +void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj, + Uint pos, Eterm** hpp, Uint extra); int db_has_variable(Eterm obj); int db_is_variable(Eterm obj); +Eterm db_do_read_element(DbUpdateHandle* handle, Sint position); void db_do_update_element(DbUpdateHandle* handle, Sint position, Eterm newval); -void db_finalize_update_element(DbUpdateHandle* handle); +void db_finalize_resize(DbUpdateHandle* handle, Uint offset); Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr); Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags); Binary *db_match_set_compile(Process *p, Eterm matchexpr, @@ -365,6 +423,8 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Eterm *body, int num_matches, Uint flags, DMCErrInfo *err_info); +Eterm db_prog_match_and_copy(DbTableCommon* tb, Process* c_p, Binary* bprog, + int all, DbTerm* obj, Eterm** hpp, Uint extra); /* Returns newly allocated MatchProg binary with refc == 0*/ Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm *termp, int arity, Uint32 *return_flags /* Zeroed on enter */); diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 58d3f92f56..d7d6fcf0a2 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -261,7 +261,7 @@ static int verify_eterm(Process *p,Eterm element) return 1; for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next) { - if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) { + if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) { return 1; } } @@ -308,7 +308,7 @@ void erts_check_stack(Process *p) if (IN_HEAP(p, ptr)) continue; for (mbuf = p->mbuf; mbuf; mbuf = mbuf->next) - if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->size)) { + if (WITHIN(ptr, &mbuf->mem[0], &mbuf->mem[0] + mbuf->used_size)) { in_mbuf = 1; break; } @@ -746,7 +746,7 @@ static void print_process_memory(Process *p) PTR_SIZE, "heap fragments", dashes, dashes, dashes, dashes); while (bp) { - print_untagged_memory(bp->mem,bp->mem + bp->size); + print_untagged_memory(bp->mem,bp->mem + bp->used_size); bp = bp->next; } } diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 9733c0e5b5..13a73e01bb 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -150,6 +150,27 @@ typedef struct { #define ERL_DRV_FLAG_SOFT_BUSY (1 << 1) /* + * Integer types + */ + +typedef unsigned long ErlDrvTermData; +typedef unsigned long ErlDrvUInt; +typedef signed long ErlDrvSInt; + +#if defined(__WIN32__) +typedef unsigned __int64 ErlDrvUInt64; +typedef __int64 ErlDrvSInt64; +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlDrvUInt64; +typedef long ErlDrvSInt64; +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlDrvUInt64; +typedef long long ErlDrvSInt64; +#else +#error No 64-bit integer type +#endif + +/* * A binary as seen in a driver. Note that a binary should never be * altered by the driver when it has been sent to Erlang. */ @@ -179,26 +200,6 @@ struct erl_drv_event_data { #endif typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */ -/* - * Used in monitors... - */ -typedef unsigned long ErlDrvTermData; -typedef unsigned long ErlDrvUInt; -typedef signed long ErlDrvSInt; - -#if defined(__WIN32__) -typedef unsigned __int64 ErlDrvUInt64; -typedef __int64 ErlDrvSInt64; -#elif SIZEOF_LONG == 8 -typedef unsigned long ErlDrvUInt64; -typedef long ErlDrvSInt64; -#elif SIZEOF_LONG_LONG == 8 -typedef unsigned long long ErlDrvUInt64; -typedef long long ErlDrvSInt64; -#else -#error No 64-bit integer type -#endif - /* * A driver monitor */ @@ -394,9 +395,9 @@ EXTERN int driver_exit (ErlDrvPort port, int err); EXTERN ErlDrvPDL driver_pdl_create(ErlDrvPort); EXTERN void driver_pdl_lock(ErlDrvPDL); EXTERN void driver_pdl_unlock(ErlDrvPDL); -EXTERN long driver_pdl_get_refc(ErlDrvPDL); -EXTERN long driver_pdl_inc_refc(ErlDrvPDL); -EXTERN long driver_pdl_dec_refc(ErlDrvPDL); +EXTERN ErlDrvSInt driver_pdl_get_refc(ErlDrvPDL); +EXTERN ErlDrvSInt driver_pdl_inc_refc(ErlDrvPDL); +EXTERN ErlDrvSInt driver_pdl_dec_refc(ErlDrvPDL); /* * Process monitors @@ -432,9 +433,9 @@ EXTERN ErlDrvBinary* driver_realloc_binary(ErlDrvBinary *bin, int size); EXTERN void driver_free_binary(ErlDrvBinary *bin); /* Referenc count on driver binaries */ -EXTERN long driver_binary_get_refc(ErlDrvBinary *dbp); -EXTERN long driver_binary_inc_refc(ErlDrvBinary *dbp); -EXTERN long driver_binary_dec_refc(ErlDrvBinary *dbp); +EXTERN ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp); +EXTERN ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp); +EXTERN ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp); /* Allocation interface */ EXTERN void *driver_alloc(size_t size); diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 50d8c25c46..17b08a71d4 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -186,10 +186,9 @@ int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_trylock(&dmtx->mtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_mutex_trylock()"); - return res; + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_trylock()"); + return ethr_mutex_trylock(&dmtx->mtx); #else return 0; #endif @@ -199,9 +198,9 @@ void erl_drv_mutex_lock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_lock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_lock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_lock()"); + ethr_mutex_lock(&dmtx->mtx); #endif } @@ -209,9 +208,9 @@ void erl_drv_mutex_unlock(ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res = dmtx ? ethr_mutex_unlock(&dmtx->mtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_mutex_unlock()"); + if (!dmtx) + fatal_error(EINVAL, "erl_drv_mutex_unlock()"); + ethr_mutex_unlock(&dmtx->mtx); #endif } @@ -256,9 +255,9 @@ void erl_drv_cond_signal(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_signal(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_signal()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_signal()"); + ethr_cond_signal(&dcnd->cnd); #endif } @@ -266,9 +265,9 @@ void erl_drv_cond_broadcast(ErlDrvCond *dcnd) { #ifdef USE_THREADS - int res = dcnd ? ethr_cond_broadcast(&dcnd->cnd) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_cond_broadcast()"); + if (!dcnd) + fatal_error(EINVAL, "erl_drv_cond_broadcast()"); + ethr_cond_broadcast(&dcnd->cnd); #endif } @@ -277,18 +276,13 @@ void erl_drv_cond_wait(ErlDrvCond *dcnd, ErlDrvMutex *dmtx) { #ifdef USE_THREADS - int res; if (!dcnd || !dmtx) { - res = EINVAL; - error: - fatal_error(res, "erl_drv_cond_wait()"); + fatal_error(EINVAL, "erl_drv_cond_wait()"); } while (1) { - res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); + int res = ethr_cond_wait(&dcnd->cnd, &dmtx->mtx); if (res == 0) break; - if (res != EINTR) - goto error; } #endif } @@ -333,10 +327,9 @@ int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrlock()"); + return ethr_rwmutex_tryrlock(&drwlck->rwmtx); #else return 0; #endif @@ -346,9 +339,9 @@ void erl_drv_rwlock_rlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rlock()"); + ethr_rwmutex_rlock(&drwlck->rwmtx); #endif } @@ -356,9 +349,9 @@ void erl_drv_rwlock_runlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_runlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_runlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_runlock()"); + ethr_rwmutex_runlock(&drwlck->rwmtx); #endif } @@ -366,10 +359,9 @@ int erl_drv_rwlock_tryrwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_tryrwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0 && res != EBUSY) - fatal_error(res, "erl_drv_rwlock_tryrwlock()"); - return res; + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_tryrwlock()"); + return ethr_rwmutex_tryrwlock(&drwlck->rwmtx); #else return 0; #endif @@ -379,9 +371,9 @@ void erl_drv_rwlock_rwlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwlock()"); + ethr_rwmutex_rwlock(&drwlck->rwmtx); #endif } @@ -389,9 +381,9 @@ void erl_drv_rwlock_rwunlock(ErlDrvRWLock *drwlck) { #ifdef USE_THREADS - int res = drwlck ? ethr_rwmutex_rwunlock(&drwlck->rwmtx) : EINVAL; - if (res != 0) - fatal_error(res, "erl_drv_rwlock_rwunlock()"); + if (!drwlck) + fatal_error(EINVAL, "erl_drv_rwlock_rwunlock()"); + ethr_rwmutex_rwunlock(&drwlck->rwmtx); #endif } @@ -536,7 +528,7 @@ erl_drv_tsd_get(ErlDrvTSDKey key) if (!dtid) return NULL; #endif - if (ERL_DRV_TSD_LEN__ < key) + if (ERL_DRV_TSD_LEN__ <= key) return NULL; return ERL_DRV_TSD__[key]; } @@ -603,11 +595,7 @@ erl_drv_thread_create(char *name, dtid->name = ((char *) dtid) + sizeof(struct ErlDrvTid_); sys_strcpy(dtid->name, name); } -#ifdef ERTS_ENABLE_LOCK_COUNT - res = erts_lcnt_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts); -#else res = ethr_thr_create(&dtid->tid, erl_drv_thread_wrapper, dtid, use_opts); -#endif if (res != 0) { erts_free(ERTS_ALC_T_DRV_TID, dtid); diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 15d9538301..88947b5536 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -37,8 +37,6 @@ static erts_smp_rwmtx_t erts_fun_table_lock; #define erts_fun_read_unlock() erts_smp_rwmtx_runlock(&erts_fun_table_lock) #define erts_fun_write_lock() erts_smp_rwmtx_rwlock(&erts_fun_table_lock) #define erts_fun_write_unlock() erts_smp_rwmtx_rwunlock(&erts_fun_table_lock) -#define erts_fun_init_lock() erts_smp_rwmtx_init(&erts_fun_table_lock, \ - "fun_tab") static HashValue fun_hash(ErlFunEntry* obj); static int fun_cmp(ErlFunEntry* obj1, ErlFunEntry* obj2); @@ -57,8 +55,12 @@ void erts_init_fun_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&erts_fun_table_lock, &rwmtx_opt, "fun_tab"); - erts_fun_init_lock(); f.hash = (H_FUN) fun_hash; f.cmp = (HCMP_FUN) fun_cmp; f.alloc = (HALLOC_FUN) fun_alloc; @@ -95,7 +97,7 @@ erts_put_fun_entry(Eterm mod, int uniq, int index) { ErlFunEntry template; ErlFunEntry* fe; - long refc; + erts_aint_t refc; ASSERT(is_atom(mod)); template.old_uniq = uniq; template.old_index = index; @@ -117,7 +119,7 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, { ErlFunEntry template; ErlFunEntry* fe; - long refc; + erts_aint_t refc; ASSERT(is_atom(mod)); template.old_uniq = old_uniq; @@ -155,7 +157,7 @@ erts_get_fun_entry(Eterm mod, int uniq, int index) erts_fun_read_lock(); ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template); if (ret) { - long refc = erts_refc_inctest(&ret->refc, 1); + erts_aint_t refc = erts_refc_inctest(&ret->refc, 1); if (refc < 2) /* Pending delete */ erts_refc_inc(&ret->refc, 1); } @@ -192,20 +194,6 @@ erts_erase_fun_entry(ErlFunEntry* fe) erts_fun_write_unlock(); } -#ifndef HYBRID /* FIND ME! */ -void -erts_cleanup_funs(ErlFunThing* funp) -{ - while (funp) { - ErlFunEntry* fe = funp->fe; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - erts_erase_fun_entry(fe); - } - funp = funp->next; - } -} -#endif - void erts_cleanup_funs_on_purge(BeamInstr* start, BeamInstr* end) { @@ -269,7 +257,7 @@ erts_dump_fun_entries(int to, void *to_arg) #ifdef HIPE erts_print(to, to_arg, "Native_address: %p\n", fe->native_address); #endif - erts_print(to, to_arg, "Refc: %d\n", erts_refc_read(&fe->refc, 1)); + erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1)); b = b->next; } } diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index 944d4b3df5..2f165afa06 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -53,10 +53,10 @@ typedef struct erl_fun_entry { typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ + ErlFunEntry* fe; /* Pointer to fun entry. */ #ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* next; /* Next fun in mso list. */ + struct erl_off_heap_header* next; #endif - ErlFunEntry* fe; /* Pointer to fun entry. */ #ifdef HIPE UWord* native_address; /* Native code for the fun. */ #endif diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 9ed566e66e..2aa932e7d1 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -110,9 +110,7 @@ static Uint adjust_after_fullsweep(Process *p, int size_before, int need, Eterm *objv, int nobj); static void shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj); static void grow_new_heap(Process *p, Uint new_sz, Eterm* objv, int nobj); -static void sweep_proc_bins(Process *p, int fullsweep); -static void sweep_proc_funs(Process *p, int fullsweep); -static void sweep_proc_externals(Process *p, int fullsweep); +static void sweep_off_heap(Process *p, int fullsweep); static void offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size); static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, @@ -145,6 +143,16 @@ erts_init_gc(void) { int i = 0; + ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); + ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); + ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); + ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size)); + ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size)); + ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size)); + ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); + ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); + ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); + erts_smp_spinlock_init(&info_lck, "gc_info"); garbage_cols = 0; reclaimed = 0; @@ -286,25 +294,14 @@ erts_offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, offset_heap_ptr(hp, sz, offs, (char *) low, ((char *)high)-((char *)low)); } + #define ptr_within(ptr, low, high) ((ptr) < (high) && (ptr) >= (low)) void erts_offset_off_heap(ErlOffHeap *ohp, Sint offs, Eterm* low, Eterm* high) { - if (ohp->mso && ptr_within((Eterm *)ohp->mso, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->mso; - *uptr += offs; - } - -#ifndef HYBRID /* FIND ME! */ - if (ohp->funs && ptr_within((Eterm *)ohp->funs, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->funs; - *uptr += offs; - } -#endif - - if (ohp->externals && ptr_within((Eterm *)ohp->externals, low, high)) { - Eterm** uptr = (Eterm**) (void *) &ohp->externals; + if (ohp->first && ptr_within((Eterm *)ohp->first, low, high)) { + Eterm** uptr = (Eterm**) (void *) &ohp->first; *uptr += offs; } } @@ -504,14 +501,8 @@ erts_garbage_collect_hibernate(Process* p) cleanup_rootset(&rootset); - if (MSO(p).mso) { - sweep_proc_bins(p, 1); - } - if (MSO(p).funs) { - sweep_proc_funs(p, 1); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 1); + if (MSO(p).first) { + sweep_off_heap(p, 1); } /* @@ -667,7 +658,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) case TAG_PRIMARY_BOXED: ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, area, area_size)) { @@ -679,7 +670,7 @@ erts_garbage_collect_literals(Process* p, Eterm* literals, Uint lit_size) case TAG_PRIMARY_LIST: ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { /* Moved */ + if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; } else if (in_area(ptr, area, area_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); @@ -752,7 +743,10 @@ minor_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) * is large enough. */ - if (OLD_HEAP(p) && mature <= OLD_HEND(p) - OLD_HTOP(p)) { + if (OLD_HEAP(p) && + ((mature <= OLD_HEND(p) - OLD_HTOP(p)) && + ((BIN_VHEAP_MATURE(p) < ( BIN_OLD_VHEAP_SZ(p) - BIN_OLD_VHEAP(p)))) && + ((BIN_OLD_VHEAP_SZ(p) > BIN_OLD_VHEAP(p))) ) ) { ErlMessage *msgp; Uint size_after; Uint need_after; @@ -913,7 +907,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, heap, mature_size)) { @@ -929,7 +923,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { /* Moved */ + if (IS_MOVED_CONS(val)) { /* Moved */ *g_ptr++ = ptr[1]; } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr,val,old_htop,g_ptr++); @@ -972,7 +966,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, heap, mature_size)) { @@ -987,7 +981,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr,val,old_htop,n_hp++); @@ -1008,7 +1002,7 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) Eterm* origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(val); } else if (in_area(ptr, heap, mature_size)) { @@ -1041,15 +1035,8 @@ do_minor(Process *p, int new_sz, Eterm* objv, int nobj) OLD_HTOP(p) = old_htop; HIGH_WATER(p) = (HEAP_START(p) != HIGH_WATER(p)) ? n_heap : n_htop; - if (MSO(p).mso) { - sweep_proc_bins(p, 0); - } - - if (MSO(p).funs) { - sweep_proc_funs(p, 0); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 0); + if (MSO(p).first) { + sweep_off_heap(p, 0); } #ifdef HARDDEBUG @@ -1161,7 +1148,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { @@ -1175,7 +1162,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,g_ptr++); @@ -1216,7 +1203,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { @@ -1229,7 +1216,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { MOVE_CONS(ptr,val,n_htop,n_hp++); @@ -1249,7 +1236,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(*origptr); } else if (in_area(ptr, src, src_size) || @@ -1271,17 +1258,11 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) } } - if (MSO(p).mso) { - sweep_proc_bins(p, 1); - } - if (MSO(p).funs) { - sweep_proc_funs(p, 1); - } - if (MSO(p).externals) { - sweep_proc_externals(p, 1); + if (MSO(p).first) { + sweep_off_heap(p, 1); } - if (OLD_HEAP(p) != NULL) { + if (OLD_HEAP(p) != NULL) { ERTS_HEAP_FREE(ERTS_ALC_T_OLD_HEAP, OLD_HEAP(p), (OLD_HEND(p) - OLD_HEAP(p)) * sizeof(Eterm)); @@ -1305,6 +1286,7 @@ major_collection(Process* p, int need, Eterm* objv, int nobj, Uint *recl) HIGH_WATER(p) = HEAP_TOP(p); ErtsGcQuickSanityCheck(p); + /* * Copy newly received message onto the end of the new heap. */ @@ -1392,17 +1374,12 @@ combined_message_size(Process* p) static void remove_message_buffers(Process* p) { - ErlHeapFragment* bp = MBUF(p); - - MBUF(p) = NULL; - MBUF_SIZE(p) = 0; - while (bp != NULL) { - ErlHeapFragment* next_bp = bp->next; - free_message_buffer(bp); - bp = next_bp; - } + if (MBUF(p) != NULL) { + free_message_buffer(MBUF(p)); + MBUF(p) = NULL; + } + MBUF_SIZE(p) = 0; } - #ifdef HARDDEBUG /* @@ -1433,12 +1410,12 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj) case TAG_PRIMARY_BOXED: { ptr = _unchecked_boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); objv++; } else { for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1450,11 +1427,11 @@ disallow_heap_frag_ref(Process* p, Eterm* n_htop, Eterm* objv, int nobj) case TAG_PRIMARY_LIST: { ptr = _unchecked_list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { objv++; } else { for (qb = mbuf; qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1499,7 +1476,7 @@ disallow_heap_frag_ref_in_heap(Process* p) ptr = _unchecked_boxed_val(val); if (!in_area(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1509,7 +1486,7 @@ disallow_heap_frag_ref_in_heap(Process* p) ptr = _unchecked_list_val(val); if (!in_area(ptr, heap, heap_size)) { for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1557,7 +1534,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p) abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1570,7 +1547,7 @@ disallow_heap_frag_ref_in_old_heap(Process* p) abort(); } for (qb = MBUF(p); qb != NULL; qb = qb->next) { - if (in_area(ptr, qb->mem, qb->size*sizeof(Eterm))) { + if (in_area(ptr, qb->mem, qb->alloc_size*sizeof(Eterm))) { abort(); } } @@ -1610,7 +1587,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *g_ptr++ = val; } else if (in_area(ptr, src, src_size)) { @@ -1623,7 +1600,7 @@ sweep_rootset(Rootset* rootset, Eterm* htop, char* src, Uint src_size) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { /* Moved */ + if (IS_MOVED_CONS(val)) { *g_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size)) { MOVE_CONS(ptr,val,htop,g_ptr++); @@ -1657,7 +1634,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *n_hp++ = val; } else if (in_area(ptr, src, src_size)) { @@ -1670,7 +1647,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *n_hp++ = ptr[1]; } else if (in_area(ptr, src, src_size)) { MOVE_CONS(ptr,val,n_htop,n_hp++); @@ -1690,7 +1667,7 @@ sweep_one_area(Eterm* n_hp, Eterm* n_htop, char* src, Uint src_size) origptr = &(mb->orig); ptr = boxed_val(*origptr); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { *origptr = val; mb->base = binary_bytes(*origptr); } else if (in_area(ptr, src, src_size)) { @@ -1722,7 +1699,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr case TAG_PRIMARY_BOXED: { ptr = boxed_val(gval); val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *heap_ptr++ = val; } else if (in_area(ptr, src, src_size)) { @@ -1735,7 +1712,7 @@ sweep_one_heap(Eterm* heap_ptr, Eterm* heap_end, Eterm* htop, char* src, Uint sr case TAG_PRIMARY_LIST: { ptr = list_val(gval); val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *heap_ptr++ = ptr[1]; } else if (in_area(ptr, src, src_size)) { MOVE_CONS(ptr,val,htop,heap_ptr++); @@ -1830,28 +1807,6 @@ collect_heap_frags(Process* p, Eterm* n_hstart, Eterm* n_htop, return n_htop; } -#ifdef DEBUG -static Eterm follow_moved(Eterm term) -{ - Eterm* ptr; - switch (primary_tag(term)) { - case TAG_PRIMARY_IMMED1: - break; - case TAG_PRIMARY_BOXED: - ptr = boxed_val(term); - if (IS_MOVED(*ptr)) term = *ptr; - break; - case TAG_PRIMARY_LIST: - ptr = list_val(term); - if (is_non_value(ptr[0])) term = ptr[1]; - break; - default: - abort(); - } - return term; -} -#endif - static Uint setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) { @@ -2030,8 +1985,8 @@ shrink_new_heap(Process *p, Uint new_sz, Eterm *objv, int nobj) HEAP_SIZE(p) = new_sz; } -static Uint -do_next_vheap_size(Uint vheap, Uint vheap_sz) { +static Uint64 +do_next_vheap_size(Uint64 vheap, Uint64 vheap_sz) { /* grow * @@ -2048,131 +2003,53 @@ do_next_vheap_size(Uint vheap, Uint vheap_sz) { * ---------------------- */ - if (vheap > (Uint) (vheap_sz*3/4)) { + if ((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) { + Uint64 new_vheap_sz = vheap_sz; - while(vheap > (Uint) (vheap_sz*3/4)) { - vheap_sz = vheap_sz*2; + while((Uint64) vheap/3 > (Uint64) (vheap_sz/4)) { + /* the golden ratio = 1.618 */ + new_vheap_sz = (Uint64) vheap_sz * 1.618; + if (new_vheap_sz < vheap_sz ) { + return vheap_sz; + } + vheap_sz = new_vheap_sz; } - return erts_next_heap_size(vheap_sz, 0); + return vheap_sz; } - if (vheap < (Uint) (vheap_sz/4)) { - return erts_next_heap_size((Uint) (vheap_sz / 2), 0); + if (vheap < (Uint64) (vheap_sz/4)) { + return (vheap_sz >> 1); } return vheap_sz; } -static Uint -next_vheap_size(Process* p, Uint vheap, Uint vheap_sz) { - vheap_sz = do_next_vheap_size(vheap, vheap_sz); - return vheap_sz < p->min_vheap_size ? p->min_vheap_size : vheap_sz; -} - -static void -sweep_proc_externals(Process *p, int fullsweep) -{ - ExternalThing** prev; - ExternalThing* ptr; - char* oh = 0; - Uint oh_size = 0; - - if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; - } - - prev = &MSO(p).externals; - ptr = MSO(p).externals; - - while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED(*ppt)) { /* Object is alive */ - ExternalThing* ro = external_thing_ptr(*ppt); - - *prev = ro; /* Patch to moved pos */ - prev = &ro->next; - ptr = ro->next; - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - prev = &ptr->next; - ptr = ptr->next; - } else { /* Object has not been moved - deref it */ - erts_deref_node_entry(ptr->node); - *prev = ptr = ptr->next; - } - } - ASSERT(*prev == NULL); -} - -static void -sweep_proc_funs(Process *p, int fullsweep) -{ - ErlFunThing** prev; - ErlFunThing* ptr; - char* oh = 0; - Uint oh_size = 0; - - if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; - } - - prev = &MSO(p).funs; - ptr = MSO(p).funs; - - while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED(*ppt)) { /* Object is alive */ - ErlFunThing* ro = (ErlFunThing *) fun_val(*ppt); - - *prev = ro; /* Patch to moved pos */ - prev = &ro->next; - ptr = ro->next; - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - prev = &ptr->next; - ptr = ptr->next; - } else { /* Object has not been moved - deref it */ - ErlFunEntry* fe = ptr->fe; - - *prev = ptr = ptr->next; - if (erts_refc_dectest(&fe->refc, 0) == 0) { - erts_erase_fun_entry(fe); - } - } - } - ASSERT(*prev == NULL); +static Uint64 +next_vheap_size(Process* p, Uint64 vheap, Uint64 vheap_sz) { + Uint64 new_vheap_sz = do_next_vheap_size(vheap, vheap_sz); + return new_vheap_sz < p->min_vheap_size ? p->min_vheap_size : new_vheap_sz; } struct shrink_cand_data { - ProcBin* new_candidates; - ProcBin* new_candidates_end; - ProcBin* old_candidates; + struct erl_off_heap_header* new_candidates; + struct erl_off_heap_header* new_candidates_end; + struct erl_off_heap_header* old_candidates; Uint no_of_candidates; Uint no_of_active; }; static ERTS_INLINE void link_live_proc_bin(struct shrink_cand_data *shrink, - ProcBin ***prevppp, - ProcBin **pbpp, + struct erl_off_heap_header*** prevppp, + struct erl_off_heap_header** currpp, int new_heap) { - ProcBin *pbp = *pbpp; - - *pbpp = pbp->next; + ProcBin *pbp = (ProcBin*) *currpp; + ASSERT(**prevppp == *currpp); + *currpp = pbp->next; if (pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) { ASSERT(((pbp->flags & (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) == (PB_ACTIVE_WRITER|PB_IS_WRITABLE)) @@ -2189,15 +2066,16 @@ link_live_proc_bin(struct shrink_cand_data *shrink, /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ if (unused >= 8) { /* A shrink candidate; save in candidate list */ + **prevppp = pbp->next; if (new_heap) { if (!shrink->new_candidates) - shrink->new_candidates_end = pbp; + shrink->new_candidates_end = (struct erl_off_heap_header*)pbp; pbp->next = shrink->new_candidates; - shrink->new_candidates = pbp; + shrink->new_candidates = (struct erl_off_heap_header*)pbp; } else { pbp->next = shrink->old_candidates; - shrink->old_candidates = pbp; + shrink->old_candidates = (struct erl_off_heap_header*)pbp; } shrink->no_of_candidates++; return; @@ -2205,83 +2083,117 @@ link_live_proc_bin(struct shrink_cand_data *shrink, } } - /* Not a shrink candidate; keep in original mso list */ - **prevppp = pbp; + /* Not a shrink candidate; keep in original mso list */ *prevppp = &pbp->next; - } -static void -sweep_proc_bins(Process *p, int fullsweep) +static void +sweep_off_heap(Process *p, int fullsweep) { struct shrink_cand_data shrink = {0}; - ProcBin** prev; - ProcBin* ptr; - Binary* bptr; - char* oh = NULL; - Uint oh_size = 0; - Uint bin_vheap = 0; + struct erl_off_heap_header* ptr; + struct erl_off_heap_header** prev; + char* oheap = NULL; + Uint oheap_sz = 0; + Uint64 bin_vheap = 0; +#ifdef DEBUG + int seen_mature = 0; +#endif if (fullsweep == 0) { - oh = (char *) OLD_HEAP(p); - oh_size = (char *) OLD_HEND(p) - oh; + oheap = (char *) OLD_HEAP(p); + oheap_sz = (char *) OLD_HEND(p) - oheap; } BIN_OLD_VHEAP(p) = 0; - prev = &MSO(p).mso; - ptr = MSO(p).mso; + prev = &MSO(p).first; + ptr = MSO(p).first; - /* - * Note: In R7 we no longer force a fullsweep when we find binaries - * on the old heap. The reason is that with the introduction of the - * bit syntax we can expect binaries to be used a lot more. Note that - * in earlier releases a brand new binary (or any other term) could - * be put on the old heap during a gen-gc fullsweep, but this is - * no longer the case in R7. + /* Firts part of the list will reside on the (old) new-heap. + * Keep if moved, otherwise deref. */ while (ptr) { - Eterm* ppt = (Eterm *) ptr; - - if (IS_MOVED(*ppt)) { /* Object is alive */ - bin_vheap += ptr->size / sizeof(Eterm); - ptr = (ProcBin*) binary_val(*ppt); - link_live_proc_bin(&shrink, - &prev, - &ptr, - !in_area(ptr, oh, oh_size)); - } else if (in_area(ppt, oh, oh_size)) { - /* - * Object resides on old heap, and we just did a - * generational collection - keep object in list. - */ - BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ - link_live_proc_bin(&shrink, &prev, &ptr, 0); - } else { /* Object has not been moved - deref it */ - - *prev = ptr->next; - bptr = ptr->val; - if (erts_refc_dectest(&bptr->refc, 0) == 0) - erts_bin_free(bptr); - ptr = *prev; - } + if (IS_MOVED_BOXED(ptr->thing_word)) { + ASSERT(!in_area(ptr, oheap, oheap_sz)); + *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); + ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); + if (ptr->thing_word == HEADER_PROC_BIN) { + int to_new_heap = !in_area(ptr, oheap, oheap_sz); + ASSERT(to_new_heap == !seen_mature || (!to_new_heap && (seen_mature=1))); + if (to_new_heap) { + bin_vheap += ptr->size / sizeof(Eterm); + } else { + BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ + } + link_live_proc_bin(&shrink, &prev, &ptr, to_new_heap); + } + else { + prev = &ptr->next; + ptr = ptr->next; + } + } + else if (!in_area(ptr, oheap, oheap_sz)) { + /* garbage */ + switch (thing_subtag(ptr->thing_word)) { + case REFC_BINARY_SUBTAG: + { + Binary* bptr = ((ProcBin*)ptr)->val; + if (erts_refc_dectest(&bptr->refc, 0) == 0) { + erts_bin_free(bptr); + } + break; + } + case FUN_SUBTAG: + { + ErlFunEntry* fe = ((ErlFunThing*)ptr)->fe; + if (erts_refc_dectest(&fe->refc, 0) == 0) { + erts_erase_fun_entry(fe); + } + break; + } + default: + ASSERT(is_external_header(ptr->thing_word)); + erts_deref_node_entry(((ExternalThing*)ptr)->node); + } + *prev = ptr = ptr->next; + } + else break; /* and let old-heap loop continue */ } - if (BIN_OLD_VHEAP(p) >= BIN_OLD_VHEAP_SZ(p)) { - FLAGS(p) |= F_NEED_FULLSWEEP; + /* The rest of the list resides on old-heap, and we just did a + * generational collection - keep objects in list. + */ + while (ptr) { + ASSERT(in_area(ptr, oheap, oheap_sz)); + ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); + if (ptr->thing_word == HEADER_PROC_BIN) { + BIN_OLD_VHEAP(p) += ptr->size / sizeof(Eterm); /* for binary gc (words)*/ + link_live_proc_bin(&shrink, &prev, &ptr, 0); + } + else { + ASSERT(is_fun_header(ptr->thing_word) || + is_external_header(ptr->thing_word)); + prev = &ptr->next; + ptr = ptr->next; + } } - BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); - BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p), BIN_OLD_VHEAP_SZ(p)); - MSO(p).overhead = bin_vheap; + if (fullsweep) { + BIN_OLD_VHEAP_SZ(p) = next_vheap_size(p, BIN_OLD_VHEAP(p) + MSO(p).overhead, BIN_OLD_VHEAP_SZ(p)); + } + BIN_VHEAP_SZ(p) = next_vheap_size(p, bin_vheap, BIN_VHEAP_SZ(p)); + MSO(p).overhead = bin_vheap; + BIN_VHEAP_MATURE(p) = bin_vheap; /* * If we got any shrink candidates, check them out. */ if (shrink.no_of_candidates) { - ProcBin *candlist[] = {shrink.new_candidates, shrink.old_candidates}; + ProcBin *candlist[] = { (ProcBin*)shrink.new_candidates, + (ProcBin*)shrink.old_candidates }; Uint leave_unused = 0; int i; @@ -2293,21 +2205,21 @@ sweep_proc_bins(Process *p, int fullsweep) } for (i = 0; i < sizeof(candlist)/sizeof(candlist[0]); i++) { - - for (ptr = candlist[i]; ptr; ptr = ptr->next) { - Uint new_size = ptr->size; + ProcBin* pb; + for (pb = candlist[i]; pb; pb = (ProcBin*)pb->next) { + Uint new_size = pb->size; if (leave_unused) { new_size += (new_size * 100) / leave_unused; /* Our allocators are 8 byte aligned, i.e., shrinking with less than 8 bytes will have no real effect */ - if (new_size + 8 >= ptr->val->orig_size) + if (new_size + 8 >= pb->val->orig_size) continue; } - ptr->val = erts_bin_realloc(ptr->val, new_size); - ptr->val->orig_size = new_size; - ptr->bytes = (byte *) ptr->val->orig_bytes; + pb->val = erts_bin_realloc(pb->val, new_size); + pb->val->orig_size = new_size; + pb->bytes = (byte *) pb->val->orig_bytes; } } @@ -2316,21 +2228,20 @@ sweep_proc_bins(Process *p, int fullsweep) * We now potentially have the mso list divided into three lists: * - shrink candidates on new heap (inactive writable with unused data) * - shrink candidates on old heap (inactive writable with unused data) - * - other binaries (read only + active writable ...) + * - other binaries (read only + active writable ...) + funs and externals * * Put them back together: new candidates -> other -> old candidates * This order will ensure that the list only refers from new * generation to old and never from old to new *which is important*. */ if (shrink.new_candidates) { - if (prev == &MSO(p).mso) /* empty other binaries list */ + if (prev == &MSO(p).first) /* empty other binaries list */ prev = &shrink.new_candidates_end->next; else - shrink.new_candidates_end->next = MSO(p).mso; - MSO(p).mso = shrink.new_candidates; + shrink.new_candidates_end->next = MSO(p).first; + MSO(p).first = shrink.new_candidates; } } - *prev = shrink.old_candidates; } @@ -2361,15 +2272,17 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) tari = thing_arityval(val); switch (thing_subtag(val)) { case REFC_BINARY_SUBTAG: + case FUN_SUBTAG: + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: { - ProcBin* pb = (ProcBin*) hp; - Eterm** uptr = (Eterm **) (void *) &pb->next; + struct erl_off_heap_header* oh = (struct erl_off_heap_header*) hp; - if (*uptr && in_area((Eterm *)pb->next, area, area_size)) { + if (in_area(oh->next, area, area_size)) { + Eterm** uptr = (Eterm **) (void *) &oh->next; *uptr += offs; /* Patch the mso chain */ } - sz -= tari; - hp += tari + 1; } break; case BIN_MATCHSTATE_SUBTAG: @@ -2380,40 +2293,11 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) mb->orig = offset_ptr(mb->orig, offs); mb->base = binary_bytes(mb->orig); } - sz -= tari; - hp += tari + 1; } break; - case FUN_SUBTAG: - { - ErlFunThing* funp = (ErlFunThing *) hp; - Eterm** uptr = (Eterm **) (void *) &funp->next; - - if (*uptr && in_area((Eterm *)funp->next, area, area_size)) { - *uptr += offs; - } - sz -= tari; - hp += tari + 1; - } - break; - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - { - ExternalThing* etp = (ExternalThing *) hp; - Eterm** uptr = (Eterm **) (void *) &etp->next; - - if (*uptr && in_area((Eterm *)etp->next, area, area_size)) { - *uptr += offs; - } - sz -= tari; - hp += tari + 1; - } - break; - default: - sz -= tari; - hp += tari + 1; } + sz -= tari; + hp += tari + 1; break; } default: @@ -2450,18 +2334,8 @@ offset_heap_ptr(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size) { - if (MSO(p).mso && in_area((Eterm *)MSO(p).mso, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).mso; - *uptr += offs; - } - - if (MSO(p).funs && in_area((Eterm *)MSO(p).funs, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).funs; - *uptr += offs; - } - - if (MSO(p).externals && in_area((Eterm *)MSO(p).externals, area, area_size)) { - Eterm** uptr = (Eterm**) (void *) &MSO(p).externals; + if (MSO(p).first && in_area((Eterm *)MSO(p).first, area, area_size)) { + Eterm** uptr = (Eterm**) (void *) &MSO(p).first; *uptr += offs; } } @@ -2542,7 +2416,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) return 1; } while (bp != NULL) { - if (bp->mem <= ptr && ptr < bp->mem + bp->size) { + if (bp->mem <= ptr && ptr < bp->mem + bp->used_size) { return 1; } bp = bp->next; @@ -2556,7 +2430,7 @@ within2(Eterm *ptr, Process *p, Eterm *real_htop) hfp = erts_dist_ext_trailer(mp->data.dist_ext); else hfp = NULL; - if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->size) + if (hfp && hfp->mem <= ptr && ptr < hfp->mem + hfp->used_size) return 1; } mp = mp->next; @@ -2582,8 +2456,8 @@ do { \ __FILE__, __LINE__, #EXP); \ } while (0) -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST -# define ERTS_EXTERNAL_VISITED_BIT ((Eterm) 1 << 31) +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST +# define ERTS_OFFHEAP_VISITED_BIT ((Eterm) 1 << 31) #endif @@ -2593,62 +2467,45 @@ erts_check_off_heap2(Process *p, Eterm *htop) Eterm *oheap = (Eterm *) OLD_HEAP(p); Eterm *ohtop = (Eterm *) OLD_HTOP(p); int old; - ProcBin *pb; - ErlFunThing *eft; - ExternalThing *et; + union erl_off_heap_ptr u; old = 0; - for (pb = MSO(p).mso; pb; pb = pb->next) { - Eterm *ptr = (Eterm *) pb; - long refc = erts_refc_read(&pb->val->refc, 1); + for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) { + erts_aint_t refc; + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + refc = erts_refc_read(&u.pb->val->refc, 1); + break; + case FUN_SUBTAG: + refc = erts_refc_read(&u.fun->fe->refc, 1); + break; + case EXTERNAL_PID_SUBTAG: + case EXTERNAL_PORT_SUBTAG: + case EXTERNAL_REF_SUBTAG: + refc = erts_refc_read(&u.ext->node->refc, 1); + break; + default: + ASSERT(!!"erts_check_off_heap2: Invalid thing_word"); + } ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); +#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_LIST + ERTS_CHK_OFFHEAP_ASSERT(!(u.hdr->thing_word & ERTS_EXTERNAL_VISITED_BIT)); + u.hdr->thing_word |= ERTS_OFFHEAP_VISITED_BIT; +#endif if (old) { - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); + ERTS_CHK_OFFHEAP_ASSERT(oheap <= u.ep && u.ep < ohtop); } - else if (oheap <= ptr && ptr < ohtop) + else if (oheap <= u.ep && u.ep < ohtop) old = 1; else { - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); + ERTS_CHK_OFFHEAP_ASSERT(within2(u.ep, p, htop)); } } - old = 0; - for (eft = MSO(p).funs; eft; eft = eft->next) { - Eterm *ptr = (Eterm *) eft; - long refc = erts_refc_read(&eft->fe->refc, 1); - ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); - if (old) - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); - else if (oheap <= ptr && ptr < ohtop) - old = 1; - else - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); - } - - old = 0; - for (et = MSO(p).externals; et; et = et->next) { - Eterm *ptr = (Eterm *) et; - long refc = erts_refc_read(&et->node->refc, 1); - ERTS_CHK_OFFHEAP_ASSERT(refc >= 1); -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - ERTS_CHK_OFFHEAP_ASSERT(!(et->header & ERTS_EXTERNAL_VISITED_BIT)); -#endif - if (old) - ERTS_CHK_OFFHEAP_ASSERT(oheap <= ptr && ptr < ohtop); - else if (oheap <= ptr && ptr < ohtop) - old = 1; - else - ERTS_CHK_OFFHEAP_ASSERT(within2(ptr, p, htop)); -#ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - et->header |= ERTS_EXTERNAL_VISITED_BIT; -#endif - } - #ifdef ERTS_OFFHEAP_DEBUG_CHK_CIRCULAR_EXTERNAL_LIST - for (et = MSO(p).externals; et; et = et->next) - et->header &= ~ERTS_EXTERNAL_VISITED_BIT; + for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) + u.hdr->thing_word &= ~ERTS_OFFHEAP_VISITED_BIT; #endif - } void diff --git a/erts/emulator/beam/erl_gc.h b/erts/emulator/beam/erl_gc.h index af55b6363f..807ef8ae8d 100644 --- a/erts/emulator/beam/erl_gc.h +++ b/erts/emulator/beam/erl_gc.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -22,11 +22,12 @@ /* GC declarations shared by beam/erl_gc.c and hipe/hipe_gc.c */ -#ifdef DEBUG +#if defined(DEBUG) && !ERTS_GLB_INLINE_INCL_FUNC_DEF # define HARDDEBUG 1 #endif -#define IS_MOVED(x) (!is_header((x))) +#define IS_MOVED_BOXED(x) (!is_header((x))) +#define IS_MOVED_CONS(x) (is_non_value((x))) #define MOVE_CONS(PTR,CAR,HTOP,ORIG) \ do { \ @@ -69,4 +70,28 @@ extern Uint erts_test_long_gc_sleep; int within(Eterm *ptr, Process *p); #endif +ERTS_GLB_INLINE Eterm follow_moved(Eterm term); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Eterm follow_moved(Eterm term) +{ + Eterm* ptr; + switch (primary_tag(term)) { + case TAG_PRIMARY_IMMED1: + break; + case TAG_PRIMARY_BOXED: + ptr = boxed_val(term); + if (IS_MOVED_BOXED(*ptr)) term = *ptr; + break; + case TAG_PRIMARY_LIST: + ptr = list_val(term); + if (IS_MOVED_CONS(ptr[0])) term = ptr[1]; + break; + default: + ASSERT(!"strange tag in follow_moved"); + } + return term; +} +#endif + #endif /* __ERL_GC_H__ */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index e63ec8a3cc..0a57eb6d88 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -41,6 +41,7 @@ #include "erl_printf_term.h" #include "erl_misc_utils.h" #include "packet_parser.h" +#include "erl_cpu_topology.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -63,6 +64,8 @@ extern void ConNormalExit(void); extern void ConWaitForExit(void); #endif +static void erl_init(int ncpu); + #define ERTS_MIN_COMPAT_REL 7 #ifdef ERTS_SMP @@ -76,8 +79,6 @@ int erts_initialized = 0; static erts_tid_t main_thread; #endif -erts_cpu_info_t *erts_cpuinfo; - int erts_use_sender_punish; /* @@ -99,7 +100,7 @@ int erts_backtrace_depth; /* How many functions to show in a backtrace int erts_async_max_threads; /* number of threads for async support */ int erts_async_thread_suggested_stack_size; -erts_smp_atomic_t erts_max_gen_gcs; +erts_smp_atomic32_t erts_max_gen_gcs; Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, am_info or am_warning, am_error is @@ -228,18 +229,18 @@ void erl_error(char *fmt, va_list args) erts_vfprintf(stderr, fmt, args); } -static void early_init(int *argc, char **argv); +static int early_init(int *argc, char **argv); void erts_short_init(void) { - early_init(NULL, NULL); - erl_init(); + int ncpu = early_init(NULL, NULL); + erl_init(ncpu); erts_initialized = 1; } -void -erl_init(void) +static void +erl_init(int ncpu) { init_benchmarking(); @@ -249,12 +250,13 @@ erl_init(void) erts_init_monitors(); erts_init_gc(); - init_time(); - erts_init_process(); + erts_init_time(); + erts_init_sys_common_misc(); + erts_init_process(ncpu); erts_init_scheduling(use_multi_run_queue, no_schedulers, no_schedulers_online); - + erts_init_cpu_topology(); /* Must be after init_scheduling */ H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0); @@ -281,12 +283,13 @@ erl_init(void) init_load(); erts_init_bif(); erts_init_bif_chksum(); + erts_init_bif_binary(); erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG - erts_mseg_late_init(); /* Must be after timer (init_time()) and thread + erts_mseg_late_init(); /* Must be after timer (erts_init_time()) and thread initializations */ #endif #ifdef HIPE @@ -320,7 +323,7 @@ init_shared_memory(int argc, char **argv) #endif global_gen_gcs = 0; - global_max_gen_gcs = erts_smp_atomic_read(&erts_max_gen_gcs); + global_max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); global_gc_flags = erts_default_process_flags; erts_global_offheap.mso = NULL; @@ -335,59 +338,6 @@ init_shared_memory(int argc, char **argv) #endif } - -/* - * Create the very first process. - */ - -void -erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv) -{ - int i; - Eterm args; - Eterm pid; - Eterm* hp; - Process parent; - Process* p; - ErlSpawnOpts so; - - if (erts_find_function(modname, am_start, 1) == NULL) { - char sbuf[256]; - Atom* ap; - - ap = atom_tab(atom_val(modname)); - memcpy(sbuf, ap->name, ap->len); - sbuf[ap->len] = '\0'; - erl_exit(5, "No function %s:start/1\n", sbuf); - } - - /* - * We need a dummy parent process to be able to call erl_create_process(). - */ - erts_init_empty_process(&parent); - hp = HAlloc(&parent, argc*2 + 4); - args = NIL; - for (i = argc-1; i >= 0; i--) { - int len = sys_strlen(argv[i]); - args = CONS(hp, new_binary(&parent, (byte*)argv[i], len), args); - hp += 2; - } - args = CONS(hp, new_binary(&parent, code, size), args); - hp += 2; - args = CONS(hp, args, NIL); - - so.flags = 0; - pid = erl_create_process(&parent, modname, am_start, args, &so); - p = process_tab[internal_pid_index(pid)]; - p->group_leader = pid; - - erts_cleanup_empty_process(&parent); -} - -/* - * XXX Old way of starting. Hopefully soon obsolete. - */ - static void erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** argv) { @@ -557,10 +507,13 @@ void erts_usage(void) ERTS_MIN_COMPAT_REL, this_rel_num()); erts_fprintf(stderr, "-r force ets memory block to be moved on realloc\n"); + erts_fprintf(stderr, "-rg amount set reader groups limit\n"); erts_fprintf(stderr, "-sbt type set scheduler bind type, valid types are:\n"); erts_fprintf(stderr, " u|ns|ts|ps|s|nnts|nnps|tnnps|db\n"); erts_fprintf(stderr, "-sct cput set cpu topology,\n"); erts_fprintf(stderr, " see the erl(1) documentation for more info.\n"); + erts_fprintf(stderr, "-swt val set scheduler wakeup threshold, valid values are:\n"); + erts_fprintf(stderr, " very_low|low|medium|high|very_high.\n"); erts_fprintf(stderr, "-sss size suggested stack size in kilo words for scheduler threads,\n"); erts_fprintf(stderr, " valid range is [%d-%d]\n", ERTS_SCHED_THREAD_MIN_STACK_SIZE, @@ -582,7 +535,8 @@ void erts_usage(void) erts_fprintf(stderr, "-W<i|w> set error logger warnings mapping,\n"); erts_fprintf(stderr, " see error_logger documentation for details\n"); - + erts_fprintf(stderr, "-zdbbl size set the distribution buffer busy limit in kilobytes\n"); + erts_fprintf(stderr, " valid range is [1-%d]\n", INT_MAX/1024); erts_fprintf(stderr, "\n"); erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n"); erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n"); @@ -590,7 +544,51 @@ void erts_usage(void) erl_exit(-1, ""); } -static void +#ifdef USE_THREADS +/* + * allocators for thread lib + */ + +static void *ethr_std_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_STD, (Uint) size); +} +static void *ethr_std_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_STD, ptr, (Uint) size); +} +static void ethr_std_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_STD, ptr); +} +static void *ethr_sl_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_SL, (Uint) size); +} +static void *ethr_sl_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_SL, ptr, (Uint) size); +} +static void ethr_sl_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_SL, ptr); +} +static void *ethr_ll_alloc(size_t size) +{ + return erts_alloc_fnf(ERTS_ALC_T_ETHR_LL, (Uint) size); +} +static void *ethr_ll_realloc(void *ptr, size_t size) +{ + return erts_realloc_fnf(ERTS_ALC_T_ETHR_LL, ptr, (Uint) size); +} +static void ethr_ll_free(void *ptr) +{ + erts_free(ERTS_ALC_T_ETHR_LL, ptr); +} + +#endif + +static int early_init(int *argc, char **argv) /* * Only put things here which are * really important initialize @@ -603,6 +601,10 @@ early_init(int *argc, char **argv) /* int ncpuavail; int schdlrs; int schdlrs_onln; + int max_main_threads; + int max_reader_groups; + int reader_groups; + use_multi_run_queue = 1; erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; @@ -618,13 +620,11 @@ early_init(int *argc, char **argv) /* erts_use_sender_punish = 1; - erts_cpuinfo = erts_cpu_info_create(); - -#ifdef ERTS_SMP - ncpu = erts_get_cpu_configured(erts_cpuinfo); - ncpuonln = erts_get_cpu_online(erts_cpuinfo); - ncpuavail = erts_get_cpu_available(erts_cpuinfo); -#else + erts_pre_early_init_cpu_topology(&max_reader_groups, + &ncpu, + &ncpuonln, + &ncpuavail); +#ifndef ERTS_SMP ncpu = 1; ncpuonln = 1; ncpuavail = 1; @@ -651,7 +651,7 @@ early_init(int *argc, char **argv) /* erts_writing_erl_crash_dump = 0; #endif - erts_smp_atomic_init(&erts_max_gen_gcs, (long)((Uint16) -1)); + erts_smp_atomic32_init(&erts_max_gen_gcs, (erts_aint32_t) ((Uint16) -1)); erts_pre_init_process(); #if defined(USE_THREADS) && !defined(ERTS_SMP) @@ -679,6 +679,24 @@ early_init(int *argc, char **argv) /* } if (argv[i][0] == '-') { switch (argv[i][1]) { + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + char *arg = get_arg(sub_param+1, argv[i+1], &i); + if (sscanf(arg, "%d", &max_reader_groups) != 1) { + erts_fprintf(stderr, + "bad reader groups limit: %s\n", arg); + erts_usage(); + } + if (max_reader_groups < 0) { + erts_fprintf(stderr, + "bad reader groups limit: %d\n", + max_reader_groups); + erts_usage(); + } + } + break; + } case 'S' : { int tot, onln; char *arg = get_arg(argv[i]+2, argv[i+1], &i); @@ -747,13 +765,40 @@ early_init(int *argc, char **argv) /* erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes) -M flags. */ + /* Require allocators */ + erts_early_init_scheduling(); + erts_init_utils(); + erts_early_init_cpu_topology(no_schedulers, + &max_main_threads, + max_reader_groups, + &reader_groups); - erts_early_init_scheduling(); /* Require allocators */ - erts_init_utils(); /* Require allocators */ +#ifdef USE_THREADS + { + erts_thr_late_init_data_t elid = ERTS_THR_LATE_INIT_DATA_DEF_INITER; + elid.mem.std.alloc = ethr_std_alloc; + elid.mem.std.realloc = ethr_std_realloc; + elid.mem.std.free = ethr_std_free; + elid.mem.sl.alloc = ethr_sl_alloc; + elid.mem.sl.realloc = ethr_sl_realloc; + elid.mem.sl.free = ethr_sl_free; + elid.mem.ll.alloc = ethr_ll_alloc; + elid.mem.ll.realloc = ethr_ll_realloc; + elid.mem.ll.free = ethr_ll_free; + elid.main_threads = max_main_threads; + elid.reader_groups = reader_groups; + + erts_thr_late_init(&elid); + } +#endif #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_late_init(); #endif + +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_late_init(); +#endif #if defined(HIPE) hipe_signal_init(); /* must be done very early */ @@ -763,7 +808,10 @@ early_init(int *argc, char **argv) /* erl_sys_args(argc, argv); erts_ets_realloc_always_moves = 0; + erts_ets_always_compress = 0; + erts_dist_buf_busy_limit = ERTS_DE_BUSY_LIMIT; + return ncpu; } #ifndef ERTS_SMP @@ -797,8 +845,7 @@ erl_start(int argc, char **argv) char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; int async_max_threads = erts_async_max_threads; - - early_init(&argc, argv); + int ncpu = early_init(&argc, argv); envbufsz = sizeof(envbuf); if (erts_sys_getenv(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) @@ -809,7 +856,7 @@ erl_start(int argc, char **argv) envbufsz = sizeof(envbuf); if (erts_sys_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) { Uint16 max_gen_gcs = atoi(envbuf); - erts_smp_atomic_set(&erts_max_gen_gcs, (long) max_gen_gcs); + erts_smp_atomic32_set(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs); } envbufsz = sizeof(envbuf); @@ -861,7 +908,27 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("using display items %d\n",display_items)); break; - + case 'f': + if (!strncmp(argv[i],"-fn",3)) { + arg = get_arg(argv[i]+3, argv[i+1], &i); + switch (*arg) { + case 'u': + erts_set_user_requested_filename_encoding(ERL_FILENAME_UTF8); + break; + case 'l': + erts_set_user_requested_filename_encoding(ERL_FILENAME_LATIN1); + break; + case 'a': + erts_set_user_requested_filename_encoding(ERL_FILENAME_UNKNOWN); + default: + erts_fprintf(stderr, "bad filename encoding %s, can be (l,u or a)\n", arg); + erts_usage(); + } + break; + } else { + erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]); + erts_usage(); + } case 'l': display_loads++; break; @@ -983,15 +1050,20 @@ erl_start(int argc, char **argv) break; case 'e': - /* set maximum number of ets tables */ - arg = get_arg(argv[i]+2, argv[i+1], &i); - if (( user_requested_db_max_tabs = atoi(arg) ) < 0) { - erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg); - erts_usage(); + if (sys_strcmp("c", argv[i]+2) == 0) { + erts_ets_always_compress = 1; + } + else { + /* set maximum number of ets tables */ + arg = get_arg(argv[i]+2, argv[i+1], &i); + if (( user_requested_db_max_tabs = atoi(arg) ) < 0) { + erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("using maximum number of ets tables %d\n", + user_requested_db_max_tabs)); } - VERBOSE(DEBUG_SYSTEM, - ("using maximum number of ets tables %d\n", - user_requested_db_max_tabs)); break; case 'i': @@ -1055,7 +1127,7 @@ erl_start(int argc, char **argv) char *sub_param = argv[i]+2; if (has_prefix("bt", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); - res = erts_init_scheduler_bind_type(arg); + res = erts_init_scheduler_bind_type_string(arg); if (res != ERTS_INIT_SCHED_BIND_TYPE_SUCCESS) { switch (res) { case ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED: @@ -1080,7 +1152,7 @@ erl_start(int argc, char **argv) } else if (has_prefix("ct", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); - res = erts_init_cpu_topology(arg); + res = erts_init_cpu_topology_string(arg); if (res != ERTS_INIT_CPU_TOPOLOGY_OK) { switch (res) { case ERTS_INIT_CPU_TOPOLOGY_INVALID_ID: @@ -1123,10 +1195,20 @@ erl_start(int argc, char **argv) } else if (sys_strcmp("mrq", sub_param) == 0) use_multi_run_queue = 1; - else if (sys_strcmp("srq", sub_param) == 0) - use_multi_run_queue = 0; else if (sys_strcmp("nsp", sub_param) == 0) erts_use_sender_punish = 0; + else if (sys_strcmp("srq", sub_param) == 0) + use_multi_run_queue = 0; + else if (sys_strcmp("wt", sub_param) == 0) { + arg = get_arg(sub_param+2, argv[i+1], &i); + if (erts_sched_set_wakeup_limit(arg) != 0) { + erts_fprintf(stderr, "scheduler wakeup threshold: %s\n", + arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("scheduler wakup threshold: %s\n", arg)); + } else if (has_prefix("ss", sub_param)) { /* suggested stack size (Kilo Words) for scheduler threads */ arg = get_arg(sub_param+2, argv[i+1], &i); @@ -1241,9 +1323,17 @@ erl_start(int argc, char **argv) erts_async_thread_suggested_stack_size)); break; - case 'r': - erts_ets_realloc_always_moves = 1; + case 'r': { + char *sub_param = argv[i]+2; + if (has_prefix("g", sub_param)) { + get_arg(sub_param+1, argv[i+1], &i); + /* already handled */ + } + else { + erts_ets_realloc_always_moves = 1; + } break; + } case 'n': /* XXX obsolete */ break; case 'c': @@ -1273,6 +1363,26 @@ erl_start(int argc, char **argv) } break; + case 'z': { + char *sub_param = argv[i]+2; + int new_limit; + + if (has_prefix("dbbl", sub_param)) { + arg = get_arg(sub_param+4, argv[i+1], &i); + new_limit = atoi(arg); + if (new_limit < 1 || INT_MAX/1024 < new_limit) { + erts_fprintf(stderr, "Invalid dbbl limit: %d\n", new_limit); + erts_usage(); + } else { + erts_dist_buf_busy_limit = new_limit*1024; + } + } else { + erts_fprintf(stderr, "bad -z option %s\n", argv[i]); + erts_usage(); + } + break; + } + default: erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]); erts_usage(); @@ -1313,7 +1423,7 @@ erl_start(int argc, char **argv) boot_argc = argc - i; /* Number of arguments to init */ boot_argv = &argv[i]; - erl_init(); + erl_init(ncpu); init_shared_memory(boot_argc, boot_argv); load_preloaded(); @@ -1328,6 +1438,7 @@ erl_start(int argc, char **argv) erts_sys_main_thread(); /* May or may not return! */ #else + erts_thr_set_main_status(1, 1); set_main_stack_size(); process_main(); #endif @@ -1401,7 +1512,7 @@ system_cleanup(int exit_code) erts_cleanup_incgc(); #endif -#if defined(USE_THREADS) && !defined(ERTS_SMP) +#if defined(USE_THREADS) exit_async(); #endif #if HAVE_ERTS_MSEG diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 6ff5c1b9da..0185baee6b 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -96,16 +96,15 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "proc_status", "pid" }, { "proc_tab", NULL }, { "ports_snapshot", NULL }, - { "db_tab", "address" }, - { "db_tab_fix", "address" }, { "meta_name_tab", "address" }, { "meta_main_tab_slot", "address" }, + { "db_tab", "address" }, + { "db_tab_fix", "address" }, { "meta_main_tab_main", NULL }, { "db_hash_slot", "address" }, { "node_table", NULL }, { "dist_table", NULL }, { "sys_tracers", NULL }, - { "trace_pattern", NULL }, { "module_tab", NULL }, { "export_tab", NULL }, { "fun_tab", NULL }, @@ -120,17 +119,17 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "child_status", NULL }, #endif #ifdef __WIN32__ - { "sys_driver_data_lock", NULL }, + { "sys_driver_data_lock", NULL }, #endif - { "drv_ev_state_grow", NULL, }, + { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, { "safe_hash", "address" }, { "pollset_rm_list", NULL }, { "removed_fd_pre_alloc_lock", NULL }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, - { "cpu_bind", NULL }, { "run_queue", "address" }, + { "cpu_info", NULL }, { "pollset", "address" }, #ifdef __WIN32__ { "pollwaiter", "address" }, @@ -154,6 +153,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "instr", NULL }, { "fix_alloc", "index" }, { "alcu_allocator", "index" }, + { "alcu_delayed_free", "index" }, { "mseg", NULL }, #ifdef HALFWORD_HEAP { "pmmap", NULL }, @@ -177,18 +177,22 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "async_id", NULL }, { "pix_lock", "address" }, { "run_queues_lists", NULL }, + { "misc_aux_work_queue", "index" }, + { "misc_aux_work_pre_alloc_lock", "address" }, { "sched_stat", NULL }, + { "run_queue_sleep_list", "address" }, #endif { "alloc_thr_ix_lock", NULL }, #ifdef ERTS_SMP - { "proc_lck_wtr_alloc", NULL }, + { "proc_lck_qs_alloc", NULL }, #endif #ifdef __WIN32__ #ifdef DEBUG { "save_ops_lock", NULL }, #endif #endif - { "mtrace_buf", NULL } + { "mtrace_buf", NULL }, + { "erts_alloc_hard_debug", NULL } }; #define ERTS_LOCK_ORDER_SIZE \ @@ -200,6 +204,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { & ERTS_LC_FLG_LT_ALL \ & ~(ERTS_LC_FLG_LT_SPINLOCK|ERTS_LC_FLG_LT_RWSPINLOCK))) +static __decl_noreturn void __noreturn lc_abort(void); + static char * lock_type(Uint16 flags) { @@ -223,7 +229,7 @@ rw_op_str(Uint16 flags) return " (r)"; case ERTS_LC_FLG_LO_WRITE: erts_fprintf(stderr, "\nInternal error\n"); - abort(); + lc_abort(); default: break; } @@ -272,28 +278,18 @@ static erts_lc_free_block_t *free_blocks; #define ERTS_LC_FB_CHUNK_SIZE 10 #endif -#ifdef ETHR_HAVE_NATIVE_LOCKS static ethr_spinlock_t free_blocks_lock; -#define ERTS_LC_LOCK ethr_spin_lock -#define ERTS_LC_UNLOCK ethr_spin_unlock -#else -static ethr_mutex free_blocks_lock; -#define ERTS_LC_LOCK ethr_mutex_lock -#define ERTS_LC_UNLOCK ethr_mutex_unlock -#endif static ERTS_INLINE void lc_lock(void) { - if (ERTS_LC_LOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_lock(&free_blocks_lock); } static ERTS_INLINE void lc_unlock(void) { - if (ERTS_LC_UNLOCK(&free_blocks_lock) != 0) - abort(); + ethr_spin_unlock(&free_blocks_lock); } static ERTS_INLINE void lc_free(void *p) @@ -314,7 +310,7 @@ static void *lc_core_alloc(void) { lc_unlock(); erts_fprintf(stderr, "Lock checker out of memory!\n"); - abort(); + lc_abort(); } #else @@ -328,7 +324,7 @@ static void *lc_core_alloc(void) * ERTS_LC_FB_CHUNK_SIZE); if (!fbs) { erts_fprintf(stderr, "Lock checker failed to allocate memory!\n"); - abort(); + lc_abort(); } for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) { #ifdef DEBUG @@ -368,11 +364,11 @@ create_locked_locks(char *thread_name) { erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t)); if (!l_lcks) - abort(); + lc_abort(); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); l_lcks->tid = erts_thr_self(); l_lcks->required.first = NULL; @@ -514,7 +510,7 @@ uninitialized_lock(void) { erts_fprintf(stderr, "Performing operations on uninitialized lock!\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } static void @@ -524,7 +520,7 @@ lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "%s%s", prefix, rw_op_str(op_flags)); print_lock(" ", lck, " lock which is already locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -534,7 +530,7 @@ unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck, erts_fprintf(stderr, "Unlocking%s ", rw_op_str(op_flags)); print_lock("", lck, " lock which mismatch previous lock operation!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -542,7 +538,7 @@ unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking ", lck, " lock which is not locked by thread!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -551,7 +547,7 @@ lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) print_lock("Lock order violation occured when locking ", lck, "!\n"); print_curr_locks(l_lcks); print_lock_order(); - abort(); + lc_abort(); } static void @@ -562,7 +558,7 @@ type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks, print_lock(op, lck, "!\n"); ASSERT(l_lcks); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -614,7 +610,7 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact, } } print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -622,7 +618,7 @@ unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Unlocking required ", lck, " lock!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -630,7 +626,7 @@ unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *l { print_lock("Unrequire on ", lck, " lock not required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -638,7 +634,7 @@ require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Require on ", lck, " lock already required!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } static void @@ -646,7 +642,7 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck) { print_lock("Required ", lck, " lock not locked!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } @@ -659,13 +655,23 @@ thread_exit_handler(void) erts_fprintf(stderr, "Thread exiting while having locked locks!\n"); print_curr_locks(l_lcks); - abort(); + lc_abort(); } destroy_locked_locks(l_lcks); /* erts_tsd_set(locks_key, NULL); */ } } +static __decl_noreturn void +lc_abort(void) +{ +#ifdef __WIN32__ + DebugBreak(); +#else + abort(); +#endif +} + void erts_lc_set_thread_name(char *thread_name) { @@ -677,7 +683,7 @@ erts_lc_set_thread_name(char *thread_name) free((void *) l_lcks->thread_name); l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown"); if (!l_lcks->thread_name) - abort(); + lc_abort(); } } @@ -687,7 +693,7 @@ erts_lc_assert_failed(char *file, int line, char *assertion) erts_fprintf(stderr, "%s:%d: Lock check assertion \"%s\" failed!\n", file, line, assertion); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); return 0; } @@ -700,7 +706,7 @@ void erts_lc_fail(char *fmt, ...) va_end(args); erts_fprintf(stderr, "\n"); print_curr_locks(get_my_locked_locks()); - abort(); + lc_abort(); } @@ -720,7 +726,7 @@ erts_lc_get_lock_order_id(char *name) "(update erl_lock_check.c)\n", name); } - abort(); + lc_abort(); return (Sint16) -1; } @@ -896,6 +902,25 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len) } } +void +erts_lc_check_no_locked_of_type(Uint16 flags) +{ + erts_lc_locked_locks_t *l_lcks = get_my_locked_locks(); + if (l_lcks) { + erts_lc_locked_lock_t *l_lck = l_lcks->locked.first; + for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) { + if (l_lck->flags & flags) { + erts_fprintf(stderr, + "Locked lock of type %s found which isn't " + "allowed here!\n", + lock_type(l_lck->flags)); + print_curr_locks(l_lcks); + lc_abort(); + } + } + } +} + int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) { @@ -955,10 +980,10 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) /* We only force busy if a lock order violation would occur and when on an even millisecond. */ { - erts_thr_timeval_t time; - erts_thr_time_now(&time); + SysTimeval tv; + sys_gettimeofday(&tv); - if ((time.tv_nsec / 1000000) & 1) + if ((tv.tv_usec / 1000) & 1) return 0; } #endif @@ -1235,7 +1260,6 @@ erts_lc_init_lock(erts_lc_lock_t *lck, char *name, Uint16 flags) { lck->id = erts_lc_get_lock_order_id(name); - /* XXX:PaN What to do with the extra information? */ lck->extra = make_boxed(&lck->extra); lck->flags = flags; lck->inited = ERTS_LC_INITITALIZED; @@ -1284,13 +1308,8 @@ erts_lc_init(void) free_blocks = NULL; #endif /* #ifdef ERTS_LC_STATIC_ALLOC */ -#ifdef ETHR_HAVE_NATIVE_LOCKS if (ethr_spinlock_init(&free_blocks_lock) != 0) - abort(); -#else - if (ethr_mutex_init(&free_blocks_lock) != 0) - abort(); -#endif + lc_abort(); erts_tsd_key_create(&locks_key); } diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index d5e2ede9ac..0372e6850d 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -77,6 +77,7 @@ void erts_lc_check(erts_lc_lock_t *have, int have_len, void erts_lc_check_exact(erts_lc_lock_t *have, int have_len); void erts_lc_have_locks(int *resv, erts_lc_lock_t *lcks, int len); void erts_lc_have_lock_ids(int *resv, int *ids, int len); +void erts_lc_check_no_locked_of_type(Uint16 flags); int erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_trylock_flg(int locked, erts_lc_lock_t *lck, Uint16 op_flags); void erts_lc_lock_flg(erts_lc_lock_t *lck, Uint16 op_flags); diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 0d7e1335c1..a36c53560e 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -159,15 +159,15 @@ static char* lock_opt(Uint16 flag) { } static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char *extra) { - long int colls, tries, w_state, r_state; + erts_aint_t colls, tries, w_state, r_state; erts_lcnt_lock_stats_t *stats = NULL; char *type; int i; type = lcnt_lock_type(lock->flag); - ethr_atomic_read(&lock->r_state, &r_state); - ethr_atomic_read(&lock->w_state, &w_state); + r_state = ethr_atomic_read(&lock->r_state); + w_state = ethr_atomic_read(&lock->w_state); if (lock->flag & flag) { @@ -257,6 +257,10 @@ void erts_lcnt_init() { erts_lcnt_clear_counters(); } +void erts_lcnt_late_init() { + erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler); +} + /* list operations */ /* BEGIN ASSUMPTION: lcnt_data_lock taken */ @@ -381,7 +385,7 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { /* lock */ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { - long r_state = 0, w_state = 0; + erts_aint_t r_state = 0, w_state = 0; erts_lcnt_thread_data_t *eltd; if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; @@ -390,10 +394,10 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { ASSERT(eltd); - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); if (option & ERTS_LCNT_LO_WRITE) { - ethr_atomic_read(&lock->r_state, &r_state); + r_state = ethr_atomic_read(&lock->r_state); ethr_atomic_inc( &lock->w_state); } if (option & ERTS_LCNT_LO_READ) { @@ -414,12 +418,12 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { } void erts_lcnt_lock(erts_lcnt_lock_t *lock) { - long w_state; + erts_aint_t w_state; erts_lcnt_thread_data_t *eltd; if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ethr_atomic_inc( &lock->w_state); eltd = lcnt_get_thread_data(); @@ -467,14 +471,14 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line erts_lcnt_time_t time_wait; erts_lcnt_lock_stats_t *stats; #ifdef DEBUG - long flowstate; + erts_aint_t flowstate; #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; #ifdef DEBUG if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); } @@ -512,18 +516,18 @@ void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) { void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { #ifdef DEBUG - long w_state; - long flowstate; + erts_aint_t w_state; + erts_aint_t flowstate; #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; #ifdef DEBUG /* flowstate */ - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 1); ethr_atomic_dec( &lock->flowstate); /* write state */ - ethr_atomic_read(&lock->w_state, &w_state); + w_state = ethr_atomic_read(&lock->w_state); ASSERT(w_state > 0) #endif ethr_atomic_dec(&lock->w_state); @@ -548,13 +552,13 @@ void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) { void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { /* Determine lock_state via res instead of state */ #ifdef DEBUG - long flowstate; + erts_aint_t flowstate; #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; if (res != EBUSY) { #ifdef DEBUG - ethr_atomic_read(&lock->flowstate, &flowstate); + flowstate = ethr_atomic_read(&lock->flowstate); ASSERT(flowstate == 0); ethr_atomic_inc( &lock->flowstate); #endif @@ -570,36 +574,26 @@ void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { /* thread operations */ -static void *lcnt_thr_init(erts_lcnt_thread_data_t *eltd) { - void *(*function)(void *); - void *argument; - void *res; - function = eltd->function; - argument = eltd->argument; - - ethr_tsd_set(lcnt_thr_data_key, eltd); - - res = (void *)function(argument); - free(eltd); - return (void *)res; -} - - - -int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts) { +void erts_lcnt_thread_setup(void) { erts_lcnt_thread_data_t *eltd; - + lcnt_lock(); /* lock for thread id global update */ eltd = lcnt_thread_data_alloc(); lcnt_unlock(); - - eltd->function = function; - eltd->argument = arg; - - return ethr_thr_create(tid, (void *)lcnt_thr_init, (void *)eltd, opts); + ASSERT(eltd); + ethr_tsd_set(lcnt_thr_data_key, eltd); } +void erts_lcnt_thread_exit_handler() { + erts_lcnt_thread_data_t *eltd; + + eltd = ethr_tsd_get(lcnt_thr_data_key); + + if (eltd) { + free(eltd); + } +} /* bindings for bifs */ diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index e3044c371f..6306580ae4 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -155,11 +155,6 @@ typedef struct { erts_lcnt_time_t timer; /* timer */ int timer_set; /* bool */ int lock_in_conflict; /* bool */ - - /* function pointer */ - void *(*function)(void *); - void *argument; - } erts_lcnt_thread_data_t; /* globals */ @@ -169,6 +164,11 @@ extern Uint16 erts_lcnt_rt_options; /* function declerations */ void erts_lcnt_init(void); +void erts_lcnt_late_init(void); + +/* thread operations */ +void erts_lcnt_thread_setup(void); +void erts_lcnt_thread_exit_handler(void); /* list operations (local) */ erts_lcnt_lock_list_t *erts_lcnt_list_init(void); @@ -194,12 +194,7 @@ void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option); void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option); void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res); -/* thread operations */ - -int erts_lcnt_thr_create(ethr_tid *tid, void * (*function)(void *), void *arg, ethr_thr_opts *opts); - /* bif interface */ - Uint16 erts_lcnt_set_rt_opt(Uint16 opt); Uint16 erts_lcnt_clear_rt_opt(Uint16 opt); void erts_lcnt_clear_counters(void); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index a056fce0c5..82f272d28a 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -30,6 +30,7 @@ #include "erl_message.h" #include "erl_process.h" #include "erl_nmgc.h" +#include "erl_binary.h" ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, ErlMessage, @@ -42,6 +43,15 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(message, #undef HARD_DEBUG #endif + + + +static ERTS_INLINE int in_heapfrag(const Eterm* ptr, const ErlHeapFragment *bp) +{ + return ((unsigned)(ptr - bp->mem) < bp->used_size); +} + + void init_message(void) { @@ -81,9 +91,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, #endif ErlHeapFragment* nbp; + /* ToDo: Make use of 'used_size' to avoid realloc + when shrinking just a few words */ + #ifdef DEBUG { - Uint off_sz = size < bp->size ? size : bp->size; + Uint off_sz = size < bp->used_size ? size : bp->used_size; for (i = 0; i < brefs_size; i++) { Eterm *ptr; if (is_immed(brefs[i])) @@ -95,12 +108,12 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif - if (size == bp->size) + if (size == bp->used_size) return bp; #ifdef HARD_DEBUG dbg_brefs = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(Eterm *)*brefs_size); - dbg_bp = new_message_buffer(bp->size); + dbg_bp = new_message_buffer(bp->used_size); dbg_hp = dbg_bp->mem; dbg_tot_size = 0; for (i = 0; i < brefs_size; i++) { @@ -109,15 +122,15 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, dbg_brefs[i] = copy_struct(brefs[i], dbg_size, &dbg_hp, &dbg_bp->off_heap); } - ASSERT(dbg_tot_size == (size < bp->size ? size : bp->size)); + ASSERT(dbg_tot_size == (size < bp->used_size ? size : bp->used_size)); #endif nbp = (ErlHeapFragment*) ERTS_HEAP_REALLOC(ERTS_ALC_T_HEAP_FRAG, (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size), + ERTS_HEAP_FRAG_SIZE(bp->alloc_size), ERTS_HEAP_FRAG_SIZE(size)); if (bp != nbp) { - Uint off_sz = size < nbp->size ? size : nbp->size; + Uint off_sz = size < nbp->used_size ? size : nbp->used_size; Eterm *sp = &bp->mem[0]; Eterm *ep = sp + off_sz; Sint offs = &nbp->mem[0] - sp; @@ -135,7 +148,7 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, } #endif } - nbp->size = size; + nbp->alloc_size = size; nbp->used_size = size; #ifdef HARD_DEBUG @@ -152,26 +165,40 @@ erts_resize_message_buffer(ErlHeapFragment *bp, Uint size, void erts_cleanup_offheap(ErlOffHeap *offheap) { - if (offheap->mso) { - erts_cleanup_mso(offheap->mso); - } -#ifndef HYBRID /* FIND ME! */ - if (offheap->funs) { - erts_cleanup_funs(offheap->funs); - } -#endif - if (offheap->externals) { - erts_cleanup_externals(offheap->externals); + union erl_off_heap_ptr u; + + for (u.hdr = offheap->first; u.hdr; u.hdr = u.hdr->next) { + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { + erts_bin_free(u.pb->val); + } + break; + case FUN_SUBTAG: + if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { + erts_erase_fun_entry(u.fun->fe); + } + break; + default: + ASSERT(is_external_header(u.hdr->thing_word)); + erts_deref_node_entry(u.ext->node); + break; + } } } void free_message_buffer(ErlHeapFragment* bp) { - erts_cleanup_offheap(&bp->off_heap); - ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, - (void *) bp, - ERTS_HEAP_FRAG_SIZE(bp->size)); + ASSERT(bp != NULL); + do { + ErlHeapFragment* next_bp = bp->next; + + erts_cleanup_offheap(&bp->off_heap); + ERTS_HEAP_FREE(ERTS_ALC_T_HEAP_FRAG, (void *) bp, + ERTS_HEAP_FRAG_SIZE(bp->size)); + bp = next_bp; + }while (bp != NULL); } static ERTS_INLINE void @@ -181,43 +208,19 @@ link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp) /* Link the message buffer */ bp->next = MBUF(proc); MBUF(proc) = bp; - MBUF_SIZE(proc) += bp->size; + MBUF_SIZE(proc) += bp->used_size; FLAGS(proc) |= F_FORCE_GC; - /* Move any binaries into the process */ - if (bp->off_heap.mso != NULL) { - ProcBin** next_p = &bp->off_heap.mso; - while (*next_p != NULL) { - next_p = &((*next_p)->next); - } - *next_p = MSO(proc).mso; - MSO(proc).mso = bp->off_heap.mso; - bp->off_heap.mso = NULL; - MSO(proc).overhead += bp->off_heap.overhead; - } - - /* Move any funs into the process */ -#ifndef HYBRID - if (bp->off_heap.funs != NULL) { - ErlFunThing** next_p = &bp->off_heap.funs; + /* Move any off_heap's into the process */ + if (bp->off_heap.first != NULL) { + struct erl_off_heap_header** next_p = &bp->off_heap.first; while (*next_p != NULL) { next_p = &((*next_p)->next); } - *next_p = MSO(proc).funs; - MSO(proc).funs = bp->off_heap.funs; - bp->off_heap.funs = NULL; - } -#endif - - /* Move any external things into the process */ - if (bp->off_heap.externals != NULL) { - ExternalThing** next_p = &bp->off_heap.externals; - while (*next_p != NULL) { - next_p = &((*next_p)->next); - } - *next_p = MSO(proc).externals; - MSO(proc).externals = bp->off_heap.externals; - bp->off_heap.externals = NULL; + *next_p = MSO(proc).first; + MSO(proc).first = bp->off_heap.first; + bp->off_heap.first = NULL; + OH_OVERHEAD(&(MSO(proc)), bp->off_heap.overhead); } } } @@ -242,7 +245,7 @@ erts_msg_distext2heap(Process *pp, goto decode_error; if (is_not_nil(*tokenp)) { ErlHeapFragment *heap_frag = erts_dist_ext_trailer(dist_extp); - tok_sz = heap_frag->size; + tok_sz = heap_frag->used_size; sz += tok_sz; } if (pp) @@ -283,12 +286,13 @@ erts_msg_distext2heap(Process *pp, erts_cleanup_offheap(&heap_frag->off_heap); } erts_free_dist_ext_copy(dist_extp); - if (*bpp) + if (*bpp) { free_message_buffer(*bpp); + *bpp = NULL; + } else if (hp) { HRelease(pp, hp_end, hp); } - *bpp = NULL; return THE_NON_VALUE; } @@ -436,11 +440,10 @@ erts_queue_message(Process* receiver, ERL_MESSAGE_TERM(mp) = message; ERL_MESSAGE_TOKEN(mp) = seq_trace_token; mp->next = NULL; + mp->data.heap_frag = bp; #ifdef ERTS_SMP if (*receiver_locks & ERTS_PROC_LOCK_MAIN) { - mp->data.heap_frag = bp; - /* * We move 'in queue' to 'private queue' and place * message at the end of 'private queue' in order @@ -453,11 +456,9 @@ erts_queue_message(Process* receiver, LINK_MESSAGE_PRIVQ(receiver, mp); } else { - mp->data.heap_frag = bp; LINK_MESSAGE(receiver, mp); } #else - mp->data.heap_frag = bp; LINK_MESSAGE(receiver, mp); #endif @@ -491,19 +492,7 @@ erts_link_mbuf_to_proc(struct process *proc, ErlHeapFragment *bp) void erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) { - /* Unions for typecasts avoids warnings about type-punned pointers and aliasing */ - union { - Uint** upp; - ProcBin **pbpp; - ErlFunThing **efpp; - ExternalThing **etpp; - } oh_list_pp, oh_el_next_pp; - union { - Uint *up; - ProcBin *pbp; - ErlFunThing *efp; - ExternalThing *etp; - } oh_el_p; + struct erl_off_heap_header* oh; Eterm term, token, *fhp, *hp; Sint offs; Uint sz; @@ -530,40 +519,33 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #ifdef HARD_DEBUG dbg_term_sz = size_object(term); dbg_token_sz = size_object(token); - ASSERT(bp->size == dbg_term_sz + dbg_token_sz); - - dbg_bp = new_message_buffer(bp->size); + /*ASSERT(dbg_term_sz + dbg_token_sz == erts_msg_used_frag_sz(msg)); + Copied size may be smaller due to removed SubBins's or garbage. + Copied size may be larger due to duplicated shared terms. + */ + dbg_bp = new_message_buffer(dbg_term_sz + dbg_token_sz); dbg_hp = dbg_bp->mem; dbg_term = copy_struct(term, dbg_term_sz, &dbg_hp, &dbg_bp->off_heap); dbg_token = copy_struct(token, dbg_token_sz, &dbg_hp, &dbg_bp->off_heap); dbg_thp_start = *hpp; #endif - ASSERT(bp); - msg->data.attached = NULL; - - off_heap->overhead += bp->off_heap.overhead; - sz = bp->size; - -#ifdef DEBUG - if (is_not_immed(term)) { - ASSERT(bp->mem <= ptr_val(term)); - ASSERT(bp->mem + bp->size > ptr_val(term)); + if (bp->next != NULL) { + move_multi_frags(hpp, off_heap, bp, msg->m, 2); + goto copy_done; } - if (is_not_immed(token)) { - ASSERT(bp->mem <= ptr_val(token)); - ASSERT(bp->mem + bp->size > ptr_val(token)); - } -#endif + OH_OVERHEAD(off_heap, bp->off_heap.overhead); + sz = bp->used_size; + + ASSERT(is_immed(term) || in_heapfrag(ptr_val(term),bp)); + ASSERT(is_immed(token) || in_heapfrag(ptr_val(token),bp)); fhp = bp->mem; hp = *hpp; offs = hp - fhp; - oh_list_pp.upp = NULL; - oh_el_next_pp.upp = NULL; /* Shut up compiler warning */ - oh_el_p.up = NULL; /* Shut up compiler warning */ + oh = NULL; while (sz--) { Uint cpy_sz; Eterm val = *fhp++; @@ -574,8 +556,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: - ASSERT(bp->mem <= ptr_val(val)); - ASSERT(bp->mem + bp->size > ptr_val(val)); + ASSERT(in_heapfrag(ptr_val(val), bp)); *hp++ = offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: @@ -584,31 +565,18 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) case ARITYVAL_SUBTAG: break; case REFC_BINARY_SUBTAG: - oh_list_pp.pbpp = &off_heap->mso; - oh_el_p.up = (hp-1); - oh_el_next_pp.pbpp = &(oh_el_p.pbp)->next; - cpy_sz = thing_arityval(val); - goto cpy_words; case FUN_SUBTAG: -#ifndef HYBRID - oh_list_pp.efpp = &off_heap->funs; - oh_el_p.up = (hp-1); - oh_el_next_pp.efpp = &(oh_el_p.efp)->next; -#endif - cpy_sz = thing_arityval(val); - goto cpy_words; case EXTERNAL_PID_SUBTAG: case EXTERNAL_PORT_SUBTAG: case EXTERNAL_REF_SUBTAG: - oh_list_pp.etpp = &off_heap->externals; - oh_el_p.up = (hp-1); - oh_el_next_pp.etpp = &(oh_el_p.etp)->next; + oh = (struct erl_off_heap_header*) (hp-1); cpy_sz = thing_arityval(val); goto cpy_words; default: cpy_sz = header_arity(val); cpy_words: + ASSERT(sz >= cpy_sz); sz -= cpy_sz; while (cpy_sz >= 8) { cpy_sz -= 8; @@ -631,44 +599,13 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) case 1: *hp++ = *fhp++; default: break; } - if (oh_list_pp.upp) { -#ifdef HARD_DEBUG - Uint *dbg_old_oh_list_p = *oh_list_pp.upp; -#endif + if (oh) { /* Add to offheap list */ - *oh_el_next_pp.upp = *oh_list_pp.upp; - *oh_list_pp.upp = oh_el_p.up; - ASSERT(*hpp <= oh_el_p.up); - ASSERT(hp > oh_el_p.up); -#ifdef HARD_DEBUG - switch (val & _HEADER_SUBTAG_MASK) { - case REFC_BINARY_SUBTAG: - ASSERT(off_heap->mso == *oh_list_pp.pbpp); - ASSERT(off_heap->mso->next - == (ProcBin *) dbg_old_oh_list_p); - break; -#ifndef HYBRID - case FUN_SUBTAG: - ASSERT(off_heap->funs == *oh_list_pp.efpp); - ASSERT(off_heap->funs->next - == (ErlFunThing *) dbg_old_oh_list_p); - break; -#endif - case EXTERNAL_PID_SUBTAG: - case EXTERNAL_PORT_SUBTAG: - case EXTERNAL_REF_SUBTAG: - ASSERT(off_heap->externals - == *oh_list_pp.etpp); - ASSERT(off_heap->externals->next - == (ExternalThing *) dbg_old_oh_list_p); - break; - default: - ASSERT(0); - } -#endif - oh_list_pp.upp = NULL; - - + oh->next = off_heap->first; + off_heap->first = oh; + ASSERT(*hpp <= (Eterm*)oh); + ASSERT(hp > (Eterm*)oh); + oh = NULL; } break; } @@ -676,12 +613,11 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) } } - ASSERT(bp->size == hp - *hpp); + ASSERT(bp->used_size == hp - *hpp); *hpp = hp; if (is_not_immed(token)) { - ASSERT(bp->mem <= ptr_val(token)); - ASSERT(bp->mem + bp->size > ptr_val(token)); + ASSERT(in_heapfrag(ptr_val(token), bp)); ERL_MESSAGE_TOKEN(msg) = offset_ptr(token, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TOKEN(msg))); @@ -690,8 +626,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) } if (is_not_immed(term)) { - ASSERT(bp->mem <= ptr_val(term)); - ASSERT(bp->mem + bp->size > ptr_val(term)); + ASSERT(in_heapfrag(ptr_val(term),bp)); ERL_MESSAGE_TERM(msg) = offset_ptr(term, offs); #ifdef HARD_DEBUG ASSERT(dbg_thp_start <= ptr_val(ERL_MESSAGE_TERM(msg))); @@ -699,10 +634,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #endif } +copy_done: #ifdef HARD_DEBUG { int i, j; + ErlHeapFragment* frag; { ProcBin *mso = off_heap->mso; i = j = 0; @@ -710,10 +647,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) mso = mso->next; i++; } - mso = bp->off_heap.mso; - while (mso) { - mso = mso->next; - j++; + for (frag=bp; frag; frag=frag->next) { + mso = frag->off_heap.mso; + while (mso) { + mso = mso->next; + j++; + } } ASSERT(i == j); } @@ -724,10 +663,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) fun = fun->next; i++; } - fun = bp->off_heap.funs; - while (fun) { - fun = fun->next; - j++; + for (frag=bp; frag; frag=frag->next) { + fun = frag->off_heap.funs; + while (fun) { + fun = fun->next; + j++; + } } ASSERT(i == j); } @@ -738,10 +679,12 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) external = external->next; i++; } - external = bp->off_heap.externals; - while (external) { - external = external->next; - j++; + for (frag=bp; frag; frag=frag->next) { + external = frag->off_heap.externals; + while (external) { + external = external->next; + j++; + } } ASSERT(i == j); } @@ -749,12 +692,9 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) #endif - bp->off_heap.mso = NULL; -#ifndef HYBRID - bp->off_heap.funs = NULL; -#endif - bp->off_heap.externals = NULL; + bp->off_heap.first = NULL; free_message_buffer(bp); + msg->data.heap_frag = NULL; #ifdef HARD_DEBUG ASSERT(eq(ERL_MESSAGE_TERM(msg), dbg_term)); @@ -764,6 +704,7 @@ erts_move_msg_mbuf_to_heap(Eterm** hpp, ErlOffHeap* off_heap, ErlMessage *msg) } + Uint erts_msg_attached_data_size_aux(ErlMessage *msg) { @@ -789,7 +730,7 @@ erts_msg_attached_data_size_aux(ErlMessage *msg) if (is_not_nil(msg->m[1])) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - sz += heap_frag->size; + sz += heap_frag->used_size; } return sz; } @@ -805,7 +746,7 @@ erts_move_msg_attached_data_to_heap(Eterm **hpp, ErlOffHeap *ohp, ErlMessage *ms ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); ERL_MESSAGE_TOKEN(msg) = copy_struct(ERL_MESSAGE_TOKEN(msg), - heap_frag->size, + heap_frag->used_size, hpp, ohp); erts_cleanup_offheap(&heap_frag->off_heap); @@ -1062,3 +1003,4 @@ erts_deliver_exit_message(Eterm from, Process *to, ErtsProcLocks *to_locksp, erts_queue_message(to, to_locksp, bp, save, NIL); } } + diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index 459c6363aa..5aca0db6fe 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -28,13 +28,22 @@ struct external_thing_; * but is stored outside of any heap. */ -typedef struct erl_off_heap { - struct proc_bin* mso; /* List of associated binaries. */ -#ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* funs; /* List of funs. */ +struct erl_off_heap_header { + Eterm thing_word; + Uint size; +#if HALFWORD_HEAP + void* dummy_ptr_padding__; #endif - struct external_thing_* externals; /* List of external things. */ - int overhead; /* Administrative overhead (used to force GC). */ + struct erl_off_heap_header* next; +}; + +#define OH_OVERHEAD(oh, size) do { \ + (oh)->overhead += size; \ +} while(0) + +typedef struct erl_off_heap { + struct erl_off_heap_header* first; + Uint64 overhead; /* Administrative overhead (used to force GC). */ } ErlOffHeap; #include "external.h" @@ -49,7 +58,7 @@ typedef struct erl_heap_fragment ErlHeapFragment; struct erl_heap_fragment { ErlHeapFragment* next; /* Next heap fragment */ ErlOffHeap off_heap; /* Offset heap data. */ - unsigned size; /* Size in (half)words of mem */ + unsigned alloc_size; /* Size in (half)words of mem */ unsigned used_size; /* With terms to be moved to heap by GC */ Eterm mem[1]; /* Data */ }; @@ -75,6 +84,13 @@ typedef struct { ErlMessage** last; /* point to the last next pointer */ ErlMessage** save; int len; /* queue length */ + + /* + * The following two fields are used by the recv_mark/1 and + * recv_set/1 instructions. + */ + BeamInstr* mark; /* address to rec_loop/2 instruction */ + ErlMessage** saved_last; /* saved last pointer */ } ErlMessageQueue; #ifdef ERTS_SMP @@ -137,6 +153,7 @@ do { \ (p)->msg.len--; \ if (__mp == NULL) \ (p)->msg.last = (p)->msg.save; \ + (p)->msg.mark = 0; \ } while(0) /* Reset message save point (after receive match) */ @@ -191,11 +208,9 @@ do { \ #define ERTS_INIT_HEAP_FRAG(HEAP_FRAG_P, DATA_WORDS) \ do { \ (HEAP_FRAG_P)->next = NULL; \ - (HEAP_FRAG_P)->size = (DATA_WORDS); \ + (HEAP_FRAG_P)->alloc_size = (DATA_WORDS); \ (HEAP_FRAG_P)->used_size = (DATA_WORDS); \ - (HEAP_FRAG_P)->off_heap.mso = NULL; \ - (HEAP_FRAG_P)->off_heap.funs = NULL; \ - (HEAP_FRAG_P)->off_heap.externals = NULL; \ + (HEAP_FRAG_P)->off_heap.first = NULL; \ (HEAP_FRAG_P)->off_heap.overhead = 0; \ } while (0) @@ -219,14 +234,25 @@ void erts_move_msg_attached_data_to_heap(Eterm **, ErlOffHeap *, ErlMessage *); Eterm erts_msg_distext2heap(Process *, ErtsProcLocks *, ErlHeapFragment **, Eterm *, ErtsDistExternal *); +ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg); ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg); #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE Uint erts_msg_used_frag_sz(const ErlMessage *msg) +{ + const ErlHeapFragment *bp; + Uint sz = 0; + for (bp = msg->data.heap_frag; bp!=NULL; bp=bp->next) { + sz += bp->used_size; + } + return sz; +} + ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) { ASSERT(msg->data.attached); if (is_value(ERL_MESSAGE_TERM(msg))) - return msg->data.heap_frag->size; + return erts_msg_used_frag_sz(msg); else if (msg->data.dist_ext->heap_size < 0) return erts_msg_attached_data_size_aux(msg); else { @@ -234,7 +260,7 @@ ERTS_GLB_INLINE Uint erts_msg_attached_data_size(ErlMessage *msg) if (is_not_nil(ERL_MESSAGE_TOKEN(msg))) { ErlHeapFragment *heap_frag; heap_frag = erts_dist_ext_trailer(msg->data.dist_ext); - sz += heap_frag->size; + sz += heap_frag->used_size; } return sz; } diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index 9cf55ee319..b1478758a1 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -585,9 +585,7 @@ void erts_mtrace_init(char *receiver, char *nodename) Uint16 port; erts_mtx_init(&mtrace_buf_mutex, "mtrace_buf"); - erts_mtx_set_forksafe(&mtrace_buf_mutex); erts_mtx_init(&mtrace_op_mutex, "mtrace_op"); - erts_mtx_set_forksafe(&mtrace_op_mutex); socket_desc = erts_sock_open(); if (socket_desc == ERTS_SOCK_INVALID_SOCKET) { diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 2790020117..a680097c2d 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -48,7 +48,7 @@ struct erl_module_nif { struct enif_entry_t* entry; erts_refc_t rt_cnt; /* number of resource types */ erts_refc_t rt_dtor_cnt; /* number of resource types with destructors */ - int is_orphan; /* if erlang module has been purged */ + Module* mod; /* Can be NULL if orphan with dtor-resources left */ }; #ifdef DEBUG @@ -61,6 +61,10 @@ static void add_readonly_check(ErlNifEnv*, unsigned char* ptr, unsigned sz); # define ADD_READONLY_CHECK(ENV,PTR,SIZE) ((void)0) #endif +#ifdef DEBUG +static int is_offheap(const ErlOffHeap* off_heap); +#endif + #define MIN_HEAP_FRAG_SZ 200 static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp); @@ -84,7 +88,8 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp) HEAP_TOP(env->proc) = env->hp; } else { - HRelease(env->proc, env->hp_end, env->hp); + env->heap_frag->used_size = hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } frag_sz = need + MIN_HEAP_FRAG_SZ; hp = erts_heap_alloc(env->proc, frag_sz); @@ -94,6 +99,16 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp) return hp; } +#if SIZEOF_LONG != ERTS_SIZEOF_ETERM +static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need) +{ + if (env->hp + may_need > env->hp_end) { + alloc_heap_heavy(env, may_need, env->hp); + env->hp -= may_need; + } +} +#endif + void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif) { env->mod_nif = mod_nif; @@ -143,8 +158,9 @@ void erts_post_nif(ErlNifEnv* env) } else { ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->size); - HRelease(env->proc, env->hp_end, env->hp); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } free_tmp_objs(env); } @@ -158,7 +174,7 @@ static void post_nif_noproc(ErlNifEnv* env) /* Flush out our cached heap pointers to allow an ordinary HAlloc */ -static void enable_halloc(ErlNifEnv* env) +static void flush_env(ErlNifEnv* env) { if (env->heap_frag == NULL) { ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); @@ -168,14 +184,15 @@ static void enable_halloc(ErlNifEnv* env) } else { ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->size); - HRelease(env->proc, env->hp_end, env->hp); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); + env->heap_frag->used_size = env->hp - env->heap_frag->mem; + ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } } -/* Restore cached heap pointers +/* Restore cached heap pointers to allow alloc_heap again. */ -static void disable_halloc(ErlNifEnv* env) +static void cache_env(ErlNifEnv* env) { if (env->heap_frag == NULL) { ASSERT(env->hp_end == HEAP_LIMIT(env->proc)); @@ -185,34 +202,190 @@ static void disable_halloc(ErlNifEnv* env) } else { ASSERT(env->hp_end != HEAP_LIMIT(env->proc)); - ASSERT(env->hp_end - env->hp <= env->heap_frag->size); + ASSERT(env->hp_end - env->hp <= env->heap_frag->alloc_size); env->heap_frag = MBUF(env->proc); ASSERT(env->heap_frag != NULL); env->hp = env->heap_frag->mem + env->heap_frag->used_size; - env->hp_end = env->heap_frag->mem + env->heap_frag->size; + env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; } } - void* enif_priv_data(ErlNifEnv* env) { return env->mod_nif->priv_data; } -void* enif_alloc(ErlNifEnv* env, size_t size) +void* enif_alloc(size_t size) { return erts_alloc_fnf(ERTS_ALC_T_NIF, (Uint) size); } -void* enif_realloc(ErlNifEnv* env, void* ptr, size_t size) +void* enif_realloc(void* ptr, size_t size) { return erts_realloc_fnf(ERTS_ALC_T_NIF, ptr, size); } -void enif_free(ErlNifEnv* env, void* ptr) +void enif_free(void* ptr) { erts_free(ERTS_ALC_T_NIF, ptr); } +struct enif_msg_environment_t +{ + ErlNifEnv env; + Process phony_proc; +}; + +ErlNifEnv* enif_alloc_env(void) +{ + struct enif_msg_environment_t* msg_env = + erts_alloc_fnf(ERTS_ALC_T_NIF, sizeof(struct enif_msg_environment_t)); + Eterm* phony_heap = (Eterm*) msg_env; /* dummy non-NULL ptr */ + + msg_env->env.hp = phony_heap; + msg_env->env.hp_end = phony_heap; + msg_env->env.heap_frag = NULL; + msg_env->env.mod_nif = NULL; + msg_env->env.tmp_obj_list = (struct enif_tmp_obj_t*) 1; /* invalid non-NULL */ + msg_env->env.proc = &msg_env->phony_proc; + memset(&msg_env->phony_proc, 0, sizeof(Process)); + HEAP_START(&msg_env->phony_proc) = phony_heap; + HEAP_TOP(&msg_env->phony_proc) = phony_heap; + HEAP_LIMIT(&msg_env->phony_proc) = phony_heap; + HEAP_END(&msg_env->phony_proc) = phony_heap; + MBUF(&msg_env->phony_proc) = NULL; + msg_env->phony_proc.id = ERTS_INVALID_PID; +#ifdef FORCE_HEAP_FRAGS + msg_env->phony_proc.space_verified = 0; + msg_env->phony_proc.space_verified_from = NULL; +#endif + return &msg_env->env; +} +void enif_free_env(ErlNifEnv* env) +{ + enif_clear_env(env); + erts_free(ERTS_ALC_T_NIF, env); +} + +static ERTS_INLINE void clear_offheap(ErlOffHeap* oh) +{ + oh->first = NULL; + oh->overhead = 0; +} + +void enif_clear_env(ErlNifEnv* env) +{ + struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)env; + Process* p = &menv->phony_proc; + ASSERT(p == menv->env.proc); + ASSERT(p->id == ERTS_INVALID_PID); + ASSERT(MBUF(p) == menv->env.heap_frag); + if (MBUF(p) != NULL) { + erts_cleanup_offheap(&MSO(p)); + clear_offheap(&MSO(p)); + free_message_buffer(MBUF(p)); + MBUF(p) = NULL; + menv->env.heap_frag = NULL; + } + ASSERT(HEAP_TOP(p) == HEAP_END(p)); + menv->env.hp = menv->env.hp_end = HEAP_TOP(p); + + ASSERT(!is_offheap(&MSO(p))); +} +int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, + ErlNifEnv* msg_env, ERL_NIF_TERM msg) +{ + struct enif_msg_environment_t* menv = (struct enif_msg_environment_t*)msg_env; + ErtsProcLocks rp_locks = 0; + Process* rp; + Process* c_p; + ErlHeapFragment* frags; +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) + ErtsProcLocks rp_had_locks; +#endif + Eterm receiver = to_pid->pid; + int flush_me = 0; + + if (env != NULL) { + c_p = env->proc; + if (receiver == c_p->id) { + rp_locks = ERTS_PROC_LOCK_MAIN; + flush_me = 1; + } + } + else { +#ifdef ERTS_SMP + c_p = NULL; +#else + erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); +#endif + } + +#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) + rp_had_locks = rp_locks; +#endif + rp = erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, + receiver, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); + if (rp == NULL) { + ASSERT(env == NULL || receiver != c_p->id); + return 0; + } + flush_env(msg_env); + frags = menv->env.heap_frag; + ASSERT(frags == MBUF(&menv->phony_proc)); + if (frags != NULL) { + /* Move all offheap's from phony proc to the first fragment. + Quick and dirty, but erts_move_msg_mbuf_to_heap doesn't care. */ + ASSERT(!is_offheap(&frags->off_heap)); + frags->off_heap = MSO(&menv->phony_proc); + clear_offheap(&MSO(&menv->phony_proc)); + menv->env.heap_frag = NULL; + MBUF(&menv->phony_proc) = NULL; + } + ASSERT(!is_offheap(&MSO(&menv->phony_proc))); + + if (flush_me) { + flush_env(env); /* Needed for ERTS_HOLE_CHECK */ + } + erts_queue_message(rp, &rp_locks, frags, msg, am_undefined); + if (rp_locks) { + ERTS_SMP_LC_ASSERT(rp_locks == (rp_had_locks | (ERTS_PROC_LOCK_MSGQ | + ERTS_PROC_LOCK_STATUS))); + erts_smp_proc_unlock(rp, (ERTS_PROC_LOCK_MSGQ | ERTS_PROC_LOCK_STATUS)); + } + erts_smp_proc_dec_refc(rp); + if (flush_me) { + cache_env(env); + } + return 1; +} + +ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) +{ + Uint sz; + Eterm* hp; + sz = size_object(src_term); + hp = alloc_heap(dst_env, sz); + return copy_struct(src_term, sz, &hp, &MSO(dst_env->proc)); +} + + +#ifdef DEBUG +static int is_offheap(const ErlOffHeap* oh) +{ + return oh->first != NULL; +} +#endif + +ErlNifPid* enif_self(ErlNifEnv* caller_env, ErlNifPid* pid) +{ + pid->pid = caller_env->proc->id; + return pid; +} +int enif_get_local_pid(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifPid* pid) +{ + return is_internal_pid(term) ? (pid->pid=term, 1) : 0; +} + int enif_is_atom(ErlNifEnv* env, ERL_NIF_TERM term) { return is_atom(term); @@ -248,9 +421,19 @@ int enif_is_ref(ErlNifEnv* env, ERL_NIF_TERM term) return is_ref(term); } +int enif_is_tuple(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_tuple(term); +} + +int enif_is_list(ErlNifEnv* env, ERL_NIF_TERM term) +{ + return is_list(term) || is_nil(term); +} + static void aligned_binary_dtor(struct enif_tmp_obj_t* obj) { - erts_free_aligned_binary_bytes((byte*)obj); + erts_free_aligned_binary_bytes_extra((byte*)obj,ERTS_ALC_T_TMP); } int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) @@ -260,7 +443,7 @@ int enif_inspect_binary(ErlNifEnv* env, Eterm bin_term, ErlNifBinary* bin) byte* raw_ptr; }u; u.tmp = NULL; - bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, + bin->data = erts_get_aligned_binary_bytes_extra(bin_term, &u.raw_ptr, ERTS_ALC_T_TMP, sizeof(struct enif_tmp_obj_t)); if (bin->data == NULL) { return 0; @@ -314,7 +497,7 @@ int enif_inspect_iolist_as_binary(ErlNifEnv* env, Eterm term, ErlNifBinary* bin) return 1; } -int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin) +int enif_alloc_binary(size_t size, ErlNifBinary* bin) { Binary* refbin; @@ -333,7 +516,7 @@ int enif_alloc_binary(ErlNifEnv* env, unsigned size, ErlNifBinary* bin) return 1; } -int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size) +int enif_realloc_binary(ErlNifBinary* bin, size_t size) { if (bin->ref_bin != NULL) { Binary* oldbin; @@ -351,15 +534,15 @@ int enif_realloc_binary(ErlNifEnv* env, ErlNifBinary* bin, unsigned size) } else { unsigned char* old_data = bin->data; - unsigned cpy_sz = (size < bin->size ? size : bin->size); - enif_alloc_binary(env, size, bin); + size_t cpy_sz = (size < bin->size ? size : bin->size); + enif_alloc_binary(size, bin); sys_memcpy(bin->data, old_data, cpy_sz); } return 1; } -void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin) +void enif_release_binary(ErlNifBinary* bin) { if (bin->ref_bin != NULL) { Binary* refbin = bin->ref_bin; @@ -375,21 +558,21 @@ void enif_release_binary(ErlNifEnv* env, ErlNifBinary* bin) #endif } -unsigned char* enif_make_new_binary(ErlNifEnv* env, unsigned size, +unsigned char* enif_make_new_binary(ErlNifEnv* env, size_t size, ERL_NIF_TERM* termp) { - enable_halloc(env); + flush_env(env); *termp = new_binary(env->proc, NULL, size); - disable_halloc(env); + cache_env(env); return binary_bytes(*termp); } -int enif_is_identical(ErlNifEnv* env, Eterm lhs, Eterm rhs) +int enif_is_identical(Eterm lhs, Eterm rhs) { return EQ(lhs,rhs); } -int enif_compare(ErlNifEnv* env, Eterm lhs, Eterm rhs) +int enif_compare(Eterm lhs, Eterm rhs) { return cmp(lhs,rhs); } @@ -452,13 +635,13 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) pb = (ProcBin *) alloc_heap(env, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = bptr->orig_size; - pb->next = MSO(env->proc).mso; - MSO(env->proc).mso = pb; + pb->next = MSO(env->proc).first; + MSO(env->proc).first = (struct erl_off_heap_header*) pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; - MSO(env->proc).overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(env->proc)), pb->size / sizeof(Eterm)); bin_term = make_binary(pb); if (erts_refc_read(&bptr->refc, 1) == 1) { /* Total ownership transfer */ @@ -468,15 +651,15 @@ Eterm enif_make_binary(ErlNifEnv* env, ErlNifBinary* bin) return bin_term; } else { - enable_halloc(env); + flush_env(env); bin->bin_term = new_binary(env->proc, bin->data, bin->size); - disable_halloc(env); + cache_env(env); return bin->bin_term; } } Eterm enif_make_sub_binary(ErlNifEnv* env, ERL_NIF_TERM bin_term, - unsigned pos, unsigned size) + size_t pos, size_t size) { ErlSubBin* sb; Eterm orig; @@ -506,9 +689,11 @@ Eterm enif_make_badarg(ErlNifEnv* env) BIF_ERROR(env->proc, BADARG); } -int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len) +int enif_get_atom(ErlNifEnv* env, Eterm atom, char* buf, unsigned len, + ErlNifCharEncoding encoding) { Atom* ap; + ASSERT(encoding == ERL_NIF_LATIN1); if (is_not_atom(atom)) { return 0; } @@ -555,11 +740,8 @@ int enif_get_long(ErlNifEnv* env, Eterm term, long* ip) { #if SIZEOF_LONG == ERTS_SIZEOF_ETERM return term_to_Sint(term, ip); -#elif SIZEOF_INT == ERTS_SIZEOF_ETERM - Uint u; - term_to_Sint(term, u); - *ip = (long) u; - return 1; +#elif SIZEOF_LONG == 8 + return term_to_Sint64(term, ip); #else # error Unknown long word size #endif @@ -569,18 +751,26 @@ int enif_get_ulong(ErlNifEnv* env, Eterm term, unsigned long* ip) { #if SIZEOF_LONG == ERTS_SIZEOF_ETERM return term_to_Uint(term, ip); -#elif SIZEOF_INT == ERTS_SIZEOF_ETERM - Uint u; - int r; - r = term_to_Uint(term, &u); - *ip = (unsigned long) u; - return r; +#elif SIZEOF_LONG == 8 + return term_to_Uint64(term, ip); #else # error Unknown long word size #endif } -int enif_get_double(ErlNifEnv* env, Eterm term, double* dp) +#if HAVE_INT64 && SIZEOF_LONG != 8 +int enif_get_int64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifSInt64* ip) +{ + return term_to_Sint64(term, ip); +} + +int enif_get_uint64(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifUInt64* ip) +{ + return term_to_Uint64(term, ip); +} +#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */ + +int enif_get_double(ErlNifEnv* env, ERL_NIF_TERM term, double* dp) { FloatDef f; if (is_not_float(term)) { @@ -591,6 +781,17 @@ int enif_get_double(ErlNifEnv* env, Eterm term, double* dp) return 1; } +int enif_get_atom_length(ErlNifEnv* env, Eterm atom, unsigned* len, + ErlNifCharEncoding enc) +{ + Atom* ap; + ASSERT(enc == ERL_NIF_LATIN1); + if (is_not_atom(atom)) return 0; + ap = atom_tab(atom_val(atom)); + *len = ap->len; + return 1; +} + int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) { Eterm* val; @@ -601,6 +802,13 @@ int enif_get_list_cell(ErlNifEnv* env, Eterm term, Eterm* head, Eterm* tail) return 1; } +int enif_get_list_length(ErlNifEnv* env, Eterm term, unsigned* len) +{ + if (is_not_list(term) && is_not_nil(term)) return 0; + *len = list_length(term); + return 1; +} + ERL_NIF_TERM enif_make_int(ErlNifEnv* env, int i) { #if SIZEOF_INT == ERTS_SIZEOF_ETERM @@ -621,14 +829,44 @@ ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i) ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i) { +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return IS_SSMALL(i) ? make_small(i) : small_to_big(i, alloc_heap(env,2)); +#elif SIZEOF_LONG == 8 + ensure_heap(env,3); + return erts_sint64_to_big(i, &env->hp); +#endif } ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i) { +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2)); +#elif SIZEOF_LONG == 8 + ensure_heap(env,3); + return erts_uint64_to_big(i, &env->hp); +#endif +} + +#if HAVE_INT64 && SIZEOF_LONG != 8 +ERL_NIF_TERM enif_make_int64(ErlNifEnv* env, ErlNifSInt64 i) +{ + Uint* hp; + Uint need = 0; + erts_bld_sint64(NULL, &need, i); + hp = alloc_heap(env, need); + return erts_bld_sint64(&hp, NULL, i); } +ERL_NIF_TERM enif_make_uint64(ErlNifEnv* env, ErlNifUInt64 i) +{ + Uint* hp; + Uint need = 0; + erts_bld_uint64(NULL, &need, i); + hp = alloc_heap(env, need); + return erts_bld_uint64(&hp, NULL, i); +} +#endif /* HAVE_INT64 && SIZEOF_LONG != 8 */ + ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) { Eterm* hp = alloc_heap(env,FLOAT_SIZE_OBJECT); @@ -640,12 +878,25 @@ ERL_NIF_TERM enif_make_double(ErlNifEnv* env, double d) ERL_NIF_TERM enif_make_atom(ErlNifEnv* env, const char* name) { - return am_atom_put(name, sys_strlen(name)); + return enif_make_atom_len(env, name, sys_strlen(name)); +} + +ERL_NIF_TERM enif_make_atom_len(ErlNifEnv* env, const char* name, size_t len) +{ + return am_atom_put(name, len); } -int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom) +int enif_make_existing_atom(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, + ErlNifCharEncoding enc) { - return erts_atom_get(name, sys_strlen(name), atom); + return enif_make_existing_atom_len(env, name, sys_strlen(name), atom, enc); +} + +int enif_make_existing_atom_len(ErlNifEnv* env, const char* name, size_t len, + ERL_NIF_TERM* atom, ErlNifCharEncoding encoding) +{ + ASSERT(encoding == ERL_NIF_LATIN1); + return erts_atom_get(name, len, atom); } ERL_NIF_TERM enif_make_tuple(ErlNifEnv* env, unsigned cnt, ...) @@ -724,11 +975,16 @@ ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], ERL_NIF_TERM enif_make_string(ErlNifEnv* env, const char* string, ErlNifCharEncoding encoding) -{ - Sint n = sys_strlen(string); - Eterm* hp = alloc_heap(env,n*2); +{ + return enif_make_string_len(env, string, sys_strlen(string), encoding); +} + +ERL_NIF_TERM enif_make_string_len(ErlNifEnv* env, const char* string, + size_t len, ErlNifCharEncoding encoding) +{ + Eterm* hp = alloc_heap(env,len*2); ASSERT(encoding == ERL_NIF_LATIN1); - return erts_bld_string_n(&hp,NULL,string,n); + return erts_bld_string_n(&hp,NULL,string,len); } ERL_NIF_TERM enif_make_ref(ErlNifEnv* env) @@ -799,7 +1055,8 @@ struct enif_resource_type_t ErlNifResourceDtor* dtor; /* user destructor function */ erts_refc_t refc; /* num of resources of this type (HOTSPOT warning) +1 for active erl_module_nif */ - char name[1]; + Eterm module; + Eterm name; }; /* dummy node in circular list */ @@ -817,14 +1074,14 @@ typedef struct enif_resource_t #define SIZEOF_ErlNifResource(SIZE) (offsetof(ErlNifResource,data) + (SIZE)) #define DATA_TO_RESOURCE(PTR) ((ErlNifResource*)((char*)(PTR) - offsetof(ErlNifResource,data))) -static ErlNifResourceType* find_resource_type(const char* name) +static ErlNifResourceType* find_resource_type(Eterm module, Eterm name) { ErlNifResourceType* type; for (type = resource_type_list.next; type != &resource_type_list; type = type->next) { - if (sys_strcmp(type->name, name) == 0) { + if (type->module == module && type->name == name) { return type; } } @@ -857,33 +1114,42 @@ static void steal_resource_type(ErlNifResourceType* type) if (type->dtor != NULL && erts_refc_dectest(&lib->rt_dtor_cnt, 0) == 0 - && lib->is_orphan) { + && lib->mod == NULL) { /* last type with destructor gone, close orphan lib */ close_lib(lib); } if (erts_refc_dectest(&lib->rt_cnt, 0) == 0 - && lib->is_orphan) { + && lib->mod == NULL) { erts_free(ERTS_ALC_T_NIF, lib); } } ErlNifResourceType* -enif_open_resource_type(ErlNifEnv* env, const char* type_name, - ErlNifResourceDtor* dtor, - enum ErlNifResourceFlags flags, - enum ErlNifResourceFlags* tried) +enif_open_resource_type(ErlNifEnv* env, + const char* module_str, + const char* name_str, + ErlNifResourceDtor* dtor, + ErlNifResourceFlags flags, + ErlNifResourceFlags* tried) { - ErlNifResourceType* type = find_resource_type(type_name); - enum ErlNifResourceFlags op = flags; + ErlNifResourceType* type = NULL; + ErlNifResourceFlags op = flags; + Eterm module_am, name_am; + ASSERT(erts_smp_is_system_blocked(0)); + ASSERT(module_str == NULL); /* for now... */ + module_am = make_atom(env->mod_nif->mod->module); + name_am = enif_make_atom(env, name_str); + + type = find_resource_type(module_am, name_am); if (type == NULL) { if (flags & ERL_NIF_RT_CREATE) { type = erts_alloc(ERTS_ALC_T_NIF, - sizeof(struct enif_resource_type_t) - + sys_strlen(type_name)); + sizeof(struct enif_resource_type_t)); type->dtor = dtor; - sys_strcpy(type->name, type_name); + type->module = module_am; + type->name = name_am; erts_refc_init(&type->refc, 1); type->owner = env->mod_nif; type->prev = &resource_type_list; @@ -931,13 +1197,13 @@ static void nif_resource_dtor(Binary* bin) if (erts_refc_dectest(&type->refc, 0) == 0) { ASSERT(type->next == NULL); ASSERT(type->owner != NULL); - ASSERT(type->owner->is_orphan); + ASSERT(type->owner->mod == NULL); steal_resource_type(type); erts_free(ERTS_ALC_T_NIF, type); } } -void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned size) +void* enif_alloc_resource(ErlNifResourceType* type, size_t size) { Binary* bin = erts_create_magic_binary(SIZEOF_ErlNifResource(size), &nif_resource_dtor); ErlNifResource* resource = ERTS_MAGIC_BIN_DATA(bin); @@ -950,7 +1216,7 @@ void* enif_alloc_resource(ErlNifEnv* env, ErlNifResourceType* type, unsigned siz return resource->data; } -void enif_release_resource(ErlNifEnv* env, void* obj) +void enif_release_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); @@ -964,6 +1230,18 @@ void enif_release_resource(ErlNifEnv* env, void* obj) } } +void enif_keep_resource(void* obj) +{ + ErlNifResource* resource = DATA_TO_RESOURCE(obj); + ErtsBinary* bin = ERTS_MAGIC_BIN_FROM_DATA(resource); + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(bin) == &nif_resource_dtor); +#ifdef DEBUG + erts_refc_inc(&resource->nif_refc, 1); +#endif + erts_refc_inc(&bin->binary.refc, 2); +} + ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); @@ -972,15 +1250,30 @@ ERL_NIF_TERM enif_make_resource(ErlNifEnv* env, void* obj) return erts_mk_magic_binary_term(&hp, &MSO(env->proc), &bin->binary); } +ERL_NIF_TERM enif_make_resource_binary(ErlNifEnv* env, void* obj, + const void* data, size_t size) +{ + Eterm bin = enif_make_resource(env, obj); + ProcBin* pb = (ProcBin*) binary_val(bin); + pb->bytes = (byte*) data; + pb->size = size; + return bin; +} + int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp) { + ProcBin* pb; Binary* mbin; ErlNifResource* resource; if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { return 0; } - mbin = ((ProcBin*) binary_val(term))->val; + pb = (ProcBin*) binary_val(term); + /*if (pb->size != 0) { + return 0; / * Or should we allow "resource binaries" as handles? * / + }*/ + mbin = pb->val; resource = (ErlNifResource*) ERTS_MAGIC_BIN_DATA(mbin); if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != &nif_resource_dtor || resource->type != type) { @@ -990,7 +1283,7 @@ int enif_get_resource(ErlNifEnv* env, ERL_NIF_TERM term, ErlNifResourceType* typ return 1; } -unsigned enif_sizeof_resource(ErlNifEnv* env, void* obj) +size_t enif_sizeof_resource(void* obj) { ErlNifResource* resource = DATA_TO_RESOURCE(obj); Binary* bin = &ERTS_MAGIC_BIN_FROM_DATA(resource)->binary; @@ -1220,7 +1513,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) lib->entry = entry; erts_refc_init(&lib->rt_cnt, 0); erts_refc_init(&lib->rt_dtor_cnt, 0); - lib->is_orphan = 0; + lib->mod = mod; env.mod_nif = lib; if (mod->nif != NULL) { /* Reload */ int k; @@ -1301,7 +1594,8 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - BpData* bp = (BpData*) code_ptr[1]; + BpData** bps = (BpData**) code_ptr[1]; + BpData* bp = (BpData*) bps[bp_sched2ix()]; bp->orig_instr = (BeamInstr) BeamOp(op_call_nif); } code_ptr[5+1] = (BeamInstr) entry->funcs[i].fptr; @@ -1334,7 +1628,7 @@ erts_unload_nif(struct erl_module_nif* lib) ErlNifResourceType* next; ASSERT(erts_smp_is_system_blocked(0)); ASSERT(lib != NULL); - ASSERT(!lib->is_orphan); + ASSERT(lib->mod != NULL); for (rt = resource_type_list.next; rt != &resource_type_list; rt = next) { @@ -1364,7 +1658,7 @@ erts_unload_nif(struct erl_module_nif* lib) else { ASSERT(erts_refc_read(&lib->rt_cnt, 1) > 0); } - lib->is_orphan = 1; + lib->mod = NULL; /* orphan lib */ } void erl_nif_init() @@ -1373,7 +1667,8 @@ void erl_nif_init() resource_type_list.prev = &resource_type_list; resource_type_list.dtor = NULL; resource_type_list.owner = NULL; - resource_type_list.name[0] = '\0'; + resource_type_list.module = THE_NON_VALUE; + resource_type_list.name = THE_NON_VALUE; } #ifdef READONLY_CHECK diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index a345837569..ee3a7cd5f4 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -23,14 +23,16 @@ #ifndef __ERL_NIF_H__ #define __ERL_NIF_H__ + #include "erl_drv_nif.h" /* Version history: ** 0.1: R13B03 ** 1.0: R13B04 +** 2.0: R14A */ -#define ERL_NIF_MAJOR_VERSION 1 -#define ERL_NIF_MINOR_VERSION 1 +#define ERL_NIF_MAJOR_VERSION 2 +#define ERL_NIF_MINOR_VERSION 0 #include <stdlib.h> @@ -60,6 +62,23 @@ #endif #include "erl_int_sizes_config.h" +#ifdef __cplusplus +extern "C" { +#endif + +#if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) +typedef unsigned __int64 ErlNifUInt64; +typedef __int64 ErlNifSInt64; +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlNifUInt64; +typedef long ErlNifSInt64; +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlNifUInt64; +typedef long long ErlNifSInt64; +#else +#error No 64-bit integer type +#endif + #ifdef HALFWORD_HEAP_EMULATOR typedef unsigned int ERL_NIF_TERM; #else @@ -93,7 +112,7 @@ typedef struct enif_entry_t typedef struct { - unsigned size; + size_t size; unsigned char* data; /* Internals (avert your eyes) */ @@ -103,17 +122,22 @@ typedef struct typedef struct enif_resource_type_t ErlNifResourceType; typedef void ErlNifResourceDtor(ErlNifEnv*, void*); -enum ErlNifResourceFlags +typedef enum { ERL_NIF_RT_CREATE = 1, ERL_NIF_RT_TAKEOVER = 2 -}; +}ErlNifResourceFlags; typedef enum { ERL_NIF_LATIN1 = 1 }ErlNifCharEncoding; +typedef struct +{ + ERL_NIF_TERM pid; /* internal, may change */ +}ErlNifPid; + typedef ErlDrvSysInfo ErlNifSysInfo; typedef struct ErlDrvTid_ *ErlNifTid; @@ -146,8 +170,6 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #endif - - #if (defined(__WIN32__) || defined(_WIN32) || defined(_WIN32_)) # define ERL_NIF_INIT_GLOB TWinDynNifCallbacks WinDynNifCallbacks; # define ERL_NIF_INIT_DECL(MODNAME) __declspec(dllexport) ErlNifEntry* nif_init(TWinDynNifCallbacks* callbacks) @@ -163,7 +185,18 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #endif +#ifdef __cplusplus +} +# define ERL_NIF_INIT_PROLOGUE extern "C" { +# define ERL_NIF_INIT_EPILOGUE } +#else +# define ERL_NIF_INIT_PROLOGUE +# define ERL_NIF_INIT_EPILOGUE +#endif + + #define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \ +ERL_NIF_INIT_PROLOGUE \ ERL_NIF_INIT_GLOB \ ERL_NIF_INIT_DECL(NAME) \ { \ @@ -178,7 +211,9 @@ ERL_NIF_INIT_DECL(NAME) \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ -} +} \ +ERL_NIF_INIT_EPILOGUE + #endif /* __ERL_NIF_H__ */ diff --git a/erts/emulator/beam/erl_nif_api_funcs.h b/erts/emulator/beam/erl_nif_api_funcs.h index fe8d2664e1..eca506593d 100644 --- a/erts/emulator/beam/erl_nif_api_funcs.h +++ b/erts/emulator/beam/erl_nif_api_funcs.h @@ -23,29 +23,29 @@ #ifdef ERL_NIF_API_FUNC_DECL ERL_NIF_API_FUNC_DECL(void*,enif_priv_data,(ErlNifEnv*)); -ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(ErlNifEnv*, size_t size)); -ERL_NIF_API_FUNC_DECL(void,enif_free,(ErlNifEnv*, void* ptr)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc,(size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_free,(void* ptr)); ERL_NIF_API_FUNC_DECL(int,enif_is_atom,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_binary,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_ref,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_inspect_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, ErlNifBinary* bin)); -ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(ErlNifEnv*, unsigned size, ErlNifBinary* bin)); -ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifEnv*, ErlNifBinary* bin, unsigned size)); -ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifEnv*, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_alloc_binary,(size_t size, ErlNifBinary* bin)); +ERL_NIF_API_FUNC_DECL(int,enif_realloc_binary,(ErlNifBinary* bin, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_binary,(ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(int,enif_get_int,(ErlNifEnv*, ERL_NIF_TERM term, int* ip)); ERL_NIF_API_FUNC_DECL(int,enif_get_ulong,(ErlNifEnv*, ERL_NIF_TERM term, unsigned long* ip)); ERL_NIF_API_FUNC_DECL(int,enif_get_double,(ErlNifEnv*, ERL_NIF_TERM term, double* dp)); ERL_NIF_API_FUNC_DECL(int,enif_get_list_cell,(ErlNifEnv* env, ERL_NIF_TERM term, ERL_NIF_TERM* head, ERL_NIF_TERM* tail)); ERL_NIF_API_FUNC_DECL(int,enif_get_tuple,(ErlNifEnv* env, ERL_NIF_TERM tpl, int* arity, const ERL_NIF_TERM** array)); -ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); -ERL_NIF_API_FUNC_DECL(int,enif_compare,(ErlNifEnv* env, ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(int,enif_is_identical,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); +ERL_NIF_API_FUNC_DECL(int,enif_compare,(ERL_NIF_TERM lhs, ERL_NIF_TERM rhs)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_binary,(ErlNifEnv* env, ErlNifBinary* bin)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_badarg,(ErlNifEnv* env)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int,(ErlNifEnv* env, int i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_ulong,(ErlNifEnv* env, unsigned long i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_double,(ErlNifEnv* env, double d)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_atom,(ErlNifEnv* env, const char* name)); -ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom)); +ERL_NIF_API_FUNC_DECL(int,enif_make_existing_atom,(ErlNifEnv* env, const char* name, ERL_NIF_TERM* atom, ErlNifCharEncoding)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple,(ErlNifEnv* env, unsigned cnt, ...)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list,(ErlNifEnv* env, unsigned cnt, ...)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_cell,(ErlNifEnv* env, ERL_NIF_TERM car, ERL_NIF_TERM cdr)); @@ -82,13 +82,13 @@ ERL_NIF_API_FUNC_DECL(int,enif_equal_tids,(ErlNifTid tid1, ErlNifTid tid2)); ERL_NIF_API_FUNC_DECL(void,enif_thread_exit,(void *resp)); ERL_NIF_API_FUNC_DECL(int,enif_thread_join,(ErlNifTid, void **respp)); -ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(ErlNifEnv*, void* ptr, size_t size)); +ERL_NIF_API_FUNC_DECL(void*,enif_realloc,(void* ptr, size_t size)); ERL_NIF_API_FUNC_DECL(void,enif_system_info,(ErlNifSysInfo *sip, size_t si_size)); ERL_NIF_API_FUNC_DECL(int,enif_fprintf,(void/* FILE* */ *filep, const char *format, ...)); ERL_NIF_API_FUNC_DECL(int,enif_inspect_iolist_as_binary,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifBinary* bin)); -ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, unsigned pos, unsigned size)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_sub_binary,(ErlNifEnv*, ERL_NIF_TERM bin_term, size_t pos, size_t size)); ERL_NIF_API_FUNC_DECL(int,enif_get_string,(ErlNifEnv*, ERL_NIF_TERM list, char* buf, unsigned len, ErlNifCharEncoding)); -ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom,(ErlNifEnv*, ERL_NIF_TERM atom, char* buf, unsigned len, ErlNifCharEncoding)); ERL_NIF_API_FUNC_DECL(int,enif_is_fun,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_pid,(ErlNifEnv*, ERL_NIF_TERM term)); ERL_NIF_API_FUNC_DECL(int,enif_is_port,(ErlNifEnv*, ERL_NIF_TERM term)); @@ -99,13 +99,35 @@ ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_long,(ErlNifEnv*, long i)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_tuple_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_list_from_array,(ErlNifEnv*, const ERL_NIF_TERM arr[], unsigned cnt)); ERL_NIF_API_FUNC_DECL(int,enif_is_empty_list,(ErlNifEnv*, ERL_NIF_TERM term)); -ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* type_name, void (*dtor)(ErlNifEnv*,void *), enum ErlNifResourceFlags flags, enum ErlNifResourceFlags* tried)); -ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifEnv*, ErlNifResourceType* type, unsigned size)); -ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(ErlNifEnv*, void* obj)); +ERL_NIF_API_FUNC_DECL(ErlNifResourceType*,enif_open_resource_type,(ErlNifEnv*, const char* module_str, const char* name_str, void (*dtor)(ErlNifEnv*,void *), ErlNifResourceFlags flags, ErlNifResourceFlags* tried)); +ERL_NIF_API_FUNC_DECL(void*,enif_alloc_resource,(ErlNifResourceType* type, size_t size)); +ERL_NIF_API_FUNC_DECL(void,enif_release_resource,(void* obj)); ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource,(ErlNifEnv*, void* obj)); ERL_NIF_API_FUNC_DECL(int,enif_get_resource,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifResourceType* type, void** objp)); -ERL_NIF_API_FUNC_DECL(unsigned,enif_sizeof_resource,(ErlNifEnv*, void* obj)); -ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,unsigned size,ERL_NIF_TERM* termp)); +ERL_NIF_API_FUNC_DECL(size_t,enif_sizeof_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,size_t size,ERL_NIF_TERM* termp)); +ERL_NIF_API_FUNC_DECL(int,enif_is_list,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_is_tuple,(ErlNifEnv*, ERL_NIF_TERM term)); +ERL_NIF_API_FUNC_DECL(int,enif_get_atom_length,(ErlNifEnv*, ERL_NIF_TERM atom, unsigned* len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(int,enif_get_list_length,(ErlNifEnv* env, ERL_NIF_TERM term, unsigned* len)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM, enif_make_atom_len,(ErlNifEnv* env, const char* name, size_t len)); +ERL_NIF_API_FUNC_DECL(int, enif_make_existing_atom_len,(ErlNifEnv* env, const char* name, size_t len, ERL_NIF_TERM* atom, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_string_len,(ErlNifEnv* env, const char* string, size_t len, ErlNifCharEncoding)); +ERL_NIF_API_FUNC_DECL(ErlNifEnv*,enif_alloc_env,(void)); +ERL_NIF_API_FUNC_DECL(void,enif_free_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(void,enif_clear_env,(ErlNifEnv* env)); +ERL_NIF_API_FUNC_DECL(int,enif_send,(ErlNifEnv* env, const ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_copy,(ErlNifEnv* dst_env, ERL_NIF_TERM src_term)); +ERL_NIF_API_FUNC_DECL(ErlNifPid*,enif_self,(ErlNifEnv* caller_env, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(int,enif_get_local_pid,(ErlNifEnv* env, ERL_NIF_TERM, ErlNifPid* pid)); +ERL_NIF_API_FUNC_DECL(void,enif_keep_resource,(void* obj)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_resource_binary,(ErlNifEnv*,void* obj,const void* data, size_t size)); +#if SIZEOF_LONG != 8 +ERL_NIF_API_FUNC_DECL(int,enif_get_int64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifSInt64* ip)); +ERL_NIF_API_FUNC_DECL(int,enif_get_uint64,(ErlNifEnv*, ERL_NIF_TERM term, ErlNifUInt64* ip)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_int64,(ErlNifEnv*, ErlNifSInt64)); +ERL_NIF_API_FUNC_DECL(ERL_NIF_TERM,enif_make_uint64,(ErlNifEnv*, ErlNifUInt64)); +#endif /* ** Add last to keep compatibility on Windows!!! @@ -198,6 +220,29 @@ ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,unsigned s # define enif_get_resource ERL_NIF_API_FUNC_MACRO(enif_get_resource) # define enif_sizeof_resource ERL_NIF_API_FUNC_MACRO(enif_sizeof_resource) # define enif_make_new_binary ERL_NIF_API_FUNC_MACRO(enif_make_new_binary) +# define enif_is_list ERL_NIF_API_FUNC_MACRO(enif_is_list) +# define enif_is_tuple ERL_NIF_API_FUNC_MACRO(enif_is_tuple) +# define enif_get_atom_length ERL_NIF_API_FUNC_MACRO(enif_get_atom_length) +# define enif_get_list_length ERL_NIF_API_FUNC_MACRO(enif_get_list_length) +# define enif_make_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_atom_len) +# define enif_make_existing_atom_len ERL_NIF_API_FUNC_MACRO(enif_make_existing_atom_len) +# define enif_make_string_len ERL_NIF_API_FUNC_MACRO(enif_make_string_len) +# define enif_alloc_env ERL_NIF_API_FUNC_MACRO(enif_alloc_env) +# define enif_free_env ERL_NIF_API_FUNC_MACRO(enif_free_env) +# define enif_clear_env ERL_NIF_API_FUNC_MACRO(enif_clear_env) +# define enif_send ERL_NIF_API_FUNC_MACRO(enif_send) +# define enif_make_copy ERL_NIF_API_FUNC_MACRO(enif_make_copy) +# define enif_self ERL_NIF_API_FUNC_MACRO(enif_self) +# define enif_get_local_pid ERL_NIF_API_FUNC_MACRO(enif_get_local_pid) +# define enif_keep_resource ERL_NIF_API_FUNC_MACRO(enif_keep_resource) +# define enif_make_resource_binary ERL_NIF_API_FUNC_MACRO(enif_make_resource_binary) +#if SIZEOF_LONG != 8 +# define enif_get_int64 ERL_NIF_API_FUNC_MACRO(enif_get_int64) +# define enif_get_uint64 ERL_NIF_API_FUNC_MACRO(enif_get_uint64) +# define enif_make_int64 ERL_NIF_API_FUNC_MACRO(enif_make_int64) +# define enif_make_uint64 ERL_NIF_API_FUNC_MACRO(enif_make_uint64) +#endif + #endif #ifndef enif_make_list1 @@ -219,9 +264,15 @@ ERL_NIF_API_FUNC_DECL(unsigned char*,enif_make_new_binary,(ErlNifEnv*,unsigned s # define enif_make_tuple7(ENV,E1,E2,E3,E4,E5,E6,E7) enif_make_tuple(ENV,7,E1,E2,E3,E4,E5,E6,E7) # define enif_make_tuple8(ENV,E1,E2,E3,E4,E5,E6,E7,E8) enif_make_tuple(ENV,8,E1,E2,E3,E4,E5,E6,E7,E8) # define enif_make_tuple9(ENV,E1,E2,E3,E4,E5,E6,E7,E8,E9) enif_make_tuple(ENV,9,E1,E2,E3,E4,E5,E6,E7,E8,E9) + +# define enif_make_pid(ENV, PID) ((const ERL_NIF_TERM)((PID)->pid)) + +#if SIZEOF_LONG == 8 +# define enif_get_int64 enif_get_long +# define enif_get_uint64 enif_get_ulong +# define enif_make_int64 enif_make_long +# define enif_make_uint64 enif_make_ulong #endif -#ifndef enif_get_data -# define enif_get_data enif_priv_data /* deprecated */ #endif diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 5865d33138..6daa127d23 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -80,6 +80,8 @@ dist_table_alloc(void *dep_tmpl) Eterm chnl_nr; Eterm sysname; DistEntry *dep; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; if(((DistEntry *) dep_tmpl) == erts_this_dist_entry) return dep_tmpl; @@ -92,7 +94,7 @@ dist_table_alloc(void *dep_tmpl) dep->prev = NULL; erts_refc_init(&dep->refc, -1); - erts_smp_rwmtx_init_x(&dep->rwmtx, "dist_entry", chnl_nr); + erts_smp_rwmtx_init_opt_x(&dep->rwmtx, &rwmtx_opt, "dist_entry", chnl_nr); dep->sysname = sysname; dep->cid = NIL; dep->connection_id = 0; @@ -105,7 +107,7 @@ dist_table_alloc(void *dep_tmpl) dep->nlinks = NULL; dep->monitors = NULL; - erts_smp_spinlock_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr); + erts_smp_mtx_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr); dep->qflgs = 0; dep->qsize = 0; dep->out_queue.first = NULL; @@ -170,7 +172,7 @@ dist_table_free(void *vdep) ASSERT(!dep->cache); erts_smp_rwmtx_destroy(&dep->rwmtx); erts_smp_mtx_destroy(&dep->lnk_mtx); - erts_smp_spinlock_destroy(&dep->qlock); + erts_smp_mtx_destroy(&dep->qlock); #ifdef DEBUG sys_memset(vdep, 0x77, sizeof(DistEntry)); @@ -233,7 +235,7 @@ erts_sysname_to_connected_dist_entry(Eterm sysname) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de); if (res_dep) { - long refc = erts_refc_inctest(&res_dep->refc, 1); + erts_aint_t refc = erts_refc_inctest(&res_dep->refc, 1); if (refc < 2) /* Pending delete */ erts_refc_inc(&res_dep->refc, 1); } @@ -255,7 +257,7 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm sysname) { DistEntry *res; DistEntry de; - long refc; + erts_aint_t refc; res = erts_find_dist_entry(sysname); if (res) return res; @@ -277,7 +279,7 @@ DistEntry *erts_find_dist_entry(Eterm sysname) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); res = hash_get(&erts_dist_table, (void *) &de); if (res) { - long refc = erts_refc_inctest(&res->refc, 1); + erts_aint_t refc = erts_refc_inctest(&res->refc, 1); if (refc < 2) /* Pending delete */ erts_refc_inc(&res->refc, 1); } @@ -580,11 +582,23 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) ErlNode ne; ne.sysname = sysname; ne.creation = creation; + + erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); + res = hash_get(&erts_node_table, (void *) &ne); + if (res && res != erts_this_node) { + erts_aint_t refc = erts_refc_inctest(&res->refc, 0); + if (refc < 2) /* New or pending delete */ + erts_refc_inc(&res->refc, 1); + } + erts_smp_rwmtx_runlock(&erts_node_table_rwmtx); + if (res) + return res; + erts_smp_rwmtx_rwlock(&erts_node_table_rwmtx); res = hash_put(&erts_node_table, (void *) &ne); ASSERT(res); if (res != erts_this_node) { - long refc = erts_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_refc_inctest(&res->refc, 0); if (refc < 2) /* New or pending delete */ erts_refc_inc(&res->refc, 1); } @@ -696,8 +710,12 @@ erts_set_this_node(Eterm sysname, Uint creation) void erts_init_node_tables(void) { + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; HashFunctions f; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + f.hash = (H_FUN) dist_table_hash; f.cmp = (HCMP_FUN) dist_table_cmp; f.alloc = (HALLOC_FUN) dist_table_alloc; @@ -719,9 +737,10 @@ void erts_init_node_tables(void) erts_this_dist_entry->prev = NULL; erts_refc_init(&erts_this_dist_entry->refc, 1); /* erts_this_node */ - erts_smp_rwmtx_init_x(&erts_this_dist_entry->rwmtx, - "dist_entry", - make_small(ERST_INTERNAL_CHANNEL_NO)); + erts_smp_rwmtx_init_opt_x(&erts_this_dist_entry->rwmtx, + &rwmtx_opt, + "dist_entry", + make_small(ERST_INTERNAL_CHANNEL_NO)); erts_this_dist_entry->sysname = am_Noname; erts_this_dist_entry->cid = NIL; erts_this_dist_entry->connection_id = 0; @@ -736,9 +755,9 @@ void erts_init_node_tables(void) erts_this_dist_entry->nlinks = NULL; erts_this_dist_entry->monitors = NULL; - erts_smp_spinlock_init_x(&erts_this_dist_entry->qlock, - "dist_entry_out_queue", - make_small(ERST_INTERNAL_CHANNEL_NO)); + erts_smp_mtx_init_x(&erts_this_dist_entry->qlock, + "dist_entry_out_queue", + make_small(ERST_INTERNAL_CHANNEL_NO)); erts_this_dist_entry->qflgs = 0; erts_this_dist_entry->qsize = 0; erts_this_dist_entry->out_queue.first = NULL; @@ -772,8 +791,8 @@ void erts_init_node_tables(void) (void) hash_put(&erts_node_table, (void *) erts_this_node); - erts_smp_rwmtx_init(&erts_node_table_rwmtx, "node_table"); - erts_smp_rwmtx_init(&erts_dist_table_rwmtx, "dist_table"); + erts_smp_rwmtx_init_opt(&erts_node_table_rwmtx, &rwmtx_opt, "node_table"); + erts_smp_rwmtx_init_opt(&erts_dist_table_rwmtx, &rwmtx_opt, "dist_table"); references_atoms_need_init = 1; } @@ -1087,30 +1106,24 @@ insert_offheap2(ErlOffHeap *oh, void *arg) static void insert_offheap(ErlOffHeap *oh, int type, Eterm id) { - if(oh->externals) { - ExternalThing *etp = oh->externals; - while (etp) { - insert_node(etp->node, type, id); - etp = etp->next; - } - } + union erl_off_heap_ptr u; + struct insert_offheap2_arg a; + a.type = BIN_REF; - if(oh->mso) { - ProcBin *pb; - struct insert_offheap2_arg a; - a.type = BIN_REF; - for(pb = oh->mso; pb; pb = pb->next) { - if(IsMatchProgBinary(pb->val)) { + for (u.hdr = oh->first; u.hdr; u.hdr = u.hdr->next) { + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if(IsMatchProgBinary(u.pb->val)) { InsertedBin *ib; int insert_bin = 1; for (ib = inserted_bins; ib; ib = ib->next) - if(ib->bin_val == pb->val) { + if(ib->bin_val == u.pb->val) { insert_bin = 0; break; } if (insert_bin) { #if HALFWORD_HEAP - UWord val = (UWord) pb->val; + UWord val = (UWord) u.pb->val; DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE*2); /* extra place allocated */ #else DeclareTmpHeapNoproc(id_heap,BIG_UINT_HEAP_SIZE); @@ -1124,13 +1137,13 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) a.id = erts_bld_uword(&hp, NULL, (UWord) val); #else UseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); - a.id = erts_bld_uint(&hp, NULL, (Uint) pb->val); + a.id = erts_bld_uint(&hp, NULL, (Uint) u.pb->val); #endif - erts_match_prog_foreach_offheap(pb->val, + erts_match_prog_foreach_offheap(u.pb->val, insert_offheap2, (void *) &a); nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin)); - nib->bin_val = pb->val; + nib->bin_val = u.pb->val; nib->next = inserted_bins; inserted_bins = nib; #if HALFWORD_HEAP @@ -1139,15 +1152,16 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) UnUseTmpHeapNoproc(BIG_UINT_HEAP_SIZE); #endif } - } + } + break; + case FUN_SUBTAG: + break; /* No need to */ + default: + ASSERT(is_external_header(u.hdr->thing_word)); + insert_node(u.ext->node, type, id); + break; } } - -#if 0 - if(oh->funs) { - /* No need to */ - } -#endif } static void doit_insert_monitor(ErtsMonitor *monitor, void *p) @@ -1289,6 +1303,7 @@ setup_reference_table(void) for (i = 0; i < erts_max_processes; i++) if (process_tab[i]) { ErlMessage *msg; + /* Insert Heap */ insert_offheap(&(process_tab[i]->off_heap), HEAP_REF, @@ -1375,21 +1390,22 @@ setup_reference_table(void) { /* Add binaries stored elsewhere ... */ ErlOffHeap oh; - ProcBin pb[2] = {{0},{0}}; - ProcBin *mso = NULL; + ProcBin pb[2]; int i = 0; Binary *default_match_spec; Binary *default_meta_match_spec; - /* Only the ProcBin members val and next will be inspected + oh.first = NULL; + /* Only the ProcBin members thing_word, val and next will be inspected (by insert_offheap()) */ #undef ADD_BINARY -#define ADD_BINARY(Bin) \ - if ((Bin)) { \ - pb[i].val = (Bin); \ - pb[i].next = mso; \ - mso = &pb[i]; \ - i++; \ +#define ADD_BINARY(Bin) \ + if ((Bin)) { \ + pb[i].thing_word = REFC_BINARY_SUBTAG; \ + pb[i].val = (Bin); \ + pb[i].next = oh.first; \ + oh.first = (struct erl_off_heap_header*) &pb[i]; \ + i++; \ } erts_get_default_trace_pattern(NULL, @@ -1401,11 +1417,6 @@ setup_reference_table(void) ADD_BINARY(default_match_spec); ADD_BINARY(default_meta_match_spec); - oh.mso = mso; - oh.externals = NULL; -#ifndef HYBRID /* FIND ME! */ - oh.funs = NULL; -#endif insert_offheap(&oh, BIN_REF, AM_match_spec); #undef ADD_BINARY } diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index eb759b87e9..b0a63ae035 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -131,7 +131,7 @@ typedef struct dist_entry_ { ErtsLink *nlinks; /* Link tree with subtrees */ ErtsMonitor *monitors; /* Monitor tree */ - erts_smp_spinlock_t qlock; /* Protects qflgs and out_queue */ + erts_smp_mtx_t qlock; /* Protects qflgs and out_queue */ Uint32 qflgs; Sint qsize; ErtsDistOutputQueue out_queue; diff --git a/erts/emulator/beam/erl_obsolete.c b/erts/emulator/beam/erl_obsolete.c deleted file mode 100644 index 9c5a7c7ff9..0000000000 --- a/erts/emulator/beam/erl_obsolete.c +++ /dev/null @@ -1,186 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "erl_driver.h" - -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * * - * ------------------------- OBSOLETE! DO NOT USE! ------------------------- * - * * -\* */ - -/* cut from ../obsolete/driver.h (since it doesn't mix well with other - * headers from the emulator). - */ -#ifdef __WIN32__ -#ifdef CONST -# undef CONST -#endif -#endif - -#if ((defined(__STDC__) || defined(SABER)) && !defined(NO_PROTOTYPE)) || defined(__cplusplus) || defined(USE_PROTOTYPE) -# define _USING_PROTOTYPES_ 1 -# define _ANSI_ARGS_(x) x -# define CONST const -#else -# define _ANSI_ARGS_(x) () -# define CONST -#endif - -typedef void* erl_mutex_t; -typedef void* erl_cond_t; -typedef void* erl_thread_t; - -EXTERN erl_mutex_t erts_mutex_create _ANSI_ARGS_((void)); -EXTERN int erts_mutex_destroy _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_lock _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_unlock _ANSI_ARGS_((erl_mutex_t)); - -EXTERN erl_cond_t erts_cond_create _ANSI_ARGS_((void)); -EXTERN int erts_cond_destroy _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_signal _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_broadcast _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_wait _ANSI_ARGS_((erl_cond_t, erl_mutex_t)); -EXTERN int erts_cond_timedwait _ANSI_ARGS_((erl_cond_t, erl_mutex_t, long)); - -EXTERN int erts_thread_create _ANSI_ARGS_((erl_thread_t*, - void* (*func)(void*), - void* arg, - int detached)); -EXTERN erl_thread_t erts_thread_self _ANSI_ARGS_((void)); -EXTERN void erts_thread_exit _ANSI_ARGS_((void*)); -EXTERN int erts_thread_join _ANSI_ARGS_((erl_thread_t, void**)); -EXTERN int erts_thread_kill _ANSI_ARGS_((erl_thread_t)); - -/* - * These functions implement the thread interface in ../obsolete/driver.h. - * Do *not* use this interface! Within the emulator, use the erl_threads.h, - * erl_smp.h, or ethread.h interface. From a driver use the thread interface - * in erl_driver.h. - */ - -erl_mutex_t -erts_mutex_create(void) -{ - return (erl_mutex_t) erl_drv_mutex_create(NULL); -} - -int -erts_mutex_destroy(erl_mutex_t mtx) -{ - erl_drv_mutex_destroy((ErlDrvMutex *) mtx); - return 0; -} - -int -erts_mutex_lock(erl_mutex_t mtx) -{ - erl_drv_mutex_lock((ErlDrvMutex *) mtx); - return 0; -} - -int -erts_mutex_unlock(erl_mutex_t mtx) -{ - erl_drv_mutex_unlock((ErlDrvMutex *) mtx); - return 0; -} - -erl_cond_t -erts_cond_create(void) -{ - return (erl_cond_t) erl_drv_cond_create(NULL); -} - -int -erts_cond_destroy(erl_cond_t cnd) -{ - erl_drv_cond_destroy((ErlDrvCond *) cnd); - return 0; -} - - -int -erts_cond_signal(erl_cond_t cnd) -{ - erl_drv_cond_signal((ErlDrvCond *) cnd); - return 0; -} - -int -erts_cond_broadcast(erl_cond_t cnd) -{ - erl_drv_cond_broadcast((ErlDrvCond *) cnd); - return 0; -} - - -int -erts_cond_wait(erl_cond_t cnd, erl_mutex_t mtx) -{ - erl_drv_cond_wait((ErlDrvCond *) cnd, (ErlDrvMutex *) mtx); - return 0; -} - -int -erts_cond_timedwait(erl_cond_t cnd, erl_mutex_t mtx, long ms) -{ - return ENOTSUP; -} - -int -erts_thread_create(erl_thread_t *tid, - void* (*func)(void*), - void* arg, - int detached) -{ - if (detached) - return ENOTSUP; - return erl_drv_thread_create(NULL, (ErlDrvTid *) tid, func, arg, NULL); -} - -erl_thread_t -erts_thread_self(void) -{ - return (erl_thread_t) erl_drv_thread_self(); -} - -void -erts_thread_exit(void *res) -{ - erl_drv_thread_exit(res); -} - -int -erts_thread_join(erl_thread_t tid, void **respp) -{ - return erl_drv_thread_join((ErlDrvTid) tid, respp); -} - -int -erts_thread_kill(erl_thread_t tid) -{ - return ENOTSUP; -} - diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 0b6bb0d8e9..1b07024ca1 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -129,7 +129,7 @@ reset_handle(ErtsPortTask *ptp) { if (ptp->handle) { ASSERT(ptp == handle2task(ptp->handle)); - erts_smp_atomic_set(ptp->handle, (long) NULL); + erts_smp_atomic_set(ptp->handle, (erts_aint_t) NULL); } } @@ -138,7 +138,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { ptp->handle = pthp; if (pthp) { - erts_smp_atomic_set(pthp, (long) ptp); + erts_smp_atomic_set(pthp, (erts_aint_t) ptp); ASSERT(ptp == handle2task(ptp->handle)); } } @@ -568,7 +568,7 @@ erts_port_task_schedule(Eterm id, ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); if (xrunq) { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (long) xrunq); + erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); runq = xrunq; } @@ -625,6 +625,7 @@ erts_port_task_schedule(Eterm id, if (!enq_port) { ERTS_PT_CHK_PRES_PORTQ(runq, pp); + erts_smp_runq_unlock(runq); } else { enqueue_port(runq, pp); @@ -634,9 +635,10 @@ erts_port_task_schedule(Eterm id, profile_runnable_port(pp, am_active); } + erts_smp_runq_unlock(runq); + erts_smp_notify_inc_runq(runq); } - erts_smp_runq_unlock(runq); return 0; } @@ -725,7 +727,8 @@ resume_after_block(void *vd) ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd; erts_smp_runq_lock(d->runq); if (d->resp) - *d->resp = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; + *d->resp = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); } /* @@ -746,7 +749,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ErtsPortTask *ptp; int res = 0; int reds = ERTS_PORT_REDS_EXECUTE; - long io_tasks_executed = 0; + erts_aint_t io_tasks_executed = 0; int fpe_was_unmasked; ErtsPortTaskExeBlockData blk_data = {runq, NULL}; @@ -940,18 +943,19 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } else { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (long) xrunq); + erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); enqueue_port(xrunq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; - erts_smp_notify_inc_runq(xrunq); erts_smp_runq_unlock(xrunq); + erts_smp_notify_inc_runq(xrunq); } #endif port_was_enqueued = 1; } - res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; + res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); ERTS_PT_CHK_PRES_PORTQ(runq, pp); @@ -969,15 +973,16 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_port_release(pp); #else { - long refc = erts_smp_atomic_dectest(&pp->refc); + erts_aint_t refc; + erts_smp_mtx_unlock(pp->lock); + refc = erts_smp_atomic_dectest(&pp->refc); ASSERT(refc >= 0); - if (refc > 0) - erts_smp_mtx_unlock(pp->lock); - else { + if (refc == 0) { erts_smp_runq_unlock(runq); erts_port_cleanup(pp); /* Might aquire runq lock */ erts_smp_runq_lock(runq); - res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; + res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); } } #endif @@ -1110,9 +1115,8 @@ erts_port_migrate(Port *prt, int *prt_locked, if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt)) return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; dequeue_port(from_rq, prt); - erts_smp_atomic_set(&prt->run_queue, (long) to_rq); + erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) to_rq); enqueue_port(to_rq, prt); - erts_smp_notify_inc_runq(to_rq); return ERTS_MIGRATE_SUCCESS; } @@ -1124,7 +1128,7 @@ erts_port_migrate(Port *prt, int *prt_locked, void erts_port_task_init(void) { - erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (long) 0); + erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (erts_aint_t) 0); init_port_task_alloc(); init_port_taskq_alloc(); } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index f12d02da0c..714b4ea7dd 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -79,7 +79,7 @@ ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void); ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp) { - erts_smp_atomic_init(pthp, (long) NULL); + erts_smp_atomic_init(pthp, (erts_aint_t) NULL); } ERTS_GLB_INLINE int diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 11ca85a41c..ddfc27a93f 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -24,7 +24,6 @@ #endif #include <stddef.h> /* offsetof() */ -#include <ctype.h> #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -38,6 +37,8 @@ #include "erl_instrument.h" #include "erl_threads.h" #include "erl_binary.h" +#include "beam_bp.h" +#include "erl_cpu_topology.h" #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ @@ -45,14 +46,23 @@ #define ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST (CONTEXT_REDS/10) -#define ERTS_SCHED_SLEEP_SPINCOUNT 10000 +#define ERTS_SCHED_SPIN_UNTIL_YIELD 100 + +#define ERTS_SCHED_SYS_SLEEP_SPINCOUNT 10 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT 1000 +#define ERTS_SCHED_TSE_SLEEP_SPINCOUNT \ + (ERTS_SCHED_SYS_SLEEP_SPINCOUNT*ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT) +#define ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT 0 + +#define ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH (200*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_HIGH (50*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_MEDIUM (10*CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_LOW (CONTEXT_REDS) +#define ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW (CONTEXT_REDS/10) -#define ERTS_WAKEUP_OTHER_LIMIT (100*CONTEXT_REDS/2) #define ERTS_WAKEUP_OTHER_DEC 10 #define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10) -#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff) - #if 0 || defined(DEBUG) #define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA #endif @@ -105,6 +115,8 @@ Uint erts_no_schedulers; Uint erts_max_processes = ERTS_DEFAULT_MAX_PROCESSES; Uint erts_process_tab_index_mask; +static int wakeup_other_limit; + int erts_sched_thread_suggested_stack_size = -1; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -115,19 +127,38 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; int erts_disable_proc_not_running_opt; -#define ERTS_SCHED_CHANGING_ONLINE 1 -#define ERTS_SCHED_CHANGING_MULTI_SCHED 2 +#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0) +#define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) +#define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) + +#ifndef DEBUG + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ + erts_smp_atomic32_set(&schdlr_sspnd.changing, (VAL)) + +#else + +#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ +do { \ + erts_aint32_t old_val__; \ + old_val__ = erts_smp_atomic32_xchg(&schdlr_sspnd.changing, \ + (VAL)); \ + ASSERT(old_val__ == (OLD_VAL)); \ +} while (0) + +#endif + static struct { erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - int changing; int online; int curr_online; int wait_curr_online; - erts_smp_atomic_t active; + erts_smp_atomic32_t changing; + erts_smp_atomic32_t active; struct { - erts_smp_atomic_t ongoing; + erts_smp_atomic32_t ongoing; long wait_active; ErtsProcList *procs; } msb; /* Multi Scheduling Block */ @@ -135,11 +166,11 @@ static struct { static struct { erts_smp_mtx_t update_mtx; - erts_smp_atomic_t active_runqs; + erts_smp_atomic32_t active_runqs; int last_active_runqs; - erts_smp_atomic_t used_runqs; + erts_smp_atomic32_t used_runqs; int forced_check_balance; - erts_smp_atomic_t checking_balance; + erts_smp_atomic32_t checking_balance; int halftime; int full_reds_history_index; struct { @@ -159,44 +190,6 @@ do { \ #endif -/* - * Cpu topology hierarchy. - */ -#define ERTS_TOPOLOGY_NODE 0 -#define ERTS_TOPOLOGY_PROCESSOR 1 -#define ERTS_TOPOLOGY_PROCESSOR_NODE 2 -#define ERTS_TOPOLOGY_CORE 3 -#define ERTS_TOPOLOGY_THREAD 4 -#define ERTS_TOPOLOGY_LOGICAL 5 - -#define ERTS_TOPOLOGY_MAX_DEPTH 6 - -typedef struct { - int bind_id; - int bound_id; -} ErtsCpuBindData; - -static ErtsCpuBindData *scheduler2cpu_map; -erts_smp_rwmtx_t erts_cpu_bind_rwmtx; - -typedef enum { - ERTS_CPU_BIND_SPREAD, - ERTS_CPU_BIND_PROCESSOR_SPREAD, - ERTS_CPU_BIND_THREAD_SPREAD, - ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD, - ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD, - ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD, - ERTS_CPU_BIND_NO_SPREAD, - ERTS_CPU_BIND_NONE -} ErtsCpuBindOrder; - -ErtsCpuBindOrder cpu_bind_order; - -static erts_cpu_topology_t *user_cpudata; -static int user_cpudata_size; -static erts_cpu_topology_t *system_cpudata; -static int system_cpudata_size; - erts_sched_stat_t erts_sched_stat; ErtsRunQueue *erts_common_run_queue; @@ -207,11 +200,11 @@ static erts_tsd_key_t sched_data_key; static erts_smp_mtx_t proc_tab_mtx; -static erts_smp_atomic_t function_calls; +static erts_smp_atomic32_t function_calls; #ifdef ERTS_SMP -static erts_smp_atomic_t doing_sys_schedule; -static erts_smp_atomic_t no_empty_run_queues; +static erts_smp_atomic32_t doing_sys_schedule; +static erts_smp_atomic32_t no_empty_run_queues; #else /* !ERTS_SMP */ ErtsSchedulerData *erts_scheduler_data; #endif @@ -219,12 +212,18 @@ ErtsSchedulerData *erts_scheduler_data; ErtsAlignedRunQueue *erts_aligned_run_queues; Uint erts_no_run_queues; +ErtsAlignedSchedulerData *erts_aligned_scheduler_data; + +#ifdef ERTS_SMP + typedef union { - ErtsSchedulerData esd; - char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; -} ErtsAlignedSchedulerData; + ErtsSchedulerSleepInfo ssi; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerSleepInfo))]; +} ErtsAlignedSchedulerSleepInfo; -ErtsAlignedSchedulerData *erts_aligned_scheduler_data; +static ErtsAlignedSchedulerSleepInfo *aligned_sched_sleep_info; + +#endif #ifndef BM_COUNTERS static int processes_busy; @@ -249,7 +248,10 @@ Uint erts_num_active_procs; Process** erts_active_procs; #endif -static erts_smp_atomic_t process_count; +#if ERTS_MAX_PROCESSES > 0x7fffffff +#error "Need to store process_count in another type" +#endif +static erts_smp_atomic32_t process_count; typedef struct ErtsTermProcElement_ ErtsTermProcElement; struct ErtsTermProcElement_ { @@ -283,8 +285,9 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, 200, ERTS_ALC_T_PROC_LIST) -#define ERTS_RUNQ_IX(IX) (&erts_aligned_run_queues[(IX)].runq) -#define ERTS_SCHEDULER_IX(IX) (&erts_aligned_scheduler_data[(IX)].esd) +#define ERTS_SCHED_SLEEP_INFO_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &aligned_sched_sleep_info[(IX)].ssi) #define ERTS_FOREACH_RUNQ(RQVAR, DO) \ do { \ @@ -340,17 +343,8 @@ static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, #ifdef ERTS_SMP static void handle_pending_exiters(ErtsProcList *); -static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata, - int size, - ErtsCpuBindOrder bind_order, - int mk_seq); -static void signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size); - #endif -static void early_cpu_bind_init(void); -static void late_cpu_bind_init(void); - #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) @@ -388,7 +382,12 @@ erts_pre_init_process(void) erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].get_locks = ERTS_PSD_DIST_ENTRY_GET_LOCKS; erts_psd_required_locks[ERTS_PSD_DIST_ENTRY].set_locks - = ERTS_PSD_DIST_ENTRY_GET_LOCKS; + = ERTS_PSD_DIST_ENTRY_SET_LOCKS; + + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].get_locks + = ERTS_PSD_CALL_TIME_BP_GET_LOCKS; + erts_psd_required_locks[ERTS_PSD_CALL_TIME_BP].set_locks + = ERTS_PSD_CALL_TIME_BP_SET_LOCKS; /* Check that we have locks for all entries */ for (ix = 0; ix < ERTS_PSD_SIZE; ix++) { @@ -401,18 +400,18 @@ erts_pre_init_process(void) /* initialize the scheduler */ void -erts_init_process(void) +erts_init_process(int ncpu) { Uint proc_bits = ERTS_PROC_BITS; #ifdef ERTS_SMP erts_disable_proc_not_running_opt = 0; - erts_init_proc_lock(); + erts_init_proc_lock(ncpu); #endif init_proclist_alloc(); - erts_smp_atomic_init(&process_count, 0); + erts_smp_atomic32_init(&process_count, 0); if (erts_use_r9_pids_ports) { proc_bits = ERTS_R9_PROC_BITS; @@ -572,6 +571,198 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) #ifdef ERTS_SMP +void +erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) +{ + switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) { + case ERTS_SSI_FLG_POLL_SLEEPING: + erts_sys_schedule_interrupt(1); + break; + case ERTS_SSI_FLG_TSE_SLEEPING: + erts_tse_set(ssi->event); + break; + case 0: + break; + default: + erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + __FILE__, __LINE__); + break; + } +} + +typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; +struct erts_misc_aux_work_t_ { + erts_misc_aux_work_t *next; + void (*func)(void *); + void *arg; +}; + +typedef struct { + erts_smp_mtx_t mtx; + erts_misc_aux_work_t *first; + erts_misc_aux_work_t *last; +} erts_misc_aux_work_q_t; + +typedef union { + erts_misc_aux_work_q_t data; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_misc_aux_work_q_t))]; +} erts_algnd_misc_aux_work_q_t; + +static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues; + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work, + erts_misc_aux_work_t, + 200, + ERTS_ALC_T_MISC_AUX_WORK) + +static void +init_misc_aux_work(void) +{ + int ix; + + init_misc_aux_work_alloc(); + + misc_aux_work_queues = erts_alloc(ERTS_ALC_T_MISC_AUX_WORK_Q, + (sizeof(erts_algnd_misc_aux_work_q_t) + *(erts_no_schedulers+1))); + if ((((UWord) misc_aux_work_queues) & ERTS_CACHE_LINE_MASK) != 0) + misc_aux_work_queues = ((erts_algnd_misc_aux_work_q_t *) + ((((UWord) misc_aux_work_queues) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + + for (ix = 0; ix < erts_no_schedulers; ix++) { + erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx, + "misc_aux_work_queue", + make_small(ix + 1)); + misc_aux_work_queues[ix].data.first = NULL; + misc_aux_work_queues[ix].data.last = NULL; + } +} + +static void +handle_misc_aux_work(ErtsSchedulerData *esdp) +{ + int ix = (int) esdp->no - 1; + erts_misc_aux_work_t *mawp; + + erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); + mawp = misc_aux_work_queues[ix].data.first; + misc_aux_work_queues[ix].data.first = NULL; + misc_aux_work_queues[ix].data.last = NULL; + erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); + + while (mawp) { + erts_misc_aux_work_t *free_mawp; + mawp->func(mawp->arg); + free_mawp = mawp; + mawp = mawp->next; + misc_aux_work_free(free_mawp); + } +} + +void +erts_smp_schedule_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg) +{ + int ix, ignore_ix = -1; + + if (ignore_self) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + ignore_ix = (int) esdp->no - 1; + } + + ASSERT(0 <= max_sched && max_sched <= erts_no_schedulers); + + for (ix = 0; ix < max_sched; ix++) { + erts_aint32_t aux_work; + erts_misc_aux_work_t *mawp; + ErtsSchedulerSleepInfo *ssi; + if (ix == ignore_ix) + continue; + + mawp = misc_aux_work_alloc(); + + mawp->func = func; + mawp->arg = arg; + mawp->next = NULL; + + erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); + if (!misc_aux_work_queues[ix].data.last) + misc_aux_work_queues[ix].data.first = mawp; + else + misc_aux_work_queues[ix].data.last->next = mawp; + misc_aux_work_queues[ix].data.last = mawp; + erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); + + ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + aux_work = erts_smp_atomic32_bor(&ssi->aux_work, + ERTS_SSI_AUX_WORK_MISC); + if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0) + erts_sched_poke(ssi); + } +} + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN +void +erts_smp_notify_check_children_needed(void) +{ + int i; + + for (i = 0; i < erts_no_schedulers; i++) { + erts_aint32_t aux_work; + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(i); + aux_work = erts_smp_atomic32_bor(&ssi->aux_work, + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) + erts_sched_poke(ssi); + } +} +#endif + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK +static ERTS_INLINE erts_aint32_t +blockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + erts_aint32_t aux_work) +{ + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + if (aux_work & ERTS_SSI_AUX_WORK_MISC) { + aux_work = erts_smp_atomic32_band(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_MISC); + aux_work &= ~ERTS_SSI_AUX_WORK_MISC; + handle_misc_aux_work(esdp); + } +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { + aux_work = erts_smp_atomic32_band(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; + erts_check_children(); + } +#endif + } + return aux_work; +} + +#endif + +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +static ERTS_INLINE erts_aint32_t +nonblockable_aux_work(ErtsSchedulerData *esdp, + ErtsSchedulerSleepInfo *ssi, + erts_aint32_t aux_work) +{ + if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) { + + } +} +#endif + static void prepare_for_block(void *vrq) { @@ -624,9 +815,33 @@ erts_active_schedulers(void) return as; } +static ERTS_INLINE int +prepare_for_sys_schedule(void) +{ +#ifdef ERTS_SMP + while (!erts_port_task_have_outstanding_io_tasks() + && !erts_smp_atomic32_xchg(&doing_sys_schedule, 1)) { + if (!erts_port_task_have_outstanding_io_tasks()) + return 1; + erts_smp_atomic32_set(&doing_sys_schedule, 0); + } + return 0; +#else + return !erts_port_task_have_outstanding_io_tasks(); +#endif +} + #ifdef ERTS_SMP static ERTS_INLINE void +sched_change_waiting_sys_to_waiting(Uint no, ErtsRunQueue *rq) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + ASSERT(rq->waiting < 0); + rq->waiting *= -1; +} + +static ERTS_INLINE void sched_waiting(Uint no, ErtsRunQueue *rq) { ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -656,265 +871,451 @@ sched_active(Uint no, ErtsRunQueue *rq) static int ERTS_INLINE ongoing_multi_scheduling_block(void) { - return erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) != 0; + return erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing) != 0; } static ERTS_INLINE void empty_runq(ErtsRunQueue *rq) { - long oifls = erts_smp_atomic_band(&rq->info_flags, ~ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_band(&rq->info_flags, + ~ERTS_RUNQ_IFLG_NONEMPTY); if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) { #ifdef DEBUG - long empty = erts_smp_atomic_read(&no_empty_run_queues); - ASSERT(0 <= empty && empty < erts_no_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); + /* + * For a short period of time no_empty_run_queues may have + * been increased twice for a specific run queue. + */ + ASSERT(0 <= empty && empty < 2*erts_no_run_queues); #endif - erts_smp_atomic_inc(&no_empty_run_queues); + erts_smp_atomic32_inc(&no_empty_run_queues); } } static ERTS_INLINE void non_empty_runq(ErtsRunQueue *rq) { - long oifls = erts_smp_atomic_bor(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_bor(&rq->info_flags, + ERTS_RUNQ_IFLG_NONEMPTY); if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) { #ifdef DEBUG - long empty = erts_smp_atomic_read(&no_empty_run_queues); - ASSERT(0 < empty && empty <= erts_no_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); + /* + * For a short period of time no_empty_run_queues may have + * been increased twice for a specific run queue. + */ + ASSERT(0 < empty && empty <= 2*erts_no_run_queues); #endif - erts_smp_atomic_dec(&no_empty_run_queues); + erts_smp_atomic32_dec(&no_empty_run_queues); } } -static ERTS_INLINE int -sched_spin_wake(ErtsRunQueue *rq) +static erts_aint32_t +sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + erts_aint32_t xflgs = 0; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_atomic_inc(&rq->spin_wake); - return 1; - } - return 0; -#endif + do { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (!(oflgs & ERTS_SSI_FLG_SUSPENDED)); + return oflgs; } -static ERTS_INLINE int -sched_spin_wake_all(ErtsRunQueue *rq) +static erts_aint32_t +sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) { -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - return 0; -#else - long val; - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING; - val = erts_smp_atomic_read(&rq->spin_waiter); - ASSERT(val >= 0); - if (val != 0) - erts_smp_atomic_add(&rq->spin_wake, val); - return val; -#endif + do { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } while (oflgs & ERTS_SSI_FLG_WAITING); + return oflgs; +} + +static erts_aint32_t +sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + erts_aint32_t flgs; + + do { + flgs = erts_smp_atomic32_read(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; } +static erts_aint32_t +sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) +{ + erts_aint32_t oflgs; + erts_aint32_t nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type; + erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + + if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING) + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) + != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { + return oflgs; + } + xflgs = oflgs; + nflgs |= oflgs & ERTS_SSI_FLG_SUSPENDED; + } +} + +#define ERTS_SCHED_WAIT_WOKEN(FLGS) \ + (((FLGS) & (ERTS_SSI_FLG_WAITING|ERTS_SSI_FLG_SUSPENDED)) \ + != ERTS_SSI_FLG_WAITING) + static void -sched_sys_wait(Uint no, ErtsRunQueue *rq) +scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { - long dt; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + int spincount; + erts_aint32_t flgs; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + erts_aint32_t aux_work; +#endif + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); + erts_smp_spin_lock(&rq->sleepers.lock); + flgs = sched_prep_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { + /* Go suspend instead... */ + erts_smp_spin_unlock(&rq->sleepers.lock); + return; + } + + ssi->prev = NULL; + ssi->next = rq->sleepers.list; + if (rq->sleepers.list) + rq->sleepers.list->prev = ssi; + rq->sleepers.list = ssi; + erts_smp_spin_unlock(&rq->sleepers.lock); + + /* + * If all schedulers are waiting, one of them *should* + * be waiting in erl_sys_schedule() + */ + + if (!prepare_for_sys_schedule()) { + + sched_waiting(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + tse_wait: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + tse_blockable_aux_work: + aux_work = blockable_aux_work(esdp, ssi, aux_work); #endif + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - sched_waiting_sys(no, rq); + while (1) { -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + + flgs = sched_spin_wait(ssi, spincount); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_TSE_SLEEPING); + if (flgs & ERTS_SSI_FLG_SLEEPING) { + int res; + ASSERT(flgs & ERTS_SSI_FLG_TSE_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } - erl_sys_schedule(1); /* Might give us something to do */ + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + flgs = sched_prep_cont_spin_wait(ssi); + spincount = ERTS_SCHED_TSE_SLEEP_SPINCOUNT; + + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + break; + } + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto tse_blockable_aux_work; + } +#endif - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_runq_unlock(rq); } - } - erts_smp_runq_lock(rq); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + + erts_smp_runq_lock(rq); + sched_active(esdp->no, rq); + } else { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); + erts_aint_t dt; + + erts_smp_atomic32_set(&function_calls, 0); + *fcalls = 0; + + sched_waiting_sys(esdp->no, rq); + + erts_smp_runq_unlock(rq); + + spincount = ERTS_SCHED_SYS_SLEEP_SPINCOUNT; + + while (spincount-- > 0) { + + sys_poll_aux_work: + + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + + erl_sys_schedule(1); /* Might give us something to do */ + + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); + + sys_aux_work: + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + + flgs = erts_smp_atomic32_read(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + } + + /* + * If we got new I/O tasks we aren't allowed to + * call erl_sys_schedule() until it is handled. + */ + if (erts_port_task_have_outstanding_io_tasks()) { + erts_smp_atomic32_set(&doing_sys_schedule, 0); + /* + * Got to check that we still got I/O tasks; otherwise + * we have to continue checking for I/O... + */ + if (!prepare_for_sys_schedule()) { + spincount *= ERTS_SCHED_TSE_SLEEP_SPINCOUNT_FACT; + goto tse_wait; + } + } + } + + erts_smp_runq_lock(rq); + /* * If we got new I/O tasks we aren't allowed to * sleep in erl_sys_schedule(). */ - if (!erts_port_task_have_outstanding_io_tasks()) { -#endif + if (erts_port_task_have_outstanding_io_tasks()) { + erts_smp_atomic32_set(&doing_sys_schedule, 0); + /* + * Got to check that we still got I/O tasks; otherwise + * we have to wait in erl_sys_schedule() after all... + */ + if (prepare_for_sys_schedule()) + goto do_sys_schedule; + + /* + * Not allowed to wait in erl_sys_schedule; + * do tse wait instead... + */ + sched_change_waiting_sys_to_waiting(esdp->no, rq); + erts_smp_runq_unlock(rq); + spincount = 0; + goto tse_wait; + } + else { + do_sys_schedule: erts_sys_schedule_interrupt(0); + flgs = sched_set_sleeptype(ssi, ERTS_SSI_FLG_POLL_SLEEPING); + if (!(flgs & ERTS_SSI_FLG_SLEEPING)) { + if (!(flgs & ERTS_SSI_FLG_WAITING)) + goto sys_locked_woken; + erts_smp_runq_unlock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (!(flgs & ERTS_SSI_FLG_WAITING)) { + ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); + goto sys_woken; + } + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + goto sys_poll_aux_work; + } + + ASSERT(flgs & ERTS_SSI_FLG_POLL_SLEEPING); + ASSERT(flgs & ERTS_SSI_FLG_WAITING); + erts_smp_runq_unlock(rq); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + erl_sys_schedule(0); - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); - erts_smp_runq_lock(rq); + flgs = sched_prep_cont_spin_wait(ssi); + if (flgs & ERTS_SSI_FLG_WAITING) + goto sys_aux_work; -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 + sys_woken: + erts_smp_runq_lock(rq); + sys_locked_woken: + erts_smp_atomic32_set(&doing_sys_schedule, 0); + if (flgs & ~ERTS_SSI_FLG_SUSPENDED) + erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + sched_active_sys(esdp->no, rq); } } -#endif - - sched_active_sys(no, rq); -} -static void -sched_cnd_wait(Uint no, ErtsRunQueue *rq) -{ -#if ERTS_SCHED_SLEEP_SPINCOUNT != 0 - int val; - int spincount = ERTS_SCHED_SLEEP_SPINCOUNT; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); -#endif - - sched_waiting(no, rq); - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); - -#if ERTS_SCHED_SLEEP_SPINCOUNT == 0 - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); -#else - erts_smp_atomic_inc(&rq->spin_waiter); - erts_smp_mtx_unlock(&rq->mtx); - - while (spincount-- > 0) { - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) { - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val != 0) - goto woken; - if (spincount == 0) - goto sleep; - erts_smp_mtx_unlock(&rq->mtx); - } - } - - erts_smp_mtx_lock(&rq->mtx); - val = erts_smp_atomic_read(&rq->spin_wake); - ASSERT(val >= 0); - if (val == 0) { - sleep: - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - erts_smp_cnd_wait(&rq->cnd, &rq->mtx); - } - else { - woken: - erts_smp_atomic_dec(&rq->spin_wake); - ASSERT(erts_smp_atomic_read(&rq->spin_wake) >= 0); - erts_smp_atomic_dec(&rq->spin_waiter); - ASSERT(erts_smp_atomic_read(&rq->spin_waiter) >= 0); - } -#endif - - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - prepare_for_block, - resume_after_block, - (void *) rq); - - sched_active(no, rq); } -static void -wake_one_scheduler(void) -{ - ASSERT(erts_common_run_queue); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(erts_common_run_queue)); - if (erts_common_run_queue->waiting) { - if (!sched_spin_wake(erts_common_run_queue)) { - if (erts_common_run_queue->waiting == -1) /* One scheduler waiting - and doing so in - sys_schedule */ - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&erts_common_run_queue->cnd); - } +static ERTS_INLINE erts_aint32_t +ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) +{ + /* reset all flags but suspended */ + erts_aint32_t oflgs; + erts_aint32_t nflgs = 0; + erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + while (1) { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return oflgs; + nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; + xflgs = oflgs; } } static void -wake_scheduler(ErtsRunQueue *rq, int incq) +wake_scheduler(ErtsRunQueue *rq, int incq, int one) { - ASSERT(!erts_common_run_queue); - ASSERT(-1 <= rq->waiting && rq->waiting <= 1); - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); - if (rq->waiting && !rq->woken) { - if (!sched_spin_wake(rq)) { - if (rq->waiting < 0) - erts_sys_schedule_interrupt(1); - else - erts_smp_cnd_signal(&rq->cnd); + int res; + ErtsSchedulerSleepInfo *ssi; + ErtsSchedulerSleepList *sl; + + /* + * The unlocked run queue is not strictly necessary + * from a thread safety or deadlock prevention + * perspective. It will, however, cost us performance + * if it is locked during wakup of another scheduler, + * so all code *should* handle this without having + * the lock on the run queue. + */ + ERTS_SMP_LC_ASSERT(!erts_smp_lc_runq_is_locked(rq)); + + sl = &rq->sleepers; + + erts_smp_spin_lock(&sl->lock); + ssi = sl->list; + if (!ssi) + erts_smp_spin_unlock(&sl->lock); + else if (one) { + erts_aint32_t flgs; + if (ssi->prev) + ssi->prev->next = ssi->next; + else { + ASSERT(sl->list == ssi); + sl->list = ssi->next; } - rq->woken = 1; - if (incq) + if (ssi->next) + ssi->next->prev = ssi->prev; + + res = sl->list != NULL; + erts_smp_spin_unlock(&sl->lock); + + flgs = ssi_flags_set_wake(ssi); + erts_sched_finish_poke(ssi, flgs); + + if (incq && !erts_common_run_queue && (flgs & ERTS_SSI_FLG_WAITING)) non_empty_runq(rq); } + else { + sl->list = NULL; + erts_smp_spin_unlock(&sl->lock); + do { + ErtsSchedulerSleepInfo *wake_ssi = ssi; + ssi = ssi->next; + erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi)); + } while (ssi); + } } static void wake_all_schedulers(void) { - if (erts_common_run_queue) { - erts_smp_runq_lock(erts_common_run_queue); - if (erts_common_run_queue->waiting) { - if (erts_common_run_queue->waiting < 0) - erts_sys_schedule_interrupt(1); - sched_spin_wake_all(erts_common_run_queue); - erts_smp_cnd_broadcast(&erts_common_run_queue->cnd); - } - erts_smp_runq_unlock(erts_common_run_queue); - } + if (erts_common_run_queue) + wake_scheduler(erts_common_run_queue, 0, 0); else { int ix; for (ix = 0; ix < erts_no_run_queues; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - erts_smp_runq_lock(rq); - wake_scheduler(rq, 0); - erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); } } } @@ -922,21 +1323,23 @@ wake_all_schedulers(void) static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { - long iflgs; + erts_aint32_t iflgs; ErtsRunQueue *wrq; if (crq->ix == ix) return 0; wrq = ERTS_RUNQ_IX(ix); - iflgs = erts_smp_atomic_read(&wrq->info_flags); + iflgs = erts_smp_atomic32_read(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { - erts_smp_xrunq_lock(crq, wrq); if (activate) { - if (ix == erts_smp_atomic_cmpxchg(&balance_info.active_runqs, ix+1, ix)) { + if (ix == erts_smp_atomic32_cmpxchg(&balance_info.active_runqs, + ix+1, + ix)) { + erts_smp_xrunq_lock(crq, wrq); wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; + erts_smp_xrunq_unlock(crq, wrq); } } - wake_scheduler(wrq, 0); - erts_smp_xrunq_unlock(crq, wrq); + wake_scheduler(wrq, 0, 1); return 1; } return 0; @@ -947,8 +1350,8 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq) { int ix = crq->ix; int stop_ix = ix; - int active_ix = erts_smp_atomic_read(&balance_info.active_runqs); - int balance_ix = erts_smp_atomic_read(&balance_info.used_runqs); + int active_ix = erts_smp_atomic32_read(&balance_info.active_runqs); + int balance_ix = erts_smp_atomic32_read(&balance_info.used_runqs); if (active_ix > balance_ix) active_ix = balance_ix; @@ -982,19 +1385,42 @@ static ERTS_INLINE void smp_notify_inc_runq(ErtsRunQueue *runq) { #ifdef ERTS_SMP - if (erts_common_run_queue) - wake_one_scheduler(); - else - wake_scheduler(runq, 1); + if (runq) + wake_scheduler(runq, 1, 1); #endif } void -erts_smp_notify_inc_runq__(ErtsRunQueue *runq) +erts_smp_notify_inc_runq(ErtsRunQueue *runq) { smp_notify_inc_runq(runq); } +void +erts_sched_notify_check_cpu_bind(void) +{ +#ifdef ERTS_SMP + int ix; + if (erts_common_run_queue) { + for (ix = 0; ix < erts_no_schedulers; ix++) + erts_smp_atomic32_set(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); + wake_all_schedulers(); + } + else { + for (ix = 0; ix < erts_no_run_queues; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); + }; + } +#else + erts_sched_check_cpu_bind(erts_get_scheduler_data()); +#endif +} + + #ifdef ERTS_SMP ErtsRunQueue * @@ -1136,20 +1562,24 @@ static void evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) { Port *prt; + int notify_to_rq = 0; int prio; int prt_locked = 0; int rq_locked = 0; int evac_rq_locked = 1; + ErtsMigrateResult mres; erts_smp_runq_lock(evac_rq); + erts_smp_atomic32_bor(&evac_rq->scheduler->ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK | ERTS_RUNQ_FLGS_EVACUATE_QMASK | ERTS_RUNQ_FLG_SUSPENDED); - erts_smp_atomic_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); - + erts_smp_atomic32_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); /* * Need to set up evacuation paths first since we * may release the run queue lock on evac_rq @@ -1177,9 +1607,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) /* Evacuate scheduled ports */ prt = evac_rq->ports.start; while (prt) { - (void) erts_port_migrate(prt, &prt_locked, + mres = erts_port_migrate(prt, &prt_locked, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (prt_locked) erts_smp_port_unlock(prt); if (!evac_rq_locked) { @@ -1208,9 +1640,11 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) goto end_of_proc; } - (void) erts_proc_migrate(proc, &proc_locks, + mres = erts_proc_migrate(proc, &proc_locks, evac_rq, &evac_rq_locked, rq, &rq_locked); + if (mres == ERTS_MIGRATE_SUCCESS) + notify_to_rq = 1; if (proc_locks) erts_smp_proc_unlock(proc, proc_locks); if (!evac_rq_locked) { @@ -1242,10 +1676,13 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) if (rq_locked) erts_smp_runq_unlock(rq); - if (!evac_rq_locked) - erts_smp_runq_lock(evac_rq); - wake_scheduler(evac_rq, 0); - erts_smp_runq_unlock(evac_rq); + if (evac_rq_locked) + erts_smp_runq_unlock(evac_rq); + + if (notify_to_rq) + smp_notify_inc_runq(rq); + + wake_scheduler(evac_rq, 0, 1); } static int @@ -1391,7 +1828,7 @@ static ERTS_INLINE int check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); - long iflgs = erts_smp_atomic_read(&vrq->info_flags); + erts_aint32_t iflgs = erts_smp_atomic32_read(&vrq->info_flags); if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY) return try_steal_task_from_victim(rq, rq_lockedp, vrq); else @@ -1421,8 +1858,8 @@ try_steal_task(ErtsRunQueue *rq) ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked); - active_rqs = erts_smp_atomic_read(&balance_info.active_runqs); - blnc_rqs = erts_smp_atomic_read(&balance_info.used_runqs); + active_rqs = erts_smp_atomic32_read(&balance_info.active_runqs); + blnc_rqs = erts_smp_atomic32_read(&balance_info.used_runqs); if (active_rqs > blnc_rqs) active_rqs = blnc_rqs; @@ -1433,7 +1870,7 @@ try_steal_task(ErtsRunQueue *rq) if (active_rqs < blnc_rqs) { int no = blnc_rqs - active_rqs; int stop_ix = vix = active_rqs + rq->ix % no; - while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { res = check_possible_steal_victim(rq, &rq_locked, vix); if (res) goto done; @@ -1448,7 +1885,7 @@ try_steal_task(ErtsRunQueue *rq) vix = rq->ix; /* ... then try to steal a job from another active queue... */ - while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { vix++; if (vix >= active_rqs) vix = 0; @@ -1473,31 +1910,6 @@ try_steal_task(ErtsRunQueue *rq) return res; } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN -void -erts_smp_notify_check_children_needed(void) -{ - int i; - for (i = 0; i < erts_no_schedulers; i++) { - erts_smp_runq_lock(ERTS_SCHEDULER_IX(i)->run_queue); - ERTS_SCHEDULER_IX(i)->check_children = 1; - if (!erts_common_run_queue) - wake_scheduler(ERTS_SCHEDULER_IX(i)->run_queue, 0); - erts_smp_runq_unlock(ERTS_SCHEDULER_IX(i)->run_queue); - } - if (ongoing_multi_scheduling_block()) { - /* Also blocked schedulers need to check children */ - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - for (i = 0; i < erts_no_schedulers; i++) - ERTS_SCHEDULER_IX(i)->blocked_check_children = 1; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } - if (erts_common_run_queue) - wake_all_schedulers(); -} -#endif - /* Run queue balancing */ typedef struct { @@ -1561,20 +1973,23 @@ do { \ static void check_balance(ErtsRunQueue *c_rq) { +#if ERTS_MAX_PROCESSES >= (1 << 27) +# error check_balance() assumes ERTS_MAX_PROCESS < (1 << 27) +#endif ErtsRunQueueBalance avg = {0}; Sint64 scheds_reds, full_scheds_reds; int forced, active, current_active, oowc, half_full_scheds, full_scheds, mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix; - if (erts_smp_atomic_xchg(&balance_info.checking_balance, 1)) { + if (erts_smp_atomic32_xchg(&balance_info.checking_balance, 1)) { c_rq->check_balance_reds = INT_MAX; return; } - blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs); + blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); if (blnc_no_rqs == 1) { c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); return; } @@ -1582,7 +1997,7 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.halftime) { balance_info.halftime = 0; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); ERTS_FOREACH_RUNQ(rq, { if (rq->waiting) @@ -1610,12 +2025,12 @@ check_balance(ErtsRunQueue *c_rq) forced = balance_info.forced_check_balance; balance_info.forced_check_balance = 0; - blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs); + blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); if (blnc_no_rqs == 1) { erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); return; } @@ -1624,7 +2039,7 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE) balance_info.full_reds_history_index = 0; - current_active = erts_smp_atomic_read(&balance_info.active_runqs); + current_active = erts_smp_atomic32_read(&balance_info.active_runqs); /* Read balance information for all run queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { @@ -1684,12 +2099,14 @@ check_balance(ErtsRunQueue *c_rq) run_queue_info[qix].prio[pix].avail = 0; } else { - int xreds = 0; - int procreds = treds; - procreds -= run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds; + Sint64 xreds = 0; + Sint64 procreds = treds; + procreds -= + ((Sint64) + run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds); for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - int av; + Sint64 av; if (xreds == 0) av = 100; @@ -1700,9 +2117,10 @@ check_balance(ErtsRunQueue *c_rq) if (av == 0) av = 1; } - run_queue_info[qix].prio[pix].avail = av; + run_queue_info[qix].prio[pix].avail = (int) av; + ASSERT(run_queue_info[qix].prio[pix].avail >= 0); if (pix < PRIORITY_NORMAL) /* ie., max or high */ - xreds += run_queue_info[qix].prio[pix].reds; + xreds += (Sint64) run_queue_info[qix].prio[pix].reds; } run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].avail = 100; } @@ -1807,7 +2225,8 @@ check_balance(ErtsRunQueue *c_rq) if (max_len != 0) { int avail = avg.prio[pix].avail; if (avail != 0) { - max_len = ((100*max_len - 1) / avail) + 1; + max_len = (int) ((100*((Sint64) max_len) - 1) + / ((Sint64) avail)) + 1; avg.prio[pix].max_len = max_len; ASSERT(max_len >= 0); } @@ -1824,9 +2243,10 @@ check_balance(ErtsRunQueue *c_rq) || run_queue_info[qix].prio[pix].avail == 0) limit = 0; else - limit = (((avg.prio[pix].max_len - * run_queue_info[qix].prio[pix].avail) - 1) - / 100 + 1); + limit = (int) (((((Sint64) avg.prio[pix].max_len) + * ((Sint64) run_queue_info[qix].prio[pix].avail)) + - 1) + / 100 + 1); run_queue_info[qix].prio[pix].migration_limit = limit; } } @@ -1954,10 +2374,10 @@ erts_fprintf(stderr, "--------------------------------\n"); } balance_info.last_active_runqs = active; - erts_smp_atomic_set(&balance_info.active_runqs, active); + erts_smp_atomic32_set(&balance_info.active_runqs, active); balance_info.halftime = 1; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); /* Write migration paths and reset balance statistics in all queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { @@ -2054,9 +2474,28 @@ erts_debug_nbalance(void) void erts_early_init_scheduling(void) { - early_cpu_bind_init(); + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; } +int +erts_sched_set_wakeup_limit(char *str) +{ + if (sys_strcmp(str, "very_high") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_HIGH; + else if (sys_strcmp(str, "high") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_HIGH; + else if (sys_strcmp(str, "medium") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; + else if (sys_strcmp(str, "low") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_LOW; + else if (sys_strcmp(str, "very_low") == 0) + wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_VERY_LOW; + else + return EINVAL; + return 0; +} + + void erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) { @@ -2078,22 +2517,26 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS, (sizeof(ErtsAlignedRunQueue)*(n+1))); - if ((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0) + if ((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) != 0) erts_aligned_run_queues = ((ErtsAlignedRunQueue *) ((((UWord) erts_aligned_run_queues) & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE)); + ASSERT((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0); + #ifdef ERTS_SMP - erts_smp_atomic_init(&no_empty_run_queues, 0); + erts_smp_atomic32_init(&no_empty_run_queues, 0); #endif + erts_no_run_queues = n; + for (ix = 0; ix < n; ix++) { int pix, rix; ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); rq->ix = ix; - erts_smp_atomic_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + erts_smp_atomic32_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); /* make sure that the "extra" id correponds to the schedulers * id if the esdp->no <-> ix+1 mapping change. @@ -2102,8 +2545,10 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init_x(&rq->mtx, "run_queue", make_small(ix + 1)); erts_smp_cnd_init(&rq->cnd); - erts_smp_atomic_init(&rq->spin_waiter, 0); - erts_smp_atomic_init(&rq->spin_wake, 0); +#ifdef ERTS_SMP + erts_smp_spinlock_init(&rq->sleepers.lock, "run_queue_sleep_list"); + rq->sleepers.list = NULL; +#endif rq->waiting = 0; rq->woken = 0; @@ -2154,7 +2599,6 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } erts_common_run_queue = !mrq ? ERTS_RUNQ_IX(0) : NULL; - erts_no_run_queues = n; #ifdef ERTS_SMP @@ -2169,22 +2613,51 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #endif + n = (int) no_schedulers; + erts_no_schedulers = n; + +#ifdef ERTS_SMP + /* Create and initialize scheduler sleep info */ + + aligned_sched_sleep_info = erts_alloc(ERTS_ALC_T_SCHDLR_SLP_INFO, + (sizeof(ErtsAlignedSchedulerSleepInfo) + *(n+1))); + if ((((UWord) aligned_sched_sleep_info) & ERTS_CACHE_LINE_MASK) == 0) + aligned_sched_sleep_info = ((ErtsAlignedSchedulerSleepInfo *) + ((((UWord) aligned_sched_sleep_info) + & ~ERTS_CACHE_LINE_MASK) + + ERTS_CACHE_LINE_SIZE)); + for (ix = 0; ix < n; ix++) { + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); +#if 0 /* no need to initialize these... */ + ssi->next = NULL; + ssi->prev = NULL; +#endif + erts_smp_atomic32_init(&ssi->flags, 0); + ssi->event = NULL; /* initialized in sched_thread_func */ + erts_smp_atomic32_init(&ssi->aux_work, 0); + } +#endif + /* Create and initialize scheduler specific data */ - n = (int) no_schedulers; erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA, (sizeof(ErtsAlignedSchedulerData) *(n+1))); - if ((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0) + if ((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) != 0) erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *) ((((UWord) erts_aligned_scheduler_data) & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE)); + + ASSERT((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0); + for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); #ifdef ERTS_SMP erts_bits_init_state(&esdp->erl_bits_state); esdp->match_pseudo_process = NULL; + esdp->ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); esdp->free_process = NULL; #if HALFWORD_HEAP /* Registers need to be heap allocated (correct memory range) for tracing to work */ @@ -2213,12 +2686,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } #ifdef ERTS_SMP -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->check_children = 0; - esdp->blocked_check_children = 0; -#endif - erts_smp_atomic_init(&esdp->suspended, 0); - erts_smp_atomic_init(&esdp->chk_cpu_bind, 0); + erts_smp_atomic32_init(&esdp->chk_cpu_bind, 0); #endif } @@ -2226,21 +2694,21 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); - schdlr_sspnd.changing = 0; + erts_smp_atomic32_init(&schdlr_sspnd.changing, 0); schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; - erts_smp_atomic_init(&schdlr_sspnd.msb.ongoing, 0); - erts_smp_atomic_init(&schdlr_sspnd.active, no_schedulers); + erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0); + erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; - erts_smp_atomic_set(&balance_info.used_runqs, - erts_common_run_queue ? 1 : no_schedulers_online); - erts_smp_atomic_init(&balance_info.active_runqs, no_schedulers); + erts_smp_atomic32_set(&balance_info.used_runqs, + erts_common_run_queue ? 1 : no_schedulers_online); + erts_smp_atomic32_init(&balance_info.active_runqs, no_schedulers); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; balance_info.halftime = 1; balance_info.full_reds_history_index = 0; - erts_smp_atomic_init(&balance_info.checking_balance, 0); + erts_smp_atomic32_init(&balance_info.checking_balance, 0); balance_info.prev_rise.active_runqs = 0; balance_info.prev_rise.max_len = 0; balance_info.prev_rise.reds = 0; @@ -2249,7 +2717,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) if (no_schedulers_online < no_schedulers) { if (erts_common_run_queue) { for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic_set(&(ERTS_SCHEDULER_IX(ix)->suspended), 1); + erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); } else { for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) @@ -2260,9 +2729,12 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) schdlr_sspnd.wait_curr_online = no_schedulers_online; schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - erts_smp_atomic_init(&doing_sys_schedule, 0); + erts_smp_atomic32_init(&doing_sys_schedule, 0); + + init_misc_aux_work(); #else /* !ERTS_SMP */ { @@ -2276,12 +2748,10 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_no_schedulers = 1; #endif - erts_smp_atomic_init(&function_calls, 0); + erts_smp_atomic32_init(&function_calls, 0); /* init port tasks */ erts_port_task_init(); - - late_cpu_bind_init(); } ErtsRunQueue * @@ -2393,6 +2863,19 @@ resume_process(Process *p) p->rstatus = P_FREE; } +int +erts_get_max_no_executing_schedulers(void) +{ +#ifdef ERTS_SMP + if (erts_smp_atomic32_read(&schdlr_sspnd.changing)) + return (int) erts_no_schedulers; + ERTS_THR_MEMORY_BARRIER; + return (int) erts_smp_atomic32_read(&schdlr_sspnd.active); +#else + return 1; +#endif +} + #ifdef ERTS_SMP static void @@ -2408,13 +2891,113 @@ susp_sched_resume_block(void *unused) } static void +scheduler_ix_resume_wake(Uint ix) +{ + ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t oflgs; + do { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, 0, xflgs); + if (oflgs == xflgs) { + erts_sched_finish_poke(ssi, oflgs); + break; + } + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); +} + +static erts_aint32_t +sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) +{ + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t xflgs = xpct; + + do { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + xflgs = oflgs; + } while (oflgs & ERTS_SSI_FLG_SUSPENDED); + + return oflgs; +} + +static erts_aint32_t +sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) +{ + int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int sc = spincount; + erts_aint32_t flgs; + + do { + flgs = erts_smp_atomic32_read(&ssi->flags); + if ((flgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + break; + } + ERTS_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + } while (--sc > 0); + return flgs; +} + +static erts_aint32_t +sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) +{ + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + + erts_tse_reset(ssi->event); + + while (1) { + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); + if (oflgs == xflgs) + return nflgs; + if ((oflgs & (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) + != (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + return oflgs; + } + xflgs = oflgs; + } +} + +static void suspend_scheduler(ErtsSchedulerData *esdp) { + erts_aint32_t flgs; + erts_aint32_t changing; long no = (long) esdp->no; - ErtsRunQueue *rq = esdp->run_queue; + ErtsSchedulerSleepInfo *ssi = esdp->ssi; long active_schedulers; int curr_online = 1; int wake = 0; +#if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) + erts_aint32_t aux_work; +#endif /* * Schedulers may be suspended in two different ways: @@ -2431,126 +3014,145 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_runq_unlock(esdp->run_queue); - /* Unbind from cpu */ - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - if (scheduler2cpu_map[esdp->no].bound_id >= 0 - && erts_unbind_from_cpu(erts_cpuinfo) == 0) { - esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; - } - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); + erts_sched_check_cpu_bind_prep_suspend(esdp); if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED) { - if (active_schedulers == schdlr_sspnd.msb.wait_active) - wake = 1; - if (active_schedulers == 1) - schdlr_sspnd.changing = 0; - } - - while (1) { + flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); + if (flgs & ERTS_SSI_FLG_SUSPENDED) { -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; - erts_smp_runq_lock(esdp->run_queue); - check_children = esdp->check_children; - esdp->check_children = 0; - erts_smp_runq_unlock(esdp->run_queue); - if (check_children) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - erts_check_children(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + active_schedulers = erts_smp_atomic32_dectest(&schdlr_sspnd.active); + ASSERT(active_schedulers >= 1); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { + if (active_schedulers == schdlr_sspnd.msb.wait_active) + wake = 1; + if (active_schedulers == 1) { + changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; + } } -#endif - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) { - int changed = 0; - if (no > schdlr_sspnd.online && curr_online) { - schdlr_sspnd.curr_online--; - curr_online = 0; - changed = 1; + while (1) { + if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + int changed = 0; + if (no > schdlr_sspnd.online && curr_online) { + schdlr_sspnd.curr_online--; + curr_online = 0; + changed = 1; + } + else if (no <= schdlr_sspnd.online && !curr_online) { + schdlr_sspnd.curr_online++; + curr_online = 1; + changed = 1; + } + if (changed + && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) + wake = 1; + if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { + changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; + } } - else if (no <= schdlr_sspnd.online && !curr_online) { - schdlr_sspnd.curr_online++; - curr_online = 1; - changed = 1; + + if (wake) { + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + wake = 0; } - if (changed - && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - wake = 1; - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) - schdlr_sspnd.changing = 0; - } - if (wake) { - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - wake = 0; - } + flgs = erts_smp_atomic32_read(&ssi->flags); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ|ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + blockable_aux_work: + blockable_aux_work(esdp, ssi, aux_work); +#endif + + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + while (1) { + erts_aint32_t flgs; +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK +#ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); +#endif + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + flgs = sched_set_suspended_sleeptype(ssi); + if (flgs == (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)) { + int res; + do { + res = erts_tse_wait(ssi->event); + } while (res == EINTR); + } + } - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (1) { + flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED)); + if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) + break; + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) + break; -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->blocked_check_children) - break; + +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = erts_smp_atomic32_read(&ssi->aux_work); + if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); + goto blockable_aux_work; + } #endif - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + } - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE) - break; + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - if (!(rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ - | ERTS_RUNQ_FLG_SUSPENDED))) - break; - if ((rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - && !erts_smp_atomic_read(&esdp->suspended)) - break; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - esdp->blocked_check_children = 0; -#endif + active_schedulers = erts_smp_atomic32_inctest(&schdlr_sspnd.active); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + && schdlr_sspnd.online == active_schedulers) { + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); + } - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - } + ASSERT(no <= schdlr_sspnd.online); + ASSERT(!erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); - active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); - if (schdlr_sspnd.changing == ERTS_SCHED_CHANGING_MULTI_SCHED - && schdlr_sspnd.online == active_schedulers) { - schdlr_sspnd.changing = 0; } + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + ASSERT(curr_online); + if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); - /* Make sure we check if we should bind to a cpu or not... */ - if (rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - erts_smp_atomic_set(&esdp->chk_cpu_bind, 1); - else - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + erts_sched_check_cpu_bind_post_suspend(esdp); } #define ERTS_RUNQ_RESET_SUSPEND_INFO(RQ, DBG_ID) \ @@ -2565,7 +3167,7 @@ do { \ (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \ (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \ - erts_smp_atomic_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED); \ + erts_smp_atomic32_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\ for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \ (RQ)->procs.prio_info[pix__].max_len = 0; \ (RQ)->procs.prio_info[pix__].reds = 0; \ @@ -2607,8 +3209,10 @@ erts_schedulers_state(Uint *total, int yield_allowed) { int res; + erts_aint32_t changing; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (yield_allowed && schdlr_sspnd.changing) + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) res = ERTS_SCHDLR_SSPND_YIELD_RESTART; else { *active = *online = schdlr_sspnd.online; @@ -2628,6 +3232,7 @@ erts_set_schedulers_online(Process *p, Sint *old_no) { int ix, res, no, have_unlocked_plocks; + erts_aint32_t changing; if (new_no < 1 || erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; @@ -2637,7 +3242,8 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 0; no = (int) new_no; - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; } else { @@ -2646,17 +3252,19 @@ erts_set_schedulers_online(Process *p, res = ERTS_SCHDLR_SSPND_DONE; } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_ONLINE; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); schdlr_sspnd.online = no; if (no > online) { int ix; schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) - /* No schedulers to resume */; + if (ongoing_multi_scheduling_block()) { + for (ix = online; ix < no; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } else if (erts_common_run_queue) { for (ix = online; ix < no; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 0); + scheduler_ix_resume_wake(ix); } else { if (plocks) { @@ -2670,6 +3278,7 @@ erts_set_schedulers_online(Process *p, erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x5); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* * Spread evacuation paths among all online @@ -2680,11 +3289,10 @@ erts_set_schedulers_online(Process *p, ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no); evacuate_run_queue(from_rq, to_rq); } - erts_smp_atomic_set(&balance_info.used_runqs, no); + erts_smp_atomic32_set(&balance_info.used_runqs, no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); res = ERTS_SCHDLR_SSPND_DONE; } else /* if (no < online) */ { @@ -2701,12 +3309,17 @@ erts_set_schedulers_online(Process *p, schdlr_sspnd.wait_curr_online = no+1; } - if (ongoing_multi_scheduling_block()) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - else if (erts_common_run_queue) { + if (ongoing_multi_scheduling_block()) { for (ix = no; ix < online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, - 1); + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + } + else if (erts_common_run_queue) { + for (ix = no; ix < online; ix++) { + ErtsSchedulerSleepInfo *ssi; + ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_bor(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } wake_all_schedulers(); } else { @@ -2730,10 +3343,13 @@ erts_set_schedulers_online(Process *p, for (ix = erts_no_run_queues-1; ix >= no; ix--) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % no)); - erts_smp_atomic_set(&balance_info.used_runqs, no); + erts_smp_atomic32_set(&balance_info.used_runqs, no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ERTS_FOREACH_OP_RUNQ(rq, wake_scheduler(rq, 0)); + for (ix = no; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq, 0, 1); + } } } @@ -2747,6 +3363,13 @@ erts_set_schedulers_online(Process *p, susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic32_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic32_read(&schdlr_sspnd.changing))); + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } } @@ -2761,37 +3384,41 @@ ErtsSchedSuspendResult erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) { int ix, res, have_unlocked_plocks = 0; + erts_aint32_t changing; ErtsProcList *plp; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - if (schdlr_sspnd.changing) { + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); + if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ } else if (on) { /* ------ BLOCK ------ */ - if (erts_is_multi_scheduling_blocked()) { + if (schdlr_sspnd.msb.procs) { plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; schdlr_sspnd.msb.procs = plp; p->flags |= F_HAVE_BLCKD_MSCHED; - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; } else { + int online = schdlr_sspnd.online; p->flags |= F_HAVE_BLCKD_MSCHED; if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 1); - if (schdlr_sspnd.online == 1) { + ASSERT(0 == erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); + erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 1); + if (online == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); } else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB + | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); if (p->scheduler_data->no == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; schdlr_sspnd.msb.wait_active = 1; @@ -2805,17 +3432,19 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) schdlr_sspnd.msb.wait_active = 2; } if (erts_common_run_queue) { - for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 1); + for (ix = 1; ix < online; ix++) + erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { erts_smp_mtx_unlock(&schdlr_sspnd.mtx); erts_smp_mtx_lock(&balance_info.update_mtx); - erts_smp_atomic_set(&balance_info.used_runqs, 1); - for (ix = 0; ix < schdlr_sspnd.online; ix++) { + erts_smp_atomic32_set(&balance_info.used_runqs, 1); + for (ix = 0; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); erts_smp_runq_lock(rq); + ASSERT(!(rq->flags & ERTS_RUNQ_FLG_SUSPENDED)); ERTS_RUNQ_RESET_MIGRATION_PATHS(rq, 0x7); erts_smp_runq_unlock(rq); } @@ -2833,13 +3462,20 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) susp_sched_prep_block, susp_sched_resume_block, NULL); - while (erts_smp_atomic_read(&schdlr_sspnd.active) + while (erts_smp_atomic32_read(&schdlr_sspnd.active) != schdlr_sspnd.msb.wait_active) erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); erts_smp_activity_end(ERTS_ACTIVITY_WAIT, susp_sched_prep_block, susp_sched_resume_block, NULL); + ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED + ? (ERTS_SCHDLR_SSPND_CHNG_WAITER + & erts_smp_atomic32_read(&schdlr_sspnd.changing)) + : (ERTS_SCHDLR_SSPND_CHNG_WAITER + == erts_smp_atomic32_read(&schdlr_sspnd.changing))); + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; @@ -2883,7 +3519,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) if (schdlr_sspnd.msb.procs) res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; else { - schdlr_sspnd.changing = ERTS_SCHED_CHANGING_MULTI_SCHED; + ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); #ifdef DEBUG ERTS_FOREACH_RUNQ(rq, { @@ -2906,17 +3542,17 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) }); #endif p->flags &= ~F_HAVE_BLCKD_MSCHED; - erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 0); + erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 0); if (schdlr_sspnd.online == 1) { /* No schedulers to resume */ - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); - schdlr_sspnd.changing = 0; + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } else if (erts_common_run_queue) { for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(ix)->suspended, 0); + erts_smp_atomic32_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ~ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } else { int online = schdlr_sspnd.online; @@ -2933,6 +3569,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_lock(rq); ERTS_RUNQ_RESET_SUSPEND_INFO(rq, 0x4); erts_smp_runq_unlock(rq); + scheduler_ix_resume_wake(ix); } /* Spread evacuation paths among all online run queues */ @@ -2940,7 +3577,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % online)); - erts_smp_atomic_set(&balance_info.used_runqs, online); + erts_smp_atomic32_set(&balance_info.used_runqs, online); /* Make sure that we balance soon... */ balance_info.forced_check_balance = 1; erts_smp_runq_lock(ERTS_RUNQ_IX(0)); @@ -2948,7 +3585,6 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) erts_smp_runq_unlock(ERTS_RUNQ_IX(0)); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); } res = ERTS_SCHDLR_SSPND_DONE; } @@ -2965,7 +3601,7 @@ void erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) { if (return_value == am_blocked) { - long active = erts_smp_atomic_read(&schdlr_sspnd.active); + erts_aint32_t active = erts_smp_atomic32_read(&schdlr_sspnd.active); ASSERT(1 <= active && active <= 2); ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1); } @@ -2975,8 +3611,11 @@ erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) int erts_is_multi_scheduling_blocked(void) { - return (erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) - && erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + int res; + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + res = schdlr_sspnd.msb.procs != NULL; + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + return res; } Eterm @@ -2985,7 +3624,7 @@ erts_multi_scheduling_blockers(Process *p) Eterm res = NIL; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (erts_is_multi_scheduling_blocked()) { + if (schdlr_sspnd.msb.procs) { Eterm *hp, *hp_end; ErtsProcList *plp1, *plp2; Uint max_size; @@ -3017,18 +3656,26 @@ erts_multi_scheduling_blockers(Process *p) static void * sched_thread_func(void *vesdp) { +#ifdef ERTS_SMP + Uint no = ((ErtsSchedulerData *) vesdp)->no; +#endif #ifdef ERTS_ENABLE_LOCK_CHECK { char buf[31]; - Uint no = ((ErtsSchedulerData *) vesdp)->no; erts_snprintf(&buf[0], 31, "scheduler %bpu", no); erts_lc_set_thread_name(&buf[0]); } #endif - erts_alloc_reg_scheduler_id(((ErtsSchedulerData *) vesdp)->no); + erts_alloc_reg_scheduler_id(no); erts_tsd_set(sched_data_key, vesdp); #ifdef ERTS_SMP + + erts_sched_init_check_cpu_bind((ErtsSchedulerData *) vesdp); + erts_proc_lock_prepare_proc_lock_waiter(); + ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); + + #endif erts_register_blockable_thread(); #ifdef HIPE @@ -3037,30 +3684,30 @@ sched_thread_func(void *vesdp) erts_thread_init_float(); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(schdlr_sspnd.changing == ERTS_SCHED_CHANGING_ONLINE); + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.changing) + & ERTS_SCHDLR_SSPND_CHNG_ONLN); - schdlr_sspnd.curr_online--; + if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + if (((ErtsSchedulerData *) vesdp)->no != 1) + erts_smp_cnd_signal(&schdlr_sspnd.cnd); + } - if (((ErtsSchedulerData *) vesdp)->no != 1) { - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { - schdlr_sspnd.changing = 0; - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); + if (((ErtsSchedulerData *) vesdp)->no == 1) { + if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { + erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); + while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) + erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + erts_smp_activity_end(ERTS_ACTIVITY_WAIT, + susp_sched_prep_block, + susp_sched_resume_block, + NULL); } - } - else if (schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - schdlr_sspnd.changing = 0; - else { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, - susp_sched_prep_block, - susp_sched_resume_block, - NULL); - ASSERT(!schdlr_sspnd.changing); + ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); @@ -3096,11 +3743,7 @@ erts_start_schedulers(void) ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(actual); actual++; ASSERT(actual == esdp->no); -#ifdef ERTS_ENABLE_LOCK_COUNT - res = erts_lcnt_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts); -#else res = ethr_thr_create(&esdp->tid,sched_thread_func,(void*)esdp,&opts); -#endif if (res != 0) { actual--; break; @@ -3129,1351 +3772,6 @@ erts_start_schedulers(void) #endif /* ERTS_SMP */ -static int -int_cmp(const void *vx, const void *vy) -{ - return *((int *) vx) - *((int *) vy); -} - -static int -cpu_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->core != y->core) - return x->core - y->core; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->node != y->node) - return x->node - y->node; - return 0; -} - -static int -cpu_processor_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - if (x->node != y->node) - return x->node - y->node; - if (x->processor != y->processor) - return x->processor - y->processor; - return 0; -} - -static int -cpu_thread_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->node != y->node) - return x->node - y->node; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - return 0; -} - -static int -cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->node != y->node) - return x->node - y->node; - if (x->core != y->core) - return x->core - y->core; - if (x->processor != y->processor) - return x->processor - y->processor; - return 0; -} - -static int -cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->node != y->node) - return x->node - y->node; - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->core != y->core) - return x->core - y->core; - if (x->processor != y->processor) - return x->processor - y->processor; - return 0; -} - -static int -cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->node != y->node) - return x->node - y->node; - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->core != y->core) - return x->core - y->core; - return 0; -} - -static int -cpu_no_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->node != y->node) - return x->node - y->node; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - if (x->thread != y->thread) - return x->thread - y->thread; - return 0; -} - -static ERTS_INLINE void -make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node) -{ - int ix; - int node = -1; - int processor = -1; - int processor_node = -1; - int processor_node_node = -1; - int core = -1; - int thread = -1; - int old_node = -1; - int old_processor = -1; - int old_processor_node = -1; - int old_core = -1; - int old_thread = -1; - - for (ix = 0; ix < size; ix++) { - if (!no_node || cpudata[ix].node >= 0) { - if (old_node == cpudata[ix].node) - cpudata[ix].node = node; - else { - old_node = cpudata[ix].node; - old_processor = processor = -1; - if (!no_node) - old_processor_node = processor_node = -1; - old_core = core = -1; - old_thread = thread = -1; - if (no_node || cpudata[ix].node >= 0) - cpudata[ix].node = ++node; - } - } - if (old_processor == cpudata[ix].processor) - cpudata[ix].processor = processor; - else { - old_processor = cpudata[ix].processor; - if (!no_node) - processor_node_node = old_processor_node = processor_node = -1; - old_core = core = -1; - old_thread = thread = -1; - cpudata[ix].processor = ++processor; - } - if (no_node && cpudata[ix].processor_node < 0) - old_processor_node = -1; - else { - if (old_processor_node == cpudata[ix].processor_node) { - if (no_node) - cpudata[ix].node = cpudata[ix].processor_node = node; - else { - if (processor_node_node >= 0) - cpudata[ix].node = processor_node_node; - cpudata[ix].processor_node = processor_node; - } - } - else { - old_processor_node = cpudata[ix].processor_node; - old_core = core = -1; - old_thread = thread = -1; - if (no_node) - cpudata[ix].node = cpudata[ix].processor_node = ++node; - else { - cpudata[ix].node = processor_node_node = ++node; - cpudata[ix].processor_node = ++processor_node; - } - } - } - if (!no_node && cpudata[ix].processor_node < 0) - cpudata[ix].processor_node = 0; - if (old_core == cpudata[ix].core) - cpudata[ix].core = core; - else { - old_core = cpudata[ix].core; - old_thread = thread = -1; - cpudata[ix].core = ++core; - } - if (old_thread == cpudata[ix].thread) - cpudata[ix].thread = thread; - else - old_thread = cpudata[ix].thread = ++thread; - } -} - -static void -cpu_bind_order_sort(erts_cpu_topology_t *cpudata, - int size, - ErtsCpuBindOrder bind_order, - int mk_seq) -{ - if (size > 1) { - int no_node = 0; - int (*cmp_func)(const void *, const void *); - switch (bind_order) { - case ERTS_CPU_BIND_SPREAD: - cmp_func = cpu_spread_order_cmp; - break; - case ERTS_CPU_BIND_PROCESSOR_SPREAD: - cmp_func = cpu_processor_spread_order_cmp; - break; - case ERTS_CPU_BIND_THREAD_SPREAD: - cmp_func = cpu_thread_spread_order_cmp; - break; - case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: - no_node = 1; - cmp_func = cpu_thread_no_node_processor_spread_order_cmp; - break; - case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: - no_node = 1; - cmp_func = cpu_no_node_processor_spread_order_cmp; - break; - case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: - no_node = 1; - cmp_func = cpu_no_node_thread_spread_order_cmp; - break; - case ERTS_CPU_BIND_NO_SPREAD: - cmp_func = cpu_no_spread_order_cmp; - break; - default: - cmp_func = NULL; - erl_exit(ERTS_ABORT_EXIT, - "Bad cpu bind type: %d\n", - (int) cpu_bind_order); - break; - } - - if (mk_seq) - make_cpudata_id_seq(cpudata, size, no_node); - - qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func); - } -} - -static int -processor_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->node != y->node) - return x->node - y->node; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - if (x->thread != y->thread) - return x->thread - y->thread; - return 0; -} - -static void -check_cpu_bind(ErtsSchedulerData *esdp) -{ - int res; - int cpu_id; - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - cpu_id = scheduler2cpu_map[esdp->no].bind_id; - if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) { - res = erts_bind_to_cpu(erts_cpuinfo, cpu_id); - if (res == 0) - esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id; - else { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n", - (int) esdp->no, cpu_id, erl_errno_id(-res)); - erts_send_error_to_logger_nogl(dsbufp); - if (scheduler2cpu_map[esdp->no].bound_id >= 0) - goto unbind; - } - } - else if (cpu_id < 0 && scheduler2cpu_map[esdp->no].bound_id >= 0) { - unbind: - /* Get rid of old binding */ - res = erts_unbind_from_cpu(erts_cpuinfo); - if (res == 0) - esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; - else { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n", - (int) esdp->no, cpu_id, erl_errno_id(-res)); - erts_send_error_to_logger_nogl(dsbufp); - } - } - erts_smp_runq_lock(esdp->run_queue); -#ifdef ERTS_SMP - if (erts_common_run_queue) - erts_smp_atomic_set(&esdp->chk_cpu_bind, 0); - else { - esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; - } -#endif - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - -} - -static void -signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) -{ - int s_ix = 1; - int cpu_ix; - - if (cpu_bind_order != ERTS_CPU_BIND_NONE) { - - cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1); - - for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++) - if (erts_is_cpu_available(erts_cpuinfo, cpudata[cpu_ix].logical)) - scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical; - } - - if (s_ix <= erts_no_schedulers) - for (; s_ix <= erts_no_schedulers; s_ix++) - scheduler2cpu_map[s_ix].bind_id = -1; - -#ifdef ERTS_SMP - if (erts_common_run_queue) { - for (s_ix = 0; s_ix < erts_no_schedulers; s_ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(s_ix)->chk_cpu_bind, 1); - wake_all_schedulers(); - } - else { - ERTS_FOREACH_RUNQ(rq, - { - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - wake_scheduler(rq, 0); - }); - } -#else - check_cpu_bind(erts_get_scheduler_data()); -#endif -} - -int -erts_init_scheduler_bind_type(char *how) -{ - if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) - return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED; - - if (!system_cpudata && !user_cpudata) - return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; - - if (sys_strcmp(how, "s") == 0) - cpu_bind_order = ERTS_CPU_BIND_SPREAD; - else if (sys_strcmp(how, "ps") == 0) - cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; - else if (sys_strcmp(how, "ts") == 0) - cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (sys_strcmp(how, "db") == 0 - || sys_strcmp(how, "tnnps") == 0) - cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; - else if (sys_strcmp(how, "nnps") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; - else if (sys_strcmp(how, "nnts") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; - else if (sys_strcmp(how, "ns") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; - else if (sys_strcmp(how, "u") == 0) - cpu_bind_order = ERTS_CPU_BIND_NONE; - else - return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE; - - return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; -} - -typedef struct { - int *id; - int used; - int size; -} ErtsCpuTopIdSeq; - -typedef struct { - ErtsCpuTopIdSeq logical; - ErtsCpuTopIdSeq thread; - ErtsCpuTopIdSeq core; - ErtsCpuTopIdSeq processor_node; - ErtsCpuTopIdSeq processor; - ErtsCpuTopIdSeq node; -} ErtsCpuTopEntry; - -static void -init_cpu_top_entry(ErtsCpuTopEntry *cte) -{ - int size = 10; - cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->logical.size = size; - cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->thread.size = size; - cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->core.size = size; - cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->processor_node.size = size; - cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->processor.size = size; - cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->node.size = size; -} - -static void -destroy_cpu_top_entry(ErtsCpuTopEntry *cte) -{ - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id); -} - -static int -get_cput_value_or_range(int *v, int *vr, char **str) -{ - long l; - char *c = *str; - errno = 0; - if (!isdigit((unsigned char)*c)) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; - l = strtol(c, &c, 10); - if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; - *v = (int) l; - if (*c == '-') { - c++; - if (!isdigit((unsigned char)*c)) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - l = strtol(c, &c, 10); - if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - *vr = (int) l; - } - *str = c; - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -static int -get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str) -{ - int ix = 0; - int need_size = 0; - char *c = *str; - - while (1) { - int res; - int val; - int nids; - int val_range = -1; - res = get_cput_value_or_range(&val, &val_range, &c); - if (res != ERTS_INIT_CPU_TOPOLOGY_OK) - return res; - if (val_range < 0 || val_range == val) - nids = 1; - else { - if (val_range > val) - nids = val_range - val + 1; - else - nids = val - val_range + 1; - } - need_size += nids; - if (need_size > idseq->size) { - idseq->size = need_size + 10; - idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS, - idseq->id, - sizeof(int)*idseq->size); - } - if (nids == 1) - idseq->id[ix++] = val; - else if (val_range > val) { - for (; val <= val_range; val++) - idseq->id[ix++] = val; - } - else { - for (; val >= val_range; val--) - idseq->id[ix++] = val; - } - if (*c != ',') - break; - c++; - } - *str = c; - idseq->used = ix; - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -static int -get_cput_entry(ErtsCpuTopEntry *cput, char **str) -{ - int h; - char *c = *str; - - cput->logical.used = 0; - cput->thread.id[0] = 0; - cput->thread.used = 1; - cput->core.id[0] = 0; - cput->core.used = 1; - cput->processor_node.id[0] = -1; - cput->processor_node.used = 1; - cput->processor.id[0] = 0; - cput->processor.used = 1; - cput->node.id[0] = -1; - cput->node.used = 1; - - h = ERTS_TOPOLOGY_MAX_DEPTH; - while (*c != ':' && *c != '\0') { - int res; - ErtsCpuTopIdSeq *idseqp; - switch (*c++) { - case 'L': - if (h <= ERTS_TOPOLOGY_LOGICAL) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->logical; - h = ERTS_TOPOLOGY_LOGICAL; - break; - case 't': - case 'T': - if (h <= ERTS_TOPOLOGY_THREAD) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->thread; - h = ERTS_TOPOLOGY_THREAD; - break; - case 'c': - case 'C': - if (h <= ERTS_TOPOLOGY_CORE) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->core; - h = ERTS_TOPOLOGY_CORE; - break; - case 'p': - case 'P': - if (h <= ERTS_TOPOLOGY_PROCESSOR) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->processor; - h = ERTS_TOPOLOGY_PROCESSOR; - break; - case 'n': - case 'N': - if (h <= ERTS_TOPOLOGY_PROCESSOR) { - do_node: - if (h <= ERTS_TOPOLOGY_NODE) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->node; - h = ERTS_TOPOLOGY_NODE; - } - else { - int p_node = 0; - char *p_chk = c; - while (*p_chk != '\0' && *p_chk != ':') { - if (*p_chk == 'p' || *p_chk == 'P') { - p_node = 1; - break; - } - p_chk++; - } - if (!p_node) - goto do_node; - if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->processor_node; - h = ERTS_TOPOLOGY_PROCESSOR_NODE; - } - break; - default: - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE; - } - res = get_cput_id_seq(idseqp, &c); - if (res != ERTS_INIT_CPU_TOPOLOGY_OK) - return res; - } - - if (cput->logical.used < 1) - return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID; - - if (*c == ':') { - c++; - } - - if (cput->thread.used != 1 - && cput->thread.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->core.used != 1 - && cput->core.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->processor_node.used != 1 - && cput->processor_node.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->processor.used != 1 - && cput->processor.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->node.used != 1 - && cput->node.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - - *str = c; - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -static int -verify_topology(erts_cpu_topology_t *cpudata, int size) -{ - if (size > 0) { - int *logical; - int node, processor, no_nodes, i; - - /* Verify logical ids */ - logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size); - - for (i = 0; i < user_cpudata_size; i++) - logical[i] = user_cpudata[i].logical; - - qsort(logical, user_cpudata_size, sizeof(int), int_cmp); - for (i = 0; i < user_cpudata_size-1; i++) { - if (logical[i] == logical[i+1]) { - erts_free(ERTS_ALC_T_TMP, logical); - return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS; - } - } - - erts_free(ERTS_ALC_T_TMP, logical); - - qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp); - - /* Verify unique entities */ - - for (i = 1; i < user_cpudata_size; i++) { - if (user_cpudata[i-1].processor == user_cpudata[i].processor - && user_cpudata[i-1].node == user_cpudata[i].node - && (user_cpudata[i-1].processor_node - == user_cpudata[i].processor_node) - && user_cpudata[i-1].core == user_cpudata[i].core - && user_cpudata[i-1].thread == user_cpudata[i].thread) { - return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES; - } - } - - /* Verify numa nodes */ - node = cpudata[0].node; - processor = cpudata[0].processor; - no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0; - for (i = 1; i < size; i++) { - if (no_nodes) { - if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - } - else { - if (cpudata[i].processor == processor && cpudata[i].node != node) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - node = cpudata[i].node; - processor = cpudata[i].processor; - if (node >= 0 && cpudata[i].processor_node >= 0) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - if (node < 0 && cpudata[i].processor_node < 0) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - } - } - } - - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -int -erts_init_cpu_topology(char *topology_str) -{ - ErtsCpuTopEntry cput; - int need_size; - char *c; - int ix; - int error = ERTS_INIT_CPU_TOPOLOGY_OK; - - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata_size = 10; - - user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(erts_cpu_topology_t) - * user_cpudata_size)); - - init_cpu_top_entry(&cput); - - ix = 0; - need_size = 0; - - c = topology_str; - if (*c == '\0') { - error = ERTS_INIT_CPU_TOPOLOGY_MISSING; - goto fail; - } - do { - int r; - error = get_cput_entry(&cput, &c); - if (error != ERTS_INIT_CPU_TOPOLOGY_OK) - goto fail; - need_size += cput.logical.used; - if (user_cpudata_size < need_size) { - user_cpudata_size = need_size + 10; - user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, - user_cpudata, - (sizeof(erts_cpu_topology_t) - * user_cpudata_size)); - } - - ASSERT(cput.thread.used == 1 - || cput.thread.used == cput.logical.used); - ASSERT(cput.core.used == 1 - || cput.core.used == cput.logical.used); - ASSERT(cput.processor_node.used == 1 - || cput.processor_node.used == cput.logical.used); - ASSERT(cput.processor.used == 1 - || cput.processor.used == cput.logical.used); - ASSERT(cput.node.used == 1 - || cput.node.used == cput.logical.used); - - for (r = 0; r < cput.logical.used; r++) { - user_cpudata[ix].logical = cput.logical.id[r]; - user_cpudata[ix].thread = - cput.thread.id[cput.thread.used == 1 ? 0 : r]; - user_cpudata[ix].core = - cput.core.id[cput.core.used == 1 ? 0 : r]; - user_cpudata[ix].processor_node = - cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r]; - user_cpudata[ix].processor = - cput.processor.id[cput.processor.used == 1 ? 0 : r]; - user_cpudata[ix].node = - cput.node.id[cput.node.used == 1 ? 0 : r]; - ix++; - } - } while (*c != '\0'); - - if (user_cpudata_size != ix) { - user_cpudata_size = ix; - user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, - user_cpudata, - (sizeof(erts_cpu_topology_t) - * user_cpudata_size)); - } - - error = verify_topology(user_cpudata, user_cpudata_size); - if (error == ERTS_INIT_CPU_TOPOLOGY_OK) { - destroy_cpu_top_entry(&cput); - return ERTS_INIT_CPU_TOPOLOGY_OK; - } - - fail: - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata_size = 0; - destroy_cpu_top_entry(&cput); - return error; -} - -#define ERTS_GET_CPU_TOPOLOGY_ERROR -1 -#define ERTS_GET_USED_CPU_TOPOLOGY 0 -#define ERTS_GET_DETECTED_CPU_TOPOLOGY 1 -#define ERTS_GET_DEFINED_CPU_TOPOLOGY 2 - -static Eterm get_cpu_topology_term(Process *c_p, int type); - -Eterm -erts_set_cpu_topology(Process *c_p, Eterm term) -{ - erts_cpu_topology_t *cpudata = NULL; - int cpudata_size = 0; - Eterm res; - - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY); - if (term == am_undefined) { - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata = NULL; - user_cpudata_size = 0; - - if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) { - cpudata_size = system_cpudata_size; - cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * cpudata_size)); - - sys_memcpy((void *) cpudata, - (void *) system_cpudata, - sizeof(erts_cpu_topology_t)*cpudata_size); - } - } - else if (is_not_list(term)) { - error: - res = THE_NON_VALUE; - goto done; - } - else { - Eterm list = term; - int ix = 0; - - cpudata_size = 100; - cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * cpudata_size)); - - while (is_list(list)) { - Eterm *lp = list_val(list); - Eterm cpu = CAR(lp); - Eterm* tp; - Sint id; - - if (is_not_tuple(cpu)) - goto error; - - tp = tuple_val(cpu); - - if (arityval(tp[0]) != 7 || tp[1] != am_cpu) - goto error; - - if (ix >= cpudata_size) { - cpudata_size += 100; - cpudata = erts_realloc(ERTS_ALC_T_TMP, - cpudata, - (sizeof(erts_cpu_topology_t) - * cpudata_size)); - } - - id = signed_val(tp[2]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].node = (int) id; - - id = signed_val(tp[3]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].processor = (int) id; - - id = signed_val(tp[4]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].processor_node = (int) id; - - id = signed_val(tp[5]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].core = (int) id; - - id = signed_val(tp[6]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].thread = (int) id; - - id = signed_val(tp[7]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].logical = (int) id; - - list = CDR(lp); - ix++; - } - - if (is_not_nil(list)) - goto error; - - cpudata_size = ix; - - if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size)) - goto error; - - if (user_cpudata_size != cpudata_size) { - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - sizeof(erts_cpu_topology_t)*cpudata_size); - user_cpudata_size = cpudata_size; - } - - sys_memcpy((void *) user_cpudata, - (void *) cpudata, - sizeof(erts_cpu_topology_t)*cpudata_size); - } - - signal_schedulers_bind_change(cpudata, cpudata_size); - - done: - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - - if (cpudata) - erts_free(ERTS_ALC_T_TMP, cpudata); - - return res; -} - -static Eterm -bound_schedulers_term(ErtsCpuBindOrder order) -{ - switch (order) { - case ERTS_CPU_BIND_SPREAD: { - ERTS_DECL_AM(spread); - return AM_spread; - } - case ERTS_CPU_BIND_PROCESSOR_SPREAD: { - ERTS_DECL_AM(processor_spread); - return AM_processor_spread; - } - case ERTS_CPU_BIND_THREAD_SPREAD: { - ERTS_DECL_AM(thread_spread); - return AM_thread_spread; - } - case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: { - ERTS_DECL_AM(thread_no_node_processor_spread); - return AM_thread_no_node_processor_spread; - } - case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: { - ERTS_DECL_AM(no_node_processor_spread); - return AM_no_node_processor_spread; - } - case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: { - ERTS_DECL_AM(no_node_thread_spread); - return AM_no_node_thread_spread; - } - case ERTS_CPU_BIND_NO_SPREAD: { - ERTS_DECL_AM(no_spread); - return AM_no_spread; - } - case ERTS_CPU_BIND_NONE: { - ERTS_DECL_AM(unbound); - return AM_unbound; - } - default: - ASSERT(0); - return THE_NON_VALUE; - } -} - -Eterm -erts_bound_schedulers_term(Process *c_p) -{ - ErtsCpuBindOrder order; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - order = cpu_bind_order; - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return bound_schedulers_term(order); -} - -static void -create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size) -{ - if (user_cpudata) { - *cpudata_size = user_cpudata_size; - *cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * (*cpudata_size))); - sys_memcpy((void *) *cpudata, - (void *) user_cpudata, - sizeof(erts_cpu_topology_t)*(*cpudata_size)); - } - else if (system_cpudata) { - *cpudata_size = system_cpudata_size; - *cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * (*cpudata_size))); - sys_memcpy((void *) *cpudata, - (void *) system_cpudata, - sizeof(erts_cpu_topology_t)*(*cpudata_size)); - } - else { - *cpudata = NULL; - *cpudata_size = 0; - } -} - -static void -destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata) -{ - if (cpudata) - erts_free(ERTS_ALC_T_TMP, cpudata); -} - -Eterm -erts_bind_schedulers(Process *c_p, Eterm how) -{ - Eterm res; - erts_cpu_topology_t *cpudata; - int cpudata_size; - ErtsCpuBindOrder old_cpu_bind_order; - - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - - if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) { - ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP); - } - else { - - old_cpu_bind_order = cpu_bind_order; - - if (ERTS_IS_ATOM_STR("spread", how)) - cpu_bind_order = ERTS_CPU_BIND_SPREAD; - else if (ERTS_IS_ATOM_STR("processor_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("thread_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("default_bind", how) - || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("no_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; - else if (ERTS_IS_ATOM_STR("unbound", how)) - cpu_bind_order = ERTS_CPU_BIND_NONE; - else { - cpu_bind_order = old_cpu_bind_order; - ERTS_BIF_PREP_ERROR(res, c_p, BADARG); - goto done; - } - - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - - if (!cpudata) { - cpu_bind_order = old_cpu_bind_order; - ERTS_BIF_PREP_ERROR(res, c_p, BADARG); - goto done; - } - - signal_schedulers_bind_change(cpudata, cpudata_size); - - destroy_tmp_cpu_topology_copy(cpudata); - - res = bound_schedulers_term(old_cpu_bind_order); - } - - done: - - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - - return res; -} - -Eterm -erts_fake_scheduler_bindings(Process *p, Eterm how) -{ - ErtsCpuBindOrder fake_cpu_bind_order; - erts_cpu_topology_t *cpudata; - int cpudata_size; - Eterm res; - - if (ERTS_IS_ATOM_STR("spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD; - else if (ERTS_IS_ATOM_STR("processor_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("thread_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("default_bind", how) - || ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("no_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; - else if (ERTS_IS_ATOM_STR("unbound", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NONE; - else { - ERTS_BIF_PREP_ERROR(res, p, BADARG); - return res; - } - - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - - if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE) - ERTS_BIF_PREP_RET(res, am_false); - else { - int i; - Eterm *hp; - - cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1); - -#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA - - erts_fprintf(stderr, "node: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].node); - erts_fprintf(stderr, "\n"); - erts_fprintf(stderr, "processor: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].processor); - erts_fprintf(stderr, "\n"); - if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD - && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD - && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) { - erts_fprintf(stderr, "processor_node:"); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].processor_node); - erts_fprintf(stderr, "\n"); - } - erts_fprintf(stderr, "core: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].core); - erts_fprintf(stderr, "\n"); - erts_fprintf(stderr, "thread: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].thread); - erts_fprintf(stderr, "\n"); - erts_fprintf(stderr, "logical: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].logical); - erts_fprintf(stderr, "\n"); -#endif - - hp = HAlloc(p, cpudata_size+1); - ERTS_BIF_PREP_RET(res, make_tuple(hp)); - *hp++ = make_arityval((Uint) cpudata_size); - for (i = 0; i < cpudata_size; i++) - *hp++ = make_small((Uint) cpudata[i].logical); - } - - destroy_tmp_cpu_topology_copy(cpudata); - - return res; -} - -Eterm -erts_get_schedulers_binds(Process *c_p) -{ - int ix; - ERTS_DECL_AM(unbound); - Eterm *hp = HAlloc(c_p, erts_no_schedulers+1); - Eterm res = make_tuple(hp); - - *(hp++) = make_arityval(erts_no_schedulers); - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - for (ix = 1; ix <= erts_no_schedulers; ix++) - *(hp++) = (scheduler2cpu_map[ix].bound_id >= 0 - ? make_small(scheduler2cpu_map[ix].bound_id) - : AM_unbound); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return res; -} - -static Eterm -bld_topology_term(Eterm **hpp, - Uint *hszp, - erts_cpu_topology_t *cpudata, - int size) -{ - Eterm res = NIL; - int i; - - if (size == 0) - return am_undefined; - - for (i = size-1; i >= 0; i--) { - res = erts_bld_cons(hpp, - hszp, - erts_bld_tuple(hpp, - hszp, - 7, - am_cpu, - make_small(cpudata[i].node), - make_small(cpudata[i].processor), - make_small(cpudata[i].processor_node), - make_small(cpudata[i].core), - make_small(cpudata[i].thread), - make_small(cpudata[i].logical)), - res); - } - return res; -} - -static Eterm -get_cpu_topology_term(Process *c_p, int type) -{ -#ifdef DEBUG - Eterm *hp_end; -#endif - Eterm *hp; - Uint hsz; - Eterm res = THE_NON_VALUE; - erts_cpu_topology_t *cpudata = NULL; - int size = 0; - - switch (type) { - case ERTS_GET_USED_CPU_TOPOLOGY: - if (user_cpudata) - goto defined; - else - goto detected; - case ERTS_GET_DETECTED_CPU_TOPOLOGY: - detected: - if (!system_cpudata) - res = am_undefined; - else { - size = system_cpudata_size; - cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * size)); - sys_memcpy((void *) cpudata, - (void *) system_cpudata, - sizeof(erts_cpu_topology_t)*size); - } - break; - case ERTS_GET_DEFINED_CPU_TOPOLOGY: - defined: - if (!user_cpudata) - res = am_undefined; - else { - size = user_cpudata_size; - cpudata = user_cpudata; - } - break; - default: - erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); - break; - } - - if (res == am_undefined) { - ASSERT(!cpudata); - return res; - } - - hsz = 0; - - bld_topology_term(NULL, &hsz, - cpudata, size); - - hp = HAlloc(c_p, hsz); - -#ifdef DEBUG - hp_end = hp + hsz; -#endif - - res = bld_topology_term(&hp, NULL, - cpudata, size); - - ASSERT(hp_end == hp); - - if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata) - erts_free(ERTS_ALC_T_TMP, cpudata); - - return res; -} - -Eterm -erts_get_cpu_topology_term(Process *c_p, Eterm which) -{ - Eterm res; - int type; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - if (ERTS_IS_ATOM_STR("used", which)) - type = ERTS_GET_USED_CPU_TOPOLOGY; - else if (ERTS_IS_ATOM_STR("detected", which)) - type = ERTS_GET_DETECTED_CPU_TOPOLOGY; - else if (ERTS_IS_ATOM_STR("defined", which)) - type = ERTS_GET_DEFINED_CPU_TOPOLOGY; - else - type = ERTS_GET_CPU_TOPOLOGY_ERROR; - if (type == ERTS_GET_CPU_TOPOLOGY_ERROR) - res = THE_NON_VALUE; - else - res = get_cpu_topology_term(c_p, type); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return res; -} - -static void -early_cpu_bind_init(void) -{ - user_cpudata = NULL; - user_cpudata_size = 0; - - system_cpudata_size = erts_get_cpu_topology_size(erts_cpuinfo); - system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(erts_cpu_topology_t) - * system_cpudata_size)); - - cpu_bind_order = ERTS_CPU_BIND_NONE; - - if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata) - || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, - system_cpudata_size)) { - erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); - system_cpudata = NULL; - system_cpudata_size = 0; - } -} - -static void -late_cpu_bind_init(void) -{ - int ix; - - erts_smp_rwmtx_init(&erts_cpu_bind_rwmtx, "cpu_bind"); - - scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(ErtsCpuBindData) - * (erts_no_schedulers+1))); - for (ix = 1; ix <= erts_no_schedulers; ix++) { - scheduler2cpu_map[ix].bind_id = -1; - scheduler2cpu_map[ix].bound_id = -1; - } - - if (cpu_bind_order != ERTS_CPU_BIND_NONE) { - erts_cpu_topology_t *cpudata; - int cpudata_size; - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - ASSERT(cpudata); - signal_schedulers_bind_change(cpudata, cpudata_size); - destroy_tmp_cpu_topology_copy(cpudata); - } -} - #ifdef ERTS_SMP static void @@ -5329,7 +4627,7 @@ dequeue_process(ErtsRunQueue *runq, Process *p) } /* schedule a process */ -static ERTS_INLINE void +static ERTS_INLINE ErtsRunQueue * internal_add_to_runq(ErtsRunQueue *runq, Process *p) { Uint32 prev_status = p->status; @@ -5340,12 +4638,12 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); if (p->status_flags & ERTS_PROC_SFLG_INRUNQ) - return; + return NULL; else if (p->runq_flags & ERTS_PROC_RUNQ_FLG_RUNNING) { ASSERT(p->status != P_SUSPENDED); ERTS_DBG_CHK_PROCS_RUNQ_NOPROC(runq, p); p->status_flags |= ERTS_PROC_SFLG_PENDADD2SCHEDQ; - return; + return NULL; } ASSERT(!p->scheduler_data); #endif @@ -5384,20 +4682,23 @@ internal_add_to_runq(ErtsRunQueue *runq, Process *p) profile_runnable_proc(p, am_active); } - smp_notify_inc_runq(add_runq); - if (add_runq != runq) erts_smp_runq_unlock(add_runq); + + return add_runq; } void erts_add_to_runq(Process *p) { + ErtsRunQueue *notify_runq; ErtsRunQueue *runq = erts_get_runq_proc(p); erts_smp_runq_lock(runq); - internal_add_to_runq(runq, p); + notify_runq = internal_add_to_runq(runq, p); erts_smp_runq_unlock(runq); + smp_notify_inc_runq(notify_runq); + } /* Possibly remove a scheduled process we need to suspend */ @@ -5536,8 +4837,6 @@ erts_proc_migrate(Process *p, ErtsProcLocks *plcks, p->run_queue = to_rq; enqueue_process(to_rq, p); - smp_notify_inc_runq(to_rq); - return ERTS_MIGRATE_SUCCESS; } #endif /* ERTS_SMP */ @@ -5734,30 +5033,6 @@ erts_set_process_priority(Process *p, Eterm new_value) return old_value; } -#ifdef ERTS_SMP - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - while (!erts_port_task_have_outstanding_io_tasks() - && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { - if (!erts_port_task_have_outstanding_io_tasks()) - return 1; - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - return 0; -} - -#else - -static ERTS_INLINE int -prepare_for_sys_schedule(void) -{ - return !erts_port_task_have_outstanding_io_tasks(); -} - -#endif - /* note that P_RUNNING is only set so that we don't try to remove ** running processes from the schedule queue if they exit - a running ** process not being in the schedule queue!! @@ -5787,10 +5062,10 @@ Process *schedule(Process *p, int calls) { ErtsRunQueue *rq; ErtsRunPrioQueue *rpq; - long dt; + erts_aint_t dt; ErtsSchedulerData *esdp; int context_reds; - long fcalls; + int fcalls; int input_reductions; int actual_reds; int reds; @@ -5813,7 +5088,7 @@ Process *schedule(Process *p, int calls) esdp = erts_get_scheduler_data(); rq = erts_get_runq_current(esdp); ASSERT(esdp); - fcalls = erts_smp_atomic_read(&function_calls); + fcalls = (int) erts_smp_atomic32_read(&function_calls); actual_reds = reds = 0; erts_smp_runq_lock(rq); } else { @@ -5831,7 +5106,7 @@ Process *schedule(Process *p, int calls) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; esdp->virtual_reds = 0; - fcalls = erts_smp_atomic_addtest(&function_calls, reds); + fcalls = (int) erts_smp_atomic32_addtest(&function_calls, reds); ASSERT(esdp && esdp == erts_get_scheduler_data()); rq = erts_get_runq_current(esdp); @@ -5846,6 +5121,9 @@ Process *schedule(Process *p, int calls) } if (IS_TRACED(p)) { + if (IS_TRACED_FL(p, F_TRACE_CALLS) && p->status != P_FREE) { + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_OUT); + } switch (p->status) { case P_EXITING: if (ARE_TRACE_FLAGS_ON(p, F_TRACE_SCHED_EXIT)) @@ -5889,8 +5167,11 @@ Process *schedule(Process *p, int calls) p->status_flags &= ~ERTS_PROC_SFLG_RUNNING; if (p->status_flags & ERTS_PROC_SFLG_PENDADD2SCHEDQ) { + ErtsRunQueue *notify_runq; p->status_flags &= ~ERTS_PROC_SFLG_PENDADD2SCHEDQ; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); + if (notify_runq != rq) + smp_notify_inc_runq(notify_runq); } #endif @@ -5926,10 +5207,10 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_NO_PROC_LOCKS; - dt = do_time_read_and_reset(); + dt = erts_do_time_read_and_reset(); if (dt) { erts_smp_runq_unlock(rq); - bump_timer(dt); + erts_bump_timer(dt); erts_smp_runq_lock(rq); } BM_STOP_TIMER(system); @@ -5958,21 +5239,33 @@ Process *schedule(Process *p, int calls) | ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic32_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) - || erts_smp_atomic_read(&esdp->chk_cpu_bind)) { - check_cpu_bind(esdp); + || erts_smp_atomic32_read(&esdp->chk_cpu_bind)) { + erts_sched_check_cpu_bind(esdp); } } -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (esdp->check_children) { - esdp->check_children = 0; - erts_smp_runq_unlock(rq); - erts_check_children(); - erts_smp_runq_lock(rq); +#if defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ + || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) + { + ErtsSchedulerSleepInfo *ssi = esdp->ssi; + erts_aint32_t aux_work = erts_smp_atomic32_read(&ssi->aux_work); + if (aux_work) { + erts_smp_runq_unlock(rq); +#ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + aux_work = blockable_aux_work(esdp, ssi, aux_work); +#endif +#ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK + nonblockable_aux_work(esdp, ssi, aux_work); +#endif + erts_smp_runq_lock(rq); + } } #endif @@ -6004,7 +5297,10 @@ Process *schedule(Process *p, int calls) if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || erts_smp_atomic_read(&esdp->suspended)) { + || (erts_smp_atomic32_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED)) { + ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) + & ERTS_SSI_FLG_SUSPENDED); non_empty_runq(rq); goto continue_check_activities_to_run; } @@ -6021,17 +5317,7 @@ Process *schedule(Process *p, int calls) } } - if (prepare_for_sys_schedule()) { - erts_smp_atomic_set(&function_calls, 0); - fcalls = 0; - sched_sys_wait(esdp->no, rq); - erts_smp_atomic_set(&doing_sys_schedule, 0); - } - else { - /* If all schedulers are waiting, one of them *should* - be waiting in erl_sys_schedule() */ - sched_cnd_wait(esdp->no, rq); - } + scheduler_wait(&fcalls, esdp, rq); non_empty_runq(rq); @@ -6055,19 +5341,21 @@ Process *schedule(Process *p, int calls) * Schedule system-level activities. */ - erts_smp_atomic_set(&function_calls, 0); + erts_smp_atomic32_set(&function_calls, 0); fcalls = 0; + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + #ifdef ERTS_SMP /* erts_sys_schedule_interrupt(0); */ #endif erts_smp_runq_unlock(rq); erl_sys_schedule(runnable); - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); #ifdef ERTS_SMP erts_smp_runq_lock(rq); - erts_smp_atomic_set(&doing_sys_schedule, 0); + erts_smp_atomic32_set(&doing_sys_schedule, 0); goto continue_check_activities_to_run; #else if (!runnable) @@ -6088,14 +5376,14 @@ Process *schedule(Process *p, int calls) if (rq->wakeup_other < 0) rq->wakeup_other = 0; } - else if (rq->wakeup_other < ERTS_WAKEUP_OTHER_LIMIT) + else if (rq->wakeup_other < wakeup_other_limit) rq->wakeup_other += rq->len*wo_reds + ERTS_WAKEUP_OTHER_FIXED_INC; else { if (erts_common_run_queue) { if (erts_common_run_queue->waiting) - wake_one_scheduler(); + wake_scheduler(erts_common_run_queue, 0, 1); } - else if (erts_smp_atomic_read(&no_empty_run_queues) != 0) { + else if (erts_smp_atomic32_read(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; } @@ -6282,7 +5570,11 @@ Process *schedule(Process *p, int calls) trace_virtual_sched(p, am_in); break; } + if (IS_TRACED_FL(p, F_TRACE_CALLS)) { + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_IN); + } } + if (p->status != P_EXITING) p->status = P_RUNNING; @@ -6385,6 +5677,15 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) ErtsRunQueue *rq = erts_get_runq_current(NULL); ErtsMiscOpList *molp = misc_op_list_alloc(); + if (!rq) { + /* + * This can only happen when the sys msg dispatcher + * thread schedules misc ops (this happens *very* + * seldom; only when trace drivers are unloaded). + */ + rq = ERTS_RUNQ_IX(0); + } + erts_smp_runq_lock(rq); while (rq->misc.evac_runq) { @@ -6404,8 +5705,8 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) else rq->misc.start = molp; rq->misc.end = molp; - smp_notify_inc_runq(rq); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(rq); } static void @@ -6539,7 +5840,7 @@ erts_test_next_pid(int set, Uint next) Uint erts_process_count(void) { - long res = erts_smp_atomic_read(&process_count); + erts_aint32_t res = erts_smp_atomic32_read(&process_count); ASSERT(res >= 0); return (Uint) res; } @@ -6588,7 +5889,7 @@ alloc_process(void) ASSERT(!process_tab[p_next]); process_tab[p_next] = p; - erts_smp_atomic_inc(&process_count); + erts_smp_atomic32_inc(&process_count); p->id = make_internal_pid(p_serial << p_serial_shift | p_next); if (p->id == ERTS_INVALID_PID) { /* Do not use the invalid pid; change serial */ @@ -6647,7 +5948,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). Eterm args, /* Arguments for function (must be well-formed list). */ ErlSpawnOpts* so) /* Options for spawn. */ { - ErtsRunQueue *rq; + ErtsRunQueue *rq, *notify_runq; Process *p; Sint arity; /* Number of arguments. */ #ifndef HYBRID @@ -6714,7 +6015,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; p->prio = PRIORITY_NORMAL; - p->max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); + p->max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); } p->skipped = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); @@ -6726,11 +6027,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* * Must initialize binary lists here before copying binaries to process. */ - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; heap_need += @@ -6764,6 +6061,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->bin_vheap_sz = p->min_vheap_size; p->bin_old_vheap_sz = p->min_vheap_size; p->bin_old_vheap = 0; + p->bin_vheap_mature = 0; /* No need to initialize p->fcalls. */ @@ -6820,7 +6118,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->group_leader = IS_CONST(parent->group_leader) ? parent->group_leader - : STORE_NC(&p->htop, &p->off_heap.externals, parent->group_leader); + : STORE_NC(&p->htop, &p->off_heap, parent->group_leader); } erts_get_default_tracing(&p->trace_flags, &p->tracer_proc); @@ -6964,10 +6262,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #endif p->status = P_WAITING; - internal_add_to_runq(rq, p); + notify_runq = internal_add_to_runq(rq, p); erts_smp_runq_unlock(rq); + smp_notify_inc_runq(notify_runq); + res = p->id; erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); @@ -7014,6 +6314,7 @@ void erts_init_empty_process(Process *p) p->bin_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap_sz = BIN_VH_MIN_SIZE; p->bin_old_vheap = 0; + p->bin_vheap_mature = 0; #ifdef ERTS_SMP p->u.ptimer = NULL; p->bound_runq = NULL; @@ -7021,11 +6322,7 @@ void erts_init_empty_process(Process *p) memset(&(p->u.tm), 0, sizeof(ErlTimer)); #endif p->next = NULL; - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; p->reg = NULL; p->heap_sz = 0; @@ -7172,11 +6469,7 @@ erts_debug_verify_clean_empty_process(Process* p) /* Thing that erts_cleanup_empty_process() cleans up */ - ASSERT(p->off_heap.mso == NULL); -#ifndef HYBRID /* FIND ME! */ - ASSERT(p->off_heap.funs == NULL); -#endif - ASSERT(p->off_heap.externals == NULL); + ASSERT(p->off_heap.first == NULL); ASSERT(p->off_heap.overhead == 0); ASSERT(p->mbuf == NULL); @@ -7187,25 +6480,16 @@ erts_debug_verify_clean_empty_process(Process* p) void erts_cleanup_empty_process(Process* p) { - ErlHeapFragment* mbufp; - /* We only check fields that are known to be used... */ erts_cleanup_offheap(&p->off_heap); - p->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - p->off_heap.funs = NULL; -#endif - p->off_heap.externals = NULL; + p->off_heap.first = NULL; p->off_heap.overhead = 0; - mbufp = p->mbuf; - while (mbufp) { - ErlHeapFragment *next = mbufp->next; - free_message_buffer(mbufp); - mbufp = next; + if (p->mbuf != NULL) { + free_message_buffer(p->mbuf); + p->mbuf = NULL; } - p->mbuf = NULL; #if defined(ERTS_ENABLE_LOCK_COUNT) && defined(ERTS_SMP) erts_lcnt_proc_lock_destroy(p); #endif @@ -7221,7 +6505,6 @@ static void delete_process(Process* p) { ErlMessage* mp; - ErlHeapFragment* bp; VERBOSE(DEBUG_PROCESSES, ("Removing process: %T\n",p->id)); @@ -7237,7 +6520,7 @@ delete_process(Process* p) * The mso list should not be used anymore, but if it is, make sure that * we'll notice. */ - p->off_heap.mso = (void *) 0x8DEFFACD; + p->off_heap.first = (void *) 0x8DEFFACD; if (p->arg_reg != p->def_arg_reg) { erts_free(ERTS_ALC_T_ARG_REG, p->arg_reg); @@ -7271,11 +6554,8 @@ delete_process(Process* p) /* * Free all pending message buffers. */ - bp = p->mbuf; - while (bp != NULL) { - ErlHeapFragment* next_bp = bp->next; - free_message_buffer(bp); - bp = next_bp; + if (p->mbuf != NULL) { + free_message_buffer(p->mbuf); } erts_erase_dicts(p); @@ -8034,8 +7314,13 @@ erts_do_exit_process(Process* p, Eterm reason) ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); #endif - if (IS_TRACED_FL(p,F_TRACE_PROCS)) - trace_proc(p, p, am_exit, reason); + if (IS_TRACED(p)) { + if (IS_TRACED_FL(p, F_TRACE_CALLS)) + erts_schedule_time_break(p, ERTS_BP_CALL_TIME_SCHEDULE_EXITING); + + if (IS_TRACED_FL(p,F_TRACE_PROCS)) + trace_proc(p, p, am_exit, reason); + } erts_trace_check_exiting(p->id); @@ -8084,6 +7369,8 @@ continue_exit_process(Process *p Eterm reason = p->fvalue; DistEntry *dep; struct saved_calls *scb; + process_breakpoint_time_t *pbt; + #ifdef DEBUG int yield_allowed = 1; #endif @@ -8185,8 +7472,8 @@ continue_exit_process(Process *p p->status_flags = 0; #endif process_tab[pix] = NULL; /* Time of death! */ - ASSERT(erts_smp_atomic_read(&process_count) > 0); - erts_smp_atomic_dec(&process_count); + ASSERT(erts_smp_atomic32_read(&process_count) > 0); + erts_smp_atomic32_dec(&process_count); #ifdef ERTS_SMP erts_pix_unlock(pix_lock); @@ -8223,6 +7510,7 @@ continue_exit_process(Process *p ? ERTS_PROC_SET_DIST_ENTRY(p, ERTS_PROC_LOCKS_ALL, NULL) : NULL); scb = ERTS_PROC_SET_SAVED_CALLS_BUF(p, ERTS_PROC_LOCKS_ALL, NULL); + pbt = ERTS_PROC_SET_CALL_TIME(p, ERTS_PROC_LOCKS_ALL, NULL); erts_smp_proc_unlock(p, ERTS_PROC_LOCKS_ALL); processes_busy--; @@ -8265,6 +7553,9 @@ continue_exit_process(Process *p if (scb) erts_free(ERTS_ALC_T_CALLS_BUF, (void *) scb); + if (pbt) + erts_free(ERTS_ALC_T_BPD, (void *) pbt); + delete_process(p); erts_smp_proc_lock(p, ERTS_PROC_LOCK_MAIN); @@ -8322,7 +7613,7 @@ cancel_timer(Process* p) #ifdef ERTS_SMP erts_cancel_smp_ptimer(p->u.ptimer); #else - erl_cancel_timer(&p->u.tm); + erts_cancel_timer(&p->u.tm); #endif } @@ -8348,7 +7639,7 @@ set_timer(Process* p, Uint timeout) (ErlTimeoutProc) timeout_proc, timeout); #else - erl_set_timer(&p->u.tm, + erts_set_timer(&p->u.tm, (ErlTimeoutProc) timeout_proc, NULL, (void*) p, diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index cbcdec4ba7..d927415f37 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -174,8 +174,8 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \ ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO))) -#define ERTS_RUNQ_IFLG_SUSPENDED (((long) 1) << 0) -#define ERTS_RUNQ_IFLG_NONEMPTY (((long) 1) << 1) +#define ERTS_RUNQ_IFLG_SUSPENDED (((erts_aint32_t) 1) << 0) +#define ERTS_RUNQ_IFLG_NONEMPTY (((erts_aint32_t) 1) << 1) #ifdef DEBUG @@ -219,6 +219,49 @@ typedef enum { ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED } ErtsMigrateResult; +#define ERTS_SSI_FLG_SLEEPING (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_FLG_POLL_SLEEPING (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_FLG_TSE_SLEEPING (((erts_aint32_t) 1) << 2) +#define ERTS_SSI_FLG_WAITING (((erts_aint32_t) 1) << 3) +#define ERTS_SSI_FLG_SUSPENDED (((erts_aint32_t) 1) << 4) + +#define ERTS_SSI_FLGS_SLEEP_TYPE \ + (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) + +#define ERTS_SSI_FLGS_SLEEP \ + (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLGS_SLEEP_TYPE) + +#define ERTS_SSI_FLGS_ALL \ + (ERTS_SSI_FLGS_SLEEP \ + | ERTS_SSI_FLG_WAITING \ + | ERTS_SSI_FLG_SUSPENDED) + +#define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK + +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 1) + +#define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \ + (ERTS_SSI_AUX_WORK_CHECK_CHILDREN \ + | ERTS_SSI_AUX_WORK_MISC) +#define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \ + (0) + +typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; + +typedef struct { + erts_smp_spinlock_t lock; + ErtsSchedulerSleepInfo *list; +} ErtsSchedulerSleepList; + +struct ErtsSchedulerSleepInfo_ { + ErtsSchedulerSleepInfo *next; + ErtsSchedulerSleepInfo *prev; + erts_smp_atomic32_t flags; + erts_tse_t *event; + erts_smp_atomic32_t aux_work; +}; + /* times to reschedule low prio process before running */ #define RESCHEDULE_LOW 8 @@ -266,13 +309,14 @@ typedef struct { struct ErtsRunQueue_ { int ix; - erts_smp_atomic_t info_flags; + erts_smp_atomic32_t info_flags; erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; - erts_smp_atomic_t spin_waiter; - erts_smp_atomic_t spin_wake; +#ifdef ERTS_SMP + ErtsSchedulerSleepList sleepers; +#endif ErtsSchedulerData *scheduler; int waiting; /* < 0 in sys schedule; > 0 on cnd variable */ @@ -353,6 +397,7 @@ struct ErtsSchedulerData_ { ethr_tid tid; /* Thread id */ struct erl_bits_state erl_bits_state; /* erl_bits.c state */ void *match_pseudo_process; /* erl_db_util.c:db_prog_match() */ + ErtsSchedulerSleepInfo *ssi; Process *free_process; #endif #if !HEAP_ON_C_STACK @@ -374,15 +419,17 @@ struct ErtsSchedulerData_ { #ifdef ERTS_SMP /* NOTE: These fields are modified under held mutexes by other threads */ -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - int check_children; /* run queue mutex */ - int blocked_check_children; /* schdlr_sspnd mutex */ -#endif - erts_smp_atomic_t suspended; /* Only used when common run queue */ - erts_smp_atomic_t chk_cpu_bind; /* Only used when common run queue */ + erts_smp_atomic32_t chk_cpu_bind; /* Only used when common run queue */ #endif }; +typedef union { + ErtsSchedulerData esd; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; +} ErtsAlignedSchedulerData; + +extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; + #ifndef ERTS_SMP extern ErtsSchedulerData *erts_scheduler_data; #endif @@ -397,8 +444,9 @@ extern ErtsSchedulerData *erts_scheduler_data; #define ERTS_PSD_SAVED_CALLS_BUF 1 #define ERTS_PSD_SCHED_ID 2 #define ERTS_PSD_DIST_ENTRY 3 +#define ERTS_PSD_CALL_TIME_BP 4 -#define ERTS_PSD_SIZE 4 +#define ERTS_PSD_SIZE 5 typedef struct { void *data[ERTS_PSD_SIZE]; @@ -419,6 +467,9 @@ typedef struct { #define ERTS_PSD_DIST_ENTRY_GET_LOCKS ERTS_PROC_LOCK_MAIN #define ERTS_PSD_DIST_ENTRY_SET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_CALL_TIME_BP_GET_LOCKS ERTS_PROC_LOCK_MAIN +#define ERTS_PSD_CALL_TIME_BP_SET_LOCKS ERTS_PROC_LOCK_MAIN + typedef struct { ErtsProcLocks get_locks; ErtsProcLocks set_locks; @@ -490,6 +541,7 @@ struct ErtsPendingSuspend_ { # define MIN_VHEAP_SIZE(p) (p)->min_vheap_size # define BIN_VHEAP_SZ(p) (p)->bin_vheap_sz +# define BIN_VHEAP_MATURE(p) (p)->bin_vheap_mature # define BIN_OLD_VHEAP_SZ(p) (p)->bin_old_vheap_sz # define BIN_OLD_VHEAP(p) (p)->bin_old_vheap @@ -607,9 +659,10 @@ struct process { Uint mbuf_sz; /* Size of all message buffers */ ErtsPSD *psd; /* Rarely used process specific data */ - Uint bin_vheap_sz; /* Virtual heap block size for binaries */ - Uint bin_old_vheap_sz; /* Virtual old heap block size for binaries */ - Uint bin_old_vheap; /* Virtual old heap size for binaries */ + Uint64 bin_vheap_sz; /* Virtual heap block size for binaries */ + Uint64 bin_vheap_mature; /* Virtual heap block size for binaries */ + Uint64 bin_old_vheap_sz; /* Virtual old heap block size for binaries */ + Uint64 bin_old_vheap; /* Virtual old heap size for binaries */ union { #ifdef ERTS_SMP @@ -765,7 +818,7 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp) { ErlHeapFragment* hf = MBUF(p); - ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->size)); + ASSERT(hf!=NULL && (hp - hf->mem < (unsigned long)hf->alloc_size)); hf->used_size = hp - hf->mem; } @@ -824,7 +877,7 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; #define F_INSLPQUEUE (1 << 1) /* Set if in timer queue */ #define F_TIMO (1 << 2) /* Set if timeout */ #define F_HEAP_GROW (1 << 3) -#define F_NEED_FULLSWEEP (1 << 4) /* If process has old binaries & funs. */ +#define F_NEED_FULLSWEEP (1 << 4) #define F_USING_DB (1 << 5) /* If have created tables */ #define F_DISTRIBUTION (1 << 6) /* Process used in distribution */ #define F_USING_DDLL (1 << 7) /* Process has used the DDLL interface */ @@ -958,26 +1011,12 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; (p)->flags &= ~F_TIMO; \ } while (0) - -#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS 0 -#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED 1 -#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY 2 -#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE 3 - -int erts_init_scheduler_bind_type(char *how); - -#define ERTS_INIT_CPU_TOPOLOGY_OK 0 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID 1 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE 2 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY 3 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE 4 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES 5 -#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID 6 -#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS 7 -#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES 8 -#define ERTS_INIT_CPU_TOPOLOGY_MISSING 9 - -int erts_init_cpu_topology(char *topology_str); +#define ERTS_RUNQ_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues), \ + &erts_aligned_run_queues[(IX)].runq) +#define ERTS_SCHEDULER_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &erts_aligned_scheduler_data[(IX)].esd) void erts_pre_init_process(void); void erts_late_init_process(void); @@ -988,9 +1027,12 @@ ErtsProcList *erts_proclist_create(Process *); void erts_proclist_destroy(ErtsProcList *); int erts_proclist_same(ErtsProcList *, Process *); +int erts_sched_set_wakeup_limit(char *str); + #ifdef DEBUG void erts_dbg_multi_scheduling_return_trap(Process *, Eterm); #endif +int erts_get_max_no_executing_schedulers(void); #ifdef ERTS_SMP ErtsSchedSuspendResult erts_schedulers_state(Uint *, Uint *, Uint *, int); @@ -1005,9 +1047,15 @@ int erts_is_multi_scheduling_blocked(void); Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); void erts_smp_notify_check_children_needed(void); +void +erts_smp_schedule_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg); #endif +void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); -void erts_init_process(void); +void erts_init_process(int); Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm); Uint erts_run_queues_len(Uint *); void erts_add_to_runq(Process *); @@ -1081,6 +1129,9 @@ void erts_handle_pending_exit(Process *, ErtsProcLocks); void erts_deep_process_dump(int, void *); +Eterm erts_get_reader_groups_map(Process *c_p); +Eterm erts_debug_reader_groups_map(Process *c_p, int groups); + Sint erts_test_next_pid(int, Uint); Eterm erts_debug_processes(Process *c_p); Eterm erts_debug_processes_bif_info(Process *c_p); @@ -1208,6 +1259,12 @@ erts_psd_set(Process *p, ErtsProcLocks plocks, int ix, void *data) #define ERTS_PROC_SET_SAVED_CALLS_BUF(P, L, SCB) \ ((struct saved_calls *) erts_psd_set((P), (L), ERTS_PSD_SAVED_CALLS_BUF, (void *) (SCB))) +#define ERTS_PROC_GET_CALL_TIME(P) \ + ((process_breakpoint_time_t *) erts_psd_get((P), ERTS_PSD_CALL_TIME_BP)) +#define ERTS_PROC_SET_CALL_TIME(P, L, PBT) \ + ((process_breakpoint_time_t *) erts_psd_set((P), (L), ERTS_PSD_CALL_TIME_BP, (void *) (PBT))) + + ERTS_GLB_INLINE Eterm erts_proc_get_error_handler(Process *p); ERTS_GLB_INLINE Eterm erts_proc_set_error_handler(Process *p, ErtsProcLocks plocks, @@ -1499,29 +1556,30 @@ extern int erts_disable_proc_not_running_opt; #define ERTS_MIN_PROCESSES 16 #endif -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS -ERTS_GLB_INLINE void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -void erts_smp_notify_inc_runq__(ErtsRunQueue *runq); -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ +void erts_smp_notify_inc_runq(ErtsRunQueue *runq); -#if ERTS_GLB_INLINE_INCL_FUNC_DEF +#ifdef ERTS_SMP +void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); +ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); -#ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS +#if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_smp_notify_inc_runq(ErtsRunQueue *runq) +erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { -#ifdef ERTS_SMP - ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); - if (runq->waiting) - erts_smp_notify_inc_runq__(runq); -#endif + erts_aint32_t flags = erts_smp_atomic32_read(&ssi->flags); + ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) + || (flags & ERTS_SSI_FLG_WAITING)); + if (flags & ERTS_SSI_FLG_SLEEPING) { + flags = erts_smp_atomic32_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); + erts_sched_finish_poke(ssi, flags); + } } -#endif /* ERTS_INCLUDE_SCHEDULER_INTERNALS */ - #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* #ifdef ERTS_SMP */ + #include "erl_process_lock.h" #undef ERTS_INCLUDE_SCHEDULER_INTERNALS diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index 52440fb635..72560aa124 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -71,9 +71,12 @@ const Process erts_proc_lock_busy; #ifdef ERTS_SMP -/*#define ERTS_PROC_LOCK_SPIN_ON_GATE*/ -#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 16000 +#define ERTS_PROC_LOCK_SPIN_COUNT_MAX 2000 +#define ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC 32 #define ERTS_PROC_LOCK_SPIN_COUNT_BASE 1000 +#define ERTS_PROC_LOCK_AUX_SPIN_COUNT 50 + +#define ERTS_PROC_LOCK_SPIN_UNTIL_YIELD 25 #ifdef ERTS_PROC_LOCK_DEBUG #define ERTS_PROC_LOCK_HARD_DEBUG @@ -83,32 +86,19 @@ const Process erts_proc_lock_busy; static void check_queue(erts_proc_lock_t *lck); #endif - -typedef struct erts_proc_lock_waiter_t_ erts_proc_lock_waiter_t; -struct erts_proc_lock_waiter_t_ { - erts_proc_lock_waiter_t *next; - erts_proc_lock_waiter_t *prev; - ErtsProcLocks wait_locks; - erts_smp_gate_t gate; - erts_proc_lock_queues_t *queues; -}; +#if SIZEOF_INT < 4 +#error "The size of the 'uflgs' field of the erts_tse_t type is too small" +#endif struct erts_proc_lock_queues_t_ { erts_proc_lock_queues_t *next; - erts_proc_lock_waiter_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; -}; - -struct erts_proc_lock_thr_spec_data_t_ { - erts_proc_lock_queues_t *qs; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; }; static erts_proc_lock_queues_t zeroqs = {0}; -static erts_smp_spinlock_t wtr_lock; -static erts_proc_lock_waiter_t *waiter_free_list; +static erts_smp_spinlock_t qs_lock; static erts_proc_lock_queues_t *queue_free_list; -static erts_tsd_key_t waiter_key; #ifdef ERTS_ENABLE_LOCK_CHECK static struct { @@ -122,122 +112,131 @@ static struct { erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; static int proc_lock_spin_count; -static int proc_lock_trans_spin_cost; +static int aux_thr_proc_lock_spin_count; -static void cleanup_waiter(void); +static void cleanup_tse(void); void -erts_init_proc_lock(void) +erts_init_proc_lock(int cpus) { int i; - int cpus; - erts_smp_spinlock_init(&wtr_lock, "proc_lck_wtr_alloc"); + erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc"); for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { -#if ERTS_PROC_LOCK_MUTEX_IMPL -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_mtx_init_x(&erts_pix_locks[i].u.mtx, "pix_lock", make_small(i)); -#else - erts_smp_mtx_init(&erts_pix_locks[i].u.mtx, "pix_lock"); -#endif -#else #ifdef ERTS_ENABLE_LOCK_COUNT - erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, "pix_lock", make_small(i)); + erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, + "pix_lock", make_small(i)); #else erts_smp_spinlock_init(&erts_pix_locks[i].u.spnlck, "pix_lock"); #endif -#endif } - waiter_free_list = NULL; queue_free_list = NULL; - erts_tsd_key_create(&waiter_key); - erts_thr_install_exit_handler(cleanup_waiter); + erts_thr_install_exit_handler(cleanup_tse); #ifdef ERTS_ENABLE_LOCK_CHECK lc_id.proc_lock_main = erts_lc_get_lock_order_id("proc_main"); lc_id.proc_lock_link = erts_lc_get_lock_order_id("proc_link"); lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); #endif - cpus = erts_get_cpu_configured(erts_cpuinfo); - if (cpus > 1) - proc_lock_spin_count = (ERTS_PROC_LOCK_SPIN_COUNT_BASE - * ((int) erts_no_schedulers)); - else if (cpus == 1) - proc_lock_spin_count = 0; - else /* No of cpus unknown. Assume multi proc, but be conservative. */ + if (cpus > 1) { proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE; - if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) - proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; - proc_lock_trans_spin_cost = proc_lock_spin_count/20; -} - -static ERTS_INLINE erts_proc_lock_waiter_t * -alloc_wtr(void) -{ - erts_proc_lock_waiter_t *wtr; - erts_smp_spin_lock(&wtr_lock); - wtr = waiter_free_list; - if (wtr) { - waiter_free_list = wtr->next; - ERTS_LC_ASSERT(queue_free_list); - wtr->queues = queue_free_list; - queue_free_list = wtr->queues->next; - erts_smp_spin_unlock(&wtr_lock); + proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC + * ((int) erts_no_schedulers)); + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT; } - else { - erts_smp_spin_unlock(&wtr_lock); - wtr = erts_alloc(ERTS_ALC_T_PROC_LCK_WTR, - sizeof(erts_proc_lock_waiter_t)); - erts_smp_gate_init(&wtr->gate); - wtr->wait_locks = (ErtsProcLocks) 0; - wtr->queues = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, - sizeof(erts_proc_lock_queues_t)); - sys_memcpy((void *) wtr->queues, - (void *) &zeroqs, - sizeof(erts_proc_lock_queues_t)); + else if (cpus == 1) { + proc_lock_spin_count = 0; + aux_thr_proc_lock_spin_count = 0; } - return wtr; + else { /* No of cpus unknown. Assume multi proc, but be conservative. */ + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE/2; + aux_thr_proc_lock_spin_count = ERTS_PROC_LOCK_AUX_SPIN_COUNT/2; + } + if (proc_lock_spin_count > ERTS_PROC_LOCK_SPIN_COUNT_MAX) + proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_MAX; } #ifdef ERTS_ENABLE_LOCK_CHECK static void -check_unused_waiter(erts_proc_lock_waiter_t *wtr) +check_unused_tse(erts_tse_t *wtr) { int i; - ERTS_LC_ASSERT(wtr->wait_locks == 0); + erts_proc_lock_queues_t *queues = wtr->udata; + ERTS_LC_ASSERT(wtr->uflgs == 0); for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - ERTS_LC_ASSERT(!wtr->queues->queue[i]); + ERTS_LC_ASSERT(!queues->queue[i]); } -#define CHECK_UNUSED_WAITER(W) check_unused_waiter((W)) +#define CHECK_UNUSED_TSE(W) check_unused_tse((W)) #else -#define CHECK_UNUSED_WAITER(W) +#define CHECK_UNUSED_TSE(W) #endif +static ERTS_INLINE erts_tse_t * +tse_fetch(erts_pix_lock_t *pix_lock) +{ + erts_tse_t *tse = erts_tse_fetch(); + if (!tse->udata) { + erts_proc_lock_queues_t *qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_unlock(pix_lock); +#endif + erts_smp_spin_lock(&qs_lock); + qs = queue_free_list; + if (qs) { + queue_free_list = queue_free_list->next; + erts_smp_spin_unlock(&qs_lock); + } + else { + erts_smp_spin_unlock(&qs_lock); + qs = erts_alloc(ERTS_ALC_T_PROC_LCK_QS, + sizeof(erts_proc_lock_queues_t)); + sys_memcpy((void *) qs, + (void *) &zeroqs, + sizeof(erts_proc_lock_queues_t)); + } + tse->udata = qs; +#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL + if (pix_lock) + erts_pix_lock(pix_lock); +#endif + } + tse->uflgs = 0; + return tse; +} static ERTS_INLINE void -free_wtr(erts_proc_lock_waiter_t *wtr) +tse_return(erts_tse_t *tse, int force_free_q) { - CHECK_UNUSED_WAITER(wtr); - erts_smp_spin_lock(&wtr_lock); - wtr->next = waiter_free_list; - waiter_free_list = wtr; - wtr->queues->next = queue_free_list; - queue_free_list = wtr->queues; - erts_smp_spin_unlock(&wtr_lock); + CHECK_UNUSED_TSE(tse); + if (force_free_q || erts_tse_is_tmp(tse)) { + erts_proc_lock_queues_t *qs = tse->udata; + ASSERT(qs); + erts_smp_spin_lock(&qs_lock); + qs->next = queue_free_list; + queue_free_list = qs; + erts_smp_spin_unlock(&qs_lock); + tse->udata = NULL; + } + erts_tse_return(tse); } void erts_proc_lock_prepare_proc_lock_waiter(void) { - erts_tsd_set(waiter_key, (void *) alloc_wtr()); + tse_return(tse_fetch(NULL), 0); } static void -cleanup_waiter(void) +cleanup_tse(void) { - erts_proc_lock_waiter_t *wtr = erts_tsd_get(waiter_key); - if (wtr) - free_wtr(wtr); + erts_tse_t *tse = erts_tse_fetch(); + if (tse) { + if (tse->udata) + tse_return(tse, 1); + else + erts_tse_return(tse); + } } @@ -250,7 +249,7 @@ cleanup_waiter(void) static ERTS_INLINE void enqueue_waiter(erts_proc_lock_queues_t *qs, int ix, - erts_proc_lock_waiter_t *wtr) + erts_tse_t *wtr) { if (!qs->queue[ix]) { qs->queue[ix] = wtr; @@ -266,10 +265,10 @@ enqueue_waiter(erts_proc_lock_queues_t *qs, } } -static erts_proc_lock_waiter_t * +static erts_tse_t * dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) { - erts_proc_lock_waiter_t *wtr = qs->queue[ix]; + erts_tse_t *wtr = qs->queue[ix]; ERTS_LC_ASSERT(qs->queue[ix]); if (wtr->next == wtr) { ERTS_LC_ASSERT(qs->queue[ix]->prev == wtr); @@ -295,10 +294,10 @@ dequeue_waiter(erts_proc_lock_queues_t *qs, int ix) * lock. */ static ERTS_INLINE void -try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) +try_aquire(erts_proc_lock_t *lck, erts_tse_t *wtr) { ErtsProcLocks got_locks = (ErtsProcLocks) 0; - ErtsProcLocks locks = wtr->wait_locks; + ErtsProcLocks locks = wtr->uflgs; int lock_no; ERTS_LC_ASSERT(lck->queues); @@ -334,7 +333,7 @@ try_aquire(erts_proc_lock_t *lck, erts_proc_lock_waiter_t *wtr) } } - wtr->wait_locks &= ~got_locks; + wtr->uflgs &= ~got_locks; } /* @@ -350,8 +349,8 @@ transfer_locks(Process *p, int unlock) { int transferred = 0; - erts_proc_lock_waiter_t *wake = NULL; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wake = NULL; + erts_tse_t *wtr; ErtsProcLocks unset_waiter = 0; ErtsProcLocks tlocks = trnsfr_lcks; int lock_no; @@ -377,11 +376,11 @@ transfer_locks(Process *p, ERTS_LC_ASSERT(wtr); if (!qs->queue[lock_no]) unset_waiter |= lock; - ERTS_LC_ASSERT(wtr->wait_locks & lock); - wtr->wait_locks &= ~lock; - if (wtr->wait_locks) + ERTS_LC_ASSERT(wtr->uflgs & lock); + wtr->uflgs &= ~lock; + if (wtr->uflgs) try_aquire(&p->lock, wtr); - if (!wtr->wait_locks) { + if (!wtr->uflgs) { /* * The other thread got all locks it needs; * need to wake it up. @@ -412,9 +411,10 @@ transfer_locks(Process *p, erts_pix_unlock(pix_lock); do { - erts_proc_lock_waiter_t *tmp = wake; + erts_tse_t *tmp = wake; wake = wake->next; - erts_smp_gate_let_through(&tmp->gate, 1); + erts_atomic32_set(&tmp->uaflgs, 0); + erts_tse_set(tmp); } while (wake); if (!unlock) @@ -462,26 +462,16 @@ wait_for_locks(Process *p, ErtsProcLocks olflgs) { erts_pix_lock_t *pix_lock = pixlck ? pixlck : ERTS_PID2PIXLOCK(p->id); - int tsd; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; + erts_proc_lock_queues_t *qs; /* Acquire a waiter object on which this thread can wait. */ - wtr = erts_tsd_get(waiter_key); - if (wtr) - tsd = 1; - else { -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_unlock(pix_lock); -#endif - wtr = alloc_wtr(); - tsd = 0; -#if ERTS_PROC_LOCK_SPINLOCK_IMPL && !ERTS_PROC_LOCK_ATOMIC_IMPL - erts_pix_lock(pix_lock); -#endif - } + wtr = tse_fetch(pix_lock); /* Record which locks this waiter needs. */ - wtr->wait_locks = need_locks; + wtr->uflgs = need_locks; + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); #if ERTS_PROC_LOCK_ATOMIC_IMPL erts_pix_lock(pix_lock); @@ -489,14 +479,16 @@ wait_for_locks(Process *p, ERTS_LC_ASSERT(erts_lc_pix_lock_is_locked(pix_lock)); + qs = wtr->udata; + ASSERT(qs); /* Provide the process with waiter queues, if it doesn't have one. */ if (!p->lock.queues) { - wtr->queues->next = NULL; - p->lock.queues = wtr->queues; + qs->next = NULL; + p->lock.queues = qs; } else { - wtr->queues->next = p->lock.queues->next; - p->lock.queues->next = wtr->queues; + qs->next = p->lock.queues->next; + p->lock.queues->next = qs; } #ifdef ERTS_PROC_LOCK_HARD_DEBUG @@ -506,46 +498,59 @@ wait_for_locks(Process *p, /* Try to aquire locks one at a time in lock order and set wait flag */ try_aquire(&p->lock, wtr); + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + #ifdef ERTS_PROC_LOCK_HARD_DEBUG check_queue(&p->lock); #endif - if (wtr->wait_locks) { /* We didn't get them all; need to wait... */ - /* Got to wait for locks... */ + if (wtr->uflgs) { + /* We didn't get them all; need to wait... */ + + ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); + + erts_atomic32_set(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); - /* - * Wait for needed locks. When we return all needed locks have - * have been acquired by other threads and transfered to us. - */ -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - erts_smp_gate_swait(&wtr->gate, proc_lock_spin_count); -#else - erts_smp_gate_wait(&wtr->gate); -#endif + while (1) { + int res; + erts_tse_reset(wtr); + + if (erts_atomic32_read(&wtr->uaflgs) == 0) + break; + + /* + * Wait for needed locks. When we are woken all needed locks have + * have been acquired by other threads and transfered to us. + * However, we need to be prepared for spurious wakeups. + */ + do { + res = erts_tse_wait(wtr); /* might return EINTR */ + } while (res != 0); + } erts_pix_lock(pix_lock); + + ASSERT(wtr->uflgs == 0); } /* Recover some queues to store in the waiter. */ ERTS_LC_ASSERT(p->lock.queues); if (p->lock.queues->next) { - wtr->queues = p->lock.queues->next; - p->lock.queues->next = wtr->queues->next; + qs = p->lock.queues->next; + p->lock.queues->next = qs->next; } else { - wtr->queues = p->lock.queues; + qs = p->lock.queues; p->lock.queues = NULL; } + wtr->udata = qs; erts_pix_unlock(pix_lock); ERTS_LC_ASSERT(locks == (ERTS_PROC_LOCK_FLGS_READ_(&p->lock) & locks)); - if (tsd) - CHECK_UNUSED_WAITER(wtr); - else - free_wtr(wtr); + tse_return(wtr, 0); } /* @@ -563,52 +568,57 @@ erts_proc_lock_failed(Process *p, ErtsProcLocks locks, ErtsProcLocks old_lflgs) { -#ifdef ERTS_PROC_LOCK_SPIN_ON_GATE - int spin_count = 0; -#else - int spin_count = proc_lock_spin_count; -#endif - + int until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + int thr_spin_count; + int spin_count; ErtsProcLocks need_locks = locks; ErtsProcLocks olflgs = old_lflgs; - while (need_locks != 0) - { - ErtsProcLocks can_grab = in_order_locks(olflgs, need_locks); + if (erts_thr_get_main_status()) + thr_spin_count = proc_lock_spin_count; + else + thr_spin_count = aux_thr_proc_lock_spin_count; + + spin_count = thr_spin_count; + + while (need_locks != 0) { + ErtsProcLocks can_grab; + + can_grab = in_order_locks(olflgs, need_locks); - if (can_grab == 0) - { + if (can_grab == 0) { /* Someone already has the lowest-numbered lock we want. */ - if (spin_count-- <= 0) - { + if (spin_count-- <= 0) { /* Too many retries, give up and sleep for the lock. */ wait_for_locks(p, pixlck, locks, need_locks, olflgs); return; } + ERTS_SPIN_BODY; + + if (--until_yield == 0) { + until_yield = ERTS_PROC_LOCK_SPIN_UNTIL_YIELD; + erts_thr_yield(); + } + olflgs = ERTS_PROC_LOCK_FLGS_READ_(&p->lock); } - else - { + else { /* Try to grab all of the grabbable locks at once with cmpxchg. */ ErtsProcLocks grabbed = olflgs | can_grab; ErtsProcLocks nflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, grabbed, olflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, grabbed, olflgs); - if (nflgs == olflgs) - { + if (nflgs == olflgs) { /* Success! We grabbed the 'can_grab' locks. */ olflgs = grabbed; need_locks &= ~can_grab; -#ifndef ERTS_PROC_LOCK_SPIN_ON_GATE /* Since we made progress, reset the spin count. */ - spin_count = proc_lock_spin_count; -#endif + spin_count = thr_spin_count; } - else - { + else { /* Compare-and-exchange failed, try again. */ olflgs = nflgs; } @@ -945,7 +955,7 @@ erts_proc_lock_init(Process *p) { /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic_init(&p->lock.flags, (long) ERTS_PROC_LOCKS_ALL); + erts_smp_atomic32_init(&p->lock.flags, (erts_aint32_t) ERTS_PROC_LOCKS_ALL); #else p->lock.flags = ERTS_PROC_LOCKS_ALL; #endif @@ -964,7 +974,7 @@ erts_proc_lock_init(Process *p) { int i; for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - erts_smp_atomic_init(&p->lock.locked[i], (long) 1); + erts_smp_atomic32_init(&p->lock.locked[i], (erts_aint32_t) 1); } #endif } @@ -1407,7 +1417,7 @@ check_queue(erts_proc_lock_t *lck) wtr = (((ErtsProcLocks) 1) << lock_no) << ERTS_PROC_LOCK_WAITER_SHIFT; if (lflgs & wtr) { int n; - erts_proc_lock_waiter_t *wtr; + erts_tse_t *wtr; ERTS_LC_ASSERT(lck->queues && lck->queues->queue[lock_no]); wtr = lck->queues->queue[lock_no]; n = 0; diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index d71e5a0a6e..355179f084 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. + * Copyright Ericsson AB 2007-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -54,20 +54,20 @@ #define ERTS_PROC_LOCK_MAX_BIT 3 -typedef Uint32 ErtsProcLocks; +typedef erts_aint32_t ErtsProcLocks; typedef struct erts_proc_lock_queues_t_ erts_proc_lock_queues_t; typedef struct erts_proc_lock_t_ { #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic_t flags; + erts_smp_atomic32_t flags; #else ErtsProcLocks flags; #endif erts_proc_lock_queues_t *queues; - long refc; + Sint32 refc; #ifdef ERTS_PROC_LOCK_DEBUG - erts_smp_atomic_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; + erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_t lcnt_main; @@ -255,11 +255,7 @@ void erts_proc_lc_unrequire_lock(Process *p, ErtsProcLocks locks); typedef struct { union { -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_t mtx; -#else erts_smp_spinlock_t spnlck; -#endif char buf[64]; /* Try to get locks in different cache lines */ } u; } erts_pix_lock_t; @@ -274,14 +270,19 @@ typedef struct { #if ERTS_PROC_LOCK_ATOMIC_IMPL #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic_band(&(L)->flags, (long) (MSK))) + ((ErtsProcLocks) erts_smp_atomic32_band(&(L)->flags, (erts_aint32_t) (MSK))) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic_bor(&(L)->flags, (long) (MSK))) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ - ((ErtsProcLocks) erts_smp_atomic_cmpxchg(&(L)->flags, \ - (long) (NEW), (long) (EXPECTED))) + ((ErtsProcLocks) erts_smp_atomic32_bor(&(L)->flags, (erts_aint32_t) (MSK))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \ + (erts_aint32_t) (NEW), \ + (erts_aint32_t) (EXPECTED))) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ + ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_relb(&(L)->flags, \ + (erts_aint32_t) (NEW), \ + (erts_aint32_t) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_READ_(L) \ - ((ErtsProcLocks) erts_smp_atomic_read(&(L)->flags)) + ((ErtsProcLocks) erts_smp_atomic32_read(&(L)->flags)) #else /* no opt atomic ops */ @@ -289,6 +290,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_band(erts_proc_lock_t *, ErtsProcLocks); ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_bor(erts_proc_lock_t *, ErtsProcLocks); +ERTS_GLB_INLINE ErtsProcLocks erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *, + ErtsProcLocks, + ErtsProcLocks); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -322,7 +326,9 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) erts_proc_lock_flags_band((L), (MSK)) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) erts_proc_lock_flags_bor((L), (MSK)) -#define ERTS_PROC_LOCK_FLGS_CMPXCHG_(L, NEW, EXPECTED) \ +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ + erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) +#define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ erts_proc_lock_flags_cmpxchg((L), (NEW), (EXPECTED)) #define ERTS_PROC_LOCK_FLGS_READ_(L) ((L)->flags) @@ -330,7 +336,7 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; -void erts_init_proc_lock(void); +void erts_init_proc_lock(int cpus); void erts_proc_lock_prepare_proc_lock_waiter(void); void erts_proc_lock_failed(Process *, erts_pix_lock_t *, @@ -348,9 +354,9 @@ ERTS_GLB_INLINE ErtsProcLocks erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_proc_lock_x__(Process *, - erts_pix_lock_t *, - ErtsProcLocks, - char *file, unsigned int line); + erts_pix_lock_t *, + ErtsProcLocks, + char *file, unsigned int line); #else ERTS_GLB_INLINE void erts_smp_proc_lock__(Process *, erts_pix_lock_t *, @@ -372,30 +378,18 @@ ERTS_GLB_INLINE void erts_proc_lock_op_debug(Process *, ErtsProcLocks, int); ERTS_GLB_INLINE void erts_pix_lock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_lock(&pixlck->u.mtx); -#else erts_smp_spin_lock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE void erts_pix_unlock(erts_pix_lock_t *pixlck) { ERTS_LC_ASSERT(pixlck); -#if ERTS_PROC_LOCK_MUTEX_IMPL - erts_smp_mtx_unlock(&pixlck->u.mtx); -#else erts_smp_spin_unlock(&pixlck->u.spnlck); -#endif } ERTS_GLB_INLINE int erts_lc_pix_lock_is_locked(erts_pix_lock_t *pixlck) { -#if ERTS_PROC_LOCK_MUTEX_IMPL - return erts_smp_lc_mtx_is_locked(&pixlck->u.mtx); -#else return erts_smp_lc_spinlock_is_locked(&pixlck->u.spnlck); -#endif } /* @@ -417,9 +411,9 @@ erts_smp_proc_raw_trylock__(Process *p, ErtsProcLocks locks) ErtsProcLocks expct_lflgs = 0; while (1) { - ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, - expct_lflgs | locks, - expct_lflgs); + ErtsProcLocks lflgs = ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(&p->lock, + expct_lflgs | locks, + expct_lflgs); if (ERTS_LIKELY(lflgs == expct_lflgs)) { /* We successfully grabbed all locks. */ return 0; @@ -535,7 +529,7 @@ erts_smp_proc_unlock__(Process *p, if (want_lflgs != old_lflgs) { ErtsProcLocks new_lflgs = - ERTS_PROC_LOCK_FLGS_CMPXCHG_(&p->lock, want_lflgs, old_lflgs); + ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(&p->lock, want_lflgs, old_lflgs); if (new_lflgs != old_lflgs) { /* cmpxchg failed, try again. */ @@ -627,13 +621,13 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked) for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) { ErtsProcLocks lock = ((ErtsProcLocks) 1) << i; if (locks & lock) { - long lock_count; + erts_aint32_t lock_count; if (locked) { - lock_count = erts_smp_atomic_inctest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_inctest(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 1); } else { - lock_count = erts_smp_atomic_dectest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_dectest(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 0); } } diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 03d2a586e3..287327bfe1 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -43,27 +43,48 @@ typedef erts_thr_init_data_t erts_smp_thr_init_data_t; typedef erts_tid_t erts_smp_tid_t; typedef erts_mtx_t erts_smp_mtx_t; typedef erts_cnd_t erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER ERTS_RWMTX_OPT_DEFAULT_INITER +#define ERTS_SMP_RWMTX_TYPE_NORMAL ERTS_RWMTX_TYPE_NORMAL +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ ERTS_RWMTX_TYPE_FREQUENT_READ +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_SMP_RWMTX_LONG_LIVED ERTS_RWMTX_LONG_LIVED +#define ERTS_SMP_RWMTX_SHORT_LIVED ERTS_RWMTX_SHORT_LIVED +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED ERTS_RWMTX_UNKNOWN_LIVED +typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; typedef erts_rwmtx_t erts_smp_rwmtx_t; typedef erts_tsd_key_t erts_smp_tsd_key_t; -typedef erts_gate_t erts_smp_gate_t; -typedef ethr_atomic_t erts_smp_atomic_t; +typedef erts_atomic_t erts_smp_atomic_t; +typedef erts_atomic32_t erts_smp_atomic32_t; typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; -typedef erts_thr_timeval_t erts_smp_thr_timeval_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ #else /* #ifdef ERTS_SMP */ -#define ERTS_SMP_THR_OPTS_DEFAULT_INITER 0 +#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0} typedef int erts_smp_thr_opts_t; typedef int erts_smp_thr_init_data_t; typedef int erts_smp_tid_t; typedef int erts_smp_mtx_t; typedef int erts_smp_cnd_t; +#define ERTS_SMP_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_SMP_RWMTX_TYPE_NORMAL 0 +#define ERTS_SMP_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_SMP_RWMTX_LONG_LIVED 0 +#define ERTS_SMP_RWMTX_SHORT_LIVED 0 +#define ERTS_SMP_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_smp_rwmtx_opt_t; typedef int erts_smp_rwmtx_t; typedef int erts_smp_tsd_key_t; -typedef int erts_smp_gate_t; -typedef long erts_smp_atomic_t; +typedef SWord erts_smp_atomic_t; +typedef Uint32 erts_smp_atomic32_t; #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -72,11 +93,6 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t; typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t; #endif -typedef struct { - long tv_sec; - long tv_nsec; -} erts_smp_thr_timeval_t; - #endif /* #ifdef ERTS_SMP */ ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id); @@ -103,8 +119,6 @@ ERTS_GLB_INLINE void erts_smp_mtx_init_locked_x(erts_smp_mtx_t *mtx, ERTS_GLB_INLINE void erts_smp_mtx_init(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_init_locked(erts_smp_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_smp_mtx_destroy(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx); ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_smp_mtx_lock_x(erts_smp_mtx_t *mtx, char *file, int line); @@ -119,9 +133,17 @@ ERTS_GLB_INLINE void erts_smp_cnd_wait(erts_smp_cnd_t *cnd, erts_smp_mtx_t *mtx); ERTS_GLB_INLINE void erts_smp_cnd_signal(erts_smp_cnd_t *cnd); ERTS_GLB_INLINE void erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd); +ERTS_GLB_INLINE void erts_smp_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_smp_rwmtx_destroy(erts_smp_rwmtx_t *rwmtx); @@ -138,23 +160,82 @@ ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, long i); -ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, long i); -ERTS_GLB_INLINE long erts_smp_atomic_read(erts_smp_atomic_t *var); -ERTS_GLB_INLINE long erts_smp_atomic_inctest(erts_smp_atomic_t *incp); -ERTS_GLB_INLINE long erts_smp_atomic_dectest(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, + erts_aint_t i); +ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp); ERTS_GLB_INLINE void erts_smp_atomic_inc(erts_smp_atomic_t *incp); ERTS_GLB_INLINE void erts_smp_atomic_dec(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE long erts_smp_atomic_addtest(erts_smp_atomic_t *addp, - long i); -ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, long i); -ERTS_GLB_INLINE long erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, - long new); -ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, - long new, - long expected); -ERTS_GLB_INLINE long erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask); -ERTS_GLB_INLINE long erts_smp_atomic_band(erts_smp_atomic_t *var, long mask); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_addtest(erts_smp_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, + erts_aint_t new); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_bor(erts_smp_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_band(erts_smp_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read_acqb(erts_smp_atomic_t *var); +ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var, + erts_aint_t i); +ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE void +erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read(erts_smp_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE void +erts_smp_atomic32_inc(erts_smp_atomic32_t *incp); +ERTS_GLB_INLINE void +erts_smp_atomic32_dec(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var); +ERTS_GLB_INLINE void +erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra); @@ -185,17 +266,10 @@ ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock); -ERTS_GLB_INLINE void erts_smp_thr_time_now(erts_smp_thr_timeval_t *time); ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key); ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_smp_tsd_get(erts_smp_tsd_key_t key); -ERTS_GLB_INLINE void erts_smp_gate_init(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_destroy(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_close(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_smp_gate_wait(erts_smp_gate_t *gp); -ERTS_GLB_INLINE void erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount); #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 @@ -331,22 +405,6 @@ erts_smp_mtx_destroy(erts_smp_mtx_t *mtx) #endif } -ERTS_GLB_INLINE void -erts_smp_mtx_set_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_set_forksafe(mtx); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_mtx_unset_forksafe(erts_smp_mtx_t *mtx) -{ -#ifdef ERTS_SMP - erts_mtx_unset_forksafe(mtx); -#endif -} - ERTS_GLB_INLINE int erts_smp_mtx_trylock(erts_smp_mtx_t *mtx) { @@ -433,6 +491,25 @@ erts_smp_cnd_broadcast(erts_smp_cnd_t *cnd) } ERTS_GLB_INLINE void +erts_smp_rwmtx_set_reader_group(int no) +{ +#ifdef ERTS_SMP + erts_rwmtx_set_reader_group(no); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt_x(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name, + Eterm extra) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt_x(rwmtx, opt, name, extra); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) { #ifdef ERTS_SMP @@ -441,6 +518,16 @@ erts_smp_rwmtx_init_x(erts_smp_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_smp_rwmtx_init_opt(erts_smp_rwmtx_t *rwmtx, + erts_smp_rwmtx_opt_t *opt, + char *name) +{ +#ifdef ERTS_SMP + erts_rwmtx_init_opt(rwmtx, opt, name); +#endif +} + +ERTS_GLB_INLINE void erts_smp_rwmtx_init(erts_smp_rwmtx_t *rwmtx, char *name) { #ifdef ERTS_SMP @@ -568,7 +655,7 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx) } ERTS_GLB_INLINE void -erts_smp_atomic_init(erts_smp_atomic_t *var, long i) +erts_smp_atomic_init(erts_smp_atomic_t *var, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_init(var, i); @@ -578,7 +665,7 @@ erts_smp_atomic_init(erts_smp_atomic_t *var, long i) } ERTS_GLB_INLINE void -erts_smp_atomic_set(erts_smp_atomic_t *var, long i) +erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_set(var, i); @@ -587,7 +674,7 @@ erts_smp_atomic_set(erts_smp_atomic_t *var, long i) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var) { #ifdef ERTS_SMP @@ -597,7 +684,7 @@ erts_smp_atomic_read(erts_smp_atomic_t *var) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp) { #ifdef ERTS_SMP @@ -607,7 +694,7 @@ erts_smp_atomic_inctest(erts_smp_atomic_t *incp) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp) { #ifdef ERTS_SMP @@ -637,8 +724,8 @@ erts_smp_atomic_dec(erts_smp_atomic_t *decp) #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_addtest(erts_smp_atomic_t *addp, long i) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_addtest(erts_smp_atomic_t *addp, erts_aint_t i) { #ifdef ERTS_SMP return erts_atomic_addtest(addp, i); @@ -648,7 +735,7 @@ erts_smp_atomic_addtest(erts_smp_atomic_t *addp, long i) } ERTS_GLB_INLINE void -erts_smp_atomic_add(erts_smp_atomic_t *addp, long i) +erts_smp_atomic_add(erts_smp_atomic_t *addp, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_add(addp, i); @@ -657,58 +744,344 @@ erts_smp_atomic_add(erts_smp_atomic_t *addp, long i) #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, long new) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, erts_aint_t new) { #ifdef ERTS_SMP return erts_atomic_xchg(xchgp, new); #else - long old; + erts_aint_t old; old = *xchgp; *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, long new, long expected) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected) { #ifdef ERTS_SMP return erts_atomic_cmpxchg(xchgp, new, expected); #else - long old = *xchgp; + erts_aint_t old = *xchgp; if (old == expected) *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_bor(erts_smp_atomic_t *var, erts_aint_t mask) { #ifdef ERTS_SMP return erts_atomic_bor(var, mask); #else - long old; + erts_aint_t old; old = *var; *var |= mask; return old; #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_band(erts_smp_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_band(erts_smp_atomic_t *var, erts_aint_t mask) { #ifdef ERTS_SMP return erts_atomic_band(var, mask); #else - long old; + erts_aint_t old; + old = *var; + *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_read_acqb(erts_smp_atomic_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic_set_relb(erts_smp_atomic_t *var, erts_aint_t i) +{ +#ifdef ERTS_SMP + erts_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic_dectest_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_init(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_set(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read(erts_smp_atomic32_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic32_read(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp) +{ +#ifdef ERTS_SMP + return erts_atomic32_inctest(incp); +#else + return ++(*incp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic32_dectest(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_inc(erts_smp_atomic32_t *incp) +{ +#ifdef ERTS_SMP + erts_atomic32_inc(incp); +#else + ++(*incp); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_dec(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic32_dec(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef ERTS_SMP + return erts_atomic32_addtest(addp, i); +#else + return *addp += i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_add(addp, i); +#else + *addp += i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new) +{ +#ifdef ERTS_SMP + return erts_atomic32_xchg(xchgp, new); +#else + erts_aint32_t old; + old = *xchgp; + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg(xchgp, new, expected); +#else + erts_aint32_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef ERTS_SMP + return erts_atomic32_bor(var, mask); +#else + erts_aint32_t old; + old = *var; + *var |= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef ERTS_SMP + return erts_atomic32_band(var, mask); +#else + erts_aint32_t old; old = *var; *var &= mask; return old; #endif } +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic32_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic32_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic32_dectest_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra) { @@ -878,14 +1251,6 @@ erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock) } ERTS_GLB_INLINE void -erts_smp_thr_time_now(erts_smp_thr_timeval_t *time) -{ -#ifdef ERTS_SMP - erts_thr_time_now(time); -#endif -} - -ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp) { #ifdef ERTS_SMP @@ -919,54 +1284,6 @@ erts_smp_tsd_get(erts_smp_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_smp_gate_init(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_init((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_destroy(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_destroy((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_close(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_close((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_let_through(erts_smp_gate_t *gp, unsigned no) -{ -#ifdef ERTS_SMP - erts_gate_let_through((erts_gate_t *) gp, no); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_wait(erts_smp_gate_t *gp) -{ -#ifdef ERTS_SMP - erts_gate_wait((erts_gate_t *) gp); -#endif -} - -ERTS_GLB_INLINE void -erts_smp_gate_swait(erts_smp_gate_t *gp, int spincount) -{ -#ifdef ERTS_SMP - erts_gate_swait((erts_gate_t *) gp, spincount); -#endif -} - #ifdef ERTS_THR_HAVE_SIG_FUNCS #define ERTS_SMP_THR_HAVE_SIG_FUNCS 1 diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index 3a8c30fe6a..815cc1beae 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -193,7 +193,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #endif #define _is_aligned(x) (((Uint)(x) & 0x3) == 0) #define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) -_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*); +_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 #define _is_not_boxed(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED)) @@ -204,12 +204,12 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm) #define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED) #endif #define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED))) -_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm); +_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm) #define boxed_val(x) _ET_APPLY(boxed_val,(x)) /* cons cell ("list") access methods */ #define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) -_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*); +_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*) #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 #define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST)) @@ -226,7 +226,7 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm) #define _list_precond(x) (is_list(x)) #endif #define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST)) -_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm); +_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm) #define list_val(x) _ET_APPLY(list_val,(x)) #define CONS(hp, car, cdr) \ @@ -821,10 +821,10 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * |A A A A A A A A A A A A A A A A A A A A A A A A A A|t t t t|0 0| Thing * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - * |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next - * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * |E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E E| ErlNode * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N N| Next + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * |X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X X| Data 0 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * . . . @@ -835,7 +835,7 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) * t : External pid thing tag (1100) * t : External port thing tag (1101) * t : External ref thing tag (1110) - * N : Next (external thing) pointer + * N : Next (off_heap) pointer * E : ErlNode pointer * X : Type specific data * @@ -852,13 +852,16 @@ _ET_DECLARE_CHECKED(struct erl_node_*,internal_ref_node,Eterm) /* XXX:PaN - this structure is not perfect for halfword heap, it takes a lot of memory due to padding, and the array will not begin at the end of the structure, as otherwise expected. Be sure to access data.ui32 array and not try - to do pointer manipulation on an Eterm * to reach the actual data... */ + to do pointer manipulation on an Eterm * to reach the actual data... + XXX:Sverk - Problem made worse by "one off-heap list" when 'next' pointer + must align with 'next' in ProcBin, erl_fun_thing and erl_off_heap_header. +*/ typedef struct external_thing_ { /* ----+ */ Eterm header; /* | */ - struct external_thing_ *next; /* > External thing head */ - struct erl_node_ *node; /* | */ + struct erl_node_* node; /* > External thing head */ + struct erl_off_heap_header* next; /* | */ /* ----+ */ union { Uint32 ui32[1]; @@ -992,14 +995,14 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #endif #define _unchecked_make_cp(x) ((Eterm) COMPRESS_POINTER(x)) -_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*); +_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*) #define make_cp(x) _ET_APPLY(make_cp,(x)) #define is_not_CP(x) ((x) & _CPMASK) #define is_CP(x) (!is_not_CP(x)) #define _unchecked_cp_val(x) ((BeamInstr*) EXPAND_POINTER(x)) -_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm); +_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm) #define cp_val(x) _ET_APPLY(cp_val,(x)) #define make_catch(x) (((x) << _TAG_IMMED2_SIZE) | _TAG_IMMED2_CATCH) diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index d635916dd8..84a20b51f2 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -25,6 +25,8 @@ #ifndef ERL_THREAD_H__ #define ERL_THREAD_H__ +#define ERTS_SPIN_BODY ETHR_SPIN_BODY + #include "sys.h" #ifdef USE_THREADS @@ -34,6 +36,8 @@ #include "erl_lock_count.h" #include "erl_term.h" +#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER + #ifdef ERTS_ENABLE_LOCK_COUNT #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__) #define erts_spin_lock(L) erts_spin_lock_x(L, __FILE__, __LINE__) @@ -46,6 +50,7 @@ #define ERTS_THR_OPTS_DEFAULT_INITER ETHR_THR_OPTS_DEFAULT_INITER typedef ethr_thr_opts erts_thr_opts_t; typedef ethr_init_data erts_thr_init_data_t; +typedef ethr_late_init_data erts_thr_late_init_data_t; typedef ethr_tid erts_tid_t; /* mutex */ @@ -71,9 +76,23 @@ typedef struct { erts_lcnt_lock_t lcnt; #endif } erts_rwmtx_t; + +#define ERTS_RWMTX_OPT_DEFAULT_INITER ETHR_RWMUTEX_OPT_DEFAULT_INITER +#define ERTS_RWMTX_TYPE_NORMAL ETHR_RWMUTEX_TYPE_NORMAL +#define ERTS_RWMTX_TYPE_FREQUENT_READ ETHR_RWMUTEX_TYPE_FREQUENT_READ +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ \ + ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ +#define ERTS_RWMTX_LONG_LIVED ETHR_RWMUTEX_LONG_LIVED +#define ERTS_RWMTX_SHORT_LIVED ETHR_RWMUTEX_SHORT_LIVED +#define ERTS_RWMTX_UNKNOWN_LIVED ETHR_RWMUTEX_UNKNOWN_LIVED +typedef ethr_rwmutex_opt erts_rwmtx_opt_t; + typedef ethr_tsd_key erts_tsd_key_t; -typedef ethr_gate erts_gate_t; +typedef ethr_ts_event erts_tse_t; +typedef ethr_sint_t erts_aint_t; typedef ethr_atomic_t erts_atomic_t; +typedef ethr_sint32_t erts_aint32_t; +typedef ethr_atomic32_t erts_atomic32_t; /* spinlock */ typedef struct { @@ -97,41 +116,48 @@ typedef struct { #endif } erts_rwlock_t; -typedef ethr_timeval erts_thr_timeval_t; __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ -#ifdef ERTS_ENABLE_LOCK_CHECK -#define ERTS_REC_MTX_INITER \ - {ETHR_REC_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1,THE_NON_VALUE,ERTS_LC_FLG_LT_MUTEX)} -#define ERTS_MTX_INITER \ - {ETHR_MUTEX_INITER, \ - ERTS_LC_LOCK_INIT(-1, THE_NON_VALUE, ERTS_LC_FLG_LT_MUTEX)} -#else -#define ERTS_REC_MTX_INITER {ETHR_REC_MUTEX_INITER} -#define ERTS_MTX_INITER {ETHR_MUTEX_INITER} -#endif -#define ERTS_CND_INITER ETHR_COND_INITER #define ERTS_THR_INIT_DATA_DEF_INITER ETHR_INIT_DATA_DEFAULT_INITER +#define ERTS_THR_LATE_INIT_DATA_DEF_INITER \ + ETHR_LATE_INIT_DATA_DEFAULT_INITER #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT # define ERTS_HAVE_REC_MTX_INIT ETHR_HAVE_ETHR_REC_MUTEX_INIT #endif - #else /* #ifdef USE_THREADS */ +#define ERTS_THR_MEMORY_BARRIER + #define ERTS_THR_OPTS_DEFAULT_INITER 0 typedef int erts_thr_opts_t; typedef int erts_thr_init_data_t; +typedef int erts_thr_late_init_data_t; typedef int erts_tid_t; typedef int erts_mtx_t; typedef int erts_cnd_t; +#define ERTS_RWMTX_OPT_DEFAULT_INITER {0} +#define ERTS_RWMTX_TYPE_NORMAL 0 +#define ERTS_RWMTX_TYPE_FREQUENT_READ 0 +#define ERTS_RWMTX_TYPE_EXTREMELY_FREQUENT_READ 0 +#define ERTS_RWMTX_LONG_LIVED 0 +#define ERTS_RWMTX_SHORT_LIVED 0 +#define ERTS_RWMTX_UNKNOWN_LIVED 0 +typedef struct { + char type; + char lived; + int main_spincount; + int aux_spincount; +} erts_rwmtx_opt_t; typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; -typedef int erts_gate_t; -typedef long erts_atomic_t; +typedef int erts_tse_t; +typedef SWord erts_aint_t; +typedef SWord erts_atomic_t; +typedef SWord erts_aint32_t; +typedef SWord erts_atomic32_t; #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; typedef struct { } erts_rwlock_t; @@ -139,12 +165,7 @@ typedef struct { } erts_rwlock_t; typedef struct { int gcc_is_buggy; } erts_spinlock_t; typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif -typedef struct { - long tv_sec; - long tv_nsec; -} erts_thr_timeval_t; -#define ERTS_REC_MTX_INITER 0 #define ERTS_MTX_INITER 0 #define ERTS_CND_INITER 0 #define ERTS_THR_INIT_DATA_DEF_INITER 0 @@ -153,7 +174,10 @@ typedef struct { #endif /* #ifdef USE_THREADS */ +#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) + ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id); +ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id); ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts); ERTS_GLB_INLINE void erts_thr_join(erts_tid_t tid, void **thr_res); @@ -162,9 +186,6 @@ ERTS_GLB_INLINE void erts_thr_exit(void *res); ERTS_GLB_INLINE void erts_thr_install_exit_handler(void (*exit_handler)(void)); ERTS_GLB_INLINE erts_tid_t erts_thr_self(void); ERTS_GLB_INLINE int erts_equal_tids(erts_tid_t x, erts_tid_t y); -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void erts_rec_mtx_init(erts_mtx_t *mtx); -#endif ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra); ERTS_GLB_INLINE void erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt); ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, @@ -173,8 +194,6 @@ ERTS_GLB_INLINE void erts_mtx_init_locked_x(erts_mtx_t *mtx, ERTS_GLB_INLINE void erts_mtx_init(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_init_locked(erts_mtx_t *mtx, char *name); ERTS_GLB_INLINE void erts_mtx_destroy(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_set_forksafe(erts_mtx_t *mtx); -ERTS_GLB_INLINE void erts_mtx_unset_forksafe(erts_mtx_t *mtx); ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx); #ifdef ERTS_ENABLE_LOCK_COUNT ERTS_GLB_INLINE void erts_mtx_lock_x(erts_mtx_t *mtx, char *file, unsigned int line); @@ -188,9 +207,17 @@ ERTS_GLB_INLINE void erts_cnd_destroy(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_wait(erts_cnd_t *cnd, erts_mtx_t *mtx); ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd); ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd); +ERTS_GLB_INLINE void erts_rwmtx_set_reader_group(int no); +ERTS_GLB_INLINE void erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra); ERTS_GLB_INLINE void erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra); +ERTS_GLB_INLINE void erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name); ERTS_GLB_INLINE void erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name); ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx); @@ -207,23 +234,69 @@ ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, long i); -ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, long i); -ERTS_GLB_INLINE long erts_atomic_read(erts_atomic_t *var); -ERTS_GLB_INLINE long erts_atomic_inctest(erts_atomic_t *incp); -ERTS_GLB_INLINE long erts_atomic_dectest(erts_atomic_t *decp); +ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var); +ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp); ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp); ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp); -ERTS_GLB_INLINE long erts_atomic_addtest(erts_atomic_t *addp, - long i); -ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, long i); -ERTS_GLB_INLINE long erts_atomic_xchg(erts_atomic_t *xchgp, - long new); -ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp, - long new, - long expected); -ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask); -ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask); +ERTS_GLB_INLINE erts_aint_t erts_atomic_addtest(erts_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_atomic_xchg(erts_atomic_t *xchgp, + erts_aint_t new); +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected); +ERTS_GLB_INLINE erts_aint_t erts_atomic_bor(erts_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_atomic_band(erts_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_atomic_read_acqb(erts_atomic_t *var); +ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE void erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read(erts_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_inctest(erts_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest(erts_atomic32_t *decp); +ERTS_GLB_INLINE void erts_atomic32_inc(erts_atomic32_t *incp); +ERTS_GLB_INLINE void erts_atomic32_dec(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_addtest(erts_atomic32_t *addp, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_xchg(erts_atomic32_t *xchgp, + erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_bor(erts_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_band(erts_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_acqb(erts_atomic32_t *var); +ERTS_GLB_INLINE void erts_atomic32_set_relb(erts_atomic32_t *var, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_dec_relb(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest_relb(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, + char *name, + Eterm extra, + Uint16 opt); ERTS_GLB_INLINE void erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra); @@ -254,17 +327,20 @@ ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock); ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock); -ERTS_GLB_INLINE void erts_thr_time_now(erts_thr_timeval_t *time); ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key); ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); ERTS_GLB_INLINE void * erts_tsd_get(erts_tsd_key_t key); -ERTS_GLB_INLINE void erts_gate_init(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_destroy(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_close(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_let_through(erts_gate_t *gp, unsigned no); -ERTS_GLB_INLINE void erts_gate_wait(erts_gate_t *gp); -ERTS_GLB_INLINE void erts_gate_swait(erts_gate_t *gp, int spincount); +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void); +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep); +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount); +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep); +ERTS_GLB_INLINE void erts_thr_set_main_status(int, int); +ERTS_GLB_INLINE int erts_thr_get_main_status(void); +ERTS_GLB_INLINE void erts_thr_yield(void); #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #define ERTS_THR_HAVE_SIG_FUNCS 1 @@ -286,15 +362,21 @@ erts_thr_init(erts_thr_init_data_t *id) } ERTS_GLB_INLINE void +erts_thr_late_init(erts_thr_late_init_data_t *id) +{ +#ifdef USE_THREADS + int res = ethr_late_init(id); + if (res) + erts_thr_fatal_error(res, "complete initialization of thread library"); +#endif +} + +ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), void *arg, erts_thr_opts_t *opts) { #ifdef USE_THREADS -#ifdef ERTS_ENABLE_LOCK_COUNT - int res = erts_lcnt_thr_create(tid, func, arg, opts); -#else int res = ethr_thr_create(tid, func, arg, opts); -#endif if (res) erts_thr_fatal_error(res, "create thread"); #endif @@ -362,20 +444,6 @@ erts_equal_tids(erts_tid_t x, erts_tid_t y) #endif } - -#ifdef ERTS_HAVE_REC_MTX_INIT -ERTS_GLB_INLINE void -erts_rec_mtx_init(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_rec_mutex_init(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "initialize recursive mutex"); -#endif -} -#endif - - ERTS_GLB_INLINE void erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) { @@ -422,9 +490,7 @@ erts_mtx_init_locked_x(erts_mtx_t *mtx, char *name, Eterm extra) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -463,9 +529,7 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); #endif - res = ethr_mutex_lock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "lock mutex"); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_trylock(1, &mtx->lc); #endif @@ -492,26 +556,6 @@ erts_mtx_destroy(erts_mtx_t *mtx) #endif } -ERTS_GLB_INLINE void -erts_mtx_set_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_set_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "set mutex forksafe"); -#endif -} - -ERTS_GLB_INLINE void -erts_mtx_unset_forksafe(erts_mtx_t *mtx) -{ -#ifdef USE_THREADS - int res = ethr_mutex_unset_forksafe(&mtx->mtx); - if (res != 0 && res != ENOTSUP) - erts_thr_fatal_error(res, "unset mutex forksafe"); -#endif -} - ERTS_GLB_INLINE int erts_mtx_trylock(erts_mtx_t *mtx) { @@ -531,11 +575,7 @@ erts_mtx_trylock(erts_mtx_t *mtx) #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock(&mtx->lcnt, res); -#endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try lock mutex"); - +#endif return res; #else return 0; @@ -551,19 +591,16 @@ erts_mtx_lock(erts_mtx_t *mtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&mtx->lcnt); #endif - res = ethr_mutex_lock(&mtx->mtx); + ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&mtx->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "lock mutex"); #endif } @@ -571,16 +608,13 @@ ERTS_GLB_INLINE void erts_mtx_unlock(erts_mtx_t *mtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&mtx->lcnt); #endif - res = ethr_mutex_unlock(&mtx->mtx); - if (res) - erts_thr_fatal_error(res, "unlock mutex"); + ethr_mutex_unlock(&mtx->mtx); #endif } @@ -648,9 +682,7 @@ ERTS_GLB_INLINE void erts_cnd_signal(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_signal(cnd); - if (res) - erts_thr_fatal_error(res, "signal on condition variable"); + ethr_cond_signal(cnd); #endif } @@ -659,19 +691,34 @@ ERTS_GLB_INLINE void erts_cnd_broadcast(erts_cnd_t *cnd) { #ifdef USE_THREADS - int res = ethr_cond_broadcast(cnd); - if (res) - erts_thr_fatal_error(res, "broadcast on condition variable"); + ethr_cond_broadcast(cnd); #endif } /* rwmutex */ ERTS_GLB_INLINE void -erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) +erts_rwmtx_set_reader_group(int no) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res; +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_check_no_locked_of_type(ERTS_LC_FLG_LT_RWMUTEX); +#endif + res = ethr_rwmutex_set_reader_group(no); + if (res != 0) + erts_thr_fatal_error(res, "set reader group"); +#endif +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name, + Eterm extra) +{ +#ifdef USE_THREADS + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -684,10 +731,20 @@ erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, char *name, Eterm extra) } ERTS_GLB_INLINE void -erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +erts_rwmtx_init_x(erts_rwmtx_t *rwmtx, + char *name, + Eterm extra) +{ + erts_rwmtx_init_opt_x(rwmtx, NULL, name, extra); +} + +ERTS_GLB_INLINE void +erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, + erts_rwmtx_opt_t *opt, + char *name) { #ifdef USE_THREADS - int res = ethr_rwmutex_init(&rwmtx->rwmtx); + int res = ethr_rwmutex_init_opt(&rwmtx->rwmtx, opt); if (res != 0) erts_thr_fatal_error(res, "initialize rwmutex"); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -700,6 +757,12 @@ erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) } ERTS_GLB_INLINE void +erts_rwmtx_init(erts_rwmtx_t *rwmtx, char *name) +{ + erts_rwmtx_init_opt(rwmtx, NULL, name); +} + +ERTS_GLB_INLINE void erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS @@ -736,9 +799,6 @@ erts_rwmtx_tryrlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try read lock rwmutex"); return res; #else @@ -754,19 +814,16 @@ erts_rwmtx_rlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_rlock(&rwmtx->rwmtx); + ethr_rwmutex_rlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "read lock rwmutex"); #endif } @@ -774,16 +831,13 @@ ERTS_GLB_INLINE void erts_rwmtx_runlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_rwmutex_runlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "read unlock rwmutex"); + ethr_rwmutex_runlock(&rwmtx->rwmtx); #endif } @@ -808,9 +862,6 @@ erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx) #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_trylock_opt(&rwmtx->lcnt, res, ERTS_LCNT_LO_READ_WRITE); #endif - - if (res != 0 && res != EBUSY) - erts_thr_fatal_error(res, "try write lock rwmutex"); return res; #else @@ -826,19 +877,16 @@ erts_rwmtx_rwlock(erts_rwmtx_t *rwmtx) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwlock(&rwmtx->rwmtx); + ethr_rwmutex_rwlock(&rwmtx->rwmtx); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&rwmtx->lcnt, file, line); #endif - if (res != 0) - erts_thr_fatal_error(res, "write lock rwmutex"); #endif } @@ -846,16 +894,13 @@ ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&rwmtx->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&rwmtx->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_rwmutex_rwunlock(&rwmtx->rwmtx); - if (res != 0) - erts_thr_fatal_error(res, "write unlock rwmutex"); + ethr_rwmutex_rwunlock(&rwmtx->rwmtx); #endif } @@ -914,66 +959,50 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx) } ERTS_GLB_INLINE void -erts_atomic_init(erts_atomic_t *var, long i) +erts_atomic_init(erts_atomic_t *var, erts_aint_t i) { #ifdef USE_THREADS - int res = ethr_atomic_init(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic init"); + ethr_atomic_init(var, i); #else *var = i; #endif } ERTS_GLB_INLINE void -erts_atomic_set(erts_atomic_t *var, long i) +erts_atomic_set(erts_atomic_t *var, erts_aint_t i) { #ifdef USE_THREADS - int res = ethr_atomic_set(var, i); - if (res) - erts_thr_fatal_error(res, "perform atomic set"); + ethr_atomic_set(var, i); #else *var = i; #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var) { #ifdef USE_THREADS - long i; - int res = ethr_atomic_read(var, &i); - if (res) - erts_thr_fatal_error(res, "perform atomic read"); - return i; + return ethr_atomic_read(var); #else return *var; #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_inctest(incp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic increment and test"); - return test; + return ethr_atomic_inc_read(incp); #else return ++(*incp); #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_dectest(decp, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement and test"); - return test; + return ethr_atomic_dec_read(decp); #else return --(*decp); #endif @@ -983,9 +1012,7 @@ ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp) { #ifdef USE_THREADS - int res = ethr_atomic_inc(incp); - if (res) - erts_thr_fatal_error(res, "perform atomic increment"); + ethr_atomic_inc(incp); #else ++(*incp); #endif @@ -995,100 +1022,364 @@ ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp) { #ifdef USE_THREADS - int res = ethr_atomic_dec(decp); - if (res) - erts_thr_fatal_error(res, "perform atomic decrement"); + ethr_atomic_dec(decp); #else --(*decp); #endif } -ERTS_GLB_INLINE long -erts_atomic_addtest(erts_atomic_t *addp, long i) +ERTS_GLB_INLINE erts_aint_t +erts_atomic_addtest(erts_atomic_t *addp, erts_aint_t i) { #ifdef USE_THREADS - long test; - int res = ethr_atomic_addtest(addp, i, &test); - if (res) - erts_thr_fatal_error(res, "perform atomic addition and test"); - return test; + return ethr_atomic_add_read(addp, i); #else return *addp += i; #endif } ERTS_GLB_INLINE void -erts_atomic_add(erts_atomic_t *addp, long i) +erts_atomic_add(erts_atomic_t *addp, erts_aint_t i) { #ifdef USE_THREADS - int res = ethr_atomic_add(addp, i); - if (res) - erts_thr_fatal_error(res, "perform atomic addition"); + ethr_atomic_add(addp, i); #else *addp += i; #endif } -ERTS_GLB_INLINE long -erts_atomic_xchg(erts_atomic_t *xchgp, long new) +ERTS_GLB_INLINE erts_aint_t +erts_atomic_xchg(erts_atomic_t *xchgp, erts_aint_t new) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_xchg(xchgp, new, &old); - if (res) - erts_thr_fatal_error(res, "perform atomic exchange"); + return ethr_atomic_xchg(xchgp, new); #else - old = *xchgp; + erts_aint_t old = *xchgp; *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_cmpxchg(erts_atomic_t *xchgp, erts_aint_t new, erts_aint_t expected) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg(xchgp, new, expected); +#else + erts_aint_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_bor(erts_atomic_t *var, erts_aint_t mask) +{ +#ifdef USE_THREADS + return ethr_atomic_read_bor(var, mask); +#else + erts_aint_t old; + old = *var; + *var |= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_band(erts_atomic_t *var, erts_aint_t mask) +{ +#ifdef USE_THREADS + return ethr_atomic_read_band(var, mask); +#else + erts_aint_t old; + old = *var; + *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_read_acqb(erts_atomic_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i) +{ +#ifdef USE_THREADS + ethr_atomic_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic_dec_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint_t +erts_atomic_dectest_relb(erts_atomic_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic_dec_read_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; #endif +} + +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; return old; +#endif +} + +/* atomic32 */ + +ERTS_GLB_INLINE void +erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_init(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_set(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read(erts_atomic32_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic32_read(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_inctest(erts_atomic32_t *incp) +{ +#ifdef USE_THREADS + return ethr_atomic32_inc_read(incp); +#else + return ++(*incp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_dectest(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic32_dec_read(decp); +#else + return --(*decp); +#endif } -ERTS_GLB_INLINE long -erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected) +ERTS_GLB_INLINE void +erts_atomic32_inc(erts_atomic32_t *incp) +{ +#ifdef USE_THREADS + ethr_atomic32_inc(incp); +#else + ++(*incp); +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_dec(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic32_dec(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_addtest(erts_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef USE_THREADS + return ethr_atomic32_add_read(addp, i); +#else + return *addp += i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i) { #ifdef USE_THREADS - long old; - int res = ethr_atomic_cmpxchg(xchgp, new, expected, &old); - if (ERTS_UNLIKELY(res != 0)) - erts_thr_fatal_error(res, "perform atomic exchange"); + ethr_atomic32_add(addp, i); +#else + *addp += i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_xchg(erts_atomic32_t *xchgp, erts_aint32_t new) +{ +#ifdef USE_THREADS + return ethr_atomic32_xchg(xchgp, new); +#else + erts_aint32_t old = *xchgp; + *xchgp = new; return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg(xchgp, new, expected); #else - long old = *xchgp; + erts_aint32_t old = *xchgp; if (old == expected) *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long -erts_atomic_bor(erts_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_bor(erts_atomic32_t *var, erts_aint32_t mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_or_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise or"); + return ethr_atomic32_read_bor(var, mask); #else + erts_aint32_t old; old = *var; *var |= mask; -#endif return old; +#endif } -ERTS_GLB_INLINE long -erts_atomic_band(erts_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_band(erts_atomic32_t *var, erts_aint32_t mask) { - long old; #ifdef USE_THREADS - int res = ethr_atomic_and_old(var, mask, &old); - if (res != 0) - erts_thr_fatal_error(res, "perform atomic bitwise and"); + return ethr_atomic32_read_band(var, mask); #else + erts_aint32_t old; old = *var; *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_acqb(erts_atomic32_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic32_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_set_relb(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_set_relb(var, i); +#else + *var = i; #endif +} + +ERTS_GLB_INLINE void +erts_atomic32_dec_relb(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic32_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_dectest_relb(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic32_dec_read_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif } /* spinlock */ @@ -1112,6 +1403,26 @@ erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra) } ERTS_GLB_INLINE void +erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, + Uint16 opt) +{ +#ifdef USE_THREADS + int res = ethr_spinlock_init(&lock->slck); + if (res) + erts_thr_fatal_error(res, "init spinlock"); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra); +#endif +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra); +#endif +#else + (void)lock; +#endif +} + + +ERTS_GLB_INLINE void erts_spinlock_init(erts_spinlock_t *lock, char *name) { #ifdef USE_THREADS @@ -1152,16 +1463,13 @@ ERTS_GLB_INLINE void erts_spin_unlock(erts_spinlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock(&lock->lcnt); #endif - res = ethr_spin_unlock(&lock->slck); - if (res) - erts_thr_fatal_error(res, "release spin lock"); + ethr_spin_unlock(&lock->slck); #else (void)lock; #endif @@ -1175,19 +1483,16 @@ erts_spin_lock(erts_spinlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock(&lock->lcnt); #endif - res = ethr_spin_lock(&lock->slck); + ethr_spin_lock(&lock->slck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take spin lock"); #else (void)lock; #endif @@ -1268,16 +1573,13 @@ ERTS_GLB_INLINE void erts_read_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release read lock"); + ethr_read_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1291,19 +1593,16 @@ erts_read_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ); #endif - res = ethr_read_lock(&lock->rwlck); + ethr_read_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take read lock"); #else (void)lock; #endif @@ -1313,16 +1612,13 @@ ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock) { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_unlock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_unlock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_unlock(&lock->rwlck); - if (res) - erts_thr_fatal_error(res, "release write lock"); + ethr_write_unlock(&lock->rwlck); #else (void)lock; #endif @@ -1336,19 +1632,16 @@ erts_write_lock(erts_rwlock_t *lock) #endif { #ifdef USE_THREADS - int res; #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_lock_flg(&lock->lc, ERTS_LC_FLG_LO_READ_WRITE); #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_opt(&lock->lcnt, ERTS_LCNT_LO_READ_WRITE); #endif - res = ethr_write_lock(&lock->rwlck); + ethr_write_lock(&lock->rwlck); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_post_x(&lock->lcnt, file, line); #endif - if (res) - erts_thr_fatal_error(res, "take write lock"); #else (void)lock; #endif @@ -1383,16 +1676,6 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock) } ERTS_GLB_INLINE void -erts_thr_time_now(erts_thr_timeval_t *time) -{ -#ifdef USE_THREADS - int res = ethr_time_now(time); - if (res) - erts_thr_fatal_error(res, "get current time"); -#endif -} - -ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp) { #ifdef USE_THREADS @@ -1432,66 +1715,95 @@ erts_tsd_get(erts_tsd_key_t key) #endif } -ERTS_GLB_INLINE void -erts_gate_init(erts_gate_t *gp) +ERTS_GLB_INLINE erts_tse_t *erts_tse_fetch(void) { #ifdef USE_THREADS - int res = ethr_gate_init((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "initialize gate"); + return (erts_tse_t *) ethr_get_ts_event(); +#else + return (erts_tse_t *) NULL; #endif } -ERTS_GLB_INLINE void -erts_gate_destroy(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_return(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_destroy((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "destroy gate"); + ethr_leave_ts_event(ep); #endif } -ERTS_GLB_INLINE void -erts_gate_close(erts_gate_t *gp) +ERTS_GLB_INLINE void erts_tse_set(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_close((ethr_gate *) gp); - if (res != 0) - erts_thr_fatal_error(res, "close gate"); + ethr_event_set(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_let_through(erts_gate_t *gp, unsigned no) +ERTS_GLB_INLINE void erts_tse_reset(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_let_through((ethr_gate *) gp, no); - if (res != 0) - erts_thr_fatal_error(res, "let through gate"); + ethr_event_reset(&((ethr_ts_event *) ep)->event); #endif } -ERTS_GLB_INLINE void -erts_gate_wait(erts_gate_t *gp) +ERTS_GLB_INLINE int erts_tse_wait(erts_tse_t *ep) +{ +#ifdef USE_THREADS + return ethr_event_wait(&((ethr_ts_event *) ep)->event); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_swait(erts_tse_t *ep, int spincount) +{ +#ifdef USE_THREADS + return ethr_event_swait(&((ethr_ts_event *) ep)->event, spincount); +#else + return ENOTSUP; +#endif +} + +ERTS_GLB_INLINE int erts_tse_is_tmp(erts_tse_t *ep) { #ifdef USE_THREADS - int res = ethr_gate_wait((ethr_gate *) gp); + return (ep->iflgs & ETHR_TS_EV_TMP) == ETHR_TS_EV_TMP; +#else + return 0; +#endif +} + +ERTS_GLB_INLINE void erts_thr_set_main_status(int on, int no) +{ +#ifdef USE_THREADS + int res = ethr_set_main_thr_status(on, no); if (res != 0) - erts_thr_fatal_error(res, "wait on gate"); + erts_thr_fatal_error(res, "set thread main status"); #endif } -ERTS_GLB_INLINE void -erts_gate_swait(erts_gate_t *gp, int spincount) +ERTS_GLB_INLINE int erts_thr_get_main_status(void) { #ifdef USE_THREADS - int res = ethr_gate_swait((ethr_gate *) gp, spincount); + int main_status; + int res = ethr_get_main_thr_status(&main_status); if (res != 0) - erts_thr_fatal_error(res, "swait on gate"); + erts_thr_fatal_error(res, "get thread main status"); + return main_status; +#else + return 1; #endif } +ERTS_GLB_INLINE void erts_thr_yield(void) +{ +#ifdef USE_THREADS + int res = ETHR_YIELD(); + if (res != 0) + erts_thr_fatal_error(res, "yield"); +#endif +} + + #ifdef ETHR_HAVE_ETHR_SIG_FUNCS ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 6f6b971d34..93d8ea4cb4 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -20,11 +20,15 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ +extern erts_smp_atomic_t do_time; /* set at clock interrupt */ +extern SysTimeval erts_first_emu_time; + /* ** Timer entry: */ typedef struct erl_timer { struct erl_timer* next; /* next entry tiw slot or chain */ + struct erl_timer* prev; /* prev entry tiw slot or chain */ Uint slot; /* slot in timer wheel */ Uint count; /* number of loops remaining */ int active; /* 1=activated, 0=deactivated */ @@ -39,7 +43,6 @@ typedef void (*ErlTimeoutProc)(void*); typedef void (*ErlCancelProc)(void*); #ifdef ERTS_SMP - /* * Process and port timer */ @@ -61,7 +64,66 @@ void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, ErlTimeoutProc timeout_func, Uint timeout); void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer); +#endif + +/* timer-wheel api */ +void erts_init_time(void); +void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); +void erts_cancel_timer(ErlTimer*); +void erts_bump_timer(erts_aint_t); +Uint erts_timer_wheel_memory_size(void); +Uint erts_time_left(ErlTimer *); +erts_aint_t erts_next_time(void); + +#ifdef DEBUG +void erts_p_slpq(void); #endif +ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void); +ERTS_GLB_INLINE void erts_do_time_add(long); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg(&do_time, 0L); } +ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add(&do_time, elapsed); } + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + + +/* time_sup */ + +#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME)) +# ifndef HAVE_ERTS_NOW_CPU +# define HAVE_ERTS_NOW_CPU +# ifdef HAVE_GETHRVTIME +# define erts_start_now_cpu() sys_start_hrvtime() +# define erts_stop_now_cpu() sys_stop_hrvtime() +# endif +# endif +void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); #endif + +void erts_get_timeval(SysTimeval *tv); +long erts_get_time(void); +void erts_get_emu_time(SysTimeval *); + +ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int +erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p) +{ + if (t1p->tv_sec == t2p->tv_sec) { + if (t1p->tv_usec < t2p->tv_usec) + return -1; + else if (t1p->tv_usec > t2p->tv_usec) + return 1; + return 0; + } + return t1p->tv_sec < t2p->tv_sec ? -1 : 1; +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* ERL_TIME_H__ */ diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index c15f85f8f1..ca4b54188e 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -358,10 +358,6 @@ static int clock_resolution; ** instead of something like select. */ -#if defined(ERTS_TIMER_THREAD) -static ERTS_INLINE void init_erts_deliver_time(const SysTimeval *inittv) { } -static ERTS_INLINE void do_erts_deliver_time(const SysTimeval *current) { } -#else static SysTimeval last_delivered; static void init_erts_deliver_time(const SysTimeval *inittv) @@ -389,11 +385,10 @@ static void do_erts_deliver_time(const SysTimeval *current) this by simply pretend as if the time stood still. :) */ if (elapsed > 0) { - do_time_add(elapsed); + erts_do_time_add(elapsed); last_delivered = cur_time; } } -#endif int erts_init_time_sup(void) @@ -650,6 +645,22 @@ local_to_univ(Sint *year, Sint *month, Sint *day, t.tm_sec = *second; t.tm_isdst = isdst; the_clock = mktime(&t); + if (the_clock == -1) { + if (isdst) { + /* If this is a timezone without DST and the OS (correctly) + refuses to give us a DST time, we simulate the Linux/Solaris + behaviour of giving the same data as if is_dst was not set. */ + t.tm_isdst = 0; + the_clock = mktime(&t); + if (the_clock == -1) { + /* Failed anyway, something else is bad - will be a badarg */ + return 0; + } + } else { + /* Something else is the matter, badarg. */ + return 0; + } + } #ifdef HAVE_GMTIME_R gmtime_r(&the_clock, (tm = &tmbuf)); #else @@ -663,6 +674,10 @@ local_to_univ(Sint *year, Sint *month, Sint *day, *second = tm->tm_sec; return 1; } +#if defined(HAVE_POSIX2TIME) && defined(HAVE_DECL_POSIX2TIME) && \ + !HAVE_DECL_POSIX2TIME +extern time_t posix2time(time_t); +#endif int univ_to_local(Sint *year, Sint *month, Sint *day, @@ -766,7 +781,6 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) to a struct timeval representing current time (to save a gettimeofday() where possible) or NULL */ -#if !defined(ERTS_TIMER_THREAD) void erts_deliver_time(void) { SysTimeval now; @@ -777,7 +791,6 @@ void erts_deliver_time(void) { erts_smp_mtx_unlock(&erts_timeofday_mtx); } -#endif /* get *real* time (not ticks) remaining until next timeout - if there isn't one, give a "long" time, that is guaranteed @@ -786,14 +799,12 @@ void erts_deliver_time(void) { void erts_time_remaining(SysTimeval *rem_time) { int ticks; -#if !defined(ERTS_TIMER_THREAD) SysTimeval cur_time; -#endif long elapsed; - /* next_time() returns no of ticks to next timeout or -1 if none */ + /* erts_next_time() returns no of ticks to next timeout or -1 if none */ - if ((ticks = next_time()) == -1) { + if ((ticks = erts_next_time()) == -1) { /* timer queue empty */ /* this will cause at most 100000000 ticks */ rem_time->tv_sec = 100000; @@ -802,9 +813,6 @@ void erts_time_remaining(SysTimeval *rem_time) /* next timeout after ticks ticks */ ticks *= CLOCK_RESOLUTION; -#if defined(ERTS_TIMER_THREAD) - elapsed = 0; -#else erts_smp_mtx_lock(&erts_timeofday_mtx); get_tolerant_timeofday(&cur_time); @@ -819,7 +827,6 @@ void erts_time_remaining(SysTimeval *rem_time) rem_time->tv_sec = rem_time->tv_usec = 0; return; } -#endif rem_time->tv_sec = (ticks - elapsed) / 1000; rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000); } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8addfcf5ad..3043bb1e8c 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -43,8 +43,9 @@ #undef DEBUG_PRINTOUTS #endif -extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ -extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern Eterm beam_return_to_trace[1]; /* OpCode(i_return_to_trace) */ +extern Eterm beam_return_trace[1]; /* OpCode(i_return_trace) */ +extern Eterm beam_return_time_trace[1]; /* OpCode(i_return_time_trace) */ /* Pseudo export entries. Never filled in with data, only used to yield unique pointers of the correct type. */ @@ -1941,11 +1942,13 @@ trace_proc(Process *c_p, Process *t_p, Eterm what, Eterm data) Eterm* hp; int need; + ERTS_SMP_LC_ASSERT((erts_proc_lc_my_proc_locks(t_p) != 0) || erts_is_system_blocked(0)); if (is_internal_port(t_p->tracer_proc)) { #define LOCAL_HEAP_SIZE (5+5) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); UseTmpHeapNoproc(LOCAL_HEAP_SIZE); + hp = local_heap; mess = TUPLE4(hp, am_trace, t_p->id, what, data); hp += 5; @@ -2114,6 +2117,7 @@ erts_bif_trace(int bif_index, Process* p, Uint32 flags = 0, flags_meta = 0; int global = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_GLOBAL); int local = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_LOCAL); + int time = !!(erts_bif_trace_flags[bif_index] & BIF_TRACE_AS_CALL_TIME); Eterm meta_tracer_pid = NIL; int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif * is actually in the @@ -2137,6 +2141,17 @@ erts_bif_trace(int bif_index, Process* p, flags_meta = erts_bif_mtrace(p, ep->code+3, args, local, &meta_tracer_pid); } + if (time) { + BpDataTime *bdt = NULL; + BeamInstr *pc = (BeamInstr *)ep->code+3; + + bdt = (BpDataTime *) erts_get_time_break(p, pc); + ASSERT(bdt); + + if (!bdt->pause) { + erts_trace_time_break(p, pc, bdt, ERTS_BP_CALL_TIME_CALL); + } + } /* Restore original continuation pointer (if changed). */ p->cp = cp; @@ -2145,8 +2160,9 @@ erts_bif_trace(int bif_index, Process* p, result = func(p, arg1, arg2, arg3, I); if (applying && (flags & MATCH_SET_RETURN_TO_TRACE)) { - Uint i_return_trace = beam_return_trace[0]; - Uint i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_trace = beam_return_trace[0]; + BeamInstr i_return_to_trace = beam_return_to_trace[0]; + BeamInstr i_return_time_trace = beam_return_time_trace[0]; Eterm *cpp; /* Maybe advance cp to skip trace stack frames */ for (cpp = p->stop; ; cp = cp_val(*cpp++)) { @@ -2154,6 +2170,10 @@ erts_bif_trace(int bif_index, Process* p, /* Skip stack frame variables */ while (is_not_CP(*cpp)) cpp++; cpp += 2; /* Skip return_trace parameters */ + } else if (*cp == i_return_time_trace) { + /* Skip stack frame variables */ + while (is_not_CP(*cpp)) cpp++; + cpp += 1; /* Skip return_time_trace parameters */ } else if (*cp == i_return_to_trace) { /* A return_to trace message is going to be generated * by normal means, so we do not have to. @@ -2727,6 +2747,8 @@ trace_port(Port *t_p, Eterm what, Eterm data) { Eterm mess; Eterm* hp; + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(t_p) || erts_is_system_blocked(0)); + if (is_internal_port(t_p->tracer_proc)) { #define LOCAL_HEAP_SIZE (5+5) DeclareTmpHeapNoproc(local_heap,LOCAL_HEAP_SIZE); diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index d01a3661f9..545b345a71 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -30,6 +30,8 @@ #include "big.h" #include "erl_unicode.h" +#include "erl_unicode_normalize.h" + typedef struct _restart_context { byte *bytes; @@ -54,13 +56,6 @@ static BIF_RETTYPE finalize_list_to_list(Process *p, Uint num_resulting_chars, int state, int left, Eterm tail); -static int analyze_utf8(byte *source, Uint size, - byte **err_pos, Uint *num_chars, int *left); -#define UTF8_OK 0 -#define UTF8_INCOMPLETE 1 -#define UTF8_ERROR 2 -#define UTF8_ANALYZE_MORE 3 - static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3); static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3); static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3); @@ -463,7 +458,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ } objp = list_val(ioterm); obj = CAR(objp); - if (!is_byte(obj)) + if (!is_small(obj)) break; } } else if (is_nil(obj)) { @@ -970,11 +965,11 @@ static int is_valid_utf8(Eterm orig_bin) bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc); } size = binary_size(orig_bin); - ret = analyze_utf8(bytes, + ret = erts_analyze_utf8(bytes, size, &endpos,&numchar,NULL); erts_free_aligned_binary_bytes(temp_alloc); - return (ret == UTF8_OK); + return (ret == ERTS_UTF8_OK); } BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2) @@ -1084,14 +1079,14 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char hp += 2; rest_term = CONS(hp,leftover_bin,rest_term); } - BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, UTF8_ERROR, left, NIL)); + BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL)); } else if (rest_term == NIL && num_leftovers != 0) { Eterm leftover_bin = new_binary(p, leftover, num_leftovers); if (check_leftovers(leftover,num_leftovers) != 0) { - BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_ERROR, + BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL)); } else { - BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_INCOMPLETE, + BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_INCOMPLETE, left, NIL)); } } else { /* All OK */ @@ -1107,11 +1102,11 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char rc.num_processed_bytes = 0; /* not used */ rc.num_bytes_to_process = pos; rc.num_resulting_chars = characters; - rc.state = UTF8_OK; /* not used */ + rc.state = ERTS_UTF8_OK; /* not used */ BIF_TRAP3(&characters_to_list_trap_1_exp, p, make_magic_bin_for_restart(p,&rc), rest_term, latin1); } else { /* Success */ - BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, UTF8_OK, left, NIL)); + BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, ERTS_UTF8_OK, left, NIL)); } } } @@ -1205,7 +1200,7 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2) * When input to characters_to_list is a plain binary and the format is 'unicode', we do * a faster analyze and size count with this function. */ -static int analyze_utf8(byte *source, Uint size, +int erts_analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left) { *err_pos = source; @@ -1216,60 +1211,60 @@ static int analyze_utf8(byte *source, Uint size, --size; } else if (((*source) & ((byte) 0xE0)) == 0xC0) { if (size < 2) { - return UTF8_INCOMPLETE; + return ERTS_UTF8_INCOMPLETE; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((*source) < 0xC2) /* overlong */) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } source += 2; size -= 2; } else if (((*source) & ((byte) 0xF0)) == 0xE0) { if (size < 3) { - return UTF8_INCOMPLETE; + return ERTS_UTF8_INCOMPLETE; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((source[2] & ((byte) 0xC0)) != 0x80) || (((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } if ((((*source) & ((byte) 0xF)) == 0xD) && ((source[1] & 0x20) != 0)) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } if (((*source) == 0xEF) && (source[1] == 0xBF) && ((source[2] == 0xBE) || (source[2] == 0xBF))) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } source += 3; size -= 3; } else if (((*source) & ((byte) 0xF8)) == 0xF0) { if (size < 4) { - return UTF8_INCOMPLETE; + return ERTS_UTF8_INCOMPLETE; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((source[2] & ((byte) 0xC0)) != 0x80) || ((source[3] & ((byte) 0xC0)) != 0x80) || (((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } if ((((*source) & ((byte)0x7)) > 0x4U) || ((((*source) & ((byte)0x7)) == 0x4U) && ((source[1] & ((byte)0x3F)) > 0xFU))) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } source += 4; size -= 4; } else { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } ++(*num_chars); *err_pos = source; if (left && --(*left) <= 0) { - return UTF8_ANALYZE_MORE; + return ERTS_UTF8_ANALYZE_MORE; } } - return UTF8_OK; + return ERTS_UTF8_OK; } /* @@ -1304,7 +1299,7 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, } else if (((*source) & ((byte) 0xE0)) == 0xC0) { unipoint = (((Uint) ((*source) & ((byte) 0x1F))) << 6) | - ((Uint) (source[1] & ((byte) 0x3F))); + ((Uint) (source[1] & ((byte) 0x3F))); } else if (((*source) & ((byte) 0xF0)) == 0xE0) { unipoint = (((Uint) ((*source) & ((byte) 0xF))) << 12) | @@ -1330,6 +1325,216 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, return ret; } +static int is_candidate(Uint cp) +{ + int index,pos; + if (cp < 768) return 0; + if (cp > 4023) { + if (cp == 12441 || cp == 12442) return 1; + return 0; + } + index = cp / 32 - COMP_CANDIDATE_MAP_OFFSET; + pos = cp % 32; + return !!(comp_candidate_map[index] & (1UL << pos)); +} + +static int hashsearch(int *htab, int htab_size, CompEntry *cv, Uint16 c) +{ + int bucket = c % htab_size; + while (htab[bucket] != -1 && cv[htab[bucket]].c != c) + bucket = (bucket + 1) % htab_size; + return htab[bucket]; +} + +#define TRANSLATE_NO 0 +#define TRANSLATE_MAYBE -1 + +/* The s array is reversed */ +static int translate(Uint16 *s, int slen, Uint16 *res) +{ + /* Go backwards through buffer and match against tree */ + int pos = 0; + CompEntry *cv = compose_tab; + int *hc = hash_compose_tab; + int cvs = compose_tab_size; + int x; + while (pos < slen) { + x = hashsearch(hc,cvs*HASH_SIZE_FACTOR,cv,s[pos]); + if (x < 0) { + return TRANSLATE_NO; + } + if (cv[x].res) { + *res = cv[x].res; + return pos; + } + cvs = cv[x].num_subs; + hc = cv[x].hash; + cv = cv[x].subs; + ++pos; + } + return TRANSLATE_MAYBE; +} + +static void handle_first_norm(Uint16 *savepoints, int *numpointsp, Uint unipoint) +{ + /*erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) *numpointsp);*/ + *numpointsp = 1; + savepoints[0] = (Uint16) unipoint; +} + +static void cleanup_norm(Eterm **hpp, Uint16 *savepoints, int numpoints, Eterm *retp) +{ + Eterm *hp = *hpp; + int res,i; + Uint16 newpoint; + Eterm ret = *retp; + + ret = CONS(hp,make_small((Uint) savepoints[0]),ret); + hp += 2; + + for (i = 1;i < numpoints;) { + if(!is_candidate(savepoints[i]) || + ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) { + ret = CONS(hp,make_small((Uint) savepoints[i]),ret); + hp += 2; + ++i; + } else { + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + i += res; + } + } + *retp = ret; +} + +static void handle_potential_norm(Eterm **hpp, Uint16 *savepoints, int *numpointsp, Uint unipoint, Eterm *retp) +{ + Eterm *hp = *hpp; + int numpoints = *numpointsp; + int res,i; + Uint16 newpoint; + Eterm ret = *retp; + + /* erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) numpoints);*/ + if ((unipoint >> 16) == 0) { /* otherwise we're done here */ + savepoints[numpoints++] = (Uint16) unipoint; + res = translate(savepoints,numpoints,&newpoint); + if (res == TRANSLATE_NO) { + ret = CONS(hp,make_small((Uint) savepoints[0]),ret); + hp += 2; + for (i = 1;i < numpoints;) { + if(!is_candidate(savepoints[i]) || + ((res = translate(savepoints+i,numpoints - i, &newpoint)) == 0)) { + ret = CONS(hp,make_small((Uint) savepoints[i]),ret); + hp += 2; + ++i; + } else if (res > 0) { + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + i += res; + } else { /* res < 0 */ + /* A "maybe", means we are not done yet */ + int j = 0; + while (i < numpoints) { + savepoints[j++] = savepoints[i++]; + } + numpoints = j; + goto breakaway; + } + } + numpoints = 0; + breakaway: + ; + } else if (res > 0) { + numpoints = 0; + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + } /* < 0 means go on */ + } else { + /* Unconditional rollup, this character is larger than 16 bit */ + ret = CONS(hp,make_small((Uint) savepoints[0]),ret); + hp += 2; + + for (i = 1;i < numpoints;) { + if(!is_candidate(savepoints[i]) || + ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) { + ret = CONS(hp,make_small((Uint) savepoints[i]),ret); + hp += 2; + ++i; + } else { + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + i += res; + } + } + ret = CONS(hp,make_small(unipoint),ret); + hp += 2; + numpoints = 0; + } + *hpp = hp; + *numpointsp = numpoints; + *retp = ret; +} + +static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint sz) +{ + Eterm *hp,*hp_end; + Eterm ret; + byte *source; + Uint unipoint; + Uint16 savepoints[4]; + int numpoints = 0; + + ASSERT(num > 0); + + hp = HAlloc(p,num * 2); /* May be to much */ + hp_end = hp + num * 2; + ret = NIL; + source = bytes + sz; + while(--source >= bytes) { + if (((*source) & ((byte) 0x80)) == 0) { + unipoint = (Uint) *source; + } else if (((*source) & ((byte) 0xE0)) == 0xC0) { + unipoint = + (((Uint) ((*source) & ((byte) 0x1F))) << 6) | + ((Uint) (source[1] & ((byte) 0x3F))); + } else if (((*source) & ((byte) 0xF0)) == 0xE0) { + unipoint = + (((Uint) ((*source) & ((byte) 0xF))) << 12) | + (((Uint) (source[1] & ((byte) 0x3F))) << 6) | + ((Uint) (source[2] & ((byte) 0x3F))); + } else if (((*source) & ((byte) 0xF8)) == 0xF0) { + unipoint = + (((Uint) ((*source) & ((byte) 0x7))) << 18) | + (((Uint) (source[1] & ((byte) 0x3F))) << 12) | + (((Uint) (source[2] & ((byte) 0x3F))) << 6) | + ((Uint) (source[3] & ((byte) 0x3F))); + } else { + /* ignore 2#10XXXXXX */ + continue; + } + if (numpoints) { + handle_potential_norm(&hp,savepoints,&numpoints,unipoint,&ret); + continue; + } + /* We are not building up any normalizations yet, look that we shouldn't start... */ + if (is_candidate(unipoint)) { + handle_first_norm(savepoints,&numpoints,unipoint); + continue; + } + ret = CONS(hp,make_small(unipoint),ret); + hp += 2; + } + /* so, we'we looped to the beginning, do we have anything saved? */ + if (numpoints) { + cleanup_norm(&hp,savepoints,numpoints,&ret); + } + if (hp_end != hp) { + HRelease(p,hp_end,hp); + } + return ret; +} + /* * The last step of characters_to_list, build a list from the buffer 'bytes' (created in the same way * as for characters_to_utf8). All sizes are known in advance and most data will be held in a @@ -1378,10 +1583,10 @@ static BIF_RETTYPE finalize_list_to_list(Process *p, */ free_restart(bytes); - if (state == UTF8_INCOMPLETE) { + if (state == ERTS_UTF8_INCOMPLETE) { hp = HAlloc(p,4); ret = TUPLE3(hp,am_incomplete,converted,rest); - } else if (state == UTF8_ERROR) { + } else if (state == ERTS_UTF8_ERROR) { hp = HAlloc(p,4); ret = TUPLE3(hp,am_error,converted,rest); } else { @@ -1408,7 +1613,7 @@ static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3) /* * Hooks into the process of decoding a binary depending on state. - * If last_state is UTF8_ANALYZE_MORE, num_bytes_to_process + * If last_state is ERTS_UTF8_ANALYZE_MORE, num_bytes_to_process * and num_resulting_chars will grow * until we're done analyzing the binary. Then we'll eat * the bytes to process, lowering num_bytes_to_process and num_resulting_chars, @@ -1465,14 +1670,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, left = allowed_iterations(p); - if (state == UTF8_ANALYZE_MORE) { - state = analyze_utf8(bytes + num_bytes_to_process, + if (state == ERTS_UTF8_ANALYZE_MORE) { + state = erts_analyze_utf8(bytes + num_bytes_to_process, size - num_bytes_to_process, &endpos,&numchar,&left); cost_to_proc(p,numchar); num_resulting_chars += numchar; num_bytes_to_process = endpos - bytes; - if (state == UTF8_ANALYZE_MORE) { + if (state == ERTS_UTF8_ANALYZE_MORE) { Eterm epos = erts_make_integer(num_bytes_to_process,p); Eterm enumchar = erts_make_integer(num_resulting_chars,p); erts_free_aligned_binary_bytes(temp_alloc); @@ -1528,7 +1733,7 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, ErlSubBin *sb; Eterm orig; Uint offset; - ASSERT(state != UTF8_OK); + ASSERT(state != ERTS_UTF8_OK); hp = HAlloc(p, ERL_SUB_BIN_SIZE); sb = (ErlSubBin *) hp; ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize); @@ -1544,14 +1749,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, /* Done */ - if (state == UTF8_INCOMPLETE) { + if (state == ERTS_UTF8_INCOMPLETE) { if (check_leftovers(bytes + num_bytes_to_process + num_processed_bytes, b_sz) != 0) { goto error_return; } hp = HAlloc(p,4); ret = TUPLE3(hp,am_incomplete,converted,rest); - } else if (state == UTF8_ERROR) { + } else if (state == ERTS_UTF8_ERROR) { error_return: hp = HAlloc(p,4); ret = TUPLE3(hp,am_error,converted,rest); @@ -1589,7 +1794,7 @@ static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3) 0U, /* nothing processed yet */ num_bytes_to_process, num_resulting_chars, - UTF8_ANALYZE_MORE, /* always this state here */ + ERTS_UTF8_ANALYZE_MORE, /* always this state here */ NIL); /* Nothing built -> no tail yet */ } @@ -1642,7 +1847,7 @@ static BIF_RETTYPE utf8_to_list(BIF_ALIST_1) BIF_ERROR(BIF_P,BADARG); } return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U, - UTF8_ANALYZE_MORE,NIL); + ERTS_UTF8_ANALYZE_MORE,NIL); } @@ -1728,8 +1933,8 @@ binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist) Uint n; int reds_left = bin_size+1; /* Number of reductions left. */ - if (analyze_utf8(bytes, bin_size, &err_pos, - &n, &reds_left) == UTF8_OK) { + if (erts_analyze_utf8(bytes, bin_size, &err_pos, + &n, &reds_left) == ERTS_UTF8_OK) { /* * Correct UTF-8 encoding, but too many characters to * fit in an atom. @@ -1813,3 +2018,616 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2) { return binary_to_atom(BIF_P, BIF_ARG_1, BIF_ARG_2, 1); } + +/********************************************************** + * Simpler non-interruptable routines for UTF-8 and + * Windowish UTF-16 (restricted) + **********************************************************/ +/* + * This function is the heart of the Unicode support for + * open_port - spawn_executable. It converts both the name + * of the executable and the arguments according to the same rules + * as for filename conversion. That means as if your arguments are + * to be raw, you supply binaries, else unicode characters are allowed up to + * the encoding maximum (256 of the unicode max). + * Depending on the filename encoding standard, the vector is then + * converted to whatever is used, which might mean win_utf16 if on windows. + * Do not peek into the argument vector or filenam with ordinary + * string routines, that will certainly fail on some OS. + */ + +char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty) +{ + int encoding = erts_get_native_filename_encoding(); + char* name_buf = NULL; + + if (is_atom(name) || is_list(name) || (allow_empty && is_nil(name))) { + Sint need; + if ((need = erts_native_filename_need(name,encoding)) < 0) { + return NULL; + } + if (encoding == ERL_FILENAME_WIN_WCHAR) { + need += 2; + } else { + ++need; + } + name_buf = (char *) erts_alloc(alloc_type, need); + erts_native_filename_put(name,encoding,(byte *)name_buf); + name_buf[need-1] = 0; + if (encoding == ERL_FILENAME_WIN_WCHAR) { + name_buf[need-2] = 0; + } + } else if (is_binary(name)) { + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + Uint size,num_chars; + + size = binary_size(name); + bytes = erts_get_aligned_binary_bytes(name, &temp_alloc); + if (encoding != ERL_FILENAME_WIN_WCHAR) { + /*Add 0 termination only*/ + name_buf = (char *) erts_alloc(alloc_type, size+1); + memcpy(name_buf,bytes,size); + name_buf[size]=0; + } else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || + erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + byte *p; + /* What to do now? Maybe latin1, so just take byte for byte instead */ + name_buf = (char *) erts_alloc(alloc_type, (size+1)*2); + p = (byte *) name_buf; + while (size--) { + *p++ = *bytes++; + *p++ = 0; + } + *p++ = 0; + *p++ = 0; + } else { /* WIN_WCHAR and valid UTF8 */ + name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2); + erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars); + name_buf[num_chars*2] = 0; + name_buf[num_chars*2+1] = 0; + } + erts_free_aligned_binary_bytes(temp_alloc); + } else { + return NULL; + } + return name_buf; +} + + +Sint erts_native_filename_need(Eterm ioterm, int encoding) +{ + Eterm *objp; + Eterm obj; + DECLARE_ESTACK(stack); + Sint need = 0; + + if (is_atom(ioterm)) { + Atom* ap; + int i; + ap = atom_tab(atom_val(ioterm)); + switch (encoding) { + case ERL_FILENAME_LATIN1: + need = ap->len; + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + for (i = 0; i < ap->len; i++) { + need += (ap->name[i] >= 0x80) ? 2 : 1; + } + break; + case ERL_FILENAME_WIN_WCHAR: + need = 2*(ap->len); + break; + default: + need = -1; + } + DESTROY_ESTACK(stack); + return need; + } + + if (is_nil(ioterm)) { + DESTROY_ESTACK(stack); + return need; + } + if (!is_list(ioterm)) { + DESTROY_ESTACK(stack); + return (Sint) -1; + } + /* OK a list, needs to be processed in order, handling each flat list-level + as they occur, just like io_list_to_binary would */ + ESTACK_PUSH(stack,ioterm); + while (!ESTACK_ISEMPTY(stack)) { + ioterm = ESTACK_POP(stack); + if (is_nil(ioterm)) { + /* ignore empty lists */ + continue; + } + if(is_list(ioterm)) { +L_Again: /* Restart with sublist, old listend was pushed on stack */ + objp = list_val(ioterm); + obj = CAR(objp); + for(;;) { /* loop over one flat list of bytes and binaries + until sublist or list end is encountered */ + if (is_small(obj)) { /* Always small */ + for(;;) { + Uint x = unsigned_val(obj); + switch (encoding) { + case ERL_FILENAME_LATIN1: + if (x > 255) { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + need += 1; + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + if (x < 0x80) { + need +=1; + } else if (x < 0x800) { + need += 2; + } else if (x < 0x10000) { + if ((x >= 0xD800 && x <= 0xDFFF) || + (x == 0xFFFE) || + (x == 0xFFFF)) { /* Invalid unicode range */ + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + need += 3; + } else if (x < 0x110000) { + need += 4; + } else { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + break; + case ERL_FILENAME_WIN_WCHAR: + if (x <= 0xffff) { + need += 2; + break; + } /* else fall throug to error */ + default: + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + + /* everything else will give badarg later + in the process, so we dont check */ + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + if (!is_small(obj)) + break; + } + } else if (is_nil(obj)) { + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + } else if (is_list(obj)) { + /* push rest of list for later processing, start + again with sublist */ + ESTACK_PUSH(stack,CDR(objp)); + ioterm = obj; + goto L_Again; + } else { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + if (is_nil(ioterm) || !is_list(ioterm)) { + break; + } + } /* for(;;) */ + } /* is_list(ioterm) */ + + if (!is_list(ioterm) && !is_nil(ioterm)) { + /* inproper list end */ + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + } /* while not estack empty */ + DESTROY_ESTACK(stack); + return need; +} + +void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) +{ + Eterm *objp; + Eterm obj; + DECLARE_ESTACK(stack); + + if (is_atom(ioterm)) { + Atom* ap; + int i; + ap = atom_tab(atom_val(ioterm)); + switch (encoding) { + case ERL_FILENAME_LATIN1: + for (i = 0; i < ap->len; i++) { + *p++ = ap->name[i]; + } + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + for (i = 0; i < ap->len; i++) { + if(ap->name[i] < 0x80) { + *p++ = ap->name[i]; + } else { + *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0)); + *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80)); + } + } + break; + case ERL_FILENAME_WIN_WCHAR: + for (i = 0; i < ap->len; i++) { + /* Little endian */ + *p++ = ap->name[i]; + *p++ = 0; + } + break; + default: + ASSERT(0); + } + DESTROY_ESTACK(stack); + return; + } + + if (is_nil(ioterm)) { + DESTROY_ESTACK(stack); + return; + } + ASSERT(is_list(ioterm)); + /* OK a list, needs to be processed in order, handling each flat list-level + as they occur, just like io_list_to_binary would */ + ESTACK_PUSH(stack,ioterm); + while (!ESTACK_ISEMPTY(stack)) { + ioterm = ESTACK_POP(stack); + if (is_nil(ioterm)) { + /* ignore empty lists */ + continue; + } + if(is_list(ioterm)) { +L_Again: /* Restart with sublist, old listend was pushed on stack */ + objp = list_val(ioterm); + obj = CAR(objp); + for(;;) { /* loop over one flat list of bytes and binaries + until sublist or list end is encountered */ + if (is_small(obj)) { /* Always small */ + for(;;) { + Uint x = unsigned_val(obj); + switch (encoding) { + case ERL_FILENAME_LATIN1: + ASSERT( x < 256); + *p++ = (byte) x; + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + if (x < 0x80) { + *p++ = (byte) x; + } + else if (x < 0x800) { + *p++ = (((byte) (x >> 6)) | + ((byte) 0xC0)); + *p++ = (((byte) (x & 0x3F)) | + ((byte) 0x80)); + } else if (x < 0x10000) { + ASSERT(!((x >= 0xD800 && x <= 0xDFFF) || + (x == 0xFFFE) || + (x == 0xFFFF))); + *p++ = (((byte) (x >> 12)) | + ((byte) 0xE0)); + *p++ = ((((byte) (x >> 6)) & 0x3F) | + ((byte) 0x80)); + *p++ = (((byte) (x & 0x3F)) | + ((byte) 0x80)); + } else { + ASSERT(x < 0x110000); + *p++ = (((byte) (x >> 18)) | + ((byte) 0xF0)); + *p++ = ((((byte) (x >> 12)) & 0x3F) | + ((byte) 0x80)); + *p++ = ((((byte) (x >> 6)) & 0x3F) | + ((byte) 0x80)); + *p++ = (((byte) (x & 0x3F)) | + ((byte) 0x80)); + } + break; + case ERL_FILENAME_WIN_WCHAR: + ASSERT(x <= 0xFFFF); + *p++ = (byte) (x & 0xFFU); + *p++ = (byte) ((x >> 8) & 0xFFU); + break; + default: + ASSERT(0); + } + + /* everything else will give badarg later + in the process, so we dont check */ + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + if (!is_small(obj)) + break; + } + } else if (is_nil(obj)) { + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + } else if (is_list(obj)) { + /* push rest of list for later processing, start + again with sublist */ + ESTACK_PUSH(stack,CDR(objp)); + ioterm = obj; + goto L_Again; + } else { + ASSERT(0); + } + if (is_nil(ioterm) || !is_list(ioterm)) { + break; + } + } /* for(;;) */ + } /* is_list(ioterm) */ + + ASSERT(is_list(ioterm) || is_nil(ioterm)); + } /* while not estack empty */ + DESTROY_ESTACK(stack); + return; +} +void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars) +{ + Uint unipoint; + + while (num_chars--) { + if (((*bytes) & ((byte) 0x80)) == 0) { + unipoint = (Uint) *bytes; + ++bytes; + } else if (((*bytes) & ((byte) 0xE0)) == 0xC0) { + unipoint = + (((Uint) ((*bytes) & ((byte) 0x1F))) << 6) | + ((Uint) (bytes[1] & ((byte) 0x3F))); + bytes += 2; + } else if (((*bytes) & ((byte) 0xF0)) == 0xE0) { + unipoint = + (((Uint) ((*bytes) & ((byte) 0xF))) << 12) | + (((Uint) (bytes[1] & ((byte) 0x3F))) << 6) | + ((Uint) (bytes[2] & ((byte) 0x3F))); + bytes +=3; + } else if (((*bytes) & ((byte) 0xF8)) == 0xF0) { + unipoint = + (((Uint) ((*bytes) & ((byte) 0x7))) << 18) | + (((Uint) (bytes[1] & ((byte) 0x3F))) << 12) | + (((Uint) (bytes[2] & ((byte) 0x3F))) << 6) | + ((Uint) (bytes[3] & ((byte) 0x3F))); + bytes += 4; + } else { + erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1"); + } + *target++ = (byte) (unipoint & 0xFF); + *target++ = (byte) ((unipoint >> 8) & 0xFF); + } +} + +/* + * This internal bif converts a filename to whatever format is suitable for the file driver + * It also adds zero termination so that prim_file needn't bother with the character encoding + * of the file driver + */ +BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) +{ + int encoding = erts_get_native_filename_encoding(); + Sint need; + Eterm bin_term; + byte* bin_p; + /* Prim file explicitly does not allow atoms, although we could + very well cope with it. Instead of letting 'file' handle them, + it would probably be more efficient to handle them here. Subject to + change in R15. */ + if (is_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + if (is_binary(BIF_ARG_1)) { + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + Uint size,num_chars; + /* Uninterpreted encoding except if windows widechar, in case we convert from + utf8 to win_wchar */ + size = binary_size(BIF_ARG_1); + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if (encoding != ERL_FILENAME_WIN_WCHAR) { + /*Add 0 termination only*/ + bin_term = new_binary(BIF_P, NULL, size+1); + bin_p = binary_bytes(bin_term); + memcpy(bin_p,bytes,size); + bin_p[size]=0; + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(bin_term); + } + /* In a wchar world, the emulator flags only affect how + binaries are interpreted when sent from the user. */ + /* Determine real length and create a new binary */ + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || + erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + /* What to do now? Maybe latin1, so just take byte for byte instead */ + bin_term = new_binary(BIF_P, 0, (size+1)*2); + bin_p = binary_bytes(bin_term); + while (size--) { + *bin_p++ = *bytes++; + *bin_p++ = 0; + } + *bin_p++ = 0; + *bin_p++ = 0; + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(bin_term); + } + /* OK, UTF8 ok, number of characters is in num_chars */ + bin_term = new_binary(BIF_P, 0, (num_chars+1)*2); + bin_p = binary_bytes(bin_term); + erts_copy_utf8_to_utf16_little(bin_p, bytes, num_chars); + /* zero termination */ + bin_p[num_chars*2] = 0; + bin_p[num_chars*2+1] = 0; + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(bin_term); + } /* binary */ + + + if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (encoding == ERL_FILENAME_WIN_WCHAR) { + need += 2; + } else { + ++need; + } + + bin_term = new_binary(BIF_P, 0, need); + bin_p = binary_bytes(bin_term); + erts_native_filename_put(BIF_ARG_1,encoding,bin_p); + bin_p[need-1] = 0; + if (encoding == ERL_FILENAME_WIN_WCHAR) { + bin_p[need-2] = 0; + } + BIF_RET(bin_term); +} + +BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1) +{ + Eterm real_bin; + Uint offset; + Uint size,num_chars; + Uint bitsize; + Uint bitoffs; + Eterm *hp; + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + Uint num_built; /* characters */ + Uint num_eaten; /* bytes */ + Eterm ret; + int mac = 0; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + size = binary_size(BIF_ARG_1); + ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (size == 0) { + BIF_RET(NIL); + } + switch (erts_get_native_filename_encoding()) { + case ERL_FILENAME_LATIN1: + hp = HAlloc(BIF_P, 2 * size); + bytes = binary_bytes(real_bin)+offset; + + BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); + case ERL_FILENAME_UTF8_MAC: + mac = 1; + case ERL_FILENAME_UTF8: + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { + erts_free_aligned_binary_bytes(temp_alloc); + goto noconvert; + } + num_built = 0; + num_eaten = 0; + if (mac) { + ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size); + } else { + ret = do_utf8_to_list(BIF_P, num_chars, bytes, size, num_chars, &num_built, &num_eaten, NIL); + } + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(ret); + case ERL_FILENAME_WIN_WCHAR: + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if ((size % 2) != 0) { /* Panic fixup to avoid crashing the emulator */ + size--; + hp = HAlloc(BIF_P, size+2); + ret = CONS(hp,make_small((Uint) bytes[size]),NIL); + hp += 2; + } else { + hp = HAlloc(BIF_P, size); + ret = NIL; + } + bytes += size-1; + while (size > 0) { + Uint x = ((Uint) *bytes--) << 8; + x |= ((Uint) *bytes--); + size -= 2; + ret = CONS(hp,make_small(x),ret); + hp += 2; + } + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(ret); + default: + goto noconvert; + } + noconvert: + BIF_RET(BIF_ARG_1); +} + +BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1) +{ + Eterm real_bin; + Uint offset; + Uint size,num_chars; + Uint bitsize; + Uint bitoffs; + Eterm ret; + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + size = binary_size(BIF_ARG_1); + ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (size == 0) { + BIF_RET(NIL); + } + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P,BADARG); + } + ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size); + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(ret); +} + +BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0) +{ + switch (erts_get_native_filename_encoding()) { + case ERL_FILENAME_LATIN1: + BIF_RET(am_latin1); + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + BIF_RET(am_utf8); + case ERL_FILENAME_WIN_WCHAR: + if (erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + BIF_RET(am_latin1); + } else { + BIF_RET(am_utf8); + } + default: + BIF_RET(am_undefined); + } +} diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h new file mode 100644 index 0000000000..fb0a111ca2 --- /dev/null +++ b/erts/emulator/beam/erl_unicode_normalize.h @@ -0,0 +1,1687 @@ +/* +* %CopyrightBegin% +* +* Copyright Ericsson AB 1999-2010. All Rights Reserved. +* +* The contents of this file are subject to the Erlang Public License, +* Version 1.1, (the "License"); you may not use this file except in +* compliance with the License. You should have received a copy of the +* Erlang Public License along with this software. If not, it can be +* retrieved online at http://www.erlang.org/. +* +* Software distributed under the License is distributed on an "AS IS" +* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +* the License for the specific language governing rights and limitations +* under the License. +* +* %CopyrightEnd% +*/ +/* +* This file is automatically generated by dec.erl, do not edit manually +*/ +#define HASH_SIZE_FACTOR 2 +typedef struct _compose_entry { + Uint16 c; + Uint16 res; + Uint16 num_subs; + struct _compose_entry *subs; + int *hash; +} CompEntry; + +static int compose_tab_size = 61; +static int hash_compose_tab_0_15[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_0_15 */ +static CompEntry compose_tab_0_15[] = { +{65, 7846, 0, NULL, NULL}, +{69, 7872, 0, NULL, NULL}, +{79, 7890, 0, NULL, NULL}, +{97, 7847, 0, NULL, NULL}, +{101, 7873, 0, NULL, NULL}, +{111, 7891, 0, NULL, NULL} +}; /* compose_tab_0_15 */ +static int hash_compose_tab_0_16[8] = +{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_0_16 */ +static CompEntry compose_tab_0_16[] = { +{69, 7700, 0, NULL, NULL}, +{79, 7760, 0, NULL, NULL}, +{101, 7701, 0, NULL, NULL}, +{111, 7761, 0, NULL, NULL} +}; /* compose_tab_0_16 */ +static int hash_compose_tab_0_17[4] = +{-1,0,1,-1}; /* hash_compose_tab_0_17 */ +static CompEntry compose_tab_0_17[] = { +{65, 7856, 0, NULL, NULL}, +{97, 7857, 0, NULL, NULL} +}; /* compose_tab_0_17 */ +static int hash_compose_tab_0_18[8] = +{-1,2,-1,-1,-1,0,1,3}; /* hash_compose_tab_0_18 */ +static CompEntry compose_tab_0_18[] = { +{85, 475, 0, NULL, NULL}, +{117, 476, 0, NULL, NULL}, +{953, 8146, 0, NULL, NULL}, +{965, 8162, 0, NULL, NULL} +}; /* compose_tab_0_18 */ +static int hash_compose_tab_0_19_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_19_0 */ +static CompEntry compose_tab_0_19_0[] = { +{913, 8074, 0, NULL, NULL}, +{919, 8090, 0, NULL, NULL}, +{937, 8106, 0, NULL, NULL}, +{945, 8066, 0, NULL, NULL}, +{951, 8082, 0, NULL, NULL}, +{969, 8098, 0, NULL, NULL} +}; /* compose_tab_0_19_0 */ +static int hash_compose_tab_0_19[28] = +{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_0_19 */ +static CompEntry compose_tab_0_19[] = { +{837, 0, 6, compose_tab_0_19_0, hash_compose_tab_0_19_0}, +{913, 7946, 0, NULL, NULL}, +{917, 7962, 0, NULL, NULL}, +{919, 7978, 0, NULL, NULL}, +{921, 7994, 0, NULL, NULL}, +{927, 8010, 0, NULL, NULL}, +{937, 8042, 0, NULL, NULL}, +{945, 7938, 0, NULL, NULL}, +{949, 7954, 0, NULL, NULL}, +{951, 7970, 0, NULL, NULL}, +{953, 7986, 0, NULL, NULL}, +{959, 8002, 0, NULL, NULL}, +{965, 8018, 0, NULL, NULL}, +{969, 8034, 0, NULL, NULL} +}; /* compose_tab_0_19 */ +static int hash_compose_tab_0_20_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_20_0 */ +static CompEntry compose_tab_0_20_0[] = { +{913, 8075, 0, NULL, NULL}, +{919, 8091, 0, NULL, NULL}, +{937, 8107, 0, NULL, NULL}, +{945, 8067, 0, NULL, NULL}, +{951, 8083, 0, NULL, NULL}, +{969, 8099, 0, NULL, NULL} +}; /* compose_tab_0_20_0 */ +static int hash_compose_tab_0_20[30] = +{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5, + 12}; /* hash_compose_tab_0_20 */ +static CompEntry compose_tab_0_20[] = { +{837, 0, 6, compose_tab_0_20_0, hash_compose_tab_0_20_0}, +{913, 7947, 0, NULL, NULL}, +{917, 7963, 0, NULL, NULL}, +{919, 7979, 0, NULL, NULL}, +{921, 7995, 0, NULL, NULL}, +{927, 8011, 0, NULL, NULL}, +{933, 8027, 0, NULL, NULL}, +{937, 8043, 0, NULL, NULL}, +{945, 7939, 0, NULL, NULL}, +{949, 7955, 0, NULL, NULL}, +{951, 7971, 0, NULL, NULL}, +{953, 7987, 0, NULL, NULL}, +{959, 8003, 0, NULL, NULL}, +{965, 8019, 0, NULL, NULL}, +{969, 8035, 0, NULL, NULL} +}; /* compose_tab_0_20 */ +static int hash_compose_tab_0_21[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_0_21 */ +static CompEntry compose_tab_0_21[] = { +{79, 7900, 0, NULL, NULL}, +{85, 7914, 0, NULL, NULL}, +{111, 7901, 0, NULL, NULL}, +{117, 7915, 0, NULL, NULL} +}; /* compose_tab_0_21 */ +static int hash_compose_tab_0_22[6] = +{-1,-1,-1,0,1,2}; /* hash_compose_tab_0_22 */ +static CompEntry compose_tab_0_22[] = { +{945, 8114, 0, NULL, NULL}, +{951, 8130, 0, NULL, NULL}, +{969, 8178, 0, NULL, NULL} +}; /* compose_tab_0_22 */ +static int hash_compose_tab_0[78] = +{38,3,29,-1,-1,-1,-1,4,19,5,20,6,14,30,31,21,32,33,37,7,-1,-1,-1,8,34,-1,-1,9, + -1,35,-1,-1,-1,10,36,-1,-1,-1,-1,11,-1,12,-1,13,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,23,-1,22,-1,24,-1,25,-1,26,-1,0,-1,-1,15,1,16,27,17,2,18,28,-1,-1}; /* hash_compose_tab_0 */ +static CompEntry compose_tab_0[] = { +{65, 192, 0, NULL, NULL}, +{69, 200, 0, NULL, NULL}, +{73, 204, 0, NULL, NULL}, +{79, 210, 0, NULL, NULL}, +{85, 217, 0, NULL, NULL}, +{87, 7808, 0, NULL, NULL}, +{89, 7922, 0, NULL, NULL}, +{97, 224, 0, NULL, NULL}, +{101, 232, 0, NULL, NULL}, +{105, 236, 0, NULL, NULL}, +{111, 242, 0, NULL, NULL}, +{117, 249, 0, NULL, NULL}, +{119, 7809, 0, NULL, NULL}, +{121, 7923, 0, NULL, NULL}, +{168, 8173, 0, NULL, NULL}, +{770, 0, 6, compose_tab_0_15, hash_compose_tab_0_15}, +{772, 0, 4, compose_tab_0_16, hash_compose_tab_0_16}, +{774, 0, 2, compose_tab_0_17, hash_compose_tab_0_17}, +{776, 0, 4, compose_tab_0_18, hash_compose_tab_0_18}, +{787, 0, 14, compose_tab_0_19, hash_compose_tab_0_19}, +{788, 0, 15, compose_tab_0_20, hash_compose_tab_0_20}, +{795, 0, 4, compose_tab_0_21, hash_compose_tab_0_21}, +{837, 0, 3, compose_tab_0_22, hash_compose_tab_0_22}, +{913, 8122, 0, NULL, NULL}, +{917, 8136, 0, NULL, NULL}, +{919, 8138, 0, NULL, NULL}, +{921, 8154, 0, NULL, NULL}, +{927, 8184, 0, NULL, NULL}, +{933, 8170, 0, NULL, NULL}, +{937, 8186, 0, NULL, NULL}, +{945, 8048, 0, NULL, NULL}, +{949, 8050, 0, NULL, NULL}, +{951, 8052, 0, NULL, NULL}, +{953, 8054, 0, NULL, NULL}, +{959, 8056, 0, NULL, NULL}, +{965, 8058, 0, NULL, NULL}, +{969, 8060, 0, NULL, NULL}, +{8127, 8141, 0, NULL, NULL}, +{8190, 8157, 0, NULL, NULL} +}; /* compose_tab_0 */ +static int hash_compose_tab_1_39[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_1_39 */ +static CompEntry compose_tab_1_39[] = { +{65, 7844, 0, NULL, NULL}, +{69, 7870, 0, NULL, NULL}, +{79, 7888, 0, NULL, NULL}, +{97, 7845, 0, NULL, NULL}, +{101, 7871, 0, NULL, NULL}, +{111, 7889, 0, NULL, NULL} +}; /* compose_tab_1_39 */ +static int hash_compose_tab_1_40[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_40 */ +static CompEntry compose_tab_1_40[] = { +{79, 7756, 0, NULL, NULL}, +{85, 7800, 0, NULL, NULL}, +{111, 7757, 0, NULL, NULL}, +{117, 7801, 0, NULL, NULL} +}; /* compose_tab_1_40 */ +static int hash_compose_tab_1_41[8] = +{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_1_41 */ +static CompEntry compose_tab_1_41[] = { +{69, 7702, 0, NULL, NULL}, +{79, 7762, 0, NULL, NULL}, +{101, 7703, 0, NULL, NULL}, +{111, 7763, 0, NULL, NULL} +}; /* compose_tab_1_41 */ +static int hash_compose_tab_1_42[4] = +{-1,0,1,-1}; /* hash_compose_tab_1_42 */ +static CompEntry compose_tab_1_42[] = { +{65, 7854, 0, NULL, NULL}, +{97, 7855, 0, NULL, NULL} +}; /* compose_tab_1_42 */ +static int hash_compose_tab_1_43[12] = +{-1,0,1,-1,-1,4,5,-1,-1,2,3,-1}; /* hash_compose_tab_1_43 */ +static CompEntry compose_tab_1_43[] = { +{73, 7726, 0, NULL, NULL}, +{85, 471, 0, NULL, NULL}, +{105, 7727, 0, NULL, NULL}, +{117, 472, 0, NULL, NULL}, +{953, 8147, 0, NULL, NULL}, +{965, 8163, 0, NULL, NULL} +}; /* compose_tab_1_43 */ +static int hash_compose_tab_1_44[4] = +{-1,0,1,-1}; /* hash_compose_tab_1_44 */ +static CompEntry compose_tab_1_44[] = { +{65, 506, 0, NULL, NULL}, +{97, 507, 0, NULL, NULL} +}; /* compose_tab_1_44 */ +static int hash_compose_tab_1_45_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_45_0 */ +static CompEntry compose_tab_1_45_0[] = { +{913, 8076, 0, NULL, NULL}, +{919, 8092, 0, NULL, NULL}, +{937, 8108, 0, NULL, NULL}, +{945, 8068, 0, NULL, NULL}, +{951, 8084, 0, NULL, NULL}, +{969, 8100, 0, NULL, NULL} +}; /* compose_tab_1_45_0 */ +static int hash_compose_tab_1_45[28] = +{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_1_45 */ +static CompEntry compose_tab_1_45[] = { +{837, 0, 6, compose_tab_1_45_0, hash_compose_tab_1_45_0}, +{913, 7948, 0, NULL, NULL}, +{917, 7964, 0, NULL, NULL}, +{919, 7980, 0, NULL, NULL}, +{921, 7996, 0, NULL, NULL}, +{927, 8012, 0, NULL, NULL}, +{937, 8044, 0, NULL, NULL}, +{945, 7940, 0, NULL, NULL}, +{949, 7956, 0, NULL, NULL}, +{951, 7972, 0, NULL, NULL}, +{953, 7988, 0, NULL, NULL}, +{959, 8004, 0, NULL, NULL}, +{965, 8020, 0, NULL, NULL}, +{969, 8036, 0, NULL, NULL} +}; /* compose_tab_1_45 */ +static int hash_compose_tab_1_46_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_46_0 */ +static CompEntry compose_tab_1_46_0[] = { +{913, 8077, 0, NULL, NULL}, +{919, 8093, 0, NULL, NULL}, +{937, 8109, 0, NULL, NULL}, +{945, 8069, 0, NULL, NULL}, +{951, 8085, 0, NULL, NULL}, +{969, 8101, 0, NULL, NULL} +}; /* compose_tab_1_46_0 */ +static int hash_compose_tab_1_46[30] = +{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5, + 12}; /* hash_compose_tab_1_46 */ +static CompEntry compose_tab_1_46[] = { +{837, 0, 6, compose_tab_1_46_0, hash_compose_tab_1_46_0}, +{913, 7949, 0, NULL, NULL}, +{917, 7965, 0, NULL, NULL}, +{919, 7981, 0, NULL, NULL}, +{921, 7997, 0, NULL, NULL}, +{927, 8013, 0, NULL, NULL}, +{933, 8029, 0, NULL, NULL}, +{937, 8045, 0, NULL, NULL}, +{945, 7941, 0, NULL, NULL}, +{949, 7957, 0, NULL, NULL}, +{951, 7973, 0, NULL, NULL}, +{953, 7989, 0, NULL, NULL}, +{959, 8005, 0, NULL, NULL}, +{965, 8021, 0, NULL, NULL}, +{969, 8037, 0, NULL, NULL} +}; /* compose_tab_1_46 */ +static int hash_compose_tab_1_47[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_47 */ +static CompEntry compose_tab_1_47[] = { +{79, 7898, 0, NULL, NULL}, +{85, 7912, 0, NULL, NULL}, +{111, 7899, 0, NULL, NULL}, +{117, 7913, 0, NULL, NULL} +}; /* compose_tab_1_47 */ +static int hash_compose_tab_1_48[4] = +{1,-1,-1,0}; /* hash_compose_tab_1_48 */ +static CompEntry compose_tab_1_48[] = { +{67, 7688, 0, NULL, NULL}, +{99, 7689, 0, NULL, NULL} +}; /* compose_tab_1_48 */ +static int hash_compose_tab_1_49[6] = +{-1,-1,-1,0,1,2}; /* hash_compose_tab_1_49 */ +static CompEntry compose_tab_1_49[] = { +{945, 8116, 0, NULL, NULL}, +{951, 8132, 0, NULL, NULL}, +{959, 8180, 0, NULL, NULL} +}; /* compose_tab_1_49 */ +static int hash_compose_tab_1[140] = +{-1,-1,-1,-1,-1,-1,-1,68,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,34,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,35,-1,-1,-1,-1,64,-1,0,-1,1,-1,2,39,3,40,4,41,5,6,7, + 8,9,10,36,11,12,42,13,43,14,44,15,16,37,45,46,50,47,51,17,52,18,53,19,54,20, + 55,21,56,22,23,24,25,26,27,38,28,29,48,30,57,31,58,32,33,59,60,61,62,65,66, + 63,67,69,-1,-1,-1,-1,-1,49,-1,-1}; /* hash_compose_tab_1 */ +static CompEntry compose_tab_1[] = { +{65, 193, 0, NULL, NULL}, +{67, 262, 0, NULL, NULL}, +{69, 201, 0, NULL, NULL}, +{71, 500, 0, NULL, NULL}, +{73, 205, 0, NULL, NULL}, +{75, 7728, 0, NULL, NULL}, +{76, 313, 0, NULL, NULL}, +{77, 7742, 0, NULL, NULL}, +{78, 323, 0, NULL, NULL}, +{79, 211, 0, NULL, NULL}, +{80, 7764, 0, NULL, NULL}, +{82, 340, 0, NULL, NULL}, +{83, 346, 0, NULL, NULL}, +{85, 218, 0, NULL, NULL}, +{87, 7810, 0, NULL, NULL}, +{89, 221, 0, NULL, NULL}, +{90, 377, 0, NULL, NULL}, +{97, 225, 0, NULL, NULL}, +{99, 263, 0, NULL, NULL}, +{101, 233, 0, NULL, NULL}, +{103, 501, 0, NULL, NULL}, +{105, 237, 0, NULL, NULL}, +{107, 7729, 0, NULL, NULL}, +{108, 314, 0, NULL, NULL}, +{109, 7743, 0, NULL, NULL}, +{110, 324, 0, NULL, NULL}, +{111, 243, 0, NULL, NULL}, +{112, 7765, 0, NULL, NULL}, +{114, 341, 0, NULL, NULL}, +{115, 347, 0, NULL, NULL}, +{117, 250, 0, NULL, NULL}, +{119, 7811, 0, NULL, NULL}, +{121, 253, 0, NULL, NULL}, +{122, 378, 0, NULL, NULL}, +{168, 8174, 0, NULL, NULL}, +{198, 508, 0, NULL, NULL}, +{216, 510, 0, NULL, NULL}, +{230, 509, 0, NULL, NULL}, +{248, 511, 0, NULL, NULL}, +{770, 0, 6, compose_tab_1_39, hash_compose_tab_1_39}, +{771, 0, 4, compose_tab_1_40, hash_compose_tab_1_40}, +{772, 0, 4, compose_tab_1_41, hash_compose_tab_1_41}, +{774, 0, 2, compose_tab_1_42, hash_compose_tab_1_42}, +{776, 0, 6, compose_tab_1_43, hash_compose_tab_1_43}, +{778, 0, 2, compose_tab_1_44, hash_compose_tab_1_44}, +{787, 0, 14, compose_tab_1_45, hash_compose_tab_1_45}, +{788, 0, 15, compose_tab_1_46, hash_compose_tab_1_46}, +{795, 0, 4, compose_tab_1_47, hash_compose_tab_1_47}, +{807, 0, 2, compose_tab_1_48, hash_compose_tab_1_48}, +{837, 0, 3, compose_tab_1_49, hash_compose_tab_1_49}, +{913, 8123, 0, NULL, NULL}, +{917, 8137, 0, NULL, NULL}, +{919, 8139, 0, NULL, NULL}, +{921, 8155, 0, NULL, NULL}, +{927, 8185, 0, NULL, NULL}, +{933, 8171, 0, NULL, NULL}, +{937, 8187, 0, NULL, NULL}, +{945, 8049, 0, NULL, NULL}, +{949, 8051, 0, NULL, NULL}, +{951, 8053, 0, NULL, NULL}, +{953, 8055, 0, NULL, NULL}, +{959, 8057, 0, NULL, NULL}, +{965, 8059, 0, NULL, NULL}, +{969, 8061, 0, NULL, NULL}, +{1043, 1027, 0, NULL, NULL}, +{1050, 1036, 0, NULL, NULL}, +{1075, 1107, 0, NULL, NULL}, +{1082, 1116, 0, NULL, NULL}, +{8127, 8142, 0, NULL, NULL}, +{8190, 8158, 0, NULL, NULL} +}; /* compose_tab_1 */ +static int hash_compose_tab_2_26[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_2_26 */ +static CompEntry compose_tab_2_26[] = { +{65, 7852, 0, NULL, NULL}, +{69, 7878, 0, NULL, NULL}, +{79, 7896, 0, NULL, NULL}, +{97, 7853, 0, NULL, NULL}, +{101, 7879, 0, NULL, NULL}, +{111, 7897, 0, NULL, NULL} +}; /* compose_tab_2_26 */ +static int hash_compose_tab_2[54] = +{-1,-1,-1,20,-1,-1,-1,21,-1,22,-1,0,23,1,24,2,25,3,4,5,6,-1,-1,-1,-1,7,-1,-1, + -1,8,-1,9,-1,10,-1,11,12,-1,-1,-1,-1,-1,-1,13,-1,14,-1,15,26,16,17,18,19,-1}; /* hash_compose_tab_2 */ +static CompEntry compose_tab_2[] = { +{65, 194, 0, NULL, NULL}, +{67, 264, 0, NULL, NULL}, +{69, 202, 0, NULL, NULL}, +{71, 284, 0, NULL, NULL}, +{72, 292, 0, NULL, NULL}, +{73, 206, 0, NULL, NULL}, +{74, 308, 0, NULL, NULL}, +{79, 212, 0, NULL, NULL}, +{83, 348, 0, NULL, NULL}, +{85, 219, 0, NULL, NULL}, +{87, 372, 0, NULL, NULL}, +{89, 374, 0, NULL, NULL}, +{90, 7824, 0, NULL, NULL}, +{97, 226, 0, NULL, NULL}, +{99, 265, 0, NULL, NULL}, +{101, 234, 0, NULL, NULL}, +{103, 285, 0, NULL, NULL}, +{104, 293, 0, NULL, NULL}, +{105, 238, 0, NULL, NULL}, +{106, 309, 0, NULL, NULL}, +{111, 244, 0, NULL, NULL}, +{115, 349, 0, NULL, NULL}, +{117, 251, 0, NULL, NULL}, +{119, 373, 0, NULL, NULL}, +{121, 375, 0, NULL, NULL}, +{122, 7825, 0, NULL, NULL}, +{803, 0, 6, compose_tab_2_26, hash_compose_tab_2_26} +}; /* compose_tab_2 */ +static int hash_compose_tab_3_16[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_3_16 */ +static CompEntry compose_tab_3_16[] = { +{65, 7850, 0, NULL, NULL}, +{69, 7876, 0, NULL, NULL}, +{79, 7894, 0, NULL, NULL}, +{97, 7851, 0, NULL, NULL}, +{101, 7877, 0, NULL, NULL}, +{111, 7895, 0, NULL, NULL} +}; /* compose_tab_3_16 */ +static int hash_compose_tab_3_17[4] = +{-1,0,1,-1}; /* hash_compose_tab_3_17 */ +static CompEntry compose_tab_3_17[] = { +{65, 7860, 0, NULL, NULL}, +{97, 7861, 0, NULL, NULL} +}; /* compose_tab_3_17 */ +static int hash_compose_tab_3_18[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_3_18 */ +static CompEntry compose_tab_3_18[] = { +{79, 7904, 0, NULL, NULL}, +{85, 7918, 0, NULL, NULL}, +{111, 7905, 0, NULL, NULL}, +{117, 7919, 0, NULL, NULL} +}; /* compose_tab_3_18 */ +static int hash_compose_tab_3[38] = +{-1,-1,3,4,13,14,-1,15,-1,5,6,16,-1,7,17,-1,-1,-1,-1,-1,-1,8,-1,-1,-1,9,-1,0, + -1,10,-1,1,-1,-1,11,2,12,18}; /* hash_compose_tab_3 */ +static CompEntry compose_tab_3[] = { +{65, 195, 0, NULL, NULL}, +{69, 7868, 0, NULL, NULL}, +{73, 296, 0, NULL, NULL}, +{78, 209, 0, NULL, NULL}, +{79, 213, 0, NULL, NULL}, +{85, 360, 0, NULL, NULL}, +{86, 7804, 0, NULL, NULL}, +{89, 7928, 0, NULL, NULL}, +{97, 227, 0, NULL, NULL}, +{101, 7869, 0, NULL, NULL}, +{105, 297, 0, NULL, NULL}, +{110, 241, 0, NULL, NULL}, +{111, 245, 0, NULL, NULL}, +{117, 361, 0, NULL, NULL}, +{118, 7805, 0, NULL, NULL}, +{121, 7929, 0, NULL, NULL}, +{770, 0, 6, compose_tab_3_16, hash_compose_tab_3_16}, +{774, 0, 2, compose_tab_3_17, hash_compose_tab_3_17}, +{795, 0, 4, compose_tab_3_18, hash_compose_tab_3_18} +}; /* compose_tab_3 */ +static int hash_compose_tab_4_14[4] = +{-1,0,1,-1}; /* hash_compose_tab_4_14 */ +static CompEntry compose_tab_4_14[] = { +{65, 480, 0, NULL, NULL}, +{97, 481, 0, NULL, NULL} +}; /* compose_tab_4_14 */ +static int hash_compose_tab_4_15[8] = +{-1,0,2,-1,-1,1,3,-1}; /* hash_compose_tab_4_15 */ +static CompEntry compose_tab_4_15[] = { +{65, 478, 0, NULL, NULL}, +{85, 469, 0, NULL, NULL}, +{97, 479, 0, NULL, NULL}, +{117, 470, 0, NULL, NULL} +}; /* compose_tab_4_15 */ +static int hash_compose_tab_4_16[8] = +{-1,-1,1,3,0,2,-1,-1}; /* hash_compose_tab_4_16 */ +static CompEntry compose_tab_4_16[] = { +{76, 7736, 0, NULL, NULL}, +{82, 7772, 0, NULL, NULL}, +{108, 7737, 0, NULL, NULL}, +{114, 7773, 0, NULL, NULL} +}; /* compose_tab_4_16 */ +static int hash_compose_tab_4_17[4] = +{1,-1,-1,0}; /* hash_compose_tab_4_17 */ +static CompEntry compose_tab_4_17[] = { +{79, 492, 0, NULL, NULL}, +{111, 493, 0, NULL, NULL} +}; /* compose_tab_4_17 */ +static int hash_compose_tab_4[56] = +{-1,22,-1,-1,-1,11,13,-1,-1,0,-1,-1,-1,1,23,2,26,3,18,16,-1,-1,-1,4,17,19,-1, + 27,-1,5,12,-1,-1,-1,-1,-1,-1,20,-1,-1,24,6,-1,-1,-1,7,-1,8,14,9,15,21,25,-1, + -1,10}; /* hash_compose_tab_4 */ +static CompEntry compose_tab_4[] = { +{65, 256, 0, NULL, NULL}, +{69, 274, 0, NULL, NULL}, +{71, 7712, 0, NULL, NULL}, +{73, 298, 0, NULL, NULL}, +{79, 332, 0, NULL, NULL}, +{85, 362, 0, NULL, NULL}, +{97, 257, 0, NULL, NULL}, +{101, 275, 0, NULL, NULL}, +{103, 7713, 0, NULL, NULL}, +{105, 299, 0, NULL, NULL}, +{111, 333, 0, NULL, NULL}, +{117, 363, 0, NULL, NULL}, +{198, 482, 0, NULL, NULL}, +{230, 483, 0, NULL, NULL}, +{775, 0, 2, compose_tab_4_14, hash_compose_tab_4_14}, +{776, 0, 4, compose_tab_4_15, hash_compose_tab_4_15}, +{803, 0, 4, compose_tab_4_16, hash_compose_tab_4_16}, +{808, 0, 2, compose_tab_4_17, hash_compose_tab_4_17}, +{913, 8121, 0, NULL, NULL}, +{921, 8153, 0, NULL, NULL}, +{933, 8169, 0, NULL, NULL}, +{945, 8113, 0, NULL, NULL}, +{953, 8145, 0, NULL, NULL}, +{965, 8161, 0, NULL, NULL}, +{1048, 1250, 0, NULL, NULL}, +{1059, 1262, 0, NULL, NULL}, +{1080, 1251, 0, NULL, NULL}, +{1091, 1263, 0, NULL, NULL} +}; /* compose_tab_4 */ +static int hash_compose_tab_5_12[4] = +{-1,0,1,-1}; /* hash_compose_tab_5_12 */ +static CompEntry compose_tab_5_12[] = { +{65, 7862, 0, NULL, NULL}, +{97, 7863, 0, NULL, NULL} +}; /* compose_tab_5_12 */ +static int hash_compose_tab_5_13[4] = +{-1,0,1,-1}; /* hash_compose_tab_5_13 */ +static CompEntry compose_tab_5_13[] = { +{69, 7708, 0, NULL, NULL}, +{101, 7709, 0, NULL, NULL} +}; /* compose_tab_5_13 */ +static int hash_compose_tab_5[60] = +{28,-1,-1,-1,-1,0,19,-1,-1,1,-1,2,29,3,14,-1,-1,-1,-1,4,20,15,-1,12,-1,5,21, + 13,22,23,-1,-1,-1,16,-1,-1,-1,6,-1,24,-1,7,-1,8,-1,9,17,-1,-1,-1,-1,10,25,18, + -1,-1,-1,11,26,27}; /* hash_compose_tab_5 */ +static CompEntry compose_tab_5[] = { +{65, 258, 0, NULL, NULL}, +{69, 276, 0, NULL, NULL}, +{71, 286, 0, NULL, NULL}, +{73, 300, 0, NULL, NULL}, +{79, 334, 0, NULL, NULL}, +{85, 364, 0, NULL, NULL}, +{97, 259, 0, NULL, NULL}, +{101, 277, 0, NULL, NULL}, +{103, 287, 0, NULL, NULL}, +{105, 301, 0, NULL, NULL}, +{111, 335, 0, NULL, NULL}, +{117, 365, 0, NULL, NULL}, +{803, 0, 2, compose_tab_5_12, hash_compose_tab_5_12}, +{807, 0, 2, compose_tab_5_13, hash_compose_tab_5_13}, +{913, 8120, 0, NULL, NULL}, +{921, 8152, 0, NULL, NULL}, +{933, 8168, 0, NULL, NULL}, +{945, 8112, 0, NULL, NULL}, +{953, 8144, 0, NULL, NULL}, +{965, 8160, 0, NULL, NULL}, +{1040, 1232, 0, NULL, NULL}, +{1045, 1238, 0, NULL, NULL}, +{1046, 1217, 0, NULL, NULL}, +{1048, 1049, 0, NULL, NULL}, +{1059, 1038, 0, NULL, NULL}, +{1072, 1233, 0, NULL, NULL}, +{1077, 1239, 0, NULL, NULL}, +{1078, 1218, 0, NULL, NULL}, +{1080, 1081, 0, NULL, NULL}, +{1091, 1118, 0, NULL, NULL} +}; /* compose_tab_5 */ +static int hash_compose_tab_6_36[4] = +{1,-1,-1,0}; /* hash_compose_tab_6_36 */ +static CompEntry compose_tab_6_36[] = { +{83, 7780, 0, NULL, NULL}, +{115, 7781, 0, NULL, NULL} +}; /* compose_tab_6_36 */ +static int hash_compose_tab_6_38[4] = +{1,-1,-1,0}; /* hash_compose_tab_6_38 */ +static CompEntry compose_tab_6_38[] = { +{83, 7782, 0, NULL, NULL}, +{115, 7783, 0, NULL, NULL} +}; /* compose_tab_6_38 */ +static int hash_compose_tab_6_39[4] = +{1,-1,-1,0}; /* hash_compose_tab_6_39 */ +static CompEntry compose_tab_6_39[] = { +{83, 7784, 0, NULL, NULL}, +{115, 7785, 0, NULL, NULL} +}; /* compose_tab_6_39 */ +static int hash_compose_tab_6[80] = +{10,-1,11,12,13,39,-1,14,15,16,17,-1,-1,-1,-1,-1,-1,-1,18,19,20,21,22,23,24, + -1,-1,-1,-1,25,26,-1,27,-1,28,29,30,-1,-1,31,32,33,34,-1,-1,-1,-1,-1,-1,36, + -1,-1,-1,-1,37,-1,-1,-1,-1,-1,38,-1,-1,35,-1,-1,0,1,2,3,4,5,6,7,-1,-1,-1,8,9, + -1}; /* hash_compose_tab_6 */ +static CompEntry compose_tab_6[] = { +{66, 7682, 0, NULL, NULL}, +{67, 266, 0, NULL, NULL}, +{68, 7690, 0, NULL, NULL}, +{69, 278, 0, NULL, NULL}, +{70, 7710, 0, NULL, NULL}, +{71, 288, 0, NULL, NULL}, +{72, 7714, 0, NULL, NULL}, +{73, 304, 0, NULL, NULL}, +{77, 7744, 0, NULL, NULL}, +{78, 7748, 0, NULL, NULL}, +{80, 7766, 0, NULL, NULL}, +{82, 7768, 0, NULL, NULL}, +{83, 7776, 0, NULL, NULL}, +{84, 7786, 0, NULL, NULL}, +{87, 7814, 0, NULL, NULL}, +{88, 7818, 0, NULL, NULL}, +{89, 7822, 0, NULL, NULL}, +{90, 379, 0, NULL, NULL}, +{98, 7683, 0, NULL, NULL}, +{99, 267, 0, NULL, NULL}, +{100, 7691, 0, NULL, NULL}, +{101, 279, 0, NULL, NULL}, +{102, 7711, 0, NULL, NULL}, +{103, 289, 0, NULL, NULL}, +{104, 7715, 0, NULL, NULL}, +{109, 7745, 0, NULL, NULL}, +{110, 7749, 0, NULL, NULL}, +{112, 7767, 0, NULL, NULL}, +{114, 7769, 0, NULL, NULL}, +{115, 7777, 0, NULL, NULL}, +{116, 7787, 0, NULL, NULL}, +{119, 7815, 0, NULL, NULL}, +{120, 7819, 0, NULL, NULL}, +{121, 7823, 0, NULL, NULL}, +{122, 380, 0, NULL, NULL}, +{383, 7835, 0, NULL, NULL}, +{769, 0, 2, compose_tab_6_36, hash_compose_tab_6_36}, +{774, 784, 0, NULL, NULL}, +{780, 0, 2, compose_tab_6_38, hash_compose_tab_6_38}, +{803, 0, 2, compose_tab_6_39, hash_compose_tab_6_39} +}; /* compose_tab_6 */ +static int hash_compose_tab_7_23[4] = +{1,-1,-1,0}; /* hash_compose_tab_7_23 */ +static CompEntry compose_tab_7_23[] = { +{79, 7758, 0, NULL, NULL}, +{111, 7759, 0, NULL, NULL} +}; /* compose_tab_7_23 */ +static int hash_compose_tab_7_24[4] = +{-1,0,1,-1}; /* hash_compose_tab_7_24 */ +static CompEntry compose_tab_7_24[] = { +{85, 7802, 0, NULL, NULL}, +{117, 7803, 0, NULL, NULL} +}; /* compose_tab_7_24 */ +static int hash_compose_tab_7[100] = +{48,10,21,-1,11,12,-1,-1,-1,-1,49,13,-1,-1,-1,20,14,15,-1,16,17,18,25,-1,-1, + -1,-1,-1,-1,22,30,-1,-1,26,-1,-1,-1,-1,-1,-1,31,-1,-1,-1,-1,32,33,34,35,-1, + -1,-1,-1,27,36,-1,-1,-1,-1,37,-1,-1,-1,38,-1,0,28,39,-1,1,-1,23,2,3,24,40,-1, + 41,29,4,42,43,44,-1,-1,5,45,6,7,8,-1,46,-1,-1,-1,47,-1,9,-1,19}; /* hash_compose_tab_7 */ +static CompEntry compose_tab_7[] = { +{65, 196, 0, NULL, NULL}, +{69, 203, 0, NULL, NULL}, +{72, 7718, 0, NULL, NULL}, +{73, 207, 0, NULL, NULL}, +{79, 214, 0, NULL, NULL}, +{85, 220, 0, NULL, NULL}, +{87, 7812, 0, NULL, NULL}, +{88, 7820, 0, NULL, NULL}, +{89, 376, 0, NULL, NULL}, +{97, 228, 0, NULL, NULL}, +{101, 235, 0, NULL, NULL}, +{104, 7719, 0, NULL, NULL}, +{105, 239, 0, NULL, NULL}, +{111, 246, 0, NULL, NULL}, +{116, 7831, 0, NULL, NULL}, +{117, 252, 0, NULL, NULL}, +{119, 7813, 0, NULL, NULL}, +{120, 7821, 0, NULL, NULL}, +{121, 255, 0, NULL, NULL}, +{399, 1242, 0, NULL, NULL}, +{415, 1258, 0, NULL, NULL}, +{601, 1243, 0, NULL, NULL}, +{629, 1259, 0, NULL, NULL}, +{771, 0, 2, compose_tab_7_23, hash_compose_tab_7_23}, +{772, 0, 2, compose_tab_7_24, hash_compose_tab_7_24}, +{921, 938, 0, NULL, NULL}, +{933, 939, 0, NULL, NULL}, +{953, 970, 0, NULL, NULL}, +{965, 971, 0, NULL, NULL}, +{978, 980, 0, NULL, NULL}, +{1030, 1031, 0, NULL, NULL}, +{1040, 1234, 0, NULL, NULL}, +{1045, 1025, 0, NULL, NULL}, +{1046, 1244, 0, NULL, NULL}, +{1047, 1246, 0, NULL, NULL}, +{1048, 1252, 0, NULL, NULL}, +{1054, 1254, 0, NULL, NULL}, +{1059, 1264, 0, NULL, NULL}, +{1063, 1268, 0, NULL, NULL}, +{1067, 1272, 0, NULL, NULL}, +{1072, 1235, 0, NULL, NULL}, +{1077, 1105, 0, NULL, NULL}, +{1078, 1245, 0, NULL, NULL}, +{1079, 1247, 0, NULL, NULL}, +{1080, 1253, 0, NULL, NULL}, +{1086, 1255, 0, NULL, NULL}, +{1091, 1265, 0, NULL, NULL}, +{1095, 1269, 0, NULL, NULL}, +{1099, 1273, 0, NULL, NULL}, +{1110, 1111, 0, NULL, NULL} +}; /* compose_tab_7 */ +static int hash_compose_tab_8_12[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_8_12 */ +static CompEntry compose_tab_8_12[] = { +{65, 7848, 0, NULL, NULL}, +{69, 7874, 0, NULL, NULL}, +{79, 7892, 0, NULL, NULL}, +{97, 7849, 0, NULL, NULL}, +{101, 7875, 0, NULL, NULL}, +{111, 7893, 0, NULL, NULL} +}; /* compose_tab_8_12 */ +static int hash_compose_tab_8_13[4] = +{-1,0,1,-1}; /* hash_compose_tab_8_13 */ +static CompEntry compose_tab_8_13[] = { +{65, 7858, 0, NULL, NULL}, +{97, 7859, 0, NULL, NULL} +}; /* compose_tab_8_13 */ +static int hash_compose_tab_8_14[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_8_14 */ +static CompEntry compose_tab_8_14[] = { +{79, 7902, 0, NULL, NULL}, +{85, 7916, 0, NULL, NULL}, +{111, 7903, 0, NULL, NULL}, +{117, 7917, 0, NULL, NULL} +}; /* compose_tab_8_14 */ +static int hash_compose_tab_8[30] = +{-1,11,-1,-1,-1,0,-1,6,-1,1,-1,7,-1,2,-1,8,14,-1,-1,3,12,9,-1,-1,13,4,-1,10, + -1,5}; /* hash_compose_tab_8 */ +static CompEntry compose_tab_8[] = { +{65, 7842, 0, NULL, NULL}, +{69, 7866, 0, NULL, NULL}, +{73, 7880, 0, NULL, NULL}, +{79, 7886, 0, NULL, NULL}, +{85, 7910, 0, NULL, NULL}, +{89, 7926, 0, NULL, NULL}, +{97, 7843, 0, NULL, NULL}, +{101, 7867, 0, NULL, NULL}, +{105, 7881, 0, NULL, NULL}, +{111, 7887, 0, NULL, NULL}, +{117, 7911, 0, NULL, NULL}, +{121, 7927, 0, NULL, NULL}, +{770, 0, 6, compose_tab_8_12, hash_compose_tab_8_12}, +{774, 0, 2, compose_tab_8_13, hash_compose_tab_8_13}, +{795, 0, 4, compose_tab_8_14, hash_compose_tab_8_14} +}; /* compose_tab_8 */ +static int hash_compose_tab_9[12] = +{-1,1,2,5,-1,0,-1,-1,-1,3,-1,4}; /* hash_compose_tab_9 */ +static CompEntry compose_tab_9[] = { +{65, 197, 0, NULL, NULL}, +{85, 366, 0, NULL, NULL}, +{97, 229, 0, NULL, NULL}, +{117, 367, 0, NULL, NULL}, +{119, 7832, 0, NULL, NULL}, +{121, 7833, 0, NULL, NULL} +}; /* compose_tab_9 */ +static int hash_compose_tab_10[12] = +{-1,1,-1,2,4,-1,-1,0,-1,3,-1,5}; /* hash_compose_tab_10 */ +static CompEntry compose_tab_10[] = { +{79, 336, 0, NULL, NULL}, +{85, 368, 0, NULL, NULL}, +{111, 337, 0, NULL, NULL}, +{117, 369, 0, NULL, NULL}, +{1059, 1266, 0, NULL, NULL}, +{1091, 1267, 0, NULL, NULL} +}; /* compose_tab_10 */ +static int hash_compose_tab_11_33[4] = +{-1,0,1,-1}; /* hash_compose_tab_11_33 */ +static CompEntry compose_tab_11_33[] = { +{85, 473, 0, NULL, NULL}, +{117, 474, 0, NULL, NULL} +}; /* compose_tab_11_33 */ +static int hash_compose_tab_11[68] = +{2,3,-1,4,-1,5,-1,6,7,-1,8,9,-1,-1,10,11,12,13,-1,-1,-1,-1,14,-1,-1,-1,-1,-1, + 33,15,-1,16,17,18,31,19,-1,20,21,22,23,-1,24,25,-1,-1,26,27,28,29,32,-1,-1, + -1,30,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,-1,1}; /* hash_compose_tab_11 */ +static CompEntry compose_tab_11[] = { +{65, 461, 0, NULL, NULL}, +{67, 268, 0, NULL, NULL}, +{68, 270, 0, NULL, NULL}, +{69, 282, 0, NULL, NULL}, +{71, 486, 0, NULL, NULL}, +{73, 463, 0, NULL, NULL}, +{75, 488, 0, NULL, NULL}, +{76, 317, 0, NULL, NULL}, +{78, 327, 0, NULL, NULL}, +{79, 465, 0, NULL, NULL}, +{82, 344, 0, NULL, NULL}, +{83, 352, 0, NULL, NULL}, +{84, 356, 0, NULL, NULL}, +{85, 467, 0, NULL, NULL}, +{90, 381, 0, NULL, NULL}, +{97, 462, 0, NULL, NULL}, +{99, 269, 0, NULL, NULL}, +{100, 271, 0, NULL, NULL}, +{101, 283, 0, NULL, NULL}, +{103, 487, 0, NULL, NULL}, +{105, 464, 0, NULL, NULL}, +{106, 496, 0, NULL, NULL}, +{107, 489, 0, NULL, NULL}, +{108, 318, 0, NULL, NULL}, +{110, 328, 0, NULL, NULL}, +{111, 466, 0, NULL, NULL}, +{114, 345, 0, NULL, NULL}, +{115, 353, 0, NULL, NULL}, +{116, 357, 0, NULL, NULL}, +{117, 468, 0, NULL, NULL}, +{122, 382, 0, NULL, NULL}, +{439, 494, 0, NULL, NULL}, +{658, 495, 0, NULL, NULL}, +{776, 0, 2, compose_tab_11_33, hash_compose_tab_11_33} +}; /* compose_tab_11 */ +static int hash_compose_tab_12_1[4] = +{-1,0,1,-1}; /* hash_compose_tab_12_1 */ +static CompEntry compose_tab_12_1[] = { +{953, 912, 0, NULL, NULL}, +{965, 944, 0, NULL, NULL} +}; /* compose_tab_12_1 */ +static int hash_compose_tab_12[34] = +{11,4,12,5,-1,-1,-1,13,-1,6,-1,-1,-1,14,-1,7,-1,15,-1,8,-1,-1,-1,-1,-1,-1,16, + 9,1,2,-1,10,0,3}; /* hash_compose_tab_12 */ +static CompEntry compose_tab_12[] = { +{168, 901, 0, NULL, NULL}, +{776, 0, 2, compose_tab_12_1, hash_compose_tab_12_1}, +{913, 902, 0, NULL, NULL}, +{917, 904, 0, NULL, NULL}, +{919, 905, 0, NULL, NULL}, +{921, 906, 0, NULL, NULL}, +{927, 908, 0, NULL, NULL}, +{933, 910, 0, NULL, NULL}, +{937, 911, 0, NULL, NULL}, +{945, 940, 0, NULL, NULL}, +{949, 941, 0, NULL, NULL}, +{951, 942, 0, NULL, NULL}, +{953, 943, 0, NULL, NULL}, +{959, 972, 0, NULL, NULL}, +{965, 973, 0, NULL, NULL}, +{969, 974, 0, NULL, NULL}, +{978, 979, 0, NULL, NULL} +}; /* compose_tab_12 */ +static int hash_compose_tab_13[28] = +{-1,5,10,-1,-1,11,-1,-1,-1,0,-1,-1,-1,1,6,-1,-1,2,7,-1,12,8,13,3,-1,-1,4,9}; /* hash_compose_tab_13 */ +static CompEntry compose_tab_13[] = { +{65, 512, 0, NULL, NULL}, +{69, 516, 0, NULL, NULL}, +{73, 520, 0, NULL, NULL}, +{79, 524, 0, NULL, NULL}, +{82, 528, 0, NULL, NULL}, +{85, 532, 0, NULL, NULL}, +{97, 513, 0, NULL, NULL}, +{101, 517, 0, NULL, NULL}, +{105, 521, 0, NULL, NULL}, +{111, 525, 0, NULL, NULL}, +{114, 529, 0, NULL, NULL}, +{117, 533, 0, NULL, NULL}, +{1140, 1142, 0, NULL, NULL}, +{1141, 1143, 0, NULL, NULL} +}; /* compose_tab_13 */ +static int hash_compose_tab_14[24] = +{-1,2,6,-1,-1,7,-1,3,-1,8,4,-1,-1,5,-1,9,-1,0,10,-1,-1,1,11,-1}; /* hash_compose_tab_14 */ +static CompEntry compose_tab_14[] = { +{65, 514, 0, NULL, NULL}, +{69, 518, 0, NULL, NULL}, +{73, 522, 0, NULL, NULL}, +{79, 526, 0, NULL, NULL}, +{82, 530, 0, NULL, NULL}, +{85, 534, 0, NULL, NULL}, +{97, 515, 0, NULL, NULL}, +{101, 519, 0, NULL, NULL}, +{105, 523, 0, NULL, NULL}, +{111, 527, 0, NULL, NULL}, +{114, 531, 0, NULL, NULL}, +{117, 535, 0, NULL, NULL} +}; /* compose_tab_14 */ +static int hash_compose_tab_15_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_15_0 */ +static CompEntry compose_tab_15_0[] = { +{913, 8072, 0, NULL, NULL}, +{919, 8088, 0, NULL, NULL}, +{937, 8104, 0, NULL, NULL}, +{945, 8064, 0, NULL, NULL}, +{951, 8080, 0, NULL, NULL}, +{969, 8096, 0, NULL, NULL} +}; /* compose_tab_15_0 */ +static int hash_compose_tab_15[30] = +{-1,12,-1,-1,-1,13,-1,6,-1,14,-1,-1,-1,1,-1,7,-1,2,-1,3,8,4,9,10,-1,-1,-1,0,5, + 11}; /* hash_compose_tab_15 */ +static CompEntry compose_tab_15[] = { +{837, 0, 6, compose_tab_15_0, hash_compose_tab_15_0}, +{913, 7944, 0, NULL, NULL}, +{917, 7960, 0, NULL, NULL}, +{919, 7976, 0, NULL, NULL}, +{921, 7992, 0, NULL, NULL}, +{927, 8008, 0, NULL, NULL}, +{937, 8040, 0, NULL, NULL}, +{945, 7936, 0, NULL, NULL}, +{949, 7952, 0, NULL, NULL}, +{951, 7968, 0, NULL, NULL}, +{953, 7984, 0, NULL, NULL}, +{959, 8000, 0, NULL, NULL}, +{961, 8164, 0, NULL, NULL}, +{965, 8016, 0, NULL, NULL}, +{969, 8032, 0, NULL, NULL} +}; /* compose_tab_15 */ +static int hash_compose_tab_16_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_16_0 */ +static CompEntry compose_tab_16_0[] = { +{913, 8073, 0, NULL, NULL}, +{919, 8089, 0, NULL, NULL}, +{937, 8105, 0, NULL, NULL}, +{945, 8065, 0, NULL, NULL}, +{951, 8081, 0, NULL, NULL}, +{969, 8097, 0, NULL, NULL} +}; /* compose_tab_16_0 */ +static int hash_compose_tab_16[34] = +{11,3,12,4,-1,-1,-1,13,-1,5,14,6,-1,15,-1,7,-1,16,-1,8,-1,0,-1,-1,-1,-1,-1,9, + -1,1,-1,10,-1,2}; /* hash_compose_tab_16 */ +static CompEntry compose_tab_16[] = { +{837, 0, 6, compose_tab_16_0, hash_compose_tab_16_0}, +{913, 7945, 0, NULL, NULL}, +{917, 7961, 0, NULL, NULL}, +{919, 7977, 0, NULL, NULL}, +{921, 7993, 0, NULL, NULL}, +{927, 8009, 0, NULL, NULL}, +{929, 8172, 0, NULL, NULL}, +{933, 8025, 0, NULL, NULL}, +{937, 8041, 0, NULL, NULL}, +{945, 7937, 0, NULL, NULL}, +{949, 7953, 0, NULL, NULL}, +{951, 7969, 0, NULL, NULL}, +{953, 7985, 0, NULL, NULL}, +{959, 8001, 0, NULL, NULL}, +{961, 8165, 0, NULL, NULL}, +{965, 8017, 0, NULL, NULL}, +{969, 8033, 0, NULL, NULL} +}; /* compose_tab_16 */ +static int hash_compose_tab_17[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_17 */ +static CompEntry compose_tab_17[] = { +{79, 416, 0, NULL, NULL}, +{85, 431, 0, NULL, NULL}, +{111, 417, 0, NULL, NULL}, +{117, 432, 0, NULL, NULL} +}; /* compose_tab_17 */ +static int hash_compose_tab_18_38[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_18_38 */ +static CompEntry compose_tab_18_38[] = { +{79, 7906, 0, NULL, NULL}, +{85, 7920, 0, NULL, NULL}, +{111, 7907, 0, NULL, NULL}, +{117, 7921, 0, NULL, NULL} +}; /* compose_tab_18_38 */ +static int hash_compose_tab_18[78] = +{9,10,-1,-1,11,12,13,14,15,16,-1,17,18,-1,-1,38,-1,-1,-1,19,20,-1,21,22,-1,-1, + 23,24,-1,25,26,27,28,29,-1,-1,30,31,32,33,34,35,-1,36,37,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,-1,2,3,-1,-1,4,5,-1,6,7,8}; /* hash_compose_tab_18 */ +static CompEntry compose_tab_18[] = { +{65, 7840, 0, NULL, NULL}, +{66, 7684, 0, NULL, NULL}, +{68, 7692, 0, NULL, NULL}, +{69, 7864, 0, NULL, NULL}, +{72, 7716, 0, NULL, NULL}, +{73, 7882, 0, NULL, NULL}, +{75, 7730, 0, NULL, NULL}, +{76, 7734, 0, NULL, NULL}, +{77, 7746, 0, NULL, NULL}, +{78, 7750, 0, NULL, NULL}, +{79, 7884, 0, NULL, NULL}, +{82, 7770, 0, NULL, NULL}, +{83, 7778, 0, NULL, NULL}, +{84, 7788, 0, NULL, NULL}, +{85, 7908, 0, NULL, NULL}, +{86, 7806, 0, NULL, NULL}, +{87, 7816, 0, NULL, NULL}, +{89, 7924, 0, NULL, NULL}, +{90, 7826, 0, NULL, NULL}, +{97, 7841, 0, NULL, NULL}, +{98, 7685, 0, NULL, NULL}, +{100, 7693, 0, NULL, NULL}, +{101, 7865, 0, NULL, NULL}, +{104, 7717, 0, NULL, NULL}, +{105, 7883, 0, NULL, NULL}, +{107, 7731, 0, NULL, NULL}, +{108, 7735, 0, NULL, NULL}, +{109, 7747, 0, NULL, NULL}, +{110, 7751, 0, NULL, NULL}, +{111, 7885, 0, NULL, NULL}, +{114, 7771, 0, NULL, NULL}, +{115, 7779, 0, NULL, NULL}, +{116, 7789, 0, NULL, NULL}, +{117, 7909, 0, NULL, NULL}, +{118, 7807, 0, NULL, NULL}, +{119, 7817, 0, NULL, NULL}, +{121, 7925, 0, NULL, NULL}, +{122, 7827, 0, NULL, NULL}, +{795, 0, 4, compose_tab_18_38, hash_compose_tab_18_38} +}; /* compose_tab_18 */ +static int hash_compose_tab_19[4] = +{-1,0,1,-1}; /* hash_compose_tab_19 */ +static CompEntry compose_tab_19[] = { +{85, 7794, 0, NULL, NULL}, +{117, 7795, 0, NULL, NULL} +}; /* compose_tab_19 */ +static int hash_compose_tab_20[4] = +{-1,0,1,-1}; /* hash_compose_tab_20 */ +static CompEntry compose_tab_20[] = { +{65, 7680, 0, NULL, NULL}, +{97, 7681, 0, NULL, NULL} +}; /* compose_tab_20 */ +static int hash_compose_tab_21[40] = +{-1,-1,7,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,10,11,-1,-1,12,13,-1, + -1,0,1,14,15,2,3,16,17,4,5,18,6,19}; /* hash_compose_tab_21 */ +static CompEntry compose_tab_21[] = { +{67, 199, 0, NULL, NULL}, +{68, 7696, 0, NULL, NULL}, +{71, 290, 0, NULL, NULL}, +{72, 7720, 0, NULL, NULL}, +{75, 310, 0, NULL, NULL}, +{76, 315, 0, NULL, NULL}, +{78, 325, 0, NULL, NULL}, +{82, 342, 0, NULL, NULL}, +{83, 350, 0, NULL, NULL}, +{84, 354, 0, NULL, NULL}, +{99, 231, 0, NULL, NULL}, +{100, 7697, 0, NULL, NULL}, +{103, 291, 0, NULL, NULL}, +{104, 7721, 0, NULL, NULL}, +{107, 311, 0, NULL, NULL}, +{108, 316, 0, NULL, NULL}, +{110, 326, 0, NULL, NULL}, +{114, 343, 0, NULL, NULL}, +{115, 351, 0, NULL, NULL}, +{116, 355, 0, NULL, NULL} +}; /* compose_tab_21 */ +static int hash_compose_tab_22[20] = +{-1,6,-1,-1,-1,0,4,7,-1,1,-1,8,-1,2,-1,-1,-1,5,9,3}; /* hash_compose_tab_22 */ +static CompEntry compose_tab_22[] = { +{65, 260, 0, NULL, NULL}, +{69, 280, 0, NULL, NULL}, +{73, 302, 0, NULL, NULL}, +{79, 490, 0, NULL, NULL}, +{85, 370, 0, NULL, NULL}, +{97, 261, 0, NULL, NULL}, +{101, 281, 0, NULL, NULL}, +{105, 303, 0, NULL, NULL}, +{111, 491, 0, NULL, NULL}, +{117, 371, 0, NULL, NULL} +}; /* compose_tab_22 */ +static int hash_compose_tab_23[24] = +{-1,-1,-1,-1,2,6,3,7,-1,-1,-1,-1,4,5,8,9,-1,-1,-1,-1,0,1,10,11}; /* hash_compose_tab_23 */ +static CompEntry compose_tab_23[] = { +{68, 7698, 0, NULL, NULL}, +{69, 7704, 0, NULL, NULL}, +{76, 7740, 0, NULL, NULL}, +{78, 7754, 0, NULL, NULL}, +{84, 7792, 0, NULL, NULL}, +{85, 7798, 0, NULL, NULL}, +{100, 7699, 0, NULL, NULL}, +{101, 7705, 0, NULL, NULL}, +{108, 7741, 0, NULL, NULL}, +{110, 7755, 0, NULL, NULL}, +{116, 7793, 0, NULL, NULL}, +{117, 7799, 0, NULL, NULL} +}; /* compose_tab_23 */ +static int hash_compose_tab_24[4] = +{0,1,-1,-1}; /* hash_compose_tab_24 */ +static CompEntry compose_tab_24[] = { +{72, 7722, 0, NULL, NULL}, +{104, 7723, 0, NULL, NULL} +}; /* compose_tab_24 */ +static int hash_compose_tab_25[12] = +{-1,1,2,-1,-1,3,-1,-1,-1,0,4,5}; /* hash_compose_tab_25 */ +static CompEntry compose_tab_25[] = { +{69, 7706, 0, NULL, NULL}, +{73, 7724, 0, NULL, NULL}, +{85, 7796, 0, NULL, NULL}, +{101, 7707, 0, NULL, NULL}, +{105, 7725, 0, NULL, NULL}, +{117, 7797, 0, NULL, NULL} +}; /* compose_tab_25 */ +static int hash_compose_tab_26[34] = +{1,-1,10,-1,-1,11,12,2,3,13,4,-1,14,-1,5,15,6,-1,-1,-1,16,-1,7,-1,-1,-1,-1,-1, + -1,-1,8,-1,0,9}; /* hash_compose_tab_26 */ +static CompEntry compose_tab_26[] = { +{66, 7686, 0, NULL, NULL}, +{68, 7694, 0, NULL, NULL}, +{75, 7732, 0, NULL, NULL}, +{76, 7738, 0, NULL, NULL}, +{78, 7752, 0, NULL, NULL}, +{82, 7774, 0, NULL, NULL}, +{84, 7790, 0, NULL, NULL}, +{90, 7828, 0, NULL, NULL}, +{98, 7687, 0, NULL, NULL}, +{100, 7695, 0, NULL, NULL}, +{104, 7830, 0, NULL, NULL}, +{107, 7733, 0, NULL, NULL}, +{108, 7739, 0, NULL, NULL}, +{110, 7753, 0, NULL, NULL}, +{114, 7775, 0, NULL, NULL}, +{116, 7791, 0, NULL, NULL}, +{122, 7829, 0, NULL, NULL} +}; /* compose_tab_26 */ +static int hash_compose_tab_27_1[4] = +{-1,0,1,-1}; /* hash_compose_tab_27_1 */ +static CompEntry compose_tab_27_1[] = { +{953, 8151, 0, NULL, NULL}, +{965, 8167, 0, NULL, NULL} +}; /* compose_tab_27_1 */ +static int hash_compose_tab_27_2_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_2_0 */ +static CompEntry compose_tab_27_2_0[] = { +{913, 8078, 0, NULL, NULL}, +{919, 8094, 0, NULL, NULL}, +{937, 8110, 0, NULL, NULL}, +{945, 8070, 0, NULL, NULL}, +{951, 8086, 0, NULL, NULL}, +{969, 8102, 0, NULL, NULL} +}; /* compose_tab_27_2_0 */ +static int hash_compose_tab_27_2[20] = +{-1,3,-1,-1,-1,5,8,-1,-1,9,-1,6,-1,1,7,-1,-1,0,4,2}; /* hash_compose_tab_27_2 */ +static CompEntry compose_tab_27_2[] = { +{837, 0, 6, compose_tab_27_2_0, hash_compose_tab_27_2_0}, +{913, 7950, 0, NULL, NULL}, +{919, 7982, 0, NULL, NULL}, +{921, 7998, 0, NULL, NULL}, +{937, 8046, 0, NULL, NULL}, +{945, 7942, 0, NULL, NULL}, +{951, 7974, 0, NULL, NULL}, +{953, 7990, 0, NULL, NULL}, +{965, 8022, 0, NULL, NULL}, +{969, 8038, 0, NULL, NULL} +}; /* compose_tab_27_2 */ +static int hash_compose_tab_27_3_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_3_0 */ +static CompEntry compose_tab_27_3_0[] = { +{913, 8079, 0, NULL, NULL}, +{919, 8095, 0, NULL, NULL}, +{937, 8111, 0, NULL, NULL}, +{945, 8071, 0, NULL, NULL}, +{951, 8087, 0, NULL, NULL}, +{969, 8103, 0, NULL, NULL} +}; /* compose_tab_27_3_0 */ +static int hash_compose_tab_27_3[22] = +{-1,0,10,-1,-1,7,-1,8,-1,4,-1,1,-1,5,-1,-1,-1,2,-1,3,9,6}; /* hash_compose_tab_27_3 */ +static CompEntry compose_tab_27_3[] = { +{837, 0, 6, compose_tab_27_3_0, hash_compose_tab_27_3_0}, +{913, 7951, 0, NULL, NULL}, +{919, 7983, 0, NULL, NULL}, +{921, 7999, 0, NULL, NULL}, +{933, 8031, 0, NULL, NULL}, +{937, 8047, 0, NULL, NULL}, +{945, 7943, 0, NULL, NULL}, +{951, 7975, 0, NULL, NULL}, +{953, 7991, 0, NULL, NULL}, +{965, 8023, 0, NULL, NULL}, +{969, 8039, 0, NULL, NULL} +}; /* compose_tab_27_3 */ +static int hash_compose_tab_27_4[6] = +{-1,-1,-1,0,1,2}; /* hash_compose_tab_27_4 */ +static CompEntry compose_tab_27_4[] = { +{945, 8119, 0, NULL, NULL}, +{951, 8135, 0, NULL, NULL}, +{969, 8183, 0, NULL, NULL} +}; /* compose_tab_27_4 */ +static int hash_compose_tab_27[24] = +{0,-1,-1,-1,-1,8,11,-1,1,5,9,-1,-1,-1,-1,6,10,7,-1,2,3,4,-1,-1}; /* hash_compose_tab_27 */ +static CompEntry compose_tab_27[] = { +{168, 8129, 0, NULL, NULL}, +{776, 0, 2, compose_tab_27_1, hash_compose_tab_27_1}, +{787, 0, 10, compose_tab_27_2, hash_compose_tab_27_2}, +{788, 0, 11, compose_tab_27_3, hash_compose_tab_27_3}, +{837, 0, 3, compose_tab_27_4, hash_compose_tab_27_4}, +{945, 8118, 0, NULL, NULL}, +{951, 8134, 0, NULL, NULL}, +{953, 8150, 0, NULL, NULL}, +{965, 8166, 0, NULL, NULL}, +{969, 8182, 0, NULL, NULL}, +{8127, 8143, 0, NULL, NULL}, +{8190, 8159, 0, NULL, NULL} +}; /* compose_tab_27 */ +static int hash_compose_tab_28[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_28 */ +static CompEntry compose_tab_28[] = { +{913, 8124, 0, NULL, NULL}, +{919, 8140, 0, NULL, NULL}, +{937, 8188, 0, NULL, NULL}, +{945, 8115, 0, NULL, NULL}, +{951, 8131, 0, NULL, NULL}, +{969, 8179, 0, NULL, NULL} +}; /* compose_tab_28 */ +static int hash_compose_tab_29[4] = +{0,-1,1,-1}; /* hash_compose_tab_29 */ +static CompEntry compose_tab_29[] = { +{1488, 64302, 0, NULL, NULL}, +{1522, 64287, 0, NULL, NULL} +}; /* compose_tab_29 */ +static int hash_compose_tab_30[2] = +{0,-1}; /* hash_compose_tab_30 */ +static CompEntry compose_tab_30[] = { +{1488, 64303, 0, NULL, NULL} +}; /* compose_tab_30 */ +static int hash_compose_tab_31[2] = +{-1,0}; /* hash_compose_tab_31 */ +static CompEntry compose_tab_31[] = { +{1493, 64331, 0, NULL, NULL} +}; /* compose_tab_31 */ +static int hash_compose_tab_32[44] = +{7,8,9,10,11,-1,12,-1,13,14,-1,15,16,-1,17,18,19,20,21,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,5,6,-1}; /* hash_compose_tab_32 */ +static CompEntry compose_tab_32[] = { +{1488, 64304, 0, NULL, NULL}, +{1489, 64305, 0, NULL, NULL}, +{1490, 64306, 0, NULL, NULL}, +{1491, 64307, 0, NULL, NULL}, +{1492, 64308, 0, NULL, NULL}, +{1493, 64309, 0, NULL, NULL}, +{1494, 64310, 0, NULL, NULL}, +{1496, 64312, 0, NULL, NULL}, +{1497, 64313, 0, NULL, NULL}, +{1498, 64314, 0, NULL, NULL}, +{1499, 64315, 0, NULL, NULL}, +{1500, 64316, 0, NULL, NULL}, +{1502, 64318, 0, NULL, NULL}, +{1504, 64320, 0, NULL, NULL}, +{1505, 64321, 0, NULL, NULL}, +{1507, 64323, 0, NULL, NULL}, +{1508, 64324, 0, NULL, NULL}, +{1510, 64326, 0, NULL, NULL}, +{1511, 64327, 0, NULL, NULL}, +{1512, 64328, 0, NULL, NULL}, +{1513, 64329, 0, NULL, NULL}, +{1514, 64330, 0, NULL, NULL} +}; /* compose_tab_32 */ +static int hash_compose_tab_33[6] = +{-1,0,2,-1,-1,1}; /* hash_compose_tab_33 */ +static CompEntry compose_tab_33[] = { +{1489, 64332, 0, NULL, NULL}, +{1499, 64333, 0, NULL, NULL}, +{1508, 64334, 0, NULL, NULL} +}; /* compose_tab_33 */ +static int hash_compose_tab_34_0[2] = +{-1,0}; /* hash_compose_tab_34_0 */ +static CompEntry compose_tab_34_0[] = { +{1513, 64300, 0, NULL, NULL} +}; /* compose_tab_34_0 */ +static int hash_compose_tab_34[4] = +{0,1,-1,-1}; /* hash_compose_tab_34 */ +static CompEntry compose_tab_34[] = { +{1468, 0, 1, compose_tab_34_0, hash_compose_tab_34_0}, +{1513, 64298, 0, NULL, NULL} +}; /* compose_tab_34 */ +static int hash_compose_tab_35_0[2] = +{-1,0}; /* hash_compose_tab_35_0 */ +static CompEntry compose_tab_35_0[] = { +{1513, 64301, 0, NULL, NULL} +}; /* compose_tab_35_0 */ +static int hash_compose_tab_35[4] = +{0,1,-1,-1}; /* hash_compose_tab_35 */ +static CompEntry compose_tab_35[] = { +{1468, 0, 1, compose_tab_35_0, hash_compose_tab_35_0}, +{1513, 64299, 0, NULL, NULL} +}; /* compose_tab_35 */ +static int hash_compose_tab_36[22] = +{3,10,-1,-1,-1,4,5,-1,-1,-1,-1,-1,6,-1,-1,0,1,2,7,8,9,-1}; /* hash_compose_tab_36 */ +static CompEntry compose_tab_36[] = { +{2325, 2392, 0, NULL, NULL}, +{2326, 2393, 0, NULL, NULL}, +{2327, 2394, 0, NULL, NULL}, +{2332, 2395, 0, NULL, NULL}, +{2337, 2396, 0, NULL, NULL}, +{2338, 2397, 0, NULL, NULL}, +{2344, 2345, 0, NULL, NULL}, +{2347, 2398, 0, NULL, NULL}, +{2351, 2399, 0, NULL, NULL}, +{2352, 2353, 0, NULL, NULL}, +{2355, 2356, 0, NULL, NULL} +}; /* compose_tab_36 */ +static int hash_compose_tab_37[8] = +{-1,0,1,-1,2,-1,-1,3}; /* hash_compose_tab_37 */ +static CompEntry compose_tab_37[] = { +{2465, 2524, 0, NULL, NULL}, +{2466, 2525, 0, NULL, NULL}, +{2476, 2480, 0, NULL, NULL}, +{2479, 2527, 0, NULL, NULL} +}; /* compose_tab_37 */ +static int hash_compose_tab_38[2] = +{-1,0}; /* hash_compose_tab_38 */ +static CompEntry compose_tab_38[] = { +{2503, 2507, 0, NULL, NULL} +}; /* compose_tab_38 */ +static int hash_compose_tab_39[2] = +{-1,0}; /* hash_compose_tab_39 */ +static CompEntry compose_tab_39[] = { +{2503, 2508, 0, NULL, NULL} +}; /* compose_tab_39 */ +static int hash_compose_tab_40[10] = +{-1,-1,0,1,3,4,-1,-1,2,-1}; /* hash_compose_tab_40 */ +static CompEntry compose_tab_40[] = { +{2582, 2649, 0, NULL, NULL}, +{2583, 2650, 0, NULL, NULL}, +{2588, 2651, 0, NULL, NULL}, +{2593, 2652, 0, NULL, NULL}, +{2603, 2654, 0, NULL, NULL} +}; /* compose_tab_40 */ +static int hash_compose_tab_41[6] = +{1,2,-1,-1,-1,0}; /* hash_compose_tab_41 */ +static CompEntry compose_tab_41[] = { +{2849, 2908, 0, NULL, NULL}, +{2850, 2909, 0, NULL, NULL}, +{2863, 2911, 0, NULL, NULL} +}; /* compose_tab_41 */ +static int hash_compose_tab_42[2] = +{-1,0}; /* hash_compose_tab_42 */ +static CompEntry compose_tab_42[] = { +{2887, 2891, 0, NULL, NULL} +}; /* compose_tab_42 */ +static int hash_compose_tab_43[2] = +{-1,0}; /* hash_compose_tab_43 */ +static CompEntry compose_tab_43[] = { +{2887, 2888, 0, NULL, NULL} +}; /* compose_tab_43 */ +static int hash_compose_tab_44[2] = +{-1,0}; /* hash_compose_tab_44 */ +static CompEntry compose_tab_44[] = { +{2887, 2892, 0, NULL, NULL} +}; /* compose_tab_44 */ +static int hash_compose_tab_45[4] = +{-1,-1,0,1}; /* hash_compose_tab_45 */ +static CompEntry compose_tab_45[] = { +{3014, 3018, 0, NULL, NULL}, +{3015, 3019, 0, NULL, NULL} +}; /* compose_tab_45 */ +static int hash_compose_tab_46[4] = +{-1,-1,0,1}; /* hash_compose_tab_46 */ +static CompEntry compose_tab_46[] = { +{2962, 2964, 0, NULL, NULL}, +{3014, 3020, 0, NULL, NULL} +}; /* compose_tab_46 */ +static int hash_compose_tab_47[2] = +{0,-1}; /* hash_compose_tab_47 */ +static CompEntry compose_tab_47[] = { +{3142, 3144, 0, NULL, NULL} +}; /* compose_tab_47 */ +static int hash_compose_tab_48[2] = +{0,-1}; /* hash_compose_tab_48 */ +static CompEntry compose_tab_48[] = { +{3270, 3274, 0, NULL, NULL} +}; /* compose_tab_48 */ +static int hash_compose_tab_49_1[2] = +{0,-1}; /* hash_compose_tab_49_1 */ +static CompEntry compose_tab_49_1[] = { +{3270, 3275, 0, NULL, NULL} +}; /* compose_tab_49_1 */ +static int hash_compose_tab_49[6] = +{2,-1,1,-1,-1,0}; /* hash_compose_tab_49 */ +static CompEntry compose_tab_49[] = { +{3263, 3264, 0, NULL, NULL}, +{3266, 0, 1, compose_tab_49_1, hash_compose_tab_49_1}, +{3270, 3271, 0, NULL, NULL} +}; /* compose_tab_49 */ +static int hash_compose_tab_50[2] = +{0,-1}; /* hash_compose_tab_50 */ +static CompEntry compose_tab_50[] = { +{3270, 3272, 0, NULL, NULL} +}; /* compose_tab_50 */ +static int hash_compose_tab_51[4] = +{-1,-1,0,1}; /* hash_compose_tab_51 */ +static CompEntry compose_tab_51[] = { +{3398, 3402, 0, NULL, NULL}, +{3399, 3403, 0, NULL, NULL} +}; /* compose_tab_51 */ +static int hash_compose_tab_52[2] = +{0,-1}; /* hash_compose_tab_52 */ +static CompEntry compose_tab_52[] = { +{3398, 3404, 0, NULL, NULL} +}; /* compose_tab_52 */ +static int hash_compose_tab_53[2] = +{-1,0}; /* hash_compose_tab_53 */ +static CompEntry compose_tab_53[] = { +{3661, 3635, 0, NULL, NULL} +}; /* compose_tab_53 */ +static int hash_compose_tab_54[2] = +{-1,0}; /* hash_compose_tab_54 */ +static CompEntry compose_tab_54[] = { +{3789, 3763, 0, NULL, NULL} +}; /* compose_tab_54 */ +static int hash_compose_tab_55_2[4] = +{-1,-1,0,1}; /* hash_compose_tab_55_2 */ +static CompEntry compose_tab_55_2[] = { +{4018, 3959, 0, NULL, NULL}, +{4019, 3961, 0, NULL, NULL} +}; /* compose_tab_55_2 */ +static int hash_compose_tab_55[6] = +{0,-1,1,2,-1,-1}; /* hash_compose_tab_55 */ +static CompEntry compose_tab_55[] = { +{3954, 3955, 0, NULL, NULL}, +{3956, 3957, 0, NULL, NULL}, +{3968, 0, 2, compose_tab_55_2, hash_compose_tab_55_2} +}; /* compose_tab_55 */ +static int hash_compose_tab_56[4] = +{-1,-1,0,1}; /* hash_compose_tab_56 */ +static CompEntry compose_tab_56[] = { +{4018, 3958, 0, NULL, NULL}, +{4019, 3960, 0, NULL, NULL} +}; /* compose_tab_56 */ +static int hash_compose_tab_57[4] = +{0,1,-1,-1}; /* hash_compose_tab_57 */ +static CompEntry compose_tab_57[] = { +{3904, 3945, 0, NULL, NULL}, +{3984, 4025, 0, NULL, NULL} +}; /* compose_tab_57 */ +static int hash_compose_tab_58[20] = +{-1,2,7,-1,-1,-1,0,3,5,8,-1,4,9,-1,-1,-1,1,6,-1,-1}; /* hash_compose_tab_58 */ +static CompEntry compose_tab_58[] = { +{3906, 3907, 0, NULL, NULL}, +{3916, 3917, 0, NULL, NULL}, +{3921, 3922, 0, NULL, NULL}, +{3926, 3927, 0, NULL, NULL}, +{3931, 3932, 0, NULL, NULL}, +{3986, 3987, 0, NULL, NULL}, +{3996, 3997, 0, NULL, NULL}, +{4001, 4002, 0, NULL, NULL}, +{4006, 4007, 0, NULL, NULL}, +{4011, 4012, 0, NULL, NULL} +}; /* compose_tab_58 */ +static int hash_compose_tab_59[96] = +{33,12,34,-1,13,35,14,36,15,37,-1,-1,-1,-1,-1,16,38,-1,17,39,-1,18,40,-1,19, + 41,-1,20,42,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,43,44,45, + 46,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,21,47,-1,-1,-1,-1,-1,-1,-1,0,22,-1,-1,-1,1, + 23,2,24,3,25,4,26,5,27,6,28,7,29,8,30,9,31,10,32,11}; /* hash_compose_tab_59 */ +static CompEntry compose_tab_59[] = { +{12358, 12436, 0, NULL, NULL}, +{12363, 12364, 0, NULL, NULL}, +{12365, 12366, 0, NULL, NULL}, +{12367, 12368, 0, NULL, NULL}, +{12369, 12370, 0, NULL, NULL}, +{12371, 12372, 0, NULL, NULL}, +{12373, 12374, 0, NULL, NULL}, +{12375, 12376, 0, NULL, NULL}, +{12377, 12378, 0, NULL, NULL}, +{12379, 12380, 0, NULL, NULL}, +{12381, 12382, 0, NULL, NULL}, +{12383, 12384, 0, NULL, NULL}, +{12385, 12386, 0, NULL, NULL}, +{12388, 12389, 0, NULL, NULL}, +{12390, 12391, 0, NULL, NULL}, +{12392, 12393, 0, NULL, NULL}, +{12399, 12400, 0, NULL, NULL}, +{12402, 12403, 0, NULL, NULL}, +{12405, 12406, 0, NULL, NULL}, +{12408, 12409, 0, NULL, NULL}, +{12411, 12412, 0, NULL, NULL}, +{12445, 12446, 0, NULL, NULL}, +{12454, 12532, 0, NULL, NULL}, +{12459, 12460, 0, NULL, NULL}, +{12461, 12462, 0, NULL, NULL}, +{12463, 12464, 0, NULL, NULL}, +{12465, 12466, 0, NULL, NULL}, +{12467, 12468, 0, NULL, NULL}, +{12469, 12470, 0, NULL, NULL}, +{12471, 12472, 0, NULL, NULL}, +{12473, 12474, 0, NULL, NULL}, +{12475, 12476, 0, NULL, NULL}, +{12477, 12478, 0, NULL, NULL}, +{12479, 12480, 0, NULL, NULL}, +{12481, 12482, 0, NULL, NULL}, +{12484, 12485, 0, NULL, NULL}, +{12486, 12487, 0, NULL, NULL}, +{12488, 12489, 0, NULL, NULL}, +{12495, 12496, 0, NULL, NULL}, +{12498, 12499, 0, NULL, NULL}, +{12501, 12502, 0, NULL, NULL}, +{12504, 12505, 0, NULL, NULL}, +{12507, 12508, 0, NULL, NULL}, +{12527, 12535, 0, NULL, NULL}, +{12528, 12536, 0, NULL, NULL}, +{12529, 12537, 0, NULL, NULL}, +{12530, 12538, 0, NULL, NULL}, +{12541, 12542, 0, NULL, NULL} +}; /* compose_tab_59 */ +static int hash_compose_tab_60[20] = +{-1,7,1,-1,8,2,-1,9,3,-1,-1,4,-1,-1,-1,5,-1,-1,6,0}; /* hash_compose_tab_60 */ +static CompEntry compose_tab_60[] = { +{12399, 12401, 0, NULL, NULL}, +{12402, 12404, 0, NULL, NULL}, +{12405, 12407, 0, NULL, NULL}, +{12408, 12410, 0, NULL, NULL}, +{12411, 12413, 0, NULL, NULL}, +{12495, 12497, 0, NULL, NULL}, +{12498, 12500, 0, NULL, NULL}, +{12501, 12503, 0, NULL, NULL}, +{12504, 12506, 0, NULL, NULL}, +{12507, 12509, 0, NULL, NULL} +}; /* compose_tab_60 */ +static int hash_compose_tab[122] = +{30,31,52,60,32,-1,-1,33,-1,34,35,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,-1,5,6,7,8,9,10,11,12,36,13,37,14, + 38,15,16,55,40,-1,-1,-1,-1,17,56,-1,-1,-1,-1,-1,41,18,19,20,42,21,22,-1,45, + 39,-1,23,24,-1,25,26,-1,-1,-1,-1,-1,-1,-1,-1,48,-1,43,44,51,53,-1,-1,27,46, + 54,28,-1,-1,47,-1,-1,-1,-1,49,50,-1,-1,57,-1,58,59,29}; /* hash_compose_tab */ +static CompEntry compose_tab[] = { +{768, 0, 39, compose_tab_0, hash_compose_tab_0}, +{769, 0, 70, compose_tab_1, hash_compose_tab_1}, +{770, 0, 27, compose_tab_2, hash_compose_tab_2}, +{771, 0, 19, compose_tab_3, hash_compose_tab_3}, +{772, 0, 28, compose_tab_4, hash_compose_tab_4}, +{774, 0, 30, compose_tab_5, hash_compose_tab_5}, +{775, 0, 40, compose_tab_6, hash_compose_tab_6}, +{776, 0, 50, compose_tab_7, hash_compose_tab_7}, +{777, 0, 15, compose_tab_8, hash_compose_tab_8}, +{778, 0, 6, compose_tab_9, hash_compose_tab_9}, +{779, 0, 6, compose_tab_10, hash_compose_tab_10}, +{780, 0, 34, compose_tab_11, hash_compose_tab_11}, +{781, 0, 17, compose_tab_12, hash_compose_tab_12}, +{783, 0, 14, compose_tab_13, hash_compose_tab_13}, +{785, 0, 12, compose_tab_14, hash_compose_tab_14}, +{787, 0, 15, compose_tab_15, hash_compose_tab_15}, +{788, 0, 17, compose_tab_16, hash_compose_tab_16}, +{795, 0, 4, compose_tab_17, hash_compose_tab_17}, +{803, 0, 39, compose_tab_18, hash_compose_tab_18}, +{804, 0, 2, compose_tab_19, hash_compose_tab_19}, +{805, 0, 2, compose_tab_20, hash_compose_tab_20}, +{807, 0, 20, compose_tab_21, hash_compose_tab_21}, +{808, 0, 10, compose_tab_22, hash_compose_tab_22}, +{813, 0, 12, compose_tab_23, hash_compose_tab_23}, +{814, 0, 2, compose_tab_24, hash_compose_tab_24}, +{816, 0, 6, compose_tab_25, hash_compose_tab_25}, +{817, 0, 17, compose_tab_26, hash_compose_tab_26}, +{834, 0, 12, compose_tab_27, hash_compose_tab_27}, +{837, 0, 6, compose_tab_28, hash_compose_tab_28}, +{1463, 0, 2, compose_tab_29, hash_compose_tab_29}, +{1464, 0, 1, compose_tab_30, hash_compose_tab_30}, +{1465, 0, 1, compose_tab_31, hash_compose_tab_31}, +{1468, 0, 22, compose_tab_32, hash_compose_tab_32}, +{1471, 0, 3, compose_tab_33, hash_compose_tab_33}, +{1473, 0, 2, compose_tab_34, hash_compose_tab_34}, +{1474, 0, 2, compose_tab_35, hash_compose_tab_35}, +{2364, 0, 11, compose_tab_36, hash_compose_tab_36}, +{2492, 0, 4, compose_tab_37, hash_compose_tab_37}, +{2494, 0, 1, compose_tab_38, hash_compose_tab_38}, +{2519, 0, 1, compose_tab_39, hash_compose_tab_39}, +{2620, 0, 5, compose_tab_40, hash_compose_tab_40}, +{2876, 0, 3, compose_tab_41, hash_compose_tab_41}, +{2878, 0, 1, compose_tab_42, hash_compose_tab_42}, +{2902, 0, 1, compose_tab_43, hash_compose_tab_43}, +{2903, 0, 1, compose_tab_44, hash_compose_tab_44}, +{3006, 0, 2, compose_tab_45, hash_compose_tab_45}, +{3031, 0, 2, compose_tab_46, hash_compose_tab_46}, +{3158, 0, 1, compose_tab_47, hash_compose_tab_47}, +{3266, 0, 1, compose_tab_48, hash_compose_tab_48}, +{3285, 0, 3, compose_tab_49, hash_compose_tab_49}, +{3286, 0, 1, compose_tab_50, hash_compose_tab_50}, +{3390, 0, 2, compose_tab_51, hash_compose_tab_51}, +{3415, 0, 1, compose_tab_52, hash_compose_tab_52}, +{3634, 0, 1, compose_tab_53, hash_compose_tab_53}, +{3762, 0, 1, compose_tab_54, hash_compose_tab_54}, +{3953, 0, 3, compose_tab_55, hash_compose_tab_55}, +{3968, 0, 2, compose_tab_56, hash_compose_tab_56}, +{4021, 0, 2, compose_tab_57, hash_compose_tab_57}, +{4023, 0, 10, compose_tab_58, hash_compose_tab_58}, +{12441, 0, 48, compose_tab_59, hash_compose_tab_59}, +{12442, 0, 10, compose_tab_60, hash_compose_tab_60} +}; /* compose_tab */ +#define COMP_CANDIDATE_MAP_OFFSET 24 +static Uint32 comp_candidate_map[] = { + 0x081ABFDFU, + 0x000361B8U, + 0x00000024U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x93800000U, + 0x00000006U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x10000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x50000000U, + 0x00800000U, + 0x00000000U, + 0x00000000U, + 0x10000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x50000000U, + 0x00C00000U, + 0x00000000U, + 0x00000000U, + 0x40000000U, + 0x00800000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00400000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00600004U, + 0x00000000U, + 0x00000000U, + 0x40000000U, + 0x00800000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00040000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00040000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00020000U, + 0x00000001U, + 0x00A00000U +}; diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index 235bd7931d..5b42a2ce11 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -47,7 +47,7 @@ #define SEQ_TRACE 1 #define CONTEXT_REDS 2000 /* Swap process out after this number */ -#define MAX_ARG 256 /* Max number of arguments allowed */ +#define MAX_ARG 255 /* Max number of arguments allowed */ #define MAX_REG 1024 /* Max number of x(N) registers used */ /* Scheduler stores data for temporary heaps if @@ -61,8 +61,9 @@ /* * The new arithmetic operations need some extra X registers in the register array. + * so does the gc_bif's (i_gc_bif3 need 3 extra). */ -#define ERTS_X_REGS_ALLOCATED (MAX_REG+2) +#define ERTS_X_REGS_ALLOCATED (MAX_REG+3) #define INPUT_REDUCTIONS (2 * CONTEXT_REDS) @@ -83,6 +84,7 @@ #define ErtsHAllocLockCheck(P) \ ERTS_SMP_LC_ASSERT((ERTS_PROC_LOCK_MAIN & erts_proc_lc_my_proc_locks((P))) \ + || ((P)->id == ERTS_INVALID_PID) \ || ((P)->scheduler_data \ && (P) == (P)->scheduler_data->match_pseudo_process) \ || erts_is_system_blocked(0)) @@ -197,6 +199,7 @@ extern int BIN_VH_MIN_SIZE; /* minimum virtual (bin) heap */ extern int erts_atom_table_size;/* Atom table size */ #define ORIG_CREATION 0 +#define INTERNAL_CREATION 255 /* macros for extracting bytes from uint16's */ diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 66b05c0e9d..5bc402fe22 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -43,8 +43,6 @@ static erts_smp_rwmtx_t export_table_lock; /* Locks the secondary export table. #define export_read_unlock() erts_smp_rwmtx_runlock(&export_table_lock) #define export_write_lock() erts_smp_rwmtx_rwlock(&export_table_lock) #define export_write_unlock() erts_smp_rwmtx_rwunlock(&export_table_lock) -#define export_init_lock() erts_smp_rwmtx_init(&export_table_lock, \ - "export_tab") extern BeamInstr* em_call_error_handler; extern BeamInstr* em_call_traced_function; @@ -111,8 +109,12 @@ void init_export_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; + + erts_smp_rwmtx_init_opt(&export_table_lock, &rwmtx_opt, "export_tab"); - export_init_lock(); f.hash = (H_FUN) export_hash; f.cmp = (HCMP_FUN) export_cmp; f.alloc = (HALLOC_FUN) export_alloc; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 87c1d483f4..c604fdf7c3 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -37,7 +37,7 @@ typedef struct export void* address; /* Pointer to code for function. */ struct binary* match_prog_set; /* Match program for tracing. */ - Eterm fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ + BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ /* * code[0]: Tagged atom for module. * code[1]: Tagged atom for function. @@ -52,11 +52,7 @@ typedef struct export * on_load function that has not been run yet. * Otherwise: 0. */ -#if HALFWORD_HEAP BeamInstr code[5]; -#else - Eterm code[5]; -#endif } Export; diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index f41b61d73d..328aa2be6a 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -49,10 +49,8 @@ #define in_area(ptr,start,nbytes) ((Uint)((char*)(ptr) - (char*)(start)) < (nbytes)) #define MAX_STRING_LEN 0xffff -#define dec_set_creation(nodename,creat) \ - (((nodename) == erts_this_node->sysname && (creat) == ORIG_CREATION) \ - ? erts_this_node->creation \ - : (creat)) + +#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION || (Cre) == INTERNAL_CREATION) #undef ERTS_DEBUG_USE_DIST_SEP #ifdef DEBUG @@ -65,11 +63,9 @@ # endif #endif -/* - * For backward compatibility reasons, only encode integers that - * fit in 28 bits (signed) using INTEGER_EXT. +/* Does Sint fit in Sint32? */ -#define IS_SSMALL28(x) (((Uint) (((x) >> (28-1)) + 1)) < 2) +#define IS_SSMALL32(x) (((Uint) (((x) >> (32-1)) + 1)) < 2) /* * Valid creations for nodes are 1, 2, or 3. 0 can also be sent @@ -85,14 +81,14 @@ * */ -static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32); +static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap); static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); -static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins); +static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins, int internal_tags); static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); @@ -463,6 +459,12 @@ Uint erts_encode_ext_size(Eterm term) + 1 /* VERSION_MAGIC */; } +Uint erts_encode_ext_size_ets(Eterm term) +{ + return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS); +} + + void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp) { byte *ep = *ext; @@ -470,7 +472,7 @@ void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) #endif *ep++ = VERSION_MAGIC; - ep = enc_term(acmp, term, ep, flags); + ep = enc_term(acmp, term, ep, flags, NULL); if (!ep) erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_dist_ext(): Internal data structure error\n", @@ -482,7 +484,7 @@ void erts_encode_ext(Eterm term, byte **ext) { byte *ep = *ext; *ep++ = VERSION_MAGIC; - ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS); + ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL); if (!ep) erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_ext(): Internal data structure error\n", @@ -490,6 +492,12 @@ void erts_encode_ext(Eterm term, byte **ext) *ext = ep; } +byte* erts_encode_ext_ets(Eterm term, byte *ep, struct erl_off_heap_header** off_heap) +{ + return enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS, + off_heap); +} + ErtsDistExternal * erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) { @@ -504,7 +512,7 @@ erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) ASSERT(edep->ext_endp >= edep->extp); ext_sz = edep->ext_endp - edep->extp; - align_sz = ERTS_WORD_ALIGN_PAD_SZ(dist_ext_sz + ext_sz); + align_sz = ERTS_EXTRA_DATA_ALIGN_SZ(dist_ext_sz + ext_sz); new_edep = erts_alloc(ERTS_ALC_T_EXT_TERM_DATA, dist_ext_sz + ext_sz + align_sz + xsize); @@ -815,7 +823,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins) goto fail; ep = edep->extp+1; } - res = decoded_size(ep, edep->ext_endp, no_refc_bins); + res = decoded_size(ep, edep->ext_endp, no_refc_bins, 0); if (res >= 0) return res; fail: @@ -827,9 +835,17 @@ Sint erts_decode_ext_size(byte *ext, Uint size, int no_refc_bins) { if (size == 0 || *ext != VERSION_MAGIC) return -1; - return decoded_size(ext+1, ext+size, no_refc_bins); + return decoded_size(ext+1, ext+size, no_refc_bins, 0); } +Sint erts_decode_ext_size_ets(byte *ext, Uint size) +{ + Sint sz = decoded_size(ext, ext+size, 0, 1); + ASSERT(sz >= 0); + return sz; +} + + /* ** hpp is set to either a &p->htop or ** a pointer to a memory pointer (form message buffers) @@ -889,7 +905,13 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) return obj; } - +Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) +{ + Eterm obj; + ext = dec_term(NULL, hpp, ext, off_heap, &obj); + ASSERT(ext); + return obj; +} /**********************************************************************/ @@ -966,6 +988,7 @@ term_to_binary_1(Process* p, Eterm Term) return erts_term_to_binary(p, Term, 0, TERM_TO_BINARY_DFLAGS); } + Eterm term_to_binary_2(Process* p, Eterm Term, Eterm Flags) { @@ -1077,7 +1100,7 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) goto error; size = (Sint) dest_len; } - res = decoded_size(state->extp, state->extp + size, 0); + res = decoded_size(state->extp, state->extp + size, 0, 0); if (res < 0) goto error; return res; @@ -1185,7 +1208,8 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) opt = CAR(list_val(opts)); if (opt == am_safe) { fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE; - } else { + } + else { goto error; } opts = CDR(list_val(opts)); @@ -1257,7 +1281,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) bytes = erts_alloc(ERTS_ALC_T_TMP, size); } - if ((endp = enc_term(NULL, Term, bytes, flags)) + if ((endp = enc_term(NULL, Term, bytes, flags, NULL)) == NULL) { erl_exit(1, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); @@ -1302,7 +1326,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); bytes[0] = VERSION_MAGIC; - if ((endp = enc_term(NULL, Term, bytes+1, flags)) + if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL)) == NULL) { erl_exit(1, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); @@ -1332,6 +1356,21 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) ASSERT(is_atom(atom)); + if (dflags & DFLAGS_INTERNAL_TAGS) { + Uint aval = atom_val(atom); + ASSERT(aval < (1<<24)); + if (aval >= (1 << 16)) { + *ep++ = ATOM_INTERNAL_REF3; + put_int24(aval, ep); + ep += 3; + } + else { + *ep++ = ATOM_INTERNAL_REF2; + put_int16(aval, ep); + ep += 2; + } + return ep; + } /* * term_to_binary/1,2 and the initial distribution message * don't use the cache. @@ -1381,7 +1420,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) ep += 4; put_int32(os, ep); ep += 4; - *ep++ = pid_creation(pid); + *ep++ = (is_internal_pid(pid) && (dflags & DFLAGS_INTERNAL_TAGS)) ? + INTERNAL_CREATION : pid_creation(pid); return ep; } @@ -1420,6 +1460,23 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) } ep += len; break; + case ATOM_INTERNAL_REF2: + n = get_int16(ep); + ep += 2; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + case ATOM_INTERNAL_REF3: + n = get_int24(ep); + ep += 3; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + default: error: *objp = NIL; /* Don't leave a hole in the heap */ @@ -1428,6 +1485,19 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) return ep; } +static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) +{ + switch (creation) { + case INTERNAL_CREATION: + return erts_this_node; + case ORIG_CREATION: + if (sysname == erts_this_node->sysname) { + creation = erts_this_node->creation; + } + } + return erts_find_or_insert_node(sysname,creation); +} + static byte* dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) { @@ -1451,18 +1521,20 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete ep += 4; if (ser > ERTS_MAX_PID_SERIAL) return NULL; - if ((cre = get_int8(ep)) >= MAX_CREATION) - return NULL; + cre = get_int8(ep); ep += 1; + if (!is_valid_creation(cre)) { + return NULL; + } + data = make_pid_data(ser, num); + /* * We are careful to create the node entry only after all * validity tests are done. */ - cre = dec_set_creation(sysname,cre); - node = erts_find_or_insert_node(sysname,cre); + node = dec_get_node(sysname, cre); - data = make_pid_data(ser, num); if(node == erts_this_node) { *objp = make_internal_pid(data); } else { @@ -1470,11 +1542,11 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete *hpp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_pid_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = data; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*) etp; *objp = make_external_pid(etp); } return ep; @@ -1487,7 +1559,8 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3) static byte* -enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) +enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, + struct erl_off_heap_header** off_heap) { DECLARE_WSTACK(s); Uint n; @@ -1571,13 +1644,15 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) case SMALL_DEF: { + /* From R14B we no longer restrict INTEGER_EXT to 28 bits, + * as done earlier for backward compatibility reasons. */ Sint val = signed_val(obj); if ((Uint)val < 256) { *ep++ = SMALL_INTEGER_EXT; put_int8(val, ep); ep++; - } else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) { + } else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) { *ep++ = INTEGER_EXT; put_int32(val, ep); ep += 4; @@ -1599,18 +1674,32 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) break; case BIG_DEF: - if ((n = big_bytes(obj)) < 256) { - *ep++ = SMALL_BIG_EXT; - put_int8(n, ep); - ep += 1; - } - else { - *ep++ = LARGE_BIG_EXT; - put_int32(n, ep); - ep += 4; + { + int sign = big_sign(obj); + n = big_bytes(obj); + if (sizeof(Sint)==4 && n<=4) { + Uint dig = big_digit(obj,0); + Sint val = sign ? -dig : dig; + if ((val<0) == sign) { + *ep++ = INTEGER_EXT; + put_int32(val, ep); + ep += 4; + break; + } + } + if (n < 256) { + *ep++ = SMALL_BIG_EXT; + put_int8(n, ep); + ep += 1; + } + else { + *ep++ = LARGE_BIG_EXT; + put_int32(n, ep); + ep += 4; + } + *ep++ = sign; + ep = big_to_bytes(obj, ep); } - *ep++ = big_sign(obj); - ep = big_to_bytes(obj, ep); break; case PID_DEF: @@ -1623,12 +1712,14 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) Uint32 *ref_num; ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); + *ep++ = NEW_REFERENCE_EXT; i = ref_no_of_numbers(obj); put_int16(i, ep); ep += 2; ep = enc_atom(acmp,ref_node_name(obj),ep,dflags); - *ep++ = ref_creation(obj); + *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_ref(obj)) ? + INTERNAL_CREATION : ref_creation(obj); ref_num = ref_numbers(obj); for (j = 0; j < i; j++) { put_int32(ref_num[j], ep); @@ -1644,7 +1735,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) j = port_number(obj); put_int32(j, ep); ep += 4; - *ep++ = port_creation(obj); + *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_port(obj)) ? + INTERNAL_CREATION : port_creation(obj); break; case LIST_DEF: @@ -1724,6 +1816,41 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) byte* bytes; ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize); + if (dflags & DFLAGS_INTERNAL_TAGS) { + ProcBin* pb = (ProcBin*) binary_val(obj); + Uint bytesize = pb->size; + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sub = (ErlSubBin*)pb; + pb = (ProcBin*) binary_val(sub->orig); + ASSERT(bytesize == sub->size); + bytesize += (bitoffs + bitsize + 7) / 8; + } + if (pb->thing_word == HEADER_PROC_BIN + && heap_bin_size(bytesize) > PROC_BIN_SIZE) { + ProcBin tmp; + if (bitoffs || bitsize) { + *ep++ = BIT_BINARY_INTERNAL_REF; + *ep++ = bitoffs; + *ep++ = bitsize; + } + else { + *ep++ = BINARY_INTERNAL_REF; + } + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + erts_refc_inc(&pb->val->refc, 2); + + sys_memcpy(&tmp, pb, sizeof(ProcBin)); + tmp.next = *off_heap; + tmp.bytes = bytes; + tmp.size = bytesize; + sys_memcpy(ep, &tmp, sizeof(ProcBin)); + *off_heap = (struct erl_off_heap_header*) ep; + ep += sizeof(ProcBin); + break; + } + } if (bitsize == 0) { /* Plain old byte-sized binary. */ *ep++ = BINARY_EXT; @@ -1759,8 +1886,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) *ep++ = SMALL_INTEGER_EXT; *ep++ = bitsize; } - break; } + break; case EXPORT_DEF: { Export* exp = *((Export **) (export_val(obj) + 1)); @@ -1768,7 +1895,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) *ep++ = EXPORT_EXT; ep = enc_atom(acmp, exp->code[0], ep, dflags); ep = enc_atom(acmp, exp->code[1], ep, dflags); - ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags); + ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap); } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -1804,8 +1931,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) put_int32(funp->num_free, ep); ep += 4; ep = enc_atom(acmp, funp->fe->module, ep, dflags); - ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags); - ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags); + ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap); + ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap); ep = enc_pid(acmp, funp->creator, ep, dflags); fun_env: @@ -1858,7 +1985,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) return ep; } -static Uint +static +Uint is_external_string(Eterm list, int* p_is_string) { Uint len = 0; @@ -1896,69 +2024,30 @@ is_external_string(Eterm list, int* p_is_string) return len; } -/* Assumes that the ones to undo are preluding the lists. */ +/* Assumes that the ones to undo are preluding the list. */ static void undo_offheap_in_area(ErlOffHeap* off_heap, Eterm* start, Eterm* end) { const Uint area_sz = (end - start) * sizeof(Eterm); - struct proc_bin* mso; - struct proc_bin** mso_nextp = NULL; -#ifndef HYBRID /* FIND ME! */ - struct erl_fun_thing* funs; - struct erl_fun_thing** funs_nextp = NULL; -#endif - struct external_thing_* ext; - struct external_thing_** ext_nextp = NULL; - - for (mso = off_heap->mso; ; mso=mso->next) { - if (!in_area(mso, start, area_sz)) { - if (mso_nextp != NULL) { - *mso_nextp = NULL; - erts_cleanup_mso(off_heap->mso); - off_heap->mso = mso; - } - break; - } - mso_nextp = &mso->next; - } - -#ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; ; funs=funs->next) { - if (!in_area(funs, start, area_sz)) { - if (funs_nextp != NULL) { - *funs_nextp = NULL; - erts_cleanup_funs(off_heap->funs); - off_heap->funs = funs; + struct erl_off_heap_header* hdr; + struct erl_off_heap_header** hdr_nextp = NULL; + + for (hdr = off_heap->first; ; hdr=hdr->next) { + if (!in_area(hdr, start, area_sz)) { + if (hdr_nextp != NULL) { + *hdr_nextp = NULL; + erts_cleanup_offheap(off_heap); + off_heap->first = hdr; } break; } - funs_nextp = &funs->next; + hdr_nextp = &hdr->next; } -#endif - for (ext = off_heap->externals; ; ext=ext->next) { - if (!in_area(ext, start, area_sz)) { - if (ext_nextp != NULL) { - *ext_nextp = NULL; - erts_cleanup_externals(off_heap->externals); - off_heap->externals = ext; - } - break; - } - ext_nextp = &ext->next; - } - /* Assert that the ones to undo were indeed preluding the lists. */ + /* Assert that the ones to undo were indeed preluding the list. */ #ifdef DEBUG - for (mso = off_heap->mso; mso != NULL; mso=mso->next) { - ASSERT(!in_area(mso, start, area_sz)); - } -# ifndef HYBRID /* FIND ME! */ - for (funs = off_heap->funs; funs != NULL; funs=funs->next) { - ASSERT(!in_area(funs, start, area_sz)); - } -# endif - for (ext = off_heap->externals; ext != NULL; ext=ext->next) { - ASSERT(!in_area(ext, start, area_sz)); + for (hdr = off_heap->first; hdr != NULL; hdr = hdr->next) { + ASSERT(!in_area(hdr, start, area_sz)); } #endif /* DEBUG */ } @@ -2187,13 +2276,13 @@ dec_term_atom_common: goto error; } ep += 4; - if ((cre = get_int8(ep)) >= MAX_CREATION) { + cre = get_int8(ep); + ep++; + if (!is_valid_creation(cre)) { goto error; } - ep++; - cre = dec_set_creation(sysname,cre); - node = erts_find_or_insert_node(sysname, cre); + node = dec_get_node(sysname, cre); if(node == erts_this_node) { *objp = make_internal_port(num); } @@ -2202,11 +2291,11 @@ dec_term_atom_common: hp += EXTERNAL_THING_HEAD_SIZE + 1; etp->header = make_external_port_header(1); - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; etp->data.ui[0] = num; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_port(etp); } @@ -2230,9 +2319,11 @@ dec_term_atom_common: goto error; ep += 4; - if ((cre = get_int8(ep)) >= MAX_CREATION) - goto error; + cre = get_int8(ep); ep += 1; + if (!is_valid_creation(cre)) { + goto error; + } goto ref_ext_common; case NEW_REFERENCE_EXT: @@ -2245,10 +2336,11 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &sysname)) == NULL) goto error; - if ((cre = get_int8(ep)) >= MAX_CREATION) - goto error; + cre = get_int8(ep); ep += 1; - + if (!is_valid_creation(cre)) { + goto error; + } r0 = get_int32(ep); ep += 4; if (r0 >= MAX_REFERENCE) @@ -2256,8 +2348,7 @@ dec_term_atom_common: ref_ext_common: - cre = dec_set_creation(sysname, cre); - node = erts_find_or_insert_node(sysname, cre); + node = dec_get_node(sysname, cre); if(node == erts_this_node) { RefThing *rtp = (RefThing *) hp; ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); @@ -2284,10 +2375,10 @@ dec_term_atom_common: #else etp->header = make_external_ref_header(ref_words); #endif - etp->next = off_heap->externals; + etp->next = off_heap->first; etp->node = node; - off_heap->externals = etp; + off_heap->first = (struct erl_off_heap_header*)etp; *objp = make_external_ref(etp); ref_num = &(etp->data.ui32[0]); } @@ -2330,8 +2421,8 @@ dec_term_atom_common: hp += PROC_BIN_SIZE; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2367,8 +2458,8 @@ dec_term_atom_common: pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = n; - pb->next = off_heap->mso; - off_heap->mso = pb; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; pb->val = dbin; pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; @@ -2488,8 +2579,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif funp->fe = erts_put_fun_entry2(module, old_uniq, old_index, @@ -2566,8 +2657,8 @@ dec_term_atom_common: * It is safe to link the fun into the fun list only when * no more validity tests can fail. */ - funp->next = off_heap->funs; - off_heap->funs = funp; + funp->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)funp; #endif old_uniq = unsigned_val(temp); @@ -2585,6 +2676,66 @@ dec_term_atom_common: } break; } + case ATOM_INTERNAL_REF2: + n = get_int16(ep); + ep += 2; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + case ATOM_INTERNAL_REF3: + n = get_int24(ep); + ep += 3; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + + case BINARY_INTERNAL_REF: + { + ProcBin* pb = (ProcBin*) hp; + sys_memcpy(pb, ep, sizeof(ProcBin)); + ep += sizeof(ProcBin); + + erts_refc_inc(&pb->val->refc, 1); + hp += PROC_BIN_SIZE; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; + pb->flags = 0; + *objp = make_binary(pb); + break; + } + case BIT_BINARY_INTERNAL_REF: + { + Sint bitoffs = *ep++; + Sint bitsize = *ep++; + ProcBin* pb = (ProcBin*) hp; + ErlSubBin* sub; + sys_memcpy(pb, ep, sizeof(ProcBin)); + ep += sizeof(ProcBin); + + erts_refc_inc(&pb->val->refc, 1); + hp += PROC_BIN_SIZE; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; + pb->flags = 0; + + sub = (ErlSubBin*)hp; + sub->thing_word = HEADER_SUB_BIN; + sub->size = pb->size - (bitoffs + bitsize + 7)/8; + sub->offs = 0; + sub->bitoffs = bitoffs; + sub->bitsize = bitsize; + sub->is_writable = 0; + sub->orig = make_binary(pb); + + hp += ERL_SUB_BIN_SIZE; + *objp = make_binary(sub); + break; + } + default: error: /* UNDO: @@ -2667,27 +2818,36 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) case NIL_DEF: result++; break; - case ATOM_DEF: { - int alen = atom_tab(atom_val(obj))->len; - if ((MAX_ATOM_LENGTH <= 255 || alen <= 255) - && (dflags & DFLAG_SMALL_ATOM_TAGS)) { - /* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */ - result += 1 + 1 + alen; + case ATOM_DEF: + if (dflags & DFLAGS_INTERNAL_TAGS) { + if (atom_val(obj) >= (1<<16)) { + result += 1 + 3; + } + else { + result += 1 + 2; + } } else { - /* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */ - result += 1 + 2 + alen; + int alen = atom_tab(atom_val(obj))->len; + if ((MAX_ATOM_LENGTH <= 255 || alen <= 255) + && (dflags & DFLAG_SMALL_ATOM_TAGS)) { + /* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */ + result += 1 + 1 + alen; + } + else { + /* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */ + result += 1 + 2 + alen; + } + insert_acache_map(acmp, obj); } - insert_acache_map(acmp, obj); break; - } case SMALL_DEF: { Sint val = signed_val(obj); if ((Uint)val < 256) result += 1 + 1; /* SMALL_INTEGER_EXT */ - else if (sizeof(Sint) == 4 || IS_SSMALL28(val)) + else if (sizeof(Sint) == 4 || IS_SSMALL32(val)) result += 1 + 4; /* INTEGER_EXT */ else { DeclareTmpHeapNoproc(tmp_big,2); @@ -2699,7 +2859,10 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } break; case BIG_DEF: - if ((i = big_bytes(obj)) < 256) + i = big_bytes(obj); + if (sizeof(Sint)==4 && i <= 4 && (big_digit(obj,0)-big_sign(obj)) < (1<<31)) + result += 1 + 4; /* INTEGER_EXT */ + else if (i < 256) result += 1 + 1 + 1 + i; /* tag,size,sign,digits */ else result += 1 + 4 + 1 + i; /* tag,size,sign,digits */ @@ -2756,8 +2919,25 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } break; case BINARY_DEF: + if (dflags & DFLAGS_INTERNAL_TAGS) { + ProcBin* pb = (ProcBin*) binary_val(obj); + Uint sub_extra = 0; + Uint tot_bytes = pb->size; + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sub = (ErlSubBin*) pb; + pb = (ProcBin*) binary_val(sub->orig); + sub_extra = 2; /* bitoffs and bitsize */ + tot_bytes += (sub->bitoffs + sub->bitsize+ 7) / 8; + } + if (pb->thing_word == HEADER_PROC_BIN + && heap_bin_size(tot_bytes) > PROC_BIN_SIZE) { + + result += 1 + sub_extra + sizeof(ProcBin); + break; + } + } result += 1 + 4 + binary_size(obj) + - 5; /* For unaligned binary */ + 5; /* For unaligned binary */ break; case FUN_DEF: { @@ -2829,7 +3009,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } static Sint -decoded_size(byte *ep, byte* endp, int no_refc_bins) +decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags) { int heap_size = 0; int terms; @@ -3039,6 +3219,29 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins) heap_size += ERL_FUN_SIZE + num_free; break; } + case ATOM_INTERNAL_REF2: + SKIP(2+atom_extra_skip); + atom_extra_skip = 0; + break; + case ATOM_INTERNAL_REF3: + SKIP(3+atom_extra_skip); + atom_extra_skip = 0; + break; + + case BINARY_INTERNAL_REF: + if (!internal_tags) { + return -1; + } + SKIP(sizeof(ProcBin)); + heap_size += PROC_BIN_SIZE; + break; + case BIT_BINARY_INTERNAL_REF: + if (!internal_tags) { + return -1; + } + SKIP(2+sizeof(ProcBin)); + heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE; + break; default: return -1; } diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index db86b4d796..d8287b96a4 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -54,6 +54,10 @@ #define DIST_HEADER 'D' #define ATOM_CACHE_REF 'R' +#define ATOM_INTERNAL_REF2 'I' +#define ATOM_INTERNAL_REF3 'K' +#define BINARY_INTERNAL_REF 'J' +#define BIT_BINARY_INTERNAL_REF 'L' #define COMPRESSED 'P' #if 0 @@ -156,7 +160,9 @@ Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *); void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *); Uint erts_encode_ext_size(Eterm); +Uint erts_encode_ext_size_ets(Eterm); void erts_encode_ext(Eterm, byte **); +byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap); #ifdef ERTS_WANT_EXTERNAL_TAGS ERTS_GLB_INLINE void erts_peek_dist_header(ErtsDistHeaderPeek *, byte *, Uint); @@ -172,7 +178,9 @@ Sint erts_decode_dist_ext_size(ErtsDistExternal *, int); Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *); Sint erts_decode_ext_size(byte*, Uint, int); +Sint erts_decode_ext_size_ets(byte*, Uint); Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**); +Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*); Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags); @@ -211,7 +219,7 @@ ERTS_GLB_INLINE void * erts_dist_ext_trailer(ErtsDistExternal *edep) { void *res = (void *) (edep->ext_endp - + ERTS_WORD_ALIGN_PAD_SZ(edep->ext_endp)); + + ERTS_EXTRA_DATA_ALIGN_SZ(edep->ext_endp)); ASSERT((((UWord) res) % sizeof(Uint)) == 0); return res; } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index fbb40e4202..e8a9d5f32f 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -464,7 +464,10 @@ typedef union { typedef struct proc_bin { Eterm thing_word; /* Subtag REFC_BINARY_SUBTAG. */ Uint size; /* Binary size in bytes. */ - struct proc_bin *next; /* Pointer to next ProcBin. */ +#if HALFWORD_HEAP + void* dummy_ptr_padding__; +#endif + struct erl_off_heap_header *next; Binary *val; /* Pointer to Binary structure. */ byte *bytes; /* Pointer to the actual data bytes. */ Uint flags; /* Flag word. */ @@ -494,8 +497,8 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp) pb->thing_word = HEADER_PROC_BIN; pb->size = 0; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*) pb; pb->val = mbp; pb->bytes = (byte *) mbp->orig_bytes; pb->flags = 0; @@ -512,6 +515,16 @@ erts_mk_magic_binary_term(Eterm **hpp, ErlOffHeap *ohp, Binary *mbp) && (thing_subtag(*binary_val((T))) == REFC_BINARY_SUBTAG) \ && (((ProcBin *) binary_val((T)))->val->flags & BIN_FLAG_MAGIC)) + +union erl_off_heap_ptr { + struct erl_off_heap_header* hdr; + ProcBin *pb; + struct erl_fun_thing* fun; + struct external_thing_* ext; + Eterm* ep; + void* voidp; +}; + /* arrays that get malloced at startup */ extern Port* erts_port; extern erts_smp_atomic_t erts_ports_alive; @@ -531,7 +544,7 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) { /* Dead ports are added from the end of the snapshot buffer */ Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr, - -(long)sizeof(Eterm)); + -(erts_aint_t)sizeof(Eterm)); ASSERT(tombstone+1 != NULL); ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1); *tombstone = prt->id; @@ -550,7 +563,7 @@ extern Uint display_items; /* no of items to display in traces etc */ extern Uint display_loads; /* print info about loaded modules */ extern int erts_backtrace_depth; -extern erts_smp_atomic_t erts_max_gen_gcs; +extern erts_smp_atomic32_t erts_max_gen_gcs; extern int erts_disable_tolerant_timeofday; @@ -823,7 +836,6 @@ Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap); Eterm erts_new_mso_binary(Process*, byte*, int); Eterm new_binary(Process*, byte*, int); Eterm erts_realloc_binary(Eterm bin, size_t size); -void erts_cleanup_mso(ProcBin* pb); /* erl_bif_info.c */ @@ -881,6 +893,8 @@ Eterm copy_object(Eterm, Process*); Uint size_object(Eterm); Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); +void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, + Eterm* refs, unsigned nrefs); #ifdef HYBRID #define RRMA_DEFAULT_SIZE 256 @@ -1013,7 +1027,7 @@ void print_pass_through(int, byte*, int); /* beam_emu.c */ int catchlevel(Process*); -void init_emulator(_VOID_); +void init_emulator(void); void process_main(void); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); @@ -1078,6 +1092,7 @@ Eterm erts_heap_sizes(Process* p); void erts_offset_off_heap(ErlOffHeap *, Sint, Eterm*, Eterm*); void erts_offset_heap_ptr(Eterm*, Uint, Sint, Eterm*, Eterm*); void erts_offset_heap(Eterm*, Uint, Sint, Eterm*, Eterm*); +void erts_free_heap_frags(Process* p); #ifdef HYBRID int erts_global_garbage_collect(Process*, int, Eterm*, int); @@ -1191,13 +1206,12 @@ ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt) { #ifdef ERTS_SMP - long refc; + erts_aint_t refc; + erts_smp_mtx_unlock(prt->lock); refc = erts_smp_atomic_dectest(&prt->refc); ASSERT(refc >= 0); if (refc == 0) erts_port_cleanup(prt); - else - erts_smp_mtx_unlock(prt->lock); #endif } @@ -1411,84 +1425,6 @@ void erl_drv_thr_init(void); /* time.c */ -ERTS_GLB_INLINE long do_time_read_and_reset(void); -#ifdef ERTS_TIMER_THREAD -ERTS_GLB_INLINE int next_time(void); -ERTS_GLB_INLINE void bump_timer(long); -#else -int next_time(void); -void bump_timer(long); -extern erts_smp_atomic_t do_time; /* set at clock interrupt */ -ERTS_GLB_INLINE void do_time_add(long); -#endif - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -#ifdef ERTS_TIMER_THREAD -ERTS_GLB_INLINE long do_time_read_and_reset(void) { return 0; } -ERTS_GLB_INLINE int next_time(void) { return -1; } -ERTS_GLB_INLINE void bump_timer(long ignore) { } -#else -ERTS_GLB_INLINE long do_time_read_and_reset(void) -{ - return erts_smp_atomic_xchg(&do_time, 0L); -} -ERTS_GLB_INLINE void do_time_add(long elapsed) -{ - erts_smp_atomic_add(&do_time, elapsed); -} -#endif - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - -void init_time(void); -void erl_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); -void erl_cancel_timer(ErlTimer*); -Uint time_left(ErlTimer *); - -Uint erts_timer_wheel_memory_size(void); - -#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME)) -# ifndef HAVE_ERTS_NOW_CPU -# define HAVE_ERTS_NOW_CPU -# ifdef HAVE_GETHRVTIME -# define erts_start_now_cpu() sys_start_hrvtime() -# define erts_stop_now_cpu() sys_stop_hrvtime() -# endif -# endif -void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); -#endif - -void erts_get_timeval(SysTimeval *tv); -long erts_get_time(void); - -extern SysTimeval erts_first_emu_time; - -void erts_get_emu_time(SysTimeval *); - -ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE int -erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p) -{ - if (t1p->tv_sec == t2p->tv_sec) { - if (t1p->tv_usec < t2p->tv_usec) - return -1; - else if (t1p->tv_usec > t2p->tv_usec) - return 1; - return 0; - } - return t1p->tv_sec < t2p->tv_sec ? -1 : 1; -} - -#endif - -#ifdef DEBUG -void p_slpq(_VOID_); -#endif - /* utils.c */ /* @@ -1497,7 +1433,6 @@ void p_slpq(_VOID_); void erts_silence_warn_unused_result(long unused); void erts_cleanup_offheap(ErlOffHeap *offheap); -void erts_cleanup_externals(ExternalThing *); Uint erts_fit_in_bits(Uint); int list_length(Eterm); @@ -1530,7 +1465,7 @@ erts_bld_atom_2uint_3tup_list(Uint **hpp, Uint *szp, Sint length, Eterm atoms[], Uint uints1[], Uint uints2[]); Eterm store_external_or_ref_in_proc_(Process *, Eterm); -Eterm store_external_or_ref_(Uint **, ExternalThing **, Eterm); +Eterm store_external_or_ref_(Uint **, ErlOffHeap*, Eterm); #define NC_HEAP_SIZE(NC) \ (ASSERT_EXPR(is_node_container((NC))), \ @@ -1576,9 +1511,26 @@ void erts_init_bif_chksum(void); /* erl_bif_re.c */ void erts_init_bif_re(void); Sint erts_re_set_loop_limit(Sint limit); +/* erl_bif_binary.c */ +void erts_init_bif_binary(void); +Sint erts_binary_set_loop_limit(Sint limit); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); + +void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ; +Sint erts_native_filename_need(Eterm ioterm, int encoding); +void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars); +int erts_analyze_utf8(byte *source, Uint size, + byte **err_pos, Uint *num_chars, int *left); +char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty); + +#define ERTS_UTF8_OK 0 +#define ERTS_UTF8_INCOMPLETE 1 +#define ERTS_UTF8_ERROR 2 +#define ERTS_UTF8_ANALYZE_MORE 3 + /* erl_trace.c */ void erts_init_trace(void); void erts_trace_check_exiting(Eterm exiting); @@ -1670,7 +1622,7 @@ int io_list_to_buf(Eterm, char*, int); int io_list_to_buf2(Eterm, char*, int); int io_list_len(Eterm); int is_string(Eterm); -void erl_at_exit(FUNCTION(void,(*),(void*)), void*); +void erl_at_exit(void (*) (void*), void*); Eterm collect_memory(Process *); void dump_memory_to_fd(int); int dump_memory_data(const char *); @@ -1705,17 +1657,13 @@ Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live); Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live); +Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live); +Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live); Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(int to, void *arg, Process *c_p); -/* - * Interface to erl_init - */ -void erl_init(void); -void erts_first_process(Eterm modname, void* code, unsigned size, int argc, char** argv); - #define seq_trace_output(token, msg, type, receiver, process) \ seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL) #define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \ @@ -1734,8 +1682,10 @@ struct trace_pattern_flags { unsigned int local : 1; /* Local call trace breakpoint */ unsigned int meta : 1; /* Metadata trace breakpoint */ unsigned int call_count : 1; /* Fast call count breakpoint */ + unsigned int call_time : 1; /* Fast call time breakpoint */ }; extern const struct trace_pattern_flags erts_trace_pattern_flags_off; +extern int erts_call_time_breakpoint_tracing; int erts_set_trace_pattern(Eterm* mfa, int specified, Binary* match_prog_set, Binary *meta_match_prog_set, int on, struct trace_pattern_flags, @@ -1779,18 +1729,20 @@ extern void erts_match_prog_foreach_offheap(Binary *b, void (*)(ErlOffHeap *, void *), void *); -#define MATCH_SET_RETURN_TRACE 0x1 /* return trace requested */ -#define MATCH_SET_RETURN_TO_TRACE 0x2 /* Misleading name, it is not actually - set by the match program, but by the - breakpoint functions */ -#define MATCH_SET_EXCEPTION_TRACE 0x4 /* exception trace requested */ +#define MATCH_SET_RETURN_TRACE (0x1) /* return trace requested */ +#define MATCH_SET_RETURN_TO_TRACE (0x2) /* Misleading name, it is not actually + set by the match program, but by the + breakpoint functions */ +#define MATCH_SET_EXCEPTION_TRACE (0x4) /* exception trace requested */ #define MATCH_SET_RX_TRACE (MATCH_SET_RETURN_TRACE|MATCH_SET_EXCEPTION_TRACE) /* * Flag values when tracing bif + * Future note: flag field is 8 bits */ -#define BIF_TRACE_AS_LOCAL 0x1 -#define BIF_TRACE_AS_GLOBAL 0x2 -#define BIF_TRACE_AS_META 0x4 +#define BIF_TRACE_AS_LOCAL (0x1) +#define BIF_TRACE_AS_GLOBAL (0x2) +#define BIF_TRACE_AS_META (0x4) +#define BIF_TRACE_AS_CALL_TIME (0x8) extern erts_driver_t vanilla_driver; extern erts_driver_t spawn_driver; @@ -1946,5 +1898,5 @@ erts_alloc_message_heap(Uint size, # define UnUseTmpHeap(Size,Proc) /* Nothing */ # define UseTmpHeapNoproc(Size) /* Nothing */ # define UnUseTmpHeapNoproc(Size) /* Nothing */ -#endif -#endif +#endif /* HEAP_ON_C_STACK */ +#endif /* !__GLOBAL_H__ */ diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 10f1082039..f21a96c754 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -280,10 +280,36 @@ erts_test_next_port(int set, Uint next) return res; } + +static void port_cleanup(Port *prt); + +#ifdef ERTS_SMP + +static void +sched_port_cleanup(void *vprt) +{ + Port *prt = (Port *) vprt; + erts_smp_mtx_lock(prt->lock); + port_cleanup(prt); +} + +#endif + void erts_port_cleanup(Port *prt) { #ifdef ERTS_SMP + if (erts_smp_mtx_trylock(prt->lock) == EBUSY) + erts_schedule_misc_op(sched_port_cleanup, (void *) prt); + else +#endif + port_cleanup(prt); +} + +void +port_cleanup(Port *prt) +{ +#ifdef ERTS_SMP Uint32 port_specific; erts_smp_mtx_t *mtx; #endif @@ -402,7 +428,7 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, old_name = prt->name; prt->name = new_name; #ifdef ERTS_SMP - erts_smp_atomic_set(&prt->run_queue, (long) runq); + erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) runq); #endif ASSERT(!prt->drv_ptr); prt->drv_ptr = driver; @@ -644,7 +670,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ #ifdef ERTS_SMP erts_cancel_smp_ptimer(port->ptimer); #else - erl_cancel_timer(&(port->tm)); + erts_cancel_timer(&(port->tm)); #endif stopq(port); kill_port(port); @@ -1271,7 +1297,7 @@ void init_io(void) erts_port[i].port_data_lock = NULL; } - erts_smp_atomic_init(&erts_ports_snapshot, (long) 0); + erts_smp_atomic_init(&erts_ports_snapshot, (erts_aint_t) 0); last_port_num = 0; erts_smp_spinlock_init(&get_free_port_lck, "get_free_port"); @@ -1549,14 +1575,14 @@ static void deliver_read_message(Port* prt, Eterm to, pb = (ProcBin *) hp; pb->thing_word = HEADER_PROC_BIN; pb->size = len; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = bptr; pb->bytes = (byte*) bptr->orig_bytes; pb->flags = 0; hp += PROC_BIN_SIZE; - ohp->overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); listp = make_binary(pb); } @@ -1699,14 +1725,14 @@ deliver_vec_message(Port* prt, /* Port */ } pb->thing_word = HEADER_PROC_BIN; pb->size = iov->iov_len; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = base; pb->flags = 0; hp += PROC_BIN_SIZE; - ohp->overhead += iov->iov_len / sizeof(Eterm); + OH_OVERHEAD(ohp, iov->iov_len / sizeof(Eterm)); if (listp == NIL) { /* compatible with deliver_bin_message */ listp = make_binary(pb); @@ -1813,7 +1839,7 @@ terminate_port(Port *prt) #ifdef ERTS_SMP erts_cancel_smp_ptimer(prt->ptimer); #else - erl_cancel_timer(&prt->tm); + erts_cancel_timer(&prt->tm); #endif drv = prt->drv_ptr; @@ -2233,12 +2259,12 @@ erts_port_control(Process* p, Port* prt, Uint command, Eterm iolist) ProcBin* pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); pb->thing_word = HEADER_PROC_BIN; pb->size = dbin->orig_size; - pb->next = MSO(p).mso; - MSO(p).mso = pb; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(dbin); pb->bytes = (byte*) dbin->orig_bytes; pb->flags = 0; - MSO(p).overhead += dbin->orig_size / sizeof(Eterm); + OH_OVERHEAD(&(MSO(p)), dbin->orig_size / sizeof(Eterm)); return make_binary(pb); } port_resp = dbin->orig_bytes; @@ -2776,17 +2802,25 @@ driver_deliver_term(ErlDrvPort port, break; case ERL_DRV_INT: /* signed int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); +#if HALFWORD_HEAP + erts_bld_sint64(NULL, &need, (Sint64)ptr[0]); +#else /* check for bignum */ if (!IS_SSMALL((Sint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ +#endif ptr++; depth++; break; case ERL_DRV_UINT: /* unsigned int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); +#if HALFWORD_HEAP + erts_bld_uint64(NULL, &need, (Uint64)ptr[0]); +#else /* check for bignum */ if (!IS_USMALL(0, (Uint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ +#endif ptr++; depth++; break; @@ -2953,22 +2987,30 @@ driver_deliver_term(ErlDrvPort port, break; case ERL_DRV_INT: /* signed int argument */ +#if HALFWORD_HEAP + mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]); +#else if (IS_SSMALL((Sint)ptr[0])) mess = make_small((Sint)ptr[0]); else { mess = small_to_big((Sint)ptr[0], hp); hp += BIG_UINT_HEAP_SIZE; } +#endif ptr++; break; case ERL_DRV_UINT: /* unsigned int argument */ +#if HALFWORD_HEAP + mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]); +#else if (IS_USMALL(0, (Uint)ptr[0])) mess = make_small((Uint)ptr[0]); else { mess = uint_to_big((Uint)ptr[0], hp); hp += BIG_UINT_HEAP_SIZE; } +#endif ptr++; break; @@ -3007,14 +3049,14 @@ driver_deliver_term(ErlDrvPort port, driver_binary_inc_refc(b); /* caller will free binary */ pb->thing_word = HEADER_PROC_BIN; pb->size = size; - pb->next = ohp->mso; - ohp->mso = pb; + pb->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pb; pb->val = ErlDrvBinary2Binary(b); pb->bytes = ((byte*) b->orig_bytes) + offset; pb->flags = 0; mess = make_binary(pb); hp += PROC_BIN_SIZE; - ohp->overhead += pb->size / sizeof(Eterm); + OH_OVERHEAD(ohp, pb->size / sizeof(Eterm)); } ptr += 3; break; @@ -3046,12 +3088,12 @@ driver_deliver_term(ErlDrvPort port, hp += PROC_BIN_SIZE; pbp->thing_word = HEADER_PROC_BIN; pbp->size = size; - pbp->next = ohp->mso; - ohp->mso = pbp; + pbp->next = ohp->first; + ohp->first = (struct erl_off_heap_header*)pbp; pbp->val = bp; pbp->bytes = (byte*) bp->orig_bytes; pbp->flags = 0; - ohp->overhead += (pbp->size / sizeof(Eterm)); + OH_OVERHEAD(ohp, pbp->size / sizeof(Eterm)); mess = make_binary(pbp); } ptr += 2; @@ -3210,7 +3252,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len)); + erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, @@ -3245,7 +3287,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len) return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len)); + erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, @@ -3322,7 +3364,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip) /* XXX handle distribution !!! */ prt->bytes_in += (hlen + size); - erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + size)); + erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + size)); deliver_vec_message(prt, prt->connected, hbuf, hlen, binv, iov, n, size); return 0; } @@ -3366,25 +3408,25 @@ int len; * reference count on driver binaries... */ -long +ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return erts_refc_read(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_read(&bp->refc, 1); } -long +ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return erts_refc_inctest(&bp->refc, 2); + return (ErlDrvSInt) erts_refc_inctest(&bp->refc, 2); } -long +ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return erts_refc_dectest(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_dectest(&bp->refc, 1); } @@ -3499,12 +3541,12 @@ pdl_init_refc(ErlDrvPDL pdl) erts_atomic_init(&pdl->refc, 1); } -static ERTS_INLINE long +static ERTS_INLINE ErlDrvSInt pdl_read_refc(ErlDrvPDL pdl) { - long refc = erts_atomic_read(&pdl->refc); + erts_aint_t refc = erts_atomic_read(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); - return refc; + return (ErlDrvSInt) refc; } static ERTS_INLINE void @@ -3514,12 +3556,12 @@ pdl_inc_refc(ErlDrvPDL pdl) ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 1); } -static ERTS_INLINE long +static ERTS_INLINE ErlDrvSInt pdl_inctest_refc(ErlDrvPDL pdl) { - long refc = erts_atomic_inctest(&pdl->refc); + erts_aint_t refc = erts_atomic_inctest(&pdl->refc); ERTS_LC_ASSERT(refc > 1); - return refc; + return (ErlDrvSInt) refc; } #if 0 /* unused */ @@ -3531,12 +3573,12 @@ pdl_dec_refc(ErlDrvPDL pdl) } #endif -static ERTS_INLINE long +static ERTS_INLINE ErlDrvSInt pdl_dectest_refc(ErlDrvPDL pdl) { - long refc = erts_atomic_dectest(&pdl->refc); + erts_aint_t refc = erts_atomic_dectest(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); - return refc; + return (ErlDrvSInt) refc; } static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl) @@ -3607,7 +3649,7 @@ driver_pdl_lock(ErlDrvPDL pdl) void driver_pdl_unlock(ErlDrvPDL pdl) { - long refc; + ErlDrvSInt refc; #ifdef HARDDEBUG erts_fprintf(stderr, "driver_pdl_unlock(0x%08X)\r\n",(unsigned) pdl); #endif @@ -3617,28 +3659,30 @@ driver_pdl_unlock(ErlDrvPDL pdl) pdl_destroy(pdl); } -long +ErlDrvSInt driver_pdl_get_refc(ErlDrvPDL pdl) { return pdl_read_refc(pdl); } -long +ErlDrvSInt driver_pdl_inc_refc(ErlDrvPDL pdl) { - long refc = pdl_inctest_refc(pdl); + ErlDrvSInt refc = pdl_inctest_refc(pdl); #ifdef HARDDEBUG - erts_fprintf(stderr, "driver_pdl_inc_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc); + erts_fprintf(stderr, "driver_pdl_inc_refc(%p) -> %bpd\r\n", + pdl, refc); #endif return refc; } -long +ErlDrvSInt driver_pdl_dec_refc(ErlDrvPDL pdl) { - long refc = pdl_dectest_refc(pdl); + ErlDrvSInt refc = pdl_dectest_refc(pdl); #ifdef HARDDEBUG - erts_fprintf(stderr, "driver_pdl_dec_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc); + erts_fprintf(stderr, "driver_pdl_dec_refc(%p) -> %bpd\r\n", + pdl, refc); #endif if (!refc) pdl_destroy(pdl); @@ -4024,7 +4068,7 @@ drv_cancel_timer(Port *prt) #ifdef ERTS_SMP erts_cancel_smp_ptimer(prt->ptimer); #else - erl_cancel_timer(&prt->tm); + erts_cancel_timer(&prt->tm); #endif if (erts_port_task_is_scheduled(&prt->timeout_task)) erts_port_task_abort(prt->id, &prt->timeout_task); @@ -4048,7 +4092,7 @@ int driver_set_timer(ErlDrvPort ix, UWord t) (ErlTimeoutProc) schedule_port_timeout, t); #else - erl_set_timer(&prt->tm, + erts_set_timer(&prt->tm, (ErlTimeoutProc) schedule_port_timeout, NULL, prt, @@ -4079,9 +4123,9 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); #ifdef ERTS_SMP - *t = prt->ptimer ? time_left(&prt->ptimer->timer.tm) : 0; + *t = prt->ptimer ? erts_time_left(&prt->ptimer->timer.tm) : 0; #else - *t = time_left(&prt->tm); + *t = erts_time_left(&prt->tm); #endif return 0; } diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 9e8ac74f40..e861f97e7a 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -60,12 +60,18 @@ func_info M=a a==am_module_info A=u==0 | label L | move n r => too_old_compiler func_info M=a a==am_module_info A=u==1 | label L | move n r => too_old_compiler # The undocumented and unsupported guard BIF is_constant/1 was removed -# in R13. The is_constant/2 operation is marked as obosolete in genop.tab, +# in R13. The is_constant/2 operation is marked as obsolete in genop.tab, # so the loader will automatically generate a too_old_compiler message # it is used, but we need to handle the is_constant/1 BIF specially here. bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler +# Since the constant pool was introduced in R12B, empty tuples ({}) +# are literals. Therefore we no longer need to allow put_tuple/2 +# with a tuple size of zero. + +put_tuple u==0 d => too_old_compiler + # # All the other instructions. # @@ -78,6 +84,8 @@ i_trace_breakpoint i_mtrace_breakpoint i_debug_breakpoint i_count_breakpoint +i_time_breakpoint +i_return_time_trace i_return_to_trace i_yield i_global_cons @@ -93,16 +101,16 @@ return %macro: test_heap TestHeap -pack allocate t t -allocate_heap I I I +allocate_heap t I t deallocate I init y allocate_zero t t -allocate_heap_zero I I I +allocate_heap_zero t I t trim N Remaining => i_trim N i_trim I -test_heap I I +test_heap I t allocate_heap S u==0 R => allocate S R allocate_heap_zero S u==0 R => allocate_zero S R @@ -116,7 +124,7 @@ init Y1 | init Y2 => init2 Y1 Y2 # Selecting values -select_val S=q Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest) +select_val S=aiq Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest) select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \ gen_jump_tab(S, Fail, Size, Rest) @@ -124,34 +132,59 @@ select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \ is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \ gen_jump_tab(S, Fail, Size, Rest) +is_integer TypeFail=f S | select_val S=s Fail=f Size=u Rest=* | \ + mixed_types(Size, Rest) => \ + gen_split_values(S, TypeFail, Fail, Size, Rest) + select_val S=s Fail=f Size=u Rest=* | mixed_types(Size, Rest) => \ - gen_split_values(S, Fail, Size, Rest) + gen_split_values(S, Fail, Fail, Size, Rest) -is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | \ +is_integer Fail=f S | select_val S=d Fail=f Size=u Rest=* | \ fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest) -is_atom Fail=f S | select_val S=s Fail=f Size=u Rest=* | \ +is_atom Fail=f S | select_val S=d Fail=f Size=u Rest=* | \ fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest) -select_val S=s Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \ - gen_select_val(S, Fail, Size, Rest) +select_val S=s Fail=f Size=u Rest=* | floats_or_bignums(Size, Rest) => \ + gen_select_literals(S, Fail, Size, Rest) -select_val S=s Fail=f Size=u Rest=* | all_values_are_big(Size, Rest) => \ - gen_select_big(S, Fail, Size, Rest) +select_val S=d Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \ + gen_select_val(S, Fail, Size, Rest) -is_tuple Fail=f S | select_tuple_arity S=s Fail=f Size=u Rest=* => \ +is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -select_tuple_arity S=s Fail=f Size=u Rest=* => \ +select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -i_select_val s f I -i_select_tuple_arity s f I -i_select_big s f -i_select_float s f I +i_select_val r f I +i_select_val x f I +i_select_val y f I + +i_select_val2 r f c f c f +i_select_val2 x f c f c f +i_select_val2 y f c f c f + +i_select_tuple_arity2 r f A f A f +i_select_tuple_arity2 x f A f A f +i_select_tuple_arity2 y f A f A f + +i_select_tuple_arity r f I +i_select_tuple_arity x f I +i_select_tuple_arity y f I + +i_jump_on_val_zero r f I +i_jump_on_val_zero x f I +i_jump_on_val_zero y f I -i_jump_on_val_zero s f I -i_jump_on_val s f I I +i_jump_on_val r f I I +i_jump_on_val x f I I +i_jump_on_val y f I I + +jump Target | label Lbl | same_label(Target, Lbl) => label Lbl + +is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \ + is_eq_exact Fail S1 S2 | label L2 %macro: get_list GetList -pack get_list x x x @@ -226,11 +259,17 @@ is_number Fail Literal=q => move Literal x | is_number Fail x jump f -case_end Literal=q => move Literal x | case_end x -badmatch Literal=q => move Literal x | badmatch x +case_end Literal=cq => move Literal x | case_end x +badmatch Literal=cq => move Literal x | badmatch x + +case_end r +case_end x +case_end y + +badmatch r +badmatch x +badmatch y -case_end s -badmatch s if_end raise s s @@ -240,12 +279,33 @@ system_limit j move R R => +move C=cxy r | jump Lbl => move_jump Lbl C + +%macro: move_jump MoveJump -nonext +move_jump f n +move_jump f c +move_jump f x +move_jump f y + move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2 +move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4 + +move C=aiq X=x==1 => move_x1 C +move C=aiq X=x==2 => move_x2 C + +move_x1 c +move_x2 c %macro: move2 Move2 -pack move2 x y x y move2 y x y x +move2 x x x x + +# The compiler almost never generates a "move Literal y(Y)" instruction, +# so let's cheat if we encounter one. +move S=n D=y => init D +move S=c D=y => move S x | move x D %macro:move Move -pack -gen_dest move x x @@ -257,15 +317,10 @@ move r x move r y move c r move c x -move c y move n x move n r move y y -%cold -move s d -%hot - # Receive operations. loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src @@ -298,58 +353,78 @@ i_wait_error_locked send # -# Comparisions. +# Optimized comparisons with one immediate/literal operand. +# + +is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C +is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C + +is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C +is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C + +%macro: i_is_eq_exact_immed EqualImmed -fail_action +i_is_eq_exact_immed f r c +i_is_eq_exact_immed f x c +i_is_eq_exact_immed f y c + +i_is_eq_exact_literal r f c +i_is_eq_exact_literal x f c +i_is_eq_exact_literal y f c + +%macro: i_is_ne_exact_immed NotEqualImmed -fail_action +i_is_ne_exact_immed f r c +i_is_ne_exact_immed f x c +i_is_ne_exact_immed f y c + +i_is_ne_exact_literal r f c +i_is_ne_exact_literal x f c +i_is_ne_exact_literal y f c + +# +# All other comparisons. # -is_eq_exact Lbl=f R=rxy C=ian => i_is_eq_immed Lbl R C -is_eq Lbl=f R=rxy C=an => i_is_eq_immed Lbl R C +is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl +is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl -is_eq_exact Lbl=f S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl -is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl - +i_is_eq_exact f +i_is_ne_exact f i_is_lt f i_is_ge f i_is_eq f i_is_ne f -i_is_eq_exact f -i_is_ne_exact f - -%macro: i_is_eq_immed EqualImmed -fail_action -i_is_eq_immed f r c -i_is_eq_immed f x c -i_is_eq_immed f y c # # Putting things. # -put_tuple u==0 Dst => i_put_tuple_only u Dst -put_tuple Arity Dst | put V => i_put_tuple Arity V Dst +put_tuple Arity Dst => i_put_tuple Dst u -i_put_tuple_only A d +i_put_tuple Dst Arity Puts=* | put S1 | put S2 | \ + put S3 | put S4 | put S5 => \ + tuple_append_put5(Arity, Dst, Puts, S1, S2, S3, S4, S5) -%macro: i_put_tuple PutTuple -pack -i_put_tuple A x x -i_put_tuple A y x -i_put_tuple A r x -i_put_tuple A n x -i_put_tuple A c x -i_put_tuple A x y -i_put_tuple A x r -i_put_tuple A y r -i_put_tuple A n r -i_put_tuple A c r +i_put_tuple Dst Arity Puts=* | put S => \ + tuple_append_put(Arity, Dst, Puts, S) -%cold -i_put_tuple A r y -i_put_tuple A y y -i_put_tuple A c y -%hot +i_put_tuple/2 + +%macro:i_put_tuple PutTuple -pack -goto:do_put_tuple +i_put_tuple r I +i_put_tuple x I +i_put_tuple y I + +# +# The instruction "put_list Const [] Dst" will not be generated by +# the current BEAM compiler. But until R15A, play it safe by handling +# that instruction with the following transformation. +# +put_list Const=c n Dst => move Const x | put_list x n Dst %macro:put_list PutList -pack -gen_dest @@ -357,10 +432,8 @@ put_list x n x put_list y n x put_list x x x put_list y x x -put_list c n x put_list x x r put_list y r r -put_list c n r put_list y y x put_list x y x @@ -371,6 +444,13 @@ put_list y y r put_list y r x put_list r n x +put_list x r x +put_list x y r +put_list y x r +put_list y x x + +put_list x r r + # put_list SrcReg Constant Dst put_list r c r put_list r c x @@ -398,17 +478,9 @@ put_list c y x put_list c y y %cold -put_list x r r put_list s s d %hot -%macro: put Put -put x -put r -put y -put c -put n - %macro: i_fetch FetchArgs -pack i_fetch c c i_fetch c r @@ -459,19 +531,20 @@ move_return n r move S r | deallocate D | return => move_deallocate_return S r D -%macro: move_deallocate_return MoveDeallocateReturn -nonext -move_deallocate_return x r P -move_deallocate_return y r P -move_deallocate_return c r P -move_deallocate_return n r P +%macro: move_deallocate_return MoveDeallocateReturn -pack -nonext +move_deallocate_return x r Q +move_deallocate_return y r Q +move_deallocate_return c r Q +move_deallocate_return n r Q deallocate D | return => deallocate_return D %macro: deallocate_return DeallocateReturn -nonext -deallocate_return P +deallocate_return Q test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y +%macro: test_heap_1_put_list TestHeapPutList -pack test_heap_1_put_list I y # Test tuple & arity (head) @@ -571,14 +644,14 @@ is_list f y is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs -%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -is_nonempty_list_allocate f x I I -is_nonempty_list_allocate f r I I +%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack +is_nonempty_list_allocate f x I t +is_nonempty_list_allocate f r I t is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2 -%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -is_non_empty_list_test_heap f r I I +%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack +is_non_empty_list_test_heap f r I t %macro: is_nonempty_list IsNonemptyList -fail_action is_nonempty_list f x @@ -907,8 +980,13 @@ node x node y %hot -i_fast_element j I s d -i_element j s s d +i_fast_element r j I d +i_fast_element x j I d +i_fast_element y j I d + +i_element r j s d +i_element x j s d +i_element y j s d bif1 f b s d bif1_body b s d @@ -935,11 +1013,11 @@ move S r | call_last Ar P=f D => move_call_last S r P D i_move_call_last f P c r -%macro:move_call_last MoveCallLast -arg_f -nonext +%macro:move_call_last MoveCallLast -arg_f -nonext -pack move_call_last/4 -move_call_last x r f P -move_call_last y r f P +move_call_last x r f Q +move_call_last y r f Q move S=c r | call_only Ar P=f => i_move_call_only P S r move S=x r | call_only Ar P=f => move_call_only S r P @@ -1177,12 +1255,6 @@ i_bs_init_bits_fail_heap I j I d i_bs_init_bits I I d i_bs_init_bits_heap I I I d -bs_bits_to_bytes Fail Src Dst => i_bs_bits_to_bytes Src Fail Dst - -i_bs_bits_to_bytes r j d -i_bs_bits_to_bytes x j d -i_bs_bits_to_bytes y j d - bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D bs_add Fail S1 S2 Unit D => i_fetch S1 S2 | i_bs_add Fail Unit D @@ -1308,6 +1380,8 @@ fconv Arg=iqan Dst=l => move Arg x | fconv x Dst fmove q l fmove d l +fmove l d + fconv d l i_fadd l l l @@ -1323,12 +1397,6 @@ fcheckerror p => i_fcheckerror i_fcheckerror fclearerror -fmove FR=l Dst=d | new_float_allocation() => fmove_new FR Dst - -# The new instruction for moving a float out of a floating point register. -# (No allocation.) -fmove_new l d - # # New apply instructions in R10B. # @@ -1337,7 +1405,21 @@ apply I apply_last I P # -# New GCing arithmetic instructions. +# Optimize addition and subtraction of small literals using +# the i_increment/4 instruction (in bodies, not in guards). +# + +gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \ + gen_increment(Reg, Int, Live, Dst) +gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \ + gen_increment(Reg, Int, Live, Dst) + +gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \ + negation_is_small(Int) => \ + gen_increment_from_minus(Reg, Int, Live, Dst) + +# +# GCing arithmetic instructions. # gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst @@ -1360,6 +1442,10 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst +i_increment r I I d +i_increment x I I d +i_increment y I I d + i_plus j I d i_minus j I d i_times j I d @@ -1390,34 +1476,60 @@ bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler # Guard BIFs. # gc_bif1 Fail I Bif=u$bif:erlang:length/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:size/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:bit_size/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:byte_size/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:abs/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:float/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:round/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) gc_bif1 Fail I Bif=u$bif:erlang:trunc/1 Src Dst=d => \ - gen_guard_bif(Fail, I, Bif, Src, Dst) + gen_guard_bif1(Fail, I, Bif, Src, Dst) + +gc_bif2 Fail I Bif=u$bif:erlang:binary_part/2 S1 S2 Dst=d => \ + gen_guard_bif2(Fail, I, Bif, S1, S2, Dst) + +gc_bif3 Fail I Bif=u$bif:erlang:binary_part/3 S1 S2 S3 Dst=d => \ + gen_guard_bif3(Fail, I, Bif, S1, S2, S3, Dst) i_gc_bif1 Fail Bif V=q Live D => move V x | i_gc_bif1 Fail Bif x Live D i_gc_bif1 j I s I d +ii_gc_bif2/6 + +ii_gc_bif2 Fail Bif S1 S2 Live D => i_fetch S1 S2 | i_gc_bif2 Fail Bif Live D + +i_gc_bif2 j I I d + +ii_gc_bif3/7 + +ii_gc_bif3 Fail Bif S1 S2 S3 Live D => move S1 x | i_fetch S2 S3 | i_gc_bif3 Fail Bif x Live D + +i_gc_bif3 j I s I d # # R13B03 # on_load + +# +# R14A. +# +recv_mark f + +recv_set Fail | label Lbl | loop_rec Lf Reg => \ + i_recv_set | label Lbl | loop_rec Lf Reg +i_recv_set diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index 8c8029d450..a66d60aa22 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -47,11 +47,6 @@ (((unsigned char*) (s))[1] << 8) | \ (((unsigned char*) (s))[0])) -#define put_int24(s, x) ((((unsigned char*)(s))[0] = ((x) >> 16) & 0xff), \ - (((unsigned char*)(s))[1] = ((x) >> 8) & 0xff), \ - (((unsigned char*)(s))[2] = (x) & 0xff)) - - #if !defined(__WIN32__) && !defined(HAVE_STRNCASECMP) #define STRNCASECMP my_strncasecmp @@ -679,7 +674,7 @@ int packet_parse_http(const char* buf, int len, int* statep, while (n && SP(ptr)) { ptr++; n--; } - if (ptr==p0) return -1; + if (ptr==p0 && n>0) return -1; /* NOTE: the syntax allows empty reason phrases */ (*statep) = !0; @@ -833,7 +828,7 @@ int packet_parse_ssl(const char* buf, int len, char prefix[4]; /* <<1:8,Length:24,Data/binary>> */ prefix[0] = 1; - put_int24(&prefix[1],len-3); + put_int24(len-3,&prefix[1]); return pcb->ssl_tls(arg, 22, major, minor, buf+3, len-3, prefix, sizeof(prefix)); } else { diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index 964c10a380..26d64887d0 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -39,8 +39,6 @@ static Hash process_reg; static erts_smp_rwmtx_t regtab_rwmtx; -#define reg_lock_init() erts_smp_rwmtx_init(®tab_rwmtx, \ - "reg_tab") #define reg_try_read_lock() erts_smp_rwmtx_tryrlock(®tab_rwmtx) #define reg_try_write_lock() erts_smp_rwmtx_tryrwlock(®tab_rwmtx) #define reg_read_lock() erts_smp_rwmtx_rlock(®tab_rwmtx) @@ -147,8 +145,11 @@ static void reg_free(RegProc *obj) void init_register_table(void) { HashFunctions f; + erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; + rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; + rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; - reg_lock_init(); + erts_smp_rwmtx_init_opt(®tab_rwmtx, &rwmtx_opt, "reg_tab"); f.hash = (H_FUN) reg_hash; f.cmp = (HCMP_FUN) reg_cmp; @@ -476,8 +477,9 @@ int erts_unregister_name(Process *c_p, * on c_prt. */ - if (!c_p) + if (!c_p) { c_p_locks = 0; + } current_c_p_locks = c_p_locks; restart: @@ -489,9 +491,15 @@ int erts_unregister_name(Process *c_p, if (is_non_value(name)) { /* Unregister current process name */ ASSERT(c_p); - if (c_p->reg) +#ifdef ERTS_SMP + if (current_c_p_locks != c_p_locks) { + erts_smp_proc_lock(c_p, c_p_locks); + current_c_p_locks = c_p_locks; + } +#endif + if (c_p->reg) { r.name = c_p->reg->name; - else { + } else { /* Name got unregistered while main lock was released */ res = 0; goto done; @@ -533,24 +541,25 @@ int erts_unregister_name(Process *c_p, } } else if (rp->p) { - Process* p = rp->p; + #ifdef ERTS_SMP erts_proc_safelock(c_p, current_c_p_locks, c_p_locks, rp->p, - 0, + (c_p == rp->p) ? current_c_p_locks : 0, ERTS_PROC_LOCK_MAIN); current_c_p_locks = c_p_locks; #endif - p->reg = NULL; + rp->p->reg = NULL; + if (IS_TRACED_FL(rp->p, F_TRACE_PROCS)) { + trace_proc(c_p, rp->p, am_unregister, r.name); + } #ifdef ERTS_SMP - if (rp->p != c_p) + if (rp->p != c_p) { erts_smp_proc_unlock(rp->p, ERTS_PROC_LOCK_MAIN); -#endif - if (IS_TRACED_FL(p, F_TRACE_PROCS)) { - trace_proc(c_p, p, am_unregister, r.name); } +#endif } hash_erase(&process_reg, (void*) &r); res = 1; @@ -560,14 +569,17 @@ int erts_unregister_name(Process *c_p, reg_write_unlock(); if (c_prt != port) { - if (port) + if (port) { erts_smp_port_unlock(port); - if (c_prt) + } + if (c_prt) { erts_smp_port_lock(c_prt); + } } #ifdef ERTS_SMP - if (c_p && !current_c_p_locks) + if (c_p && !current_c_p_locks) { erts_smp_proc_lock(c_p, c_p_locks); + } #endif return res; } diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index a1955235b7..dff2dc37a2 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -25,14 +25,6 @@ # define NO_FPE_SIGNALS #endif -/* Never use elib-malloc when purify-memory-tracing */ -#if defined(PURIFY) -#undef ENABLE_ELIB_MALLOC -#undef ELIB_HEAP_SBRK -#undef ELIB_ALLOC_IS_CLIB -#endif - - /* xxxP __VXWORKS__ */ #ifdef VXWORKS #include <vxWorks.h> @@ -47,13 +39,6 @@ #define ENABLE_CHILD_WAITER_THREAD 1 #endif -/* The ERTS_TIMER_TREAD #define must be visible to the - erl_${OS}_sys.h #include files: it controls whether - certain optional facilities should be defined or not. */ -#if defined(ERTS_SMP) && 0 -#define ERTS_TIMER_THREAD -#endif - #if defined (__WIN32__) # include "erl_win_sys.h" #elif defined (VXWORKS) @@ -171,23 +156,6 @@ void erl_assert_error(char* expr, char* file, int line); #include <stdarg.h> -#if defined(__STDC__) || defined(_MSC_VER) -# define EXTERN_FUNCTION(t, f, x) extern t f x -# define FUNCTION(t, f, x) t f x -# define _DOTS_ ... -# define _VOID_ void -#elif defined(__cplusplus) -# define EXTERN_FUNCTION(f, x) extern "C" { f x } -# define FUNCTION(t, f, x) t f x -# define _DOTS_ ... -# define _VOID_ void -#else -# define EXTERN_FUNCTION(t, f, x) extern t f (/*x*/) -# define FUNCTION(t, f, x) t f (/*x*/) -# define _DOTS_ -# define _VOID_ -#endif - /* This isn't sys-dependent, but putting it here benefits sys.c and drivers - allow use of 'const' regardless of compiler */ @@ -197,7 +165,7 @@ void erl_assert_error(char* expr, char* file, int line); #ifdef VXWORKS /* Replace VxWorks' printf with a real one that does fprintf(stdout, ...) */ -EXTERN_FUNCTION(int, real_printf, (const char *fmt, ...)); +int real_printf(const char *fmt, ...); # define printf real_printf #endif @@ -359,15 +327,8 @@ typedef unsigned char byte; #error 64-bit architecture, but no appropriate type to use for Uint64 and Sint64 found #endif -#if defined(ARCH_64) -# define ERTS_WORD_ALIGN_PAD_SZ(X) \ +# define ERTS_EXTRA_DATA_ALIGN_SZ(X) \ (((size_t) 8) - (((size_t) (X)) & ((size_t) 7))) -#elif defined(ARCH_32) -# define ERTS_WORD_ALIGN_PAD_SZ(X) \ - (((size_t) 4) - (((size_t) (X)) & ((size_t) 3))) -#else -#error "Not supported..." -#endif #include "erl_lock_check.h" #include "erl_smp.h" @@ -498,8 +459,6 @@ static const int zero_value = 0, one_value = 1; # endif /* !__WIN32__ */ #endif /* WANT_NONBLOCKING */ -extern erts_cpu_info_t *erts_cpuinfo; /* erl_init.c */ - __decl_noreturn void __noreturn erl_exit(int n, char*, ...); /* Some special erl_exit() codes: */ @@ -596,11 +555,7 @@ extern char *erts_default_arg0; extern char os_type[]; extern int sys_init_time(void); -#if defined(ERTS_TIMER_THREAD) -#define erts_deliver_time() -#else extern void erts_deliver_time(void); -#endif extern void erts_time_remaining(SysTimeval *); extern int erts_init_time_sup(void); extern void erts_sys_init_float(void); @@ -648,7 +603,7 @@ extern char *erts_sys_ddll_error(int code); /* - * System interfaces for startup/sae code (functions found in respective sys.c) + * System interfaces for startup. */ @@ -667,12 +622,12 @@ extern void erl_sys_args(int *argc, char **argv); extern void erl_sys_schedule(int); void sys_tty_reset(int); -EXTERN_FUNCTION(int, sys_max_files, (_VOID_)); +int sys_max_files(void); void sys_init_io(void); Preload* sys_preloaded(void); -EXTERN_FUNCTION(unsigned char*, sys_preload_begin, (Preload*)); -EXTERN_FUNCTION(void, sys_preload_end, (Preload*)); -EXTERN_FUNCTION(int, sys_get_key, (int)); +unsigned char* sys_preload_begin(Preload*); +void sys_preload_end(Preload*); +int sys_get_key(int); void elapsed_time_both(unsigned long *ms_user, unsigned long *ms_sys, unsigned long *ms_user_diff, unsigned long *ms_sys_diff); void wall_clock_elapsed_time_both(unsigned long *ms_total, @@ -689,7 +644,7 @@ int local_to_univ(Sint *year, Sint *month, Sint *day, Sint *hour, Sint *minute, Sint *second, int isdst); void get_now(Uint*, Uint*, Uint*); void get_sys_now(Uint*, Uint*, Uint*); -EXTERN_FUNCTION(void, set_break_quit, (void (*)(void), void (*)(void))); +void set_break_quit(void (*)(void), void (*)(void)); void os_flavor(char*, unsigned); void os_version(int*, int*, int*); @@ -729,7 +684,7 @@ int erts_write_env(char *key, char *value); #define ERTS_DEFAULT_MMAP_THRESHOLD (128 * 1024) #define ERTS_DEFAULT_MMAP_MAX 64 -EXTERN_FUNCTION(int, sys_alloc_opt, (int, int)); +int sys_alloc_opt(int, int); typedef struct { Sint trim_threshold; @@ -738,7 +693,7 @@ typedef struct { Sint mmap_max; } SysAllocStat; -EXTERN_FUNCTION(void, sys_alloc_stat, (SysAllocStat *)); +void sys_alloc_stat(SysAllocStat *); /* Block the whole system... */ @@ -762,11 +717,11 @@ typedef enum { } erts_activity_error_t; typedef struct { - erts_smp_atomic_t do_block; + erts_smp_atomic32_t do_block; struct { - erts_smp_atomic_t wait; - erts_smp_atomic_t gc; - erts_smp_atomic_t io; + erts_smp_atomic32_t wait; + erts_smp_atomic32_t gc; + erts_smp_atomic32_t io; } in_activity; } erts_system_block_state_t; @@ -917,7 +872,7 @@ ERTS_GLB_INLINE int erts_smp_pending_system_block(void) { #ifdef ERTS_SMP - return erts_smp_atomic_read(&erts_system_block_state.do_block); + return (int) erts_smp_atomic32_read(&erts_system_block_state.do_block); #else return 0; #endif @@ -953,7 +908,7 @@ erts_smp_set_activity(erts_activity_t old_activity, case ERTS_ACTIVITY_UNDEFINED: break; case ERTS_ACTIVITY_WAIT: - erts_smp_atomic_dec(&erts_system_block_state.in_activity.wait); + erts_smp_atomic32_dec(&erts_system_block_state.in_activity.wait); if (locked) { /* You are not allowed to leave activity waiting * without supplying the possibility to block @@ -964,10 +919,10 @@ erts_smp_set_activity(erts_activity_t old_activity, } break; case ERTS_ACTIVITY_GC: - erts_smp_atomic_dec(&erts_system_block_state.in_activity.gc); + erts_smp_atomic32_dec(&erts_system_block_state.in_activity.gc); break; case ERTS_ACTIVITY_IO: - erts_smp_atomic_dec(&erts_system_block_state.in_activity.io); + erts_smp_atomic32_dec(&erts_system_block_state.in_activity.io); break; default: erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, @@ -983,13 +938,13 @@ erts_smp_set_activity(erts_activity_t old_activity, case ERTS_ACTIVITY_UNDEFINED: break; case ERTS_ACTIVITY_WAIT: - erts_smp_atomic_inc(&erts_system_block_state.in_activity.wait); + erts_smp_atomic32_inc(&erts_system_block_state.in_activity.wait); break; case ERTS_ACTIVITY_GC: - erts_smp_atomic_inc(&erts_system_block_state.in_activity.gc); + erts_smp_atomic32_inc(&erts_system_block_state.in_activity.gc); break; case ERTS_ACTIVITY_IO: - erts_smp_atomic_inc(&erts_system_block_state.in_activity.io); + erts_smp_atomic32_inc(&erts_system_block_state.in_activity.io); break; default: erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY, @@ -1024,27 +979,31 @@ erts_smp_set_activity(erts_activity_t old_activity, typedef erts_smp_atomic_t erts_refc_t; -ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, long val); -ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE long erts_refc_inctest(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE long erts_refc_dectest(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, long diff, long min_val); -ERTS_GLB_INLINE long erts_refc_read(erts_refc_t *refcp, long min_val); +ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val); +ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp, + erts_aint_t min_val); +ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp, + erts_aint_t min_val); +ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, + erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp, + erts_aint_t min_val); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_refc_init(erts_refc_t *refcp, long val) +erts_refc_init(erts_refc_t *refcp, erts_aint_t val) { erts_smp_atomic_init((erts_smp_atomic_t *) refcp, val); } ERTS_GLB_INLINE void -erts_refc_inc(erts_refc_t *refcp, long min_val) +erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", @@ -1054,10 +1013,10 @@ erts_refc_inc(erts_refc_t *refcp, long min_val) #endif } -ERTS_GLB_INLINE long -erts_refc_inctest(erts_refc_t *refcp, long min_val) +ERTS_GLB_INLINE erts_aint_t +erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) { - long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1068,10 +1027,10 @@ erts_refc_inctest(erts_refc_t *refcp, long min_val) } ERTS_GLB_INLINE void -erts_refc_dec(erts_refc_t *refcp, long min_val) +erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", @@ -1081,10 +1040,10 @@ erts_refc_dec(erts_refc_t *refcp, long min_val) #endif } -ERTS_GLB_INLINE long -erts_refc_dectest(erts_refc_t *refcp, long min_val) +ERTS_GLB_INLINE erts_aint_t +erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) { - long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1095,10 +1054,10 @@ erts_refc_dectest(erts_refc_t *refcp, long min_val) } ERTS_GLB_INLINE void -erts_refc_add(erts_refc_t *refcp, long diff, long min_val) +erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - long val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff); + erts_aint_t val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", @@ -1108,10 +1067,10 @@ erts_refc_add(erts_refc_t *refcp, long diff, long min_val) #endif } -ERTS_GLB_INLINE long -erts_refc_read(erts_refc_t *refcp, long min_val) +ERTS_GLB_INLINE erts_aint_t +erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) { - long val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1127,13 +1086,10 @@ erts_refc_read(erts_refc_t *refcp, long min_val) extern int erts_use_kernel_poll; #endif -void elib_ensure_initialized(void); - - #if defined(VXWORKS) /* NOTE! sys_calloc2 does not exist on other platforms than VxWorks and OSE */ -EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); +void* sys_calloc2(Uint, Uint); #endif /* VXWORKS || OSE */ @@ -1173,14 +1129,14 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); /* Standard set of integer macros .. */ -#define get_int64(s) ((((unsigned char*) (s))[0] << 56) | \ - (((unsigned char*) (s))[1] << 48) | \ - (((unsigned char*) (s))[2] << 40) | \ - (((unsigned char*) (s))[3] << 32) | \ - (((unsigned char*) (s))[4] << 24) | \ - (((unsigned char*) (s))[5] << 16) | \ - (((unsigned char*) (s))[6] << 8) | \ - (((unsigned char*) (s))[7])) +#define get_int64(s) (((Uint64)(((unsigned char*) (s))[0]) << 56) | \ + (((Uint64)((unsigned char*) (s))[1]) << 48) | \ + (((Uint64)((unsigned char*) (s))[2]) << 40) | \ + (((Uint64)((unsigned char*) (s))[3]) << 32) | \ + (((Uint64)((unsigned char*) (s))[4]) << 24) | \ + (((Uint64)((unsigned char*) (s))[5]) << 16) | \ + (((Uint64)((unsigned char*) (s))[6]) << 8) | \ + (((Uint64)((unsigned char*) (s))[7]))) #define put_int64(i, s) do {((char*)(s))[0] = (char)((Sint64)(i) >> 56) & 0xff;\ ((char*)(s))[1] = (char)((Sint64)(i) >> 48) & 0xff;\ @@ -1203,6 +1159,15 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); ((char*)(s))[3] = (char)(i) & 0xff;} \ while (0) +#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \ + (((unsigned char*) (s))[1] << 8) | \ + (((unsigned char*) (s))[2])) + +#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff; \ + ((char*)(s))[1] = (char)((i) >> 8) & 0xff; \ + ((char*)(s))[2] = (char)(i) & 0xff;} \ + while (0) + #define get_int16(s) ((((unsigned char*) (s))[0] << 8) | \ (((unsigned char*) (s))[1])) @@ -1216,6 +1181,7 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0) + /* * Use DEBUGF as you would use printf, but use double parentheses: * @@ -1225,8 +1191,8 @@ EXTERN_FUNCTION(void*, sys_calloc2, (Uint, Uint)); */ #ifdef DEBUG -EXTERN_FUNCTION(void, erl_debug, (char* format, ...)); -EXTERN_FUNCTION(void, erl_bin_write, (unsigned char *, int, int)); +void erl_debug(char* format, ...); +void erl_bin_write(unsigned char *, int, int); # define DEBUGF(x) erl_debug x #else @@ -1280,6 +1246,22 @@ char* win32_errorstr(int); #endif +/************************************************************************ + * Find out the native filename encoding of the process (look at locale of + * Unix processes and just do UTF16 on windows + ************************************************************************/ +#define ERL_FILENAME_UNKNOWN 0 +#define ERL_FILENAME_LATIN1 1 +#define ERL_FILENAME_UTF8 2 +#define ERL_FILENAME_UTF8_MAC 3 +#define ERL_FILENAME_WIN_WCHAR 4 + +int erts_get_native_filename_encoding(void); +/* The set function is only to be used by erl_init! */ +void erts_set_user_requested_filename_encoding(int encoding); +int erts_get_user_requested_filename_encoding(void); + +void erts_init_sys_common_misc(void); #endif diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index a07d6a5327..c65cc37fc6 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -83,24 +83,8 @@ #define ASSERT_NO_LOCKED_LOCKS #endif +static erts_smp_mtx_t tiw_lock; -#if defined(ERTS_TIMER_THREAD) || 1 -/* I don't yet know why, but using a mutex instead of a spinlock - or spin-based rwlock avoids excessive delays at startup. */ -static erts_smp_rwmtx_t tiw_lock; -#define tiw_read_lock() erts_smp_rwmtx_rlock(&tiw_lock) -#define tiw_read_unlock() erts_smp_rwmtx_runlock(&tiw_lock) -#define tiw_write_lock() erts_smp_rwmtx_rwlock(&tiw_lock) -#define tiw_write_unlock() erts_smp_rwmtx_rwunlock(&tiw_lock) -#define tiw_init_lock() erts_smp_rwmtx_init(&tiw_lock, "timer_wheel") -#else -static erts_smp_rwlock_t tiw_lock; -#define tiw_read_lock() erts_smp_read_lock(&tiw_lock) -#define tiw_read_unlock() erts_smp_read_unlock(&tiw_lock) -#define tiw_write_lock() erts_smp_write_lock(&tiw_lock) -#define tiw_write_unlock() erts_smp_write_unlock(&tiw_lock) -#define tiw_init_lock() erts_smp_rwlock_init(&tiw_lock, "timer_wheel") -#endif /* BEGIN tiw_lock protected variables ** @@ -115,80 +99,37 @@ static erts_smp_rwlock_t tiw_lock; static ErlTimer** tiw; /* the timing wheel, allocated in init_time() */ static Uint tiw_pos; /* current position in wheel */ static Uint tiw_nto; /* number of timeouts in wheel */ +static Uint tiw_min; +static ErlTimer *tiw_min_ptr; /* END tiw_lock protected variables */ /* Actual interval time chosen by sys_init_time() */ static int itime; /* Constant after init */ -#if defined(ERTS_TIMER_THREAD) -static SysTimeval time_start; /* start of current time interval */ -static long ticks_end; /* time_start+ticks_end == time_wakeup */ -static long ticks_latest; /* delta from time_start at latest time update*/ - -static ERTS_INLINE long time_gettimeofday(SysTimeval *now) -{ - long elapsed; - - erts_get_timeval(now); - now->tv_usec = 1000 * (now->tv_usec / 1000); /* ms resolution */ - elapsed = (1000 * (now->tv_sec - time_start.tv_sec) + - (now->tv_usec - time_start.tv_usec) / 1000); - // elapsed /= CLOCK_RESOLUTION; - return elapsed; -} - -static long do_time_update(void) -{ - SysTimeval now; - long elapsed; - - elapsed = time_gettimeofday(&now); - ticks_latest = elapsed; - return elapsed; -} - -static ERTS_INLINE long do_time_read(void) -{ - return ticks_latest; -} - -static long do_time_reset(void) -{ - SysTimeval now; - long elapsed; - - elapsed = time_gettimeofday(&now); - time_start = now; - ticks_end = LONG_MAX; - ticks_latest = 0; - return elapsed; -} - -static ERTS_INLINE void do_time_init(void) -{ - (void)do_time_reset(); -} - -#else erts_smp_atomic_t do_time; /* set at clock interrupt */ -static ERTS_INLINE long do_time_read(void) { return erts_smp_atomic_read(&do_time); } -static ERTS_INLINE long do_time_update(void) { return do_time_read(); } +static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read(&do_time); } +static ERTS_INLINE erts_aint_t do_time_update(void) { return do_time_read(); } static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init(&do_time, 0L); } -#endif /* get the time (in units of itime) to the next timeout, or -1 if there are no timeouts */ -static int next_time_internal(void) /* PRE: tiw_lock taken by caller */ +static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ { int i, tm, nto; unsigned int min; ErlTimer* p; - long dt; + erts_aint_t dt; if (tiw_nto == 0) return -1; /* no timeouts in wheel */ + + if (tiw_min_ptr) { + min = tiw_min; + dt = do_time_read(); + return ((min >= dt) ? (min - dt) : 0); + } /* start going through wheel to find next timeout */ tm = nto = 0; @@ -201,11 +142,17 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */ if (p->count == 0) { /* found next timeout */ dt = do_time_read(); + /* p->count is zero */ + tiw_min_ptr = p; + tiw_min = tm; return ((tm >= dt) ? (tm - dt) : 0); } else { /* keep shortest time in 'min' */ - if (tm + p->count*TIW_SIZE < min) + if (tm + p->count*TIW_SIZE < min) { min = tm + p->count*TIW_SIZE; + tiw_min_ptr = p; + tiw_min = min; + } } p = p->next; } @@ -218,30 +165,53 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */ return ((min >= dt) ? (min - dt) : 0); } -#if !defined(ERTS_TIMER_THREAD) +static void remove_timer(ErlTimer *p) { + /* first */ + if (!p->prev) { + tiw[p->slot] = p->next; + if(p->next) + p->next->prev = NULL; + } else { + p->prev->next = p->next; + } + + /* last */ + if (!p->next) { + if (p->prev) + p->prev->next = NULL; + } else { + p->next->prev = p->prev; + } + + p->next = NULL; + p->prev = NULL; + /* Make sure cancel callback isn't called */ + p->active = 0; + tiw_nto--; +} + /* Private export to erl_time_sup.c */ -int next_time(void) +erts_aint_t erts_next_time(void) { - int ret; + erts_aint_t ret; - tiw_write_lock(); + erts_smp_mtx_lock(&tiw_lock); (void)do_time_update(); ret = next_time_internal(); - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return ret; } -#endif -static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-locked */ +static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is write-locked */ { Uint keep_pos; Uint count; ErlTimer *p, **prev, *timeout_head, **timeout_tail; - Uint dtime = (unsigned long)dt; + Uint dtime = (Uint) dt; /* no need to bump the position if there aren't any timeouts */ if (tiw_nto == 0) { - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return; } @@ -258,12 +228,16 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l if (tiw_pos == keep_pos) count--; prev = &tiw[tiw_pos]; while ((p = *prev) != NULL) { + ASSERT( p != p->next); if (p->count < count) { /* we have a timeout */ - *prev = p->next; /* Remove from list */ - tiw_nto--; - p->next = NULL; - p->active = 0; /* Make sure cancel callback - isn't called */ + /* remove min time */ + if (tiw_min_ptr == p) { + tiw_min_ptr = NULL; + tiw_min = 0; + } + + /* Remove from list */ + remove_timer(p); *timeout_tail = p; /* Insert in timeout queue */ timeout_tail = &p->next; } @@ -277,8 +251,10 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l dtime--; } tiw_pos = keep_pos; + if (tiw_min_ptr) + tiw_min -= dt; - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); /* Call timedout timers callbacks */ while (timeout_head) { @@ -291,24 +267,17 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l * callback is called. */ p->next = NULL; + p->prev = NULL; p->slot = 0; (*p->timeout)(p->arg); } } -#if defined(ERTS_TIMER_THREAD) -static void timer_thread_bump_timer(void) +void erts_bump_timer(erts_aint_t dt) /* dt is value from do_time */ { - tiw_write_lock(); - bump_timer_internal(do_time_reset()); -} -#else -void bump_timer(long dt) /* dt is value from do_time */ -{ - tiw_write_lock(); + erts_smp_mtx_lock(&tiw_lock); bump_timer_internal(dt); } -#endif Uint erts_timer_wheel_memory_size(void) @@ -316,82 +285,10 @@ erts_timer_wheel_memory_size(void) return (Uint) TIW_SIZE * sizeof(ErlTimer*); } -#if defined(ERTS_TIMER_THREAD) -static struct erts_iwait *timer_thread_iwait; - -static int timer_thread_setup_delay(SysTimeval *rem_time) -{ - long elapsed; - int ticks; - - tiw_write_lock(); - elapsed = do_time_update(); - ticks = next_time_internal(); - if (ticks == -1) /* timer queue empty */ - ticks = 100*1000*1000; - if (elapsed > ticks) - elapsed = ticks; - ticks -= elapsed; - //ticks *= CLOCK_RESOLUTION; - rem_time->tv_sec = ticks / 1000; - rem_time->tv_usec = 1000 * (ticks % 1000); - ticks_end = ticks; - tiw_write_unlock(); - return ticks; -} - -static void *timer_thread_start(void *ignore) -{ - SysTimeval delay; - -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_set_thread_name("timer"); -#endif - erts_register_blockable_thread(); - - for(;;) { - if (timer_thread_setup_delay(&delay)) { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - ASSERT_NO_LOCKED_LOCKS; - erts_iwait_wait(timer_thread_iwait, &delay); - ASSERT_NO_LOCKED_LOCKS; - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - } - else - erts_smp_chk_system_block(NULL, NULL, NULL); - timer_thread_bump_timer(); - ASSERT_NO_LOCKED_LOCKS; - } - /*NOTREACHED*/ - return NULL; -} - -static ERTS_INLINE void timer_thread_post_insert(Uint ticks) -{ - if ((Sint)ticks < ticks_end) - erts_iwait_interrupt(timer_thread_iwait); -} - -static void timer_thread_init(void) -{ - erts_thr_opts_t opts = ERTS_THR_OPTS_DEFAULT_INITER; - erts_tid_t tid; - - opts->detached = 1; - - timer_thread_iwait = erts_iwait_init(); - erts_thr_create(&tid, timer_thread_start, NULL, &opts); -} - -#else -static ERTS_INLINE void timer_thread_post_insert(Uint ticks) { } -static ERTS_INLINE void timer_thread_init(void) { } -#endif - /* this routine links the time cells into a free list at the start and sets the time queue as empty */ void -init_time(void) +erts_init_time(void) { int i; @@ -399,7 +296,7 @@ init_time(void) if timer thread is enabled */ itime = erts_init_time_sup(); - tiw_init_lock(); + erts_smp_mtx_init(&tiw_lock, "timer_wheel"); tiw = (ErlTimer**) erts_alloc(ERTS_ALC_T_TIMER_WHEEL, TIW_SIZE * sizeof(ErlTimer*)); @@ -407,10 +304,13 @@ init_time(void) tiw[i] = NULL; do_time_init(); tiw_pos = tiw_nto = 0; - - timer_thread_init(); + tiw_min_ptr = NULL; + tiw_min = 0; } + + + /* ** Insert a process into the time queue, with a timeout 't' */ @@ -440,20 +340,35 @@ insert_timer(ErlTimer* p, Uint t) /* insert at head of list at slot */ p->next = tiw[tm]; + p->prev = NULL; + if (p->next != NULL) + p->next->prev = p; tiw[tm] = p; - tiw_nto++; - timer_thread_post_insert(ticks); + + /* insert min time */ + if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) { + tiw_min = ticks; + tiw_min_ptr = p; + } + if ((tiw_min_ptr == p) && (ticks > tiw_min)) { + /* some other timer might be 'min' now */ + tiw_min = 0; + tiw_min_ptr = NULL; + } + + tiw_nto++; } void -erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, +erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, void* arg, Uint t) { + erts_deliver_time(); - tiw_write_lock(); + erts_smp_mtx_lock(&tiw_lock); if (p->active) { /* XXX assert ? */ - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return; } p->timeout = timeout; @@ -461,45 +376,37 @@ erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, p->arg = arg; p->active = 1; insert_timer(p, t); - tiw_write_unlock(); -#if defined(ERTS_SMP) && !defined(ERTS_TIMER_THREAD) + erts_smp_mtx_unlock(&tiw_lock); +#if defined(ERTS_SMP) if (t <= (Uint) LONG_MAX) erts_sys_schedule_interrupt_timed(1, (long) t); #endif } void -erl_cancel_timer(ErlTimer* p) +erts_cancel_timer(ErlTimer* p) { - ErlTimer *tp; - ErlTimer **prev; - - tiw_write_lock(); + erts_smp_mtx_lock(&tiw_lock); if (!p->active) { /* allow repeated cancel (drivers) */ - tiw_write_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return; } - /* find p in linked list at slot p->slot and remove it */ - prev = &tiw[p->slot]; - while ((tp = *prev) != NULL) { - if (tp == p) { - *prev = p->next; /* Remove from list */ - tiw_nto--; - p->next = NULL; - p->slot = p->count = 0; - p->active = 0; - if (p->cancel != NULL) { - tiw_write_unlock(); - (*p->cancel)(p->arg); - } else { - tiw_write_unlock(); - } - return; - } else { - prev = &tp->next; - } + + /* is it the 'min' timer, remove min */ + if (p == tiw_min_ptr) { + tiw_min_ptr = NULL; + tiw_min = 0; } - tiw_write_unlock(); + + remove_timer(p); + p->slot = p->count = 0; + + if (p->cancel != NULL) { + erts_smp_mtx_unlock(&tiw_lock); + (*p->cancel)(p->arg); + return; + } + erts_smp_mtx_unlock(&tiw_lock); } /* @@ -509,15 +416,15 @@ erl_cancel_timer(ErlTimer* p) immediately if it hadn't been cancelled). */ Uint -time_left(ErlTimer *p) +erts_time_left(ErlTimer *p) { Uint left; - long dt; + erts_aint_t dt; - tiw_read_lock(); + erts_smp_mtx_lock(&tiw_lock); if (!p->active) { - tiw_read_unlock(); + erts_smp_mtx_unlock(&tiw_lock); return 0; } @@ -531,19 +438,18 @@ time_left(ErlTimer *p) else left -= dt; - tiw_read_unlock(); + erts_smp_mtx_unlock(&tiw_lock); - return left * itime; + return (Uint) left * itime; } #ifdef DEBUG - -void p_slpq() +void erts_p_slpq() { int i; ErlTimer* p; - tiw_read_lock(); + erts_smp_mtx_lock(&tiw_lock); /* print the whole wheel, starting at the current position */ erts_printf("\ntiw_pos = %d tiw_nto %d\n", tiw_pos, tiw_nto); @@ -565,7 +471,6 @@ void p_slpq() } } - tiw_read_unlock(); + erts_smp_mtx_unlock(&tiw_lock); } - #endif /* DEBUG */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 51c12a0b69..1d60b54d21 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -32,6 +32,7 @@ #include "erl_binary.h" #include "erl_bits.h" #include "packet_parser.h" +#include "erl_gc.h" #define ERTS_WANT_DB_INTERNAL__ #include "erl_db.h" #include "erl_threads.h" @@ -47,24 +48,17 @@ #undef M_MMAP_THRESHOLD #undef M_MMAP_MAX -#if !defined(ELIB_ALLOC_IS_CLIB) && defined(__GLIBC__) && defined(HAVE_MALLOC_H) +#if defined(__GLIBC__) && defined(HAVE_MALLOC_H) #include <malloc.h> #endif -#if defined(ELIB_ALLOC_IS_CLIB) || !defined(HAVE_MALLOPT) +#if !defined(HAVE_MALLOPT) #undef HAVE_MALLOPT #define HAVE_MALLOPT 0 #endif /* profile_scheduler mini message queue */ -#ifdef ERTS_TIMER_THREAD -/* A timer thread is not welcomed with this lock violation work around. - * - Bj�rn-Egil - */ -#error Timer thread may not be enabled due to lock violation. -#endif - typedef struct { Uint scheduler_id; Uint no_schedulers; @@ -125,7 +119,7 @@ erts_heap_alloc(Process* p, Uint need) n = need; bp = MBUF(p); - if (bp != NULL && need <= (bp->size - bp->used_size)) { + if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) { Eterm* ret = bp->mem + bp->used_size; bp->used_size += need; return ret; @@ -158,16 +152,11 @@ erts_heap_alloc(Process* p, Uint need) bp->next = MBUF(p); MBUF(p) = bp; - bp->size = n; + bp->alloc_size = n; bp->used_size = n; MBUF_SIZE(p) += n; - bp->off_heap.mso = NULL; -#ifndef HYBRID /* FIND ME! */ - bp->off_heap.funs = NULL; -#endif - bp->off_heap.externals = NULL; + bp->off_heap.first = NULL; bp->off_heap.overhead = 0; - return bp->mem; } @@ -408,7 +397,7 @@ erts_bld_uint64(Uint **hpp, Uint *szp, Uint64 ui64) } else { if (szp) - *szp = ERTS_UINT64_HEAP_SIZE(ui64); + *szp += ERTS_UINT64_HEAP_SIZE(ui64); if (hpp) res = erts_uint64_to_big(ui64, hpp); } @@ -425,7 +414,7 @@ erts_bld_sint64(Uint **hpp, Uint *szp, Sint64 si64) } else { if (szp) - *szp = ERTS_SINT64_HEAP_SIZE(si64); + *szp += ERTS_SINT64_HEAP_SIZE(si64); if (hpp) res = erts_sint64_to_big(si64, hpp); } @@ -509,7 +498,7 @@ erts_bld_string_n(Uint **hpp, Uint *szp, const char *str, Sint len) if (hpp) { res = NIL; while (--i >= 0) { - res = CONS(*hpp, make_small(str[i]), res); + res = CONS(*hpp, make_small((byte) str[i]), res); *hpp += 2; } } @@ -2728,21 +2717,8 @@ not_equal: } -void -erts_cleanup_externals(ExternalThing *etp) -{ - ExternalThing *tetp; - - tetp = etp; - - while(tetp) { - erts_deref_node_entry(tetp->node); - tetp = tetp->next; - } -} - Eterm -store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns) +store_external_or_ref_(Uint **hpp, ErlOffHeap* oh, Eterm ns) { Uint i; Uint size; @@ -2761,8 +2737,8 @@ store_external_or_ref_(Uint **hpp, ExternalThing **etpp, Eterm ns) erts_refc_inc(&((ExternalThing *) to_hp)->node->refc, 2); - ((ExternalThing *) to_hp)->next = *etpp; - *etpp = (ExternalThing *) to_hp; + ((struct erl_off_heap_header*) to_hp)->next = oh->first; + oh->first = (struct erl_off_heap_header*) to_hp; return make_external(to_hp); } @@ -2791,7 +2767,7 @@ store_external_or_ref_in_proc_(Process *proc, Eterm ns) sz = NC_HEAP_SIZE(ns); ASSERT(sz > 0); hp = HAlloc(proc, sz); - return store_external_or_ref_(&hp, &MSO(proc).externals, ns); + return store_external_or_ref_(&hp, &MSO(proc), ns); } void bin_write(int to, void *to_arg, byte* buf, int sz) @@ -3200,7 +3176,7 @@ erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, *timer_ref = res; - erl_set_timer(&res->timer.tm, + erts_set_timer(&res->timer.tm, (ErlTimeoutProc) ptimer_timeout, (ErlCancelProc) ptimer_cancelled, (void*) res, @@ -3214,7 +3190,7 @@ erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer) ASSERT(*ptimer->timer.timer_ref == ptimer); *ptimer->timer.timer_ref = NULL; ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED; - erl_cancel_timer(&ptimer->timer.tm); + erts_cancel_timer(&ptimer->timer.tm); } } @@ -3654,19 +3630,19 @@ erts_set_activity_error(erts_activity_error_t error, char *file, int line) } -static ERTS_INLINE int +static ERTS_INLINE erts_aint32_t threads_not_under_control(void) { - int res = system_block_state.threads_to_block; + erts_aint32_t res = system_block_state.threads_to_block; /* Waiting is always an allowed activity... */ - res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.wait); + res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.wait); if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_GC) - res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.gc); + res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.gc); if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_IO) - res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.io); + res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.io); if (res < 0) { ASSERT(0); @@ -3726,7 +3702,7 @@ erts_block_system(Uint32 allowed_activities) } else { - erts_smp_atomic_inc(&erts_system_block_state.do_block); + erts_smp_atomic32_inc(&erts_system_block_state.do_block); /* Someone else might be waiting for us to block... */ if (do_block) { @@ -3778,11 +3754,11 @@ erts_emergency_block_system(long timeout, Uint32 allowed_activities) another_blocker = erts_smp_pending_system_block(); system_block_state.emergency = 1; - erts_smp_atomic_inc(&erts_system_block_state.do_block); + erts_smp_atomic32_inc(&erts_system_block_state.do_block); if (another_blocker) { if (is_blocker()) { - erts_smp_atomic_dec(&erts_system_block_state.do_block); + erts_smp_atomic32_dec(&erts_system_block_state.do_block); res = 0; goto done; } @@ -3839,7 +3815,7 @@ erts_release_system(void) if (system_block_state.recursive_block) system_block_state.recursive_block--; else { - do_block = erts_smp_atomic_dectest(&erts_system_block_state.do_block); + do_block = erts_smp_atomic32_dectest(&erts_system_block_state.do_block); system_block_state.have_blocker = 0; if (is_blockable_thread()) system_block_state.threads_to_block++; @@ -3974,10 +3950,10 @@ erts_system_block_init(void) /* Global state... */ - erts_smp_atomic_init(&erts_system_block_state.do_block, 0L); - erts_smp_atomic_init(&erts_system_block_state.in_activity.wait, 0L); - erts_smp_atomic_init(&erts_system_block_state.in_activity.gc, 0L); - erts_smp_atomic_init(&erts_system_block_state.in_activity.io, 0L); + erts_smp_atomic32_init(&erts_system_block_state.do_block, 0); + erts_smp_atomic32_init(&erts_system_block_state.in_activity.wait, 0); + erts_smp_atomic32_init(&erts_system_block_state.in_activity.gc, 0); + erts_smp_atomic32_init(&erts_system_block_state.in_activity.io, 0); /* Make sure blockable threads unregister when exiting... */ erts_smp_install_exit_handler(erts_unregister_blockable_thread); diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index d2b916000e..6449c6f506 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -53,6 +53,8 @@ #define FILE_IPREAD 27 #define FILE_ALTNAME 28 #define FILE_READ_LINE 29 +#define FILE_FDATASYNC 30 +#define FILE_FADVISE 31 /* Return codes */ @@ -65,6 +67,8 @@ #define FILE_RESP_LDATA 6 #define FILE_RESP_N2DATA 7 #define FILE_RESP_EOF 8 +#define FILE_RESP_FNAME 9 +#define FILE_RESP_ALL_DATA 10 /* Options */ @@ -102,16 +106,16 @@ #include <ctype.h> #include <sys/types.h> -extern void erl_exit(int n, char *fmt, _DOTS_); +void erl_exit(int n, char *fmt, ...); static ErlDrvSysInfo sys_info; -/*#define TRACE 1*/ +/* #define TRACE 1 */ #ifdef TRACE -# define TRACE_C(c) (putchar(c)) -# define TRACE_S(s) (fputs((s), stdout)) -# define TRACE_F(args) (printf args) +# define TRACE_C(c) do { putchar(c); fflush(stdout); } while (0) +# define TRACE_S(s) do { fputs((s), stdout); fflush(stdout); } while (0) +# define TRACE_F(args) do { printf args ;fflush(stdout); } while (0) #else # define TRACE_C(c) ((void)(0)) # define TRACE_S(s) ((void)(0)) @@ -135,24 +139,54 @@ static ErlDrvSysInfo sys_info; #define MUTEX_UNLOCK(m) #endif - - #if 0 /* Experimental, for forcing all file operations to use the same thread. */ -static unsigned file_fixed_key = 1; -#define KEY(desc) (&file_fixed_key) + static unsigned file_fixed_key = 1; +# define KEY(desc) (&file_fixed_key) #else -#define KEY(desc) (&(desc)->key) +# define KEY(desc) (&(desc)->key) #endif +#ifdef FILENAMES_16BIT +# define FILENAME_BYTELEN(Str) filename_len_16bit(Str) +# define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) +# define FILENAME_CHARSIZE 2 + + static int filename_len_16bit(char *str) + { + char *p = str; + while(*p != '\0' || p[1] != '\0') { + p += 2; + } + return (p - str); + } + + static void filename_cpy_16bit(char *to, char *from) + { + while(*from != '\0' || from[1] != '\0') { + *to++ = *from++; + *to++ = *from++; + } + *to++ = *from++; + *to++ = *from++; + } + +#else +# define FILENAME_BYTELEN(Str) strlen(Str) +# define FILENAME_COPY(To,From) strcpy(To,From) +# define FILENAME_CHARSIZE 1 +#endif -#if MAXPATHLEN >= BUFSIZ -#define RESBUFSIZE MAXPATHLEN+1 +#if (MAXPATHLEN+1)*FILENAME_CHARSIZE+1 > BUFSIZ +# define RESBUFSIZE ((MAXPATHLEN+1)*FILENAME_CHARSIZE+1) #else -#define RESBUFSIZE BUFSIZ +# define RESBUFSIZE BUFSIZ #endif + + + #define GET_TIME(i, b) \ (i).year = get_int32((b) + 0 * 4); \ (i).month = get_int32((b) + 1 * 4); \ @@ -284,9 +318,9 @@ struct t_preadv { }; #define READDIR_BUFSIZE (8*1024) -#if READDIR_BUFSIZE < (2*MAXPATHLEN) -#undef READDIR_BUFSIZE -#define READDIR_BUFSIZE (2*MAXPATHLEN) +#if READDIR_BUFSIZE < (FILENAME_CHARSIZE*2*(MAXPATHLEN+1)) +# undef READDIR_BUFSIZE +# define READDIR_BUFSIZE (FILENAME_CHARSIZE*2*(MAXPATHLEN+1)) #endif struct t_readdir_buf { @@ -351,17 +385,22 @@ struct t_data ErlDrvBinary *binp; int size; int offset; - char name[1]; } read_file; struct { struct t_readdir_buf *first_buf; struct t_readdir_buf *last_buf; } read_dir; + struct { + Sint64 offset; + Sint64 length; + int advise; + } fadvise; } c; char b[1]; }; + #define EF_ALLOC(S) driver_alloc((S)) #define EF_REALLOC(P, S) driver_realloc((P), (S)) #define EF_SAFE_ALLOC(S) ef_safe_alloc((S)) @@ -883,6 +922,15 @@ static void invoke_chdir(void *data) invoke_name(data, efile_chdir); } +static void invoke_fdatasync(void *data) +{ + struct t_data *d = (struct t_data *) data; + int fd = (int) d->fd; + + d->again = 0; + d->result_ok = efile_fdatasync(&d->errInfo, fd); +} + static void invoke_fsync(void *data) { struct t_data *d = (struct t_data *) data; @@ -1068,7 +1116,7 @@ static void invoke_read_file(void *data) Sint64 size; if (! (d->result_ok = - efile_openfile(&d->errInfo, d->c.read_file.name, + efile_openfile(&d->errInfo, d->b, EFILE_MODE_READ, &fd, &size))) { goto done; } @@ -1272,7 +1320,7 @@ static void invoke_writev(void *data) { p < size && iovcnt < iovlen; p += iov0[iovcnt++].iov_len) ; - iov = EF_ALLOC(sizeof(SysIOVec)*iovcnt); + iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovcnt); memcpy(iov,iov0,iovcnt*sizeof(SysIOVec)); MUTEX_UNLOCK(d->c.writev.q_mtx); /* Let go of lock until we deque from original vector */ @@ -1352,7 +1400,7 @@ static void invoke_readlink(void *data) d->result_ok = efile_readlink(&d->errInfo, d->b, resbuf+1, RESBUFSIZE-1); if (d->result_ok != 0) - strcpy((char *) d->b + 1, resbuf+1); + FILENAME_COPY((char *) d->b + 1, resbuf+1); } static void invoke_altname(void *data) @@ -1364,7 +1412,7 @@ static void invoke_altname(void *data) d->result_ok = efile_altname(&d->errInfo, d->b, resbuf+1, RESBUFSIZE-1); if (d->result_ok != 0) - strcpy((char *) d->b + 1, resbuf+1); + FILENAME_COPY((char *) d->b + 1, resbuf+1); } static void invoke_pwritev(void *data) { @@ -1389,7 +1437,7 @@ static void invoke_pwritev(void *data) { /* Lock the queue just for a while, we don't want it locked during write */ MUTEX_LOCK(c->q_mtx); iov0 = driver_peekq(c->port, &iovlen); - iov = EF_ALLOC(sizeof(SysIOVec)*iovlen); + iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovlen); memcpy(iov,iov0,sizeof(SysIOVec)*iovlen); MUTEX_UNLOCK(c->q_mtx); @@ -1483,7 +1531,7 @@ static void invoke_link(void *data) char *new_name; d->again = 0; - new_name = name+strlen(name)+1; + new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; d->result_ok = efile_link(&d->errInfo, name, new_name); } @@ -1494,7 +1542,7 @@ static void invoke_symlink(void *data) char *new_name; d->again = 0; - new_name = name+strlen(name)+1; + new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; d->result_ok = efile_symlink(&d->errInfo, name, new_name); } @@ -1505,7 +1553,7 @@ static void invoke_rename(void *data) char *new_name; d->again = 0; - new_name = name+strlen(name)+1; + new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; d->result_ok = efile_rename(&d->errInfo, name, new_name); } @@ -1553,13 +1601,15 @@ static void invoke_readdir(void *data) int s; char *p = NULL; int buf_sz = 0; + size_t tmp_bs; d->again = 0; d->errInfo.posix_errno = 0; while (1) { char *str; - if (buf_sz < (4 /* sz */ + 1 /* cmd */ + MAXPATHLEN + 1 /* '\0' */)) { + if (buf_sz < (4 /* sz */ + 1 /* cmd */ + + FILENAME_CHARSIZE*(MAXPATHLEN + 1))) { struct t_readdir_buf *b; if (p) { put_int32(0, p); /* EOB */ @@ -1575,18 +1625,18 @@ static void invoke_readdir(void *data) buf_sz = READDIR_BUFSIZE - 4/* EOB */; } - p[4] = FILE_RESP_OK; + p[4] = FILE_RESP_FNAME; buf_sz -= 4 + 1; str = p + 4 + 1; ASSERT(buf_sz >= MAXPATHLEN + 1); - s = efile_readdir(&d->errInfo, d->b, &d->dir_handle, str, buf_sz); + tmp_bs = buf_sz; + s = efile_readdir(&d->errInfo, d->b, &d->dir_handle, str, &tmp_bs); if (s) { - int str_sz = strlen(str); - int sz = str_sz + 1; - put_int32(sz, p); - p += 4 + sz; - buf_sz -= str_sz; + put_int32(tmp_bs + 1 /* 1 byte for opcode */, p); + p += 4 + tmp_bs + 1; + ASSERT(p == (str + tmp_bs)); + buf_sz -= tmp_bs; } else { put_int32(1, p); @@ -1637,6 +1687,18 @@ static void invoke_open(void *data) d->result_ok = status; } +static void invoke_fadvise(void *data) +{ + struct t_data *d = (struct t_data *) data; + int fd = (int) d->fd; + off_t offset = (off_t) d->c.fadvise.offset; + off_t length = (off_t) d->c.fadvise.length; + int advise = (int) d->c.fadvise.advise; + + d->again = 0; + d->result_ok = efile_fadvise(&d->errInfo, fd, offset, length, advise); +} + static void free_readdir(void *data) { struct t_data *d = (struct t_data *) data; @@ -1883,7 +1945,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) if (!d->result_ok) reply_error(desc, &d->errInfo); else { - header[0] = FILE_RESP_OK; + header[0] = FILE_RESP_ALL_DATA; TRACE_C('R'); driver_output_binary(desc->port, header, 1, d->c.read_file.binp, @@ -1919,12 +1981,14 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) case FILE_RMDIR: case FILE_CHDIR: case FILE_DELETE: + case FILE_FDATASYNC: case FILE_FSYNC: case FILE_TRUNCATE: case FILE_LINK: case FILE_SYMLINK: case FILE_RENAME: case FILE_WRITE_INFO: + case FILE_FADVISE: reply(desc, d->result_ok, &d->errInfo); free_data(data); break; @@ -1938,10 +2002,10 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) if (!d->result_ok) reply_error(desc, &d->errInfo); else { - resbuf[0] = FILE_RESP_OK; - length = 1+strlen((char*) resbuf+1); + resbuf[0] = FILE_RESP_FNAME; + length = 1+FILENAME_BYTELEN((char*) resbuf+1); TRACE_C('R'); - driver_output2(desc->port, resbuf, length, NULL, 0); + driver_output2(desc->port, resbuf, 1, resbuf+1, length-1); } free_data(data); break; @@ -2001,13 +2065,18 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) int sz = get_int32(p); while (sz) { /* 0 == EOB */ p += 4; - driver_output2(desc->port, p, sz, NULL, 0); + if (sz - 1 > 0) { + driver_output2(desc->port, p, 1, p+1, sz-1); + } else { + driver_output2(desc->port, p, 1, NULL, 0); + } p += sz; sz = get_int32(p); } b1 = b1->next; EF_FREE(b2); } + d->c.read_dir.first_buf = NULL; d->c.read_dir.last_buf = NULL; } @@ -2083,9 +2152,9 @@ file_output(ErlDrvData e, char* buf, int count) case FILE_MKDIR: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_mkdir; d->free = free_data; @@ -2094,9 +2163,9 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_RMDIR: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_rmdir; d->free = free_data; @@ -2105,9 +2174,9 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_DELETE: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_delete_file; d->free = free_data; @@ -2117,14 +2186,14 @@ file_output(ErlDrvData e, char* buf, int count) case FILE_RENAME: { char* new_name; - - new_name = name+strlen(name)+1; + int namelen = FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; + new_name = name+namelen; d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + strlen(name) + 1 - + strlen(new_name) + 1); + + namelen + + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE); - strcpy(d->b, name); - strcpy(d->b + strlen(name) + 1, new_name); + FILENAME_COPY(d->b, name); + FILENAME_COPY(d->b + namelen, new_name); d->flags = desc->flags; d->fd = fd; d->command = command; @@ -2135,9 +2204,9 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_CHDIR: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_chdir; d->free = free_data; @@ -2160,9 +2229,10 @@ file_output(ErlDrvData e, char* buf, int count) #ifdef USE_THREADS if (sys_info.async_threads > 0) { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->dir_handle = NULL; d->command = command; d->invoke = invoke_readdir; @@ -2175,17 +2245,19 @@ file_output(ErlDrvData e, char* buf, int count) else #endif { + size_t resbufsize; char resbuf[RESBUFSIZE+1]; EFILE_DIR_HANDLE dir_handle; /* Handle to open directory. */ errInfo.posix_errno = 0; dir_handle = NULL; - resbuf[0] = FILE_RESP_OK; + resbuf[0] = FILE_RESP_FNAME; + resbufsize = RESBUFSIZE; while (efile_readdir(&errInfo, name, &dir_handle, - resbuf+1, RESBUFSIZE)) { - int length = 1 + strlen(resbuf+1); - driver_output2(desc->port, resbuf, length, NULL, 0); + resbuf+1, &resbufsize)) { + driver_output2(desc->port, resbuf, 1, resbuf+1, resbufsize); + resbufsize = RESBUFSIZE; } if (errInfo.posix_errno != 0) { reply_error(desc, &errInfo); @@ -2197,11 +2269,12 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_OPEN: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(buf+4) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(buf+4) + + FILENAME_CHARSIZE); d->flags = get_int32((uchar*)buf); name = buf+4; - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_open; d->free = free_data; @@ -2209,33 +2282,46 @@ file_output(ErlDrvData e, char* buf, int count) goto done; } + case FILE_FDATASYNC: + { + d = EF_SAFE_ALLOC(sizeof(struct t_data)); + + d->fd = fd; + d->command = command; + d->invoke = invoke_fdatasync; + d->free = free_data; + d->level = 2; + goto done; + } + case FILE_FSYNC: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data)); - - d->fd = fd; - d->command = command; - d->invoke = invoke_fsync; - d->free = free_data; - d->level = 2; - goto done; - } + { + d = EF_SAFE_ALLOC(sizeof(struct t_data)); + + d->fd = fd; + d->command = command; + d->invoke = invoke_fsync; + d->free = free_data; + d->level = 2; + goto done; + } case FILE_FSTAT: case FILE_LSTAT: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + { + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + + FILENAME_CHARSIZE); + + FILENAME_COPY(d->b, name); + d->fd = fd; + d->command = command; + d->invoke = invoke_flstat; + d->free = free_data; + d->level = 2; + goto done; + } - strcpy(d->b, name); - d->fd = fd; - d->command = command; - d->invoke = invoke_flstat; - d->free = free_data; - d->level = 2; - goto done; - } - case FILE_TRUNCATE: { d = EF_SAFE_ALLOC(sizeof(struct t_data)); @@ -2252,7 +2338,7 @@ file_output(ErlDrvData e, char* buf, int count) case FILE_WRITE_INFO: { d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + strlen(buf+21*4) + 1); + + FILENAME_BYTELEN(buf+21*4) + FILENAME_CHARSIZE); d->info.mode = get_int32(buf + 0 * 4); d->info.uid = get_int32(buf + 1 * 4); @@ -2260,7 +2346,7 @@ file_output(ErlDrvData e, char* buf, int count) GET_TIME(d->info.accessTime, buf + 3 * 4); GET_TIME(d->info.modifyTime, buf + 9 * 4); GET_TIME(d->info.cTime, buf + 15 * 4); - strcpy(d->b, buf+21*4); + FILENAME_COPY(d->b, buf+21*4); d->command = command; d->invoke = invoke_write_info; d->free = free_data; @@ -2272,7 +2358,7 @@ file_output(ErlDrvData e, char* buf, int count) { d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_readlink; d->free = free_data; @@ -2281,28 +2367,29 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_ALTNAME: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); - strcpy(d->b, name); - d->command = command; - d->invoke = invoke_altname; - d->free = free_data; - d->level = 2; - goto done; - } + { + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); + FILENAME_COPY(d->b, name); + d->command = command; + d->invoke = invoke_altname; + d->free = free_data; + d->level = 2; + goto done; + } case FILE_LINK: { char* new_name; + int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; - new_name = name+strlen(name)+1; + new_name = name+namelen; d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + strlen(name) + 1 - + strlen(new_name) + 1); + + namelen + + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE); - strcpy(d->b, name); - strcpy(d->b + strlen(name) + 1, new_name); + FILENAME_COPY(d->b, name); + FILENAME_COPY(d->b + namelen, new_name); d->flags = desc->flags; d->fd = fd; d->command = command; @@ -2315,14 +2402,15 @@ file_output(ErlDrvData e, char* buf, int count) case FILE_SYMLINK: { char* new_name; + int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; - new_name = name+strlen(name)+1; + new_name = name+namelen; d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + strlen(name) + 1 - + strlen(new_name) + 1); + + namelen + + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE); - strcpy(d->b, name); - strcpy(d->b + strlen(name) + 1, new_name); + FILENAME_COPY(d->b, name); + FILENAME_COPY(d->b + namelen, new_name); d->flags = desc->flags; d->fd = fd; d->command = command; @@ -2332,6 +2420,21 @@ file_output(ErlDrvData e, char* buf, int count) goto done; } + case FILE_FADVISE: + { + d = EF_SAFE_ALLOC(sizeof(struct t_data)); + + d->fd = fd; + d->command = command; + d->invoke = invoke_fadvise; + d->free = free_data; + d->level = 2; + d->c.fadvise.offset = get_int64((uchar*) buf); + d->c.fadvise.length = get_int64(((uchar*) buf) + sizeof(Sint64)); + d->c.fadvise.advise = get_int32(((uchar*) buf) + 2 * sizeof(Sint64)); + goto done; + } + } /* @@ -2947,6 +3050,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { case FILE_READ_FILE: { struct t_data *d; + char *filename; if (ev->size < 1+1) { /* Buffer contains empty name */ reply_posix_error(desc, ENOENT); @@ -2957,7 +3061,8 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { reply_posix_error(desc, EINVAL); goto done; } - d = EF_ALLOC(sizeof(struct t_data) + ev->size); + filename = EV_CHAR_P(ev, p, q); + d = EF_ALLOC(sizeof(struct t_data) -1 + FILENAME_BYTELEN(filename) + FILENAME_CHARSIZE); if (! d) { reply_posix_error(desc, ENOMEM); goto done; @@ -2965,8 +3070,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->command = command; d->reply = !0; /* Copy name */ - memcpy(d->c.read_file.name, EV_CHAR_P(ev, p, q), ev->size-1); - d->c.read_file.name[ev->size-1] = '\0'; + FILENAME_COPY(d->b, filename); d->c.read_file.binp = NULL; d->invoke = invoke_read_file; d->free = free_read_file; diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index 9aa941e550..3097ded3f1 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -32,7 +32,8 @@ #define EFILE_MODE_READ_WRITE 3 #define EFILE_MODE_APPEND 4 #define EFILE_COMPRESSED 8 -#define EFILE_NO_TRUNCATE 16 /* Special for reopening on VxWorks */ +#define EFILE_MODE_EXCL 16 +#define EFILE_NO_TRUNCATE 32 /* Special for reopening on VxWorks */ /* * Seek modes for efile_seek(). @@ -58,6 +59,14 @@ #define FA_WRITE 1 #define FA_READ 2 +/* Some OS'es (i.e. Windows) has filenames in wide charaqcters. That requires special handling */ +/* Note that we do *not* honor alignment in the communication to the OS specific driver, */ +/* which is not a problem on x86, but might be on other platforms. The OS specific efile */ +/* implementation is expected to align if needed */ +#ifdef __WIN32__ +#define FILENAMES_16BIT 1 +#endif + /* * An handle to an open directory. To be cast to the correct type * in the system-dependent directory functions. @@ -122,10 +131,11 @@ int efile_getdcwd(Efile_error* errInfo, int drive, char* buffer, size_t size); int efile_readdir(Efile_error* errInfo, char* name, EFILE_DIR_HANDLE* dir_handle, - char* buffer, size_t size); + char* buffer, size_t *size); int efile_openfile(Efile_error* errInfo, char* name, int flags, int* pfd, Sint64* pSize); void efile_closefile(int fd); +int efile_fdatasync(Efile_error* errInfo, int fd); int efile_fsync(Efile_error* errInfo, int fd); int efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, char *name, int info_for_link); @@ -150,3 +160,5 @@ int efile_altname(Efile_error* errInfo, char *name, int efile_link(Efile_error* errInfo, char* old, char* new); int efile_symlink(Efile_error* errInfo, char* old, char* new); int efile_may_openfile(Efile_error* errInfo, char *name); +int efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, Sint64 length, + int advise); diff --git a/erts/emulator/drivers/common/gzio.c b/erts/emulator/drivers/common/gzio.c index 801bc61d4d..5531a275ea 100644 --- a/erts/emulator/drivers/common/gzio.c +++ b/erts/emulator/drivers/common/gzio.c @@ -28,6 +28,7 @@ #ifdef __WIN32__ #define HAVE_CONFLICTING_FREAD_DECLARATION +#define FILENAMES_16BIT 1 #endif #ifdef STDC @@ -102,6 +103,40 @@ local uLong getLong OF((gz_stream *s)); # define ERTS_GZREAD(File, Buf, Count) fread((Buf), 1, (Count), (File)) #endif +/* + * Ripped from efile_drv.c + */ + +#ifdef FILENAMES_16BIT +# define FILENAME_BYTELEN(Str) filename_len_16bit(Str) +# define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) +# define FILENAME_CHARSIZE 2 + + static int filename_len_16bit(const char *str) + { + const char *p = str; + while(*p != '\0' || p[1] != '\0') { + p += 2; + } + return (p - str); + } + + static void filename_cpy_16bit(char *to, const char *from) + { + while(*from != '\0' || from[1] != '\0') { + *to++ = *from++; + *to++ = *from++; + } + *to++ = *from++; + *to++ = *from++; + } + +#else +# define FILENAME_BYTELEN(Str) strlen(Str) +# define FILENAME_COPY(To,From) strcpy(To,From) +# define FILENAME_CHARSIZE 1 +#endif + /* =========================================================================== Opens a gzip (.gz) file for reading or writing. The mode parameter is as in fopen ("rb" or "wb"). The file is given either by file descriptor @@ -144,11 +179,11 @@ local gzFile gz_open (path, mode) s->position = 0; s->destroy = destroy; - s->path = (char*)ALLOC(strlen(path)+1); + s->path = (char*)ALLOC(FILENAME_BYTELEN(path)+FILENAME_CHARSIZE); if (s->path == NULL) { return s->destroy(s), (gzFile)Z_NULL; } - strcpy(s->path, path); /* do this early for debugging */ + FILENAME_COPY(s->path, path); /* do this early for debugging */ s->mode = '\0'; do { @@ -194,7 +229,22 @@ local gzFile gz_open (path, mode) s->stream.avail_out = Z_BUFSIZE; errno = 0; -#ifdef UNIX +#if defined(FILENAMES_16BIT) + { + char wfmode[160]; + int i=0,j; + for(j=0;fmode[j] != '\0';++j) { + wfmode[i++]=fmode[j]; + wfmode[i++]='\0'; + } + wfmode[i++] = '\0'; + wfmode[i++] = '\0'; + s->file = F_OPEN(path, wfmode); + if (s->file == NULL) { + return s->destroy(s), (gzFile)Z_NULL; + } + } +#elif defined(UNIX) if (s->mode == 'r') { s->file = open(path, O_RDONLY); } else { diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index e8456cc616..818bc6334e 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -48,6 +48,15 @@ #include <sys/uio.h> #endif +#ifdef HAVE_NET_IF_DL_H +#include <net/if_dl.h> +#endif +#ifdef HAVE_IFADDRS_H +#include <ifaddrs.h> +#endif +#ifdef HAVE_NETPACKET_PACKET_H +#include <netpacket/packet.h> +#endif /* All platforms fail on malloc errors. */ #define FATAL_MALLOC @@ -55,6 +64,21 @@ #include "erl_driver.h" +/* The IS_SOCKET_ERROR macro below is used for portability reasons. While + POSIX specifies that errors from socket-related system calls should be + indicated with a -1 return value, some users have experienced non-Windows + OS kernels that return negative values other than -1. While one can argue + that such kernels are technically broken, comparing against values less + than 0 covers their out-of-spec return values without imposing incorrect + semantics on systems that manage to correctly return -1 for errors, thus + increasing Erlang's portability. +*/ +#ifdef __WIN32__ +#define IS_SOCKET_ERROR(val) ((val) == SOCKET_ERROR) +#else +#define IS_SOCKET_ERROR(val) ((val) < 0) +#endif + #ifdef __WIN32__ #define STRNCASECMP strncasecmp @@ -64,8 +88,21 @@ #include <winsock2.h> #endif #include <windows.h> +#include <Ws2tcpip.h> /* NEED VC 6.0 or higher */ + +/* Visual studio 2008+: NTDDI_VERSION needs to be set for iphlpapi.h + to define the right structures. It needs to be set to WINXP (or LONGHORN) + for IPV6 to work and it's set lower by default, so we need to change it. */ +#ifdef HAVE_SDKDDKVER_H +# include <sdkddkver.h> +# ifdef NTDDI_VERSION +# undef NTDDI_VERSION +# endif +# define NTDDI_VERSION NTDDI_WINXP +#endif + +#include <iphlpapi.h> -#include <Ws2tcpip.h> /* NEED VC 6.0 !!! */ #undef WANT_NONBLOCKING #include "sys.h" @@ -279,7 +316,7 @@ static int (*p_sctp_bindx)(int sd, struct sockaddr *addrs, #define DEBUGF(X) printf X #endif -#if !defined(__WIN32__) && !defined(HAVE_STRNCASECMP) +#if !defined(HAVE_STRNCASECMP) #define STRNCASECMP my_strncasecmp static int my_strncasecmp(const char *s1, const char *s2, size_t n) @@ -299,6 +336,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define INVALID_SOCKET -1 #define INVALID_EVENT -1 #define SOCKET_ERROR -1 + #define SOCKET int #define HANDLE long int #define FD_READ ERL_DRV_READ @@ -445,6 +483,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define INET_REQ_IFGET 22 #define INET_REQ_IFSET 23 #define INET_REQ_SUBSCRIBE 24 +#define INET_REQ_GETIFADDRS 25 /* TCP requests */ #define TCP_REQ_ACCEPT 40 #define TCP_REQ_LISTEN 41 @@ -610,15 +649,12 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define IS_BUSY(d) \ (((d)->state & INET_F_BUSY) == INET_F_BUSY) +#define INET_MAX_OPT_BUFFER (64*1024) + #define INET_DEF_BUFFER 1460 /* default buffer size */ #define INET_MIN_BUFFER 1 /* internal min buffer */ -#define INET_MAX_BUFFER (1024*64) /* internal max buffer */ -/* Note: INET_HIGH_WATERMARK MUST be less than 2*INET_MAX_BUFFER */ #define INET_HIGH_WATERMARK (1024*8) /* 8k pending high => busy */ -/* Note: INET_LOW_WATERMARK MUST be less than INET_MAX_BUFFER and -** less than INET_HIGH_WATERMARK -*/ #define INET_LOW_WATERMARK (1024*4) /* 4k pending => allow more */ #define INET_INFINITY 0xffffffff /* infinity value */ @@ -1029,7 +1065,7 @@ struct erl_drv_entry inet_driver_entry = }; /* XXX: is this a driver interface function ??? */ -extern void erl_exit(int n, char*, _DOTS_); +void erl_exit(int n, char*, ...); /* * Malloc wrapper, @@ -1229,139 +1265,136 @@ static int load_ip_and_port LOAD_ATOM((spec), (i), (flag) ? am_true : am_false); #endif /* HAVE_SCTP */ +/* Assume a cache line size of 64 bytes */ +#define INET_DRV_CACHE_LINE_SIZE ((ErlDrvUInt) 64) +#define INET_DRV_CACHE_LINE_MASK (INET_DRV_CACHE_LINE_SIZE - 1) + /* ** Binary Buffer Managment ** We keep a stack of usable buffers */ -#define BUFFER_STACK_SIZE 16 - -static erts_smp_spinlock_t inet_buffer_stack_lock; -static ErlDrvBinary* buffer_stack[BUFFER_STACK_SIZE]; -static int buffer_stack_pos = 0; +#define BUFFER_STACK_SIZE 14 +#define BUFFER_STACK_MAX_MEM_SIZE (1024*1024) +ErlDrvTSDKey buffer_stack_key; -/* - * XXX - * The erts_smp_spin_* functions should not be used by drivers (but this - * driver is special). Replace when driver locking api has been implemented. - * /rickard - */ -#define BUFSTK_LOCK erts_smp_spin_lock(&inet_buffer_stack_lock); -#define BUFSTK_UNLOCK erts_smp_spin_unlock(&inet_buffer_stack_lock); - -#ifdef DEBUG -static int tot_buf_allocated = 0; /* memory in use for i_buf */ -static int tot_buf_stacked = 0; /* memory on stack */ -static int max_buf_allocated = 0; /* max allocated */ - -#define COUNT_BUF_ALLOC(sz) do { \ - BUFSTK_LOCK; \ - tot_buf_allocated += (sz); \ - if (tot_buf_allocated > max_buf_allocated) \ - max_buf_allocated = tot_buf_allocated; \ - BUFSTK_UNLOCK; \ -} while(0) - -#define COUNT_BUF_FREE(sz) do { \ - BUFSTK_LOCK; \ - tot_buf_allocated -= (sz); \ - BUFSTK_UNLOCK; \ - } while(0) - -#define COUNT_BUF_STACK(sz) do { \ - BUFSTK_LOCK; \ - tot_buf_stacked += (sz); \ - BUFSTK_UNLOCK; \ - } while(0) +typedef struct { + int mem_size; + int pos; + ErlDrvBinary* stk[BUFFER_STACK_SIZE]; +} InetDrvBufStkBase; -#else +typedef struct { + InetDrvBufStkBase buf; + char align[(((sizeof(InetDrvBufStkBase) - 1) / INET_DRV_CACHE_LINE_SIZE) + 1) + * INET_DRV_CACHE_LINE_SIZE]; +} InetDrvBufStk; + +static InetDrvBufStk *get_bufstk(void) +{ + InetDrvBufStk *bs = erl_drv_tsd_get(buffer_stack_key); + if (bs) + return bs; + bs = driver_alloc(sizeof(InetDrvBufStk) + + INET_DRV_CACHE_LINE_SIZE - 1); + if (!bs) + return NULL; + if ((((ErlDrvUInt) bs) & INET_DRV_CACHE_LINE_MASK) != 0) + bs = ((InetDrvBufStk *) + ((((ErlDrvUInt) bs) & ~INET_DRV_CACHE_LINE_MASK) + + INET_DRV_CACHE_LINE_SIZE)); + erl_drv_tsd_set(buffer_stack_key, bs); + bs->buf.pos = 0; + bs->buf.mem_size = 0; -#define COUNT_BUF_ALLOC(sz) -#define COUNT_BUF_FREE(sz) -#define COUNT_BUF_STACK(sz) + ASSERT(bs == erl_drv_tsd_get(buffer_stack_key)); -#endif + return bs; +} static ErlDrvBinary* alloc_buffer(long minsz) { - ErlDrvBinary* buf = NULL; + InetDrvBufStk *bs = get_bufstk(); - BUFSTK_LOCK; + DEBUGF(("alloc_buffer: %ld\r\n", minsz)); - DEBUGF(("alloc_buffer: sz = %ld, tot = %d, max = %d\r\n", - minsz, tot_buf_allocated, max_buf_allocated)); + if (bs && bs->buf.pos > 0) { + long size; + ErlDrvBinary* buf = bs->buf.stk[--bs->buf.pos]; + size = buf->orig_size; + bs->buf.mem_size -= size; + ASSERT(0 <= bs->buf.mem_size + && bs->buf.mem_size <= BUFFER_STACK_MAX_MEM_SIZE); + if (size >= minsz) + return buf; - if (buffer_stack_pos > 0) { - int origsz; + driver_free_binary(buf); + } - buf = buffer_stack[--buffer_stack_pos]; - origsz = buf->orig_size; - BUFSTK_UNLOCK; - COUNT_BUF_STACK(-origsz); - if (origsz < minsz) { - if ((buf = driver_realloc_binary(buf, minsz)) == NULL) - return NULL; - COUNT_BUF_ALLOC(buf->orig_size - origsz); + ASSERT(!bs || bs->buf.pos != 0 || bs->buf.mem_size == 0); + + return driver_alloc_binary(minsz); +} + +/*#define CHECK_DOUBLE_RELEASE 1*/ +#ifdef CHECK_DOUBLE_RELEASE +static void +check_double_release(InetDrvBufStk *bs, ErlDrvBinary* buf) +{ +#ifdef __GNUC__ +#warning CHECK_DOUBLE_RELEASE is enabled, this is a custom build emulator +#endif + int i; + for (i = 0; i < bs->buf.pos; ++i) { + if (bs->buf.stk[i] == buf) { + erl_exit(ERTS_ABORT_EXIT, + "Multiple buffer release in inet_drv, this " + "is a bug, save the core and send it to " + "[email protected]!"); } } - else { - BUFSTK_UNLOCK; - if ((buf = driver_alloc_binary(minsz)) == NULL) - return NULL; - COUNT_BUF_ALLOC(buf->orig_size); - } - return buf; } +#endif -/* -** Max buffer memory "cached" BUFFER_STACK_SIZE * INET_MAX_BUFFER -** (16 * 64k ~ 1M) -*/ -/*#define CHECK_DOUBLE_RELEASE 1*/ static void release_buffer(ErlDrvBinary* buf) { + InetDrvBufStk *bs; + long size; + DEBUGF(("release_buffer: %ld\r\n", (buf==NULL) ? 0 : buf->orig_size)); - if (buf == NULL) + + if (!buf) return; - BUFSTK_LOCK; - if ((buf->orig_size > INET_MAX_BUFFER) || - (buffer_stack_pos >= BUFFER_STACK_SIZE)) { - BUFSTK_UNLOCK; - COUNT_BUF_FREE(buf->orig_size); + + size = buf->orig_size; + + if (size > BUFFER_STACK_MAX_MEM_SIZE) + goto free_binary; + + bs = get_bufstk(); + if (!bs + || (bs->buf.mem_size + size > BUFFER_STACK_MAX_MEM_SIZE) + || (bs->buf.pos >= BUFFER_STACK_SIZE)) { + free_binary: driver_free_binary(buf); } else { #ifdef CHECK_DOUBLE_RELEASE -#ifdef __GNUC__ -#warning CHECK_DOUBLE_RELEASE is enabled, this is a custom build emulator -#endif - int i; - for (i = 0; i < buffer_stack_pos; ++i) { - if (buffer_stack[i] == buf) { - erl_exit(1,"Multiple buffer release in inet_drv, this is a " - "bug, save the core and send it to " - "[email protected]!"); - } - } + check_double_release(bs, buf); #endif - buffer_stack[buffer_stack_pos++] = buf; - BUFSTK_UNLOCK; - COUNT_BUF_STACK(buf->orig_size); + ASSERT(bs->buf.pos != 0 || bs->buf.mem_size == 0); + + bs->buf.mem_size += size; + bs->buf.stk[bs->buf.pos++] = buf; + + ASSERT(0 <= bs->buf.mem_size + && bs->buf.mem_size <= BUFFER_STACK_MAX_MEM_SIZE); } } static ErlDrvBinary* realloc_buffer(ErlDrvBinary* buf, long newsz) { - ErlDrvBinary* bin; -#ifdef DEBUG - long orig_size = buf->orig_size; -#endif - - if ((bin = driver_realloc_binary(buf,newsz)) != NULL) { - COUNT_BUF_ALLOC(newsz - orig_size); - ; - } - return bin; + return driver_realloc_binary(buf, newsz); } /* use a TRICK, access the refc field to see if any one else has @@ -1375,10 +1408,8 @@ static void free_buffer(ErlDrvBinary* buf) if (buf != NULL) { if (driver_binary_get_refc(buf) == 1) release_buffer(buf); - else { - COUNT_BUF_FREE(buf->orig_size); + else driver_free_binary(buf); - } } } @@ -2152,7 +2183,7 @@ static int http_error_inetdrv(void* arg, const char* buf, int len) ErlDrvTermData spec[19]; if (desc->inet.active == INET_PASSIVE) { - /* {inet_async,S,Ref,{error,{http_error,Line}}} */ + /* {inet_async,S,Ref,{ok,{http_error,Line}}} */ int req; int aid; ErlDrvTermData caller; @@ -2162,7 +2193,7 @@ static int http_error_inetdrv(void* arg, const char* buf, int len) i = LOAD_ATOM(spec, i, am_inet_async); i = LOAD_PORT(spec, i, desc->inet.dport); i = LOAD_INT(spec, i, aid); - i = LOAD_ATOM(spec, i, am_error); + i = LOAD_ATOM(spec, i, am_ok); i = LOAD_ATOM(spec, i, am_http_error); i = http_load_string(desc, spec, i, buf, len); i = LOAD_TUPLE(spec, i, 2); @@ -3382,20 +3413,14 @@ static int inet_init() if (!sock_init()) goto error; - buffer_stack_pos = 0; - - erts_smp_spinlock_init(&inet_buffer_stack_lock, "inet_buffer_stack_lock"); + if (0 != erl_drv_tsd_key_create("inet_buffer_stack_key", &buffer_stack_key)) + goto error; ASSERT(sizeof(struct in_addr) == 4); # if defined(HAVE_IN6) && defined(AF_INET6) ASSERT(sizeof(struct in6_addr) == 16); # endif -#ifdef DEBUG - tot_buf_allocated = 0; - max_buf_allocated = 0; - tot_buf_stacked = 0; -#endif INIT_ATOM(ok); INIT_ATOM(tcp); INIT_ATOM(udp); @@ -3684,7 +3709,7 @@ static int inet_ctl_fdopen(inet_descriptor* desc, int domain, int type, unsigned int sz = sizeof(name); /* check that it is a socket and that the socket is bound */ - if (sock_name(s, (struct sockaddr*) &name, &sz) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_name(s, (struct sockaddr*) &name, &sz))) return ctl_error(sock_errno(), rbuf, rsize); desc->s = s; if ((desc->event = sock_create_event(desc)) == INVALID_EVENT) @@ -3696,7 +3721,7 @@ static int inet_ctl_fdopen(inet_descriptor* desc, int domain, int type, desc->state = INET_STATE_BOUND; /* assume bound */ if (type == SOCK_STREAM) { /* check if connected */ sz = sizeof(name); - if (sock_peer(s, (struct sockaddr*) &name, &sz) != SOCKET_ERROR) + if (!IS_SOCKET_ERROR(sock_peer(s, (struct sockaddr*) &name, &sz))) desc->state = INET_STATE_CONNECTED; } @@ -3802,39 +3827,81 @@ do { if ((end)-(ptr) < (n)) goto error; } while(0) static char* sockaddr_to_buf(struct sockaddr* addr, char* ptr, char* end) { if (addr->sa_family == AF_INET || addr->sa_family == 0) { - struct in_addr a; - buf_check(ptr,end,sizeof(struct in_addr)); - a = ((struct sockaddr_in*) addr)->sin_addr; - sys_memcpy(ptr, (char*)&a, sizeof(struct in_addr)); - return ptr + sizeof(struct in_addr); + struct in_addr *p = &(((struct sockaddr_in*) addr)->sin_addr); + buf_check(ptr, end, 1 + sizeof(struct in_addr)); + *ptr = INET_AF_INET; + sys_memcpy(ptr+1, (char*)p, sizeof(struct in_addr)); + return ptr + 1 + sizeof(struct in_addr); } #if defined(HAVE_IN6) && defined(AF_INET6) else if (addr->sa_family == AF_INET6) { - struct in6_addr a; - buf_check(ptr,end,sizeof(struct in6_addr)); - a = ((struct sockaddr_in6*) addr)->sin6_addr; - sys_memcpy(ptr, (char*)&a, sizeof(struct in6_addr)); - return ptr + sizeof(struct in6_addr); + struct in6_addr *p = &(((struct sockaddr_in6*) addr)->sin6_addr); + buf_check(ptr, end, 1 + sizeof(struct in6_addr)); + *ptr = INET_AF_INET6; + sys_memcpy(ptr+1, (char*)p, sizeof(struct in6_addr)); + return ptr + 1 + sizeof(struct in6_addr); } #endif +#if defined(AF_LINK) + else if (addr->sa_family == AF_LINK) { + struct sockaddr_dl *sdl_p = (struct sockaddr_dl*) addr; + buf_check(ptr, end, 2 + sdl_p->sdl_alen); + put_int16(sdl_p->sdl_alen, ptr); ptr += 2; + sys_memcpy(ptr, sdl_p->sdl_data + sdl_p->sdl_nlen, sdl_p->sdl_alen); + return ptr + sdl_p->sdl_alen; + } +#endif +#if defined(AF_PACKET) && defined(HAVE_NETPACKET_PACKET_H) + else if(addr->sa_family == AF_PACKET) { + struct sockaddr_ll *sll_p = (struct sockaddr_ll*) addr; + buf_check(ptr, end, 2 + sll_p->sll_halen); + put_int16(sll_p->sll_halen, ptr); ptr += 2; + sys_memcpy(ptr, sll_p->sll_addr, sll_p->sll_halen); + return ptr + sll_p->sll_halen; + } +#endif + return ptr; error: return NULL; - } static char* buf_to_sockaddr(char* ptr, char* end, struct sockaddr* addr) { - buf_check(ptr,end,sizeof(struct in_addr)); - sys_memcpy((char*) &((struct sockaddr_in*)addr)->sin_addr, ptr, - sizeof(struct in_addr)); - addr->sa_family = AF_INET; - return ptr + sizeof(struct in_addr); - + buf_check(ptr,end,1); + switch (*ptr++) { + case INET_AF_INET: { + struct in_addr *p = &((struct sockaddr_in*)addr)->sin_addr; + buf_check(ptr,end,sizeof(struct in_addr)); + sys_memcpy((char*) p, ptr, sizeof(struct in_addr)); + addr->sa_family = AF_INET; + return ptr + sizeof(struct in_addr); + } + case INET_AF_INET6: { + struct in6_addr *p = &((struct sockaddr_in6*)addr)->sin6_addr; + buf_check(ptr,end,sizeof(struct in6_addr)); + sys_memcpy((char*) p, ptr, sizeof(struct in6_addr)); + addr->sa_family = AF_INET6; + return ptr + sizeof(struct in6_addr); + } + } error: return NULL; } +#if defined (IFF_POINTOPOINT) +#define IFGET_FLAGS(cflags) IFGET_FLAGS_P2P(cflags, IFF_POINTOPOINT) +#elif defined IFF_POINTTOPOINT +#define IFGET_FLAGS(cflags) IFGET_FLAGS_P2P(cflags, IFF_POINTTOPOINT) +#endif + +#define IFGET_FLAGS_P2P(cflags, iff_ptp) \ + ((((cflags) & IFF_UP) ? INET_IFF_UP : 0) | \ + (((cflags) & IFF_BROADCAST) ? INET_IFF_BROADCAST : 0) | \ + (((cflags) & IFF_LOOPBACK) ? INET_IFF_LOOPBACK : 0) | \ + (((cflags) & iff_ptp) ? INET_IFF_POINTTOPOINT : 0) | \ + (((cflags) & IFF_UP) ? INET_IFF_RUNNING : 0) | /* emulate running ? */ \ + (((cflags) & IFF_MULTICAST) ? INET_IFF_MULTICAST : 0)) #if defined(__WIN32__) && defined(SIO_GET_INTERFACE_LIST) @@ -3872,7 +3939,6 @@ static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize) return ctl_reply(INET_REP_OK, sbuf, sptr - sbuf, rbuf, rsize); } - /* input is an ip-address in string format i.e A.B.C.D ** scan the INTERFACE_LIST to get the options */ @@ -3889,7 +3955,7 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, INTERFACE_INFO* ifp; long namaddr; - if ((len == 0) || ((namlen = buf[0]) > len)) + if ((len == 0) || ((namlen = get_int8(buf)) > len)) goto error; if (parse_addr(buf+1, namlen, &namaddr) < 0) goto error; @@ -3958,27 +4024,12 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, break; case INET_IFOPT_FLAGS: { - long eflags = 0; int flags = ifp->iiFlags; /* just enumerate the interfaces (no names) */ - /* translate flags */ - if (flags & IFF_UP) - eflags |= INET_IFF_UP; - if (flags & IFF_BROADCAST) - eflags |= INET_IFF_BROADCAST; - if (flags & IFF_LOOPBACK) - eflags |= INET_IFF_LOOPBACK; - if (flags & IFF_POINTTOPOINT) - eflags |= INET_IFF_POINTTOPOINT; - if (flags & IFF_UP) /* emulate runnign ? */ - eflags |= INET_IFF_RUNNING; - if (flags & IFF_MULTICAST) - eflags |= INET_IFF_MULTICAST; - buf_check(sptr, s_end, 5); *sptr++ = INET_IFOPT_FLAGS; - put_int32(eflags, sptr); + put_int32(IFGET_FLAGS(flags), sptr); sptr += 4; break; } @@ -3999,7 +4050,6 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } - #elif defined(SIOCGIFCONF) && defined(SIOCSIFFLAGS) /* cygwin has SIOCGIFCONF but not SIOCSIFFLAGS (Nov 2002) */ @@ -4010,69 +4060,81 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, #define SIZEA(p) (sizeof (p)) #endif - -static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize) -{ - struct ifconf ifc; - struct ifreq *ifr; - char *buf; - int buflen, ifc_len, i; - char *sbuf, *sp; - - /* Courtesy of Per Bergqvist and W. Richard Stevens */ - - ifc_len = 0; - buflen = 100 * sizeof(struct ifreq); - buf = ALLOC(buflen); +static int get_ifconf(SOCKET s, struct ifconf *ifcp) { + int ifc_len = 0; + int buflen = 100 * sizeof(struct ifreq); + char *buf = ALLOC(buflen); for (;;) { - ifc.ifc_len = buflen; - ifc.ifc_buf = buf; - if (ioctl(desc->s, SIOCGIFCONF, (char *)&ifc) < 0) { + ifcp->ifc_len = buflen; + ifcp->ifc_buf = buf; + if (ioctl(s, SIOCGIFCONF, (char *)ifcp) < 0) { int res = sock_errno(); if (res != EINVAL || ifc_len) { FREE(buf); - return ctl_error(res, rbuf, rsize); + return -1; } } else { - if (ifc.ifc_len == ifc_len) break; /* buf large enough */ - ifc_len = ifc.ifc_len; + if (ifcp->ifc_len == ifc_len) break; /* buf large enough */ + ifc_len = ifcp->ifc_len; } buflen += 10 * sizeof(struct ifreq); buf = (char *)REALLOC(buf, buflen); } - - sp = sbuf = ALLOC(ifc_len+1); + return 0; +} + +static void free_ifconf(struct ifconf *ifcp) { + FREE(ifcp->ifc_buf); +} + +static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize) +{ + struct ifconf ifc; + struct ifreq *ifrp; + char *sbuf, *sp; + int i; + + /* Courtesy of Per Bergqvist and W. Richard Stevens */ + + if (get_ifconf(desc->s, &ifc) < 0) { + return ctl_error(sock_errno(), rbuf, rsize); + } + + sp = sbuf = ALLOC(ifc.ifc_len+1); *sp++ = INET_REP_OK; i = 0; for (;;) { int n; - - ifr = (struct ifreq *) VOIDP(buf + i); - n = sizeof(ifr->ifr_name) + SIZEA(ifr->ifr_addr); - if (n < sizeof(*ifr)) n = sizeof(*ifr); - if (i+n > ifc_len) break; + + ifrp = (struct ifreq *) VOIDP(ifc.ifc_buf + i); + n = sizeof(ifrp->ifr_name) + SIZEA(ifrp->ifr_addr); + if (n < sizeof(*ifrp)) n = sizeof(*ifrp); + if (i+n > ifc.ifc_len) break; i += n; - - switch (ifr->ifr_addr.sa_family) { + + switch (ifrp->ifr_addr.sa_family) { #if defined(HAVE_IN6) && defined(AF_INET6) case AF_INET6: #endif case AF_INET: - ASSERT(sp+IFNAMSIZ+1 < sbuf+buflen+1) - strncpy(sp, ifr->ifr_name, IFNAMSIZ); + ASSERT(sp+IFNAMSIZ+1 < sbuf+ifc.ifc_len+1) + strncpy(sp, ifrp->ifr_name, IFNAMSIZ); sp[IFNAMSIZ] = '\0'; sp += strlen(sp), ++sp; } - - if (i >= ifc_len) break; + + if (i >= ifc.ifc_len) break; } - FREE(buf); + free_ifconf(&ifc); *rbuf = sbuf; return sp - sbuf; } - +/* FIXME: temporary hack */ +#ifndef IFHWADDRLEN +#define IFHWADDRLEN 6 +#endif static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, char** rbuf, int rsize) @@ -4083,11 +4145,11 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, struct ifreq ifreq; int namlen; - if ((len == 0) || ((namlen = buf[0]) > len)) + if ((len == 0) || ((namlen = get_int8(buf)) > len)) goto error; sys_memset(ifreq.ifr_name, '\0', IFNAMSIZ); sys_memcpy(ifreq.ifr_name, buf+1, - (namlen > IFNAMSIZ) ? IFNAMSIZ : namlen); + (namlen >= IFNAMSIZ) ? IFNAMSIZ-1 : namlen); buf += (namlen+1); len -= (namlen+1); sptr = sbuf; @@ -4107,11 +4169,52 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, #ifdef SIOCGIFHWADDR if (ioctl(desc->s, SIOCGIFHWADDR, (char *)&ifreq) < 0) break; - buf_check(sptr, s_end, 1+IFHWADDRLEN); + buf_check(sptr, s_end, 1+2+IFHWADDRLEN); *sptr++ = INET_IFOPT_HWADDR; + put_int16(IFHWADDRLEN, sptr); sptr += 2; /* raw memcpy (fix include autoconf later) */ sys_memcpy(sptr, (char*)(&ifreq.ifr_hwaddr.sa_data), IFHWADDRLEN); sptr += IFHWADDRLEN; +#elif defined(SIOCGENADDR) + if (ioctl(desc->s, SIOCGENADDR, (char *)&ifreq) < 0) + break; + buf_check(sptr, s_end, 1+2+sizeof(ifreq.ifr_enaddr)); + *sptr++ = INET_IFOPT_HWADDR; + put_int16(sizeof(ifreq.ifr_enaddr), sptr); sptr += 2; + /* raw memcpy (fix include autoconf later) */ + sys_memcpy(sptr, (char*)(&ifreq.ifr_enaddr), + sizeof(ifreq.ifr_enaddr)); + sptr += sizeof(ifreq.ifr_enaddr); +#elif defined(HAVE_GETIFADDRS) && defined(AF_LINK) + struct ifaddrs *ifa, *ifp; + struct sockaddr_dl *sdlp; + int found = 0; + + if (getifaddrs(&ifa) == -1) + goto error; + + for (ifp = ifa; ifp; ifp = ifp->ifa_next) { + if ((ifp->ifa_addr->sa_family == AF_LINK) && + (sys_strcmp(ifp->ifa_name, ifreq.ifr_name) == 0)) { + found = 1; + break; + } + } + + if (found == 0) { + freeifaddrs(ifa); + break; + } + sdlp = (struct sockaddr_dl *)ifp->ifa_addr; + + buf_check(sptr, s_end, 1+2+sdlp->sdl_alen); + *sptr++ = INET_IFOPT_HWADDR; + put_int16(sdlp->sdl_alen, sptr); sptr += 2; + sys_memcpy(sptr, + sdlp->sdl_data + sdlp->sdl_nlen, + sdlp->sdl_alen); + freeifaddrs(ifa); + sptr += sdlp->sdl_alen; #endif break; } @@ -4188,29 +4291,15 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, case INET_IFOPT_FLAGS: { int flags; - int eflags = 0; if (ioctl(desc->s, SIOCGIFFLAGS, (char*)&ifreq) < 0) flags = 0; else flags = ifreq.ifr_flags; - /* translate flags */ - if (flags & IFF_UP) - eflags |= INET_IFF_UP; - if (flags & IFF_BROADCAST) - eflags |= INET_IFF_BROADCAST; - if (flags & IFF_LOOPBACK) - eflags |= INET_IFF_LOOPBACK; - if (flags & IFF_POINTOPOINT) - eflags |= INET_IFF_POINTTOPOINT; - if (flags & IFF_RUNNING) - eflags |= INET_IFF_RUNNING; - if (flags & IFF_MULTICAST) - eflags |= INET_IFF_MULTICAST; buf_check(sptr, s_end, 5); *sptr++ = INET_IFOPT_FLAGS; - put_int32(eflags, sptr); + put_int32(IFGET_FLAGS(flags), sptr); sptr += 4; break; } @@ -4224,10 +4313,6 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, return ctl_error(EINVAL, rbuf, rsize); } -/* FIXME: temporary hack */ -#ifndef IFHWADDRLEN -#define IFHWADDRLEN 6 -#endif static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, char** rbuf, int rsize) @@ -4236,11 +4321,11 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, int namlen; char* b_end = buf + len; - if ((len == 0) || ((namlen = buf[0]) > len)) + if ((len == 0) || ((namlen = get_int8(buf)) > len)) goto error; sys_memset(ifreq.ifr_name, '\0', IFNAMSIZ); sys_memcpy(ifreq.ifr_name, buf+1, - (namlen > IFNAMSIZ) ? IFNAMSIZ : namlen); + (namlen >= IFNAMSIZ) ? IFNAMSIZ-1 : namlen); buf += (namlen+1); len -= (namlen+1); @@ -4252,17 +4337,22 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, (void) ioctl(desc->s, SIOCSIFADDR, (char*)&ifreq); break; - case INET_IFOPT_HWADDR: - buf_check(buf, b_end, IFHWADDRLEN); + case INET_IFOPT_HWADDR: { + unsigned int len; + buf_check(buf, b_end, 2); + len = get_int16(buf); buf += 2; + buf_check(buf, b_end, len); #ifdef SIOCSIFHWADDR /* raw memcpy (fix include autoconf later) */ - sys_memcpy((char*)(&ifreq.ifr_hwaddr.sa_data), buf, IFHWADDRLEN); + sys_memset((char*)(&ifreq.ifr_hwaddr.sa_data), + '\0', sizeof(ifreq.ifr_hwaddr.sa_data)); + sys_memcpy((char*)(&ifreq.ifr_hwaddr.sa_data), buf, len); (void) ioctl(desc->s, SIOCSIFHWADDR, (char *)&ifreq); #endif - buf += IFHWADDRLEN; + buf += len; break; - + } case INET_IFOPT_BROADADDR: #ifdef SIOCSIFBRDADDR @@ -4367,6 +4457,557 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, #endif + + +/* Latin-1 to utf8 */ + +static int utf8_len(const char *c, int m) { + int l; + for (l = 0; m; c++, l++, m--) { + if (*c == '\0') break; + if ((*c & 0x7f) != *c) l++; + } + return l; +} + +static void utf8_encode(const char *c, int m, char *p) { + for (; m; c++, m--) { + if (*c == '\0') break; + if ((*c & 0x7f) != *c) { + *p++ = (char) (0xC0 | (0x03 & (*c >> 6))); + *p++ = (char) (0x80 | (0x3F & *c)); + } else { + *p++ = (char) *c; + } + } +} + +#if defined(__WIN32__) + +static void set_netmask_bytes(char *c, int len, int pref_len) { + int i, m; + for (i = 0, m = pref_len >> 3; i < m && i < len; i++) c[i] = '\xFF'; + if (i < len) c[i++] = 0xFF << (8 - (pref_len & 7)); + for (; i < len; i++) c[i] = '\0'; +} + + +int eq_masked_bytes(char *a, char *b, int pref_len) { + int i, m; + for (i = 0, m = pref_len >> 3; i < m; i++) { + if (a[i] != b[i]) return 0; + } + m = pref_len & 7; + if (m) { + m = 0xFF & (0xFF << (8 - m)); + if ((a[i] & m) != (b[i] & m)) return 0; + } + return !0; +} + +static int inet_ctl_getifaddrs(inet_descriptor* desc_p, + char **rbuf_pp, int rsize) +{ + int i; + DWORD ret, n; + IP_INTERFACE_INFO *info_p; + MIB_IPADDRTABLE *ip_addrs_p; + IP_ADAPTER_ADDRESSES *ip_adaddrs_p, *ia_p; + + char *buf_p; + char *buf_alloc_p; + int buf_size =512; +# define BUF_ENSURE(Size) \ + do { \ + int NEED_, GOT_ = buf_p - buf_alloc_p; \ + NEED_ = GOT_ + (Size); \ + if (NEED_ > buf_size) { \ + buf_size = NEED_ + 512; \ + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); \ + buf_p = buf_alloc_p + GOT_; \ + } \ + } while(0) +# define SOCKADDR_TO_BUF(opt, sa) \ + do { \ + if (sa) { \ + char *P_; \ + *buf_p++ = (opt); \ + while (! (P_ = sockaddr_to_buf((sa), buf_p, \ + buf_alloc_p+buf_size))) { \ + int GOT_ = buf_p - buf_alloc_p; \ + buf_size += 512; \ + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); \ + buf_p = buf_alloc_p + GOT_; \ + } \ + if (P_ == buf_p) { \ + buf_p--; \ + } else { \ + buf_p = P_; \ + } \ + } \ + } while (0) + + { + /* Try GetAdaptersAddresses, if it is available */ + unsigned long ip_adaddrs_size = 16 * 1024; + ULONG family = AF_UNSPEC; + ULONG flags = + GAA_FLAG_INCLUDE_PREFIX | GAA_FLAG_SKIP_ANYCAST | + GAA_FLAG_SKIP_DNS_SERVER | GAA_FLAG_SKIP_FRIENDLY_NAME | + GAA_FLAG_SKIP_MULTICAST; + ULONG (WINAPI *fpGetAdaptersAddresses) + (ULONG, ULONG, PVOID, PIP_ADAPTER_ADDRESSES, PULONG); + HMODULE iphlpapi = GetModuleHandle("iphlpapi"); + fpGetAdaptersAddresses = (void *) + (iphlpapi ? + GetProcAddress(iphlpapi, "GetAdaptersAddresses") : + NULL); + if (fpGetAdaptersAddresses) { + ip_adaddrs_p = ALLOC(ip_adaddrs_size); + for (i = 17; i; i--) { + ret = fpGetAdaptersAddresses( + family, flags, NULL, ip_adaddrs_p, &ip_adaddrs_size); + ip_adaddrs_p = REALLOC(ip_adaddrs_p, ip_adaddrs_size); + if (ret == NO_ERROR) break; + if (ret == ERROR_BUFFER_OVERFLOW) continue; + i = 0; + } + if (! i) { + FREE(ip_adaddrs_p); + ip_adaddrs_p = NULL; + } + } else ip_adaddrs_p = NULL; + } + + { + /* Load the IP_INTERFACE_INFO table (only IPv4 interfaces), + * reliable source of interface names on XP + */ + unsigned long info_size = 4 * 1024; + info_p = ALLOC(info_size); + for (i = 17; i; i--) { + ret = GetInterfaceInfo(info_p, &info_size); + info_p = REALLOC(info_p, info_size); + if (ret == NO_ERROR) break; + if (ret == ERROR_INSUFFICIENT_BUFFER) continue; + i = 0; + } + if (! i) { + FREE(info_p); + info_p = NULL; + } + } + + if (! ip_adaddrs_p) { + /* If GetAdaptersAddresses gave nothing we fall back to + * MIB_IPADDRTABLE (only IPv4 interfaces) + */ + unsigned long ip_addrs_size = 16 * sizeof(*ip_addrs_p); + ip_addrs_p = ALLOC(ip_addrs_size); + for (i = 17; i; i--) { + ret = GetIpAddrTable(ip_addrs_p, &ip_addrs_size, FALSE); + ip_addrs_p = REALLOC(ip_addrs_p, ip_addrs_size); + if (ret == NO_ERROR) break; + if (ret == ERROR_INSUFFICIENT_BUFFER) continue; + i = 0; + } + if (! i) { + if (info_p) FREE(info_p); + FREE(ip_addrs_p); + return ctl_reply(INET_REP_OK, NULL, 0, rbuf_pp, rsize); + } + } else ip_addrs_p = NULL; + + buf_p = buf_alloc_p = ALLOC(buf_size); + *buf_p++ = INET_REP_OK; + + /* Iterate over MIB_IPADDRTABLE or IP_ADAPTER_ADDRESSES */ + for (ia_p = NULL, ip_addrs_p ? ((void *)(i = 0)) : (ia_p = ip_adaddrs_p); + ip_addrs_p ? (i < ip_addrs_p->dwNumEntries) : (ia_p != NULL); + ip_addrs_p ? ((void *)(i++)) : (ia_p = ia_p->Next)) { + MIB_IPADDRROW *ipaddrrow_p = NULL; + DWORD flags = INET_IFF_MULTICAST; + DWORD index = 0; + WCHAR *wname_p = NULL; + MIB_IFROW ifrow; + + if (ip_addrs_p) { + ipaddrrow_p = ip_addrs_p->table + i; + index = ipaddrrow_p->dwIndex; + } else { + index = ia_p->IfIndex; + if (ia_p->Flags & IP_ADAPTER_NO_MULTICAST) { + flags &= ~INET_IFF_MULTICAST; + } + } +index: + if (! index) goto done; + sys_memzero(&ifrow, sizeof(ifrow)); + ifrow.dwIndex = index; + if (GetIfEntry(&ifrow) != NO_ERROR) break; + /* Find the interface name - first try MIB_IFROW.wzname */ + if (ifrow.wszName[0] != 0) { + wname_p = ifrow.wszName; + } else { + /* Then try IP_ADAPTER_INDEX_MAP.Name (only IPv4 adapters) */ + int j; + for (j = 0; j < info_p->NumAdapters; j++) { + if (info_p->Adapter[j].Index == (ULONG) ifrow.dwIndex) { + if (info_p->Adapter[j].Name[0] != 0) { + wname_p = info_p->Adapter[j].Name; + } + break; + } + } + } + if (wname_p) { + int len; + /* Convert interface name to UTF-8 */ + len = + WideCharToMultiByte( + CP_UTF8, 0, wname_p, -1, NULL, 0, NULL, NULL); + if (! len) break; + BUF_ENSURE(len); + WideCharToMultiByte( + CP_UTF8, 0, wname_p, -1, buf_p, len, NULL, NULL); + buf_p += len; + } else { + /* Found no name - + * use "MIB_IFROW.dwIndex: MIB_IFROW.bDescr" as name instead */ + int l; + l = utf8_len(ifrow.bDescr, ifrow.dwDescrLen); + BUF_ENSURE(9 + l+1); + buf_p += + erts_sprintf( + buf_p, "%lu: ", (unsigned long) ifrow.dwIndex); + utf8_encode(ifrow.bDescr, ifrow.dwDescrLen, buf_p); + buf_p += l; + *buf_p++ = '\0'; + } + /* Interface flags, often make up broadcast and multicast flags */ + switch (ifrow.dwType) { + case IF_TYPE_ETHERNET_CSMACD: + flags |= INET_IFF_BROADCAST; + break; + case IF_TYPE_SOFTWARE_LOOPBACK: + flags |= INET_IFF_LOOPBACK; + flags &= ~INET_IFF_MULTICAST; + break; + default: + flags &= ~INET_IFF_MULTICAST; + break; + } + if (ifrow.dwAdminStatus) { + flags |= INET_IFF_UP; + switch (ifrow.dwOperStatus) { + case IF_OPER_STATUS_CONNECTING: + flags |= INET_IFF_POINTTOPOINT; + break; + case IF_OPER_STATUS_CONNECTED: + flags |= INET_IFF_RUNNING | INET_IFF_POINTTOPOINT; + break; + case IF_OPER_STATUS_OPERATIONAL: + flags |= INET_IFF_RUNNING; + break; + } + } + BUF_ENSURE(1 + 4); + *buf_p++ = INET_IFOPT_FLAGS; + put_int32(flags, buf_p); buf_p += 4; + if (ipaddrrow_p) { + /* Legacy implementation through GetIpAddrTable */ + struct sockaddr_in sin; + /* IP Address */ + sys_memzero(&sin, sizeof(sin)); + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = ipaddrrow_p->dwAddr; + BUF_ENSURE(1); + /* Netmask */ + SOCKADDR_TO_BUF(INET_IFOPT_ADDR, (struct sockaddr *) &sin); + sin.sin_addr.s_addr = ipaddrrow_p->dwMask; + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, (struct sockaddr *) &sin); + if (flags & INET_IFF_BROADCAST) { + /* Broadcast address - fake it*/ + sin.sin_addr.s_addr = ipaddrrow_p->dwAddr; + sin.sin_addr.s_addr |= ~ipaddrrow_p->dwMask; + BUF_ENSURE(1); + SOCKADDR_TO_BUF( + INET_IFOPT_BROADADDR, (struct sockaddr *) &sin); + } + } else { + IP_ADAPTER_UNICAST_ADDRESS *p; + /* IP Address(es) */ + for (p = ia_p->FirstUnicastAddress; + p; + p = p->Next) + { + IP_ADAPTER_PREFIX *q; + ULONG shortest_length; + struct sockaddr *shortest_p, *sa_p = p->Address.lpSockaddr; + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_ADDR, sa_p); + shortest_p = NULL; + shortest_length = 0; + for (q = ia_p->FirstPrefix; + q; + q = q->Next) { + struct sockaddr *sp_p = q->Address.lpSockaddr; + if (sa_p->sa_family != sp_p->sa_family) continue; + switch (sa_p->sa_family) { + case AF_INET: { + struct sockaddr_in sin; + DWORD sa, sp, mask; + sa = ntohl((DWORD) + ((struct sockaddr_in *) + sa_p)->sin_addr.s_addr); + sp = ntohl((DWORD) + ((struct sockaddr_in *) + sp_p)->sin_addr.s_addr); + mask = 0xFFFFFFFF << (32 - q->PrefixLength); + if ((sa & mask) != (sp & mask)) continue; + if ((! shortest_p) + || q->PrefixLength < shortest_length) { + shortest_p = sp_p; + shortest_length = q->PrefixLength; + } + } break; + case AF_INET6: { + struct sockaddr_in6 sin6; + if (!eq_masked_bytes((char *) + &((struct sockaddr_in6 *) + sa_p)->sin6_addr, + (char *) + &((struct sockaddr_in6 *) + sp_p)->sin6_addr, + q->PrefixLength)) { + continue; + } + if ((! shortest_p) + || q->PrefixLength < shortest_length) { + shortest_p = sp_p; + shortest_length = q->PrefixLength; + } + } break; + } + } + if (! shortest_p) { + /* Found no shortest prefix */ + shortest_p = sa_p; + switch (shortest_p->sa_family) { + case AF_INET: { + /* Fall back to old classfull network addresses */ + DWORD addr = ntohl(((struct sockaddr_in *)shortest_p) + ->sin_addr.s_addr); + if (! (addr & 0x800000)) { + /* Class A */ + shortest_length = 8; + } else if (! (addr & 0x400000)) { + /* Class B */ + shortest_length = 16; + } else if (! (addr & 0x200000)) { + /* Class C */ + shortest_length = 24; + } else { + shortest_length = 32; + } + } break; + case AF_INET6: { + /* Just play it safe */ + shortest_length = 128; + } break; + } + } + switch (shortest_p->sa_family) { + case AF_INET: { + struct sockaddr_in sin; + DWORD mask = 0xFFFFFFFF << (32 - shortest_length); + sys_memzero(&sin, sizeof(sin)); + sin.sin_family = shortest_p->sa_family; + sin.sin_addr.s_addr = htonl(mask); + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, + (struct sockaddr *) &sin); + if (flags & INET_IFF_BROADCAST) { + DWORD sp = + ntohl((DWORD) + ((struct sockaddr_in *)shortest_p) + -> sin_addr.s_addr); + sin.sin_addr.s_addr = htonl(sp | ~mask); + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_BROADADDR, + (struct sockaddr *) &sin); + } + } break; + case AF_INET6: { + struct sockaddr_in6 sin6; + sys_memzero(&sin6, sizeof(sin6)); + sin6.sin6_family = shortest_p->sa_family; + set_netmask_bytes((char *) &sin6.sin6_addr, + 16, + shortest_length); + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, + (struct sockaddr *) &sin6); + } break; + } + } + } + if (ifrow.dwPhysAddrLen) { + /* Hardware Address */ + BUF_ENSURE(1 + 2 + ifrow.dwPhysAddrLen); + *buf_p++ = INET_IFOPT_HWADDR; + put_int16(ifrow.dwPhysAddrLen, buf_p); buf_p += 2; + sys_memcpy(buf_p, ifrow.bPhysAddr, ifrow.dwPhysAddrLen); + buf_p += ifrow.dwPhysAddrLen; + } + +done: + /* That is all for this interface */ + BUF_ENSURE(1); + *buf_p++ = '\0'; + if (ia_p && + ia_p->Ipv6IfIndex && + ia_p->Ipv6IfIndex != index) + { + /* Oops, there was an other interface for IPv6. Possible? XXX */ + index = ia_p->Ipv6IfIndex; + goto index; + } + } + + if (ip_adaddrs_p) FREE(ip_adaddrs_p); + if (info_p) FREE(info_p); + if (ip_addrs_p) FREE(ip_addrs_p); + + buf_size = buf_p - buf_alloc_p; + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); + /* buf_p is now unreliable */ + *rbuf_pp = buf_alloc_p; + return buf_size; +# undef BUF_ENSURE +} + +#elif defined(HAVE_GETIFADDRS) + +static int inet_ctl_getifaddrs(inet_descriptor* desc_p, + char **rbuf_pp, int rsize) +{ + struct ifaddrs *ifa_p, *ifa_free_p; + + int buf_size; + char *buf_p; + char *buf_alloc_p; + + buf_size = 512; + buf_alloc_p = ALLOC(buf_size); + buf_p = buf_alloc_p; +# define BUF_ENSURE(Size) \ + do { \ + int NEED_, GOT_ = buf_p - buf_alloc_p; \ + NEED_ = GOT_ + (Size); \ + if (NEED_ > buf_size) { \ + buf_size = NEED_ + 512; \ + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); \ + buf_p = buf_alloc_p + GOT_; \ + } \ + } while (0) +# define SOCKADDR_TO_BUF(opt, sa) \ + do { \ + if (sa) { \ + char *P_; \ + *buf_p++ = (opt); \ + while (! (P_ = sockaddr_to_buf((sa), buf_p, \ + buf_alloc_p+buf_size))) { \ + int GOT_ = buf_p - buf_alloc_p; \ + buf_size += 512; \ + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); \ + buf_p = buf_alloc_p + GOT_; \ + } \ + if (P_ == buf_p) { \ + buf_p--; \ + } else { \ + buf_p = P_; \ + } \ + } \ + } while (0) + + if (getifaddrs(&ifa_p) < 0) { + return ctl_error(sock_errno(), rbuf_pp, rsize); + } + ifa_free_p = ifa_p; + *buf_p++ = INET_REP_OK; + for (; ifa_p; ifa_p = ifa_p->ifa_next) { + int len = utf8_len(ifa_p->ifa_name, -1); + BUF_ENSURE(len+1 + 1+4 + 1); + utf8_encode(ifa_p->ifa_name, -1, buf_p); + buf_p += len; + *buf_p++ = '\0'; + *buf_p++ = INET_IFOPT_FLAGS; + put_int32(IFGET_FLAGS(ifa_p->ifa_flags), buf_p); buf_p += 4; + if (ifa_p->ifa_addr) { + if (ifa_p->ifa_addr->sa_family == AF_INET +#if defined(AF_INET6) + || ifa_p->ifa_addr->sa_family == AF_INET6 +#endif + ) { + SOCKADDR_TO_BUF(INET_IFOPT_ADDR, ifa_p->ifa_addr); + if (ifa_p->ifa_netmask) { + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, ifa_p->ifa_netmask); + } + if (ifa_p->ifa_dstaddr && + (ifa_p->ifa_flags & IFF_POINTOPOINT)) { + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_DSTADDR, ifa_p->ifa_dstaddr); + } else if (ifa_p->ifa_broadaddr && + (ifa_p->ifa_flags & IFF_BROADCAST)) { + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_BROADADDR, ifa_p->ifa_broadaddr); + } + } +#if defined(AF_LINK) || defined(AF_PACKET) + else if ( +#if defined(AF_LINK) + ifa_p->ifa_addr->sa_family == AF_LINK +#else + 0 +#endif +#if defined(AF_PACKET) + || ifa_p->ifa_addr->sa_family == AF_PACKET +#endif + ) { + char *bp = buf_p; + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_HWADDR, ifa_p->ifa_addr); + if (buf_p - bp < 4) buf_p = bp; /* Empty hwaddr */ + } +#endif + } + BUF_ENSURE(1); + *buf_p++ = '\0'; + } + buf_size = buf_p - buf_alloc_p; + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); + /* buf_p is now unreliable */ + freeifaddrs(ifa_free_p); + *rbuf_pp = buf_alloc_p; + return buf_size; +# undef BUF_ENSURE +} + +#else + +static int inet_ctl_getifaddrs(inet_descriptor* desc_p, + char **rbuf_pp, int rsize) +{ + return ctl_error(ENOTSUP, rbuf_pp, rsize); +} + +#endif + + + #ifdef VXWORKS /* ** THIS is a terrible creature, a bug in the TCP part @@ -4528,8 +5169,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) case INET_LOPT_BUFFER: DEBUGF(("inet_set_opts(%ld): s=%d, BUFFER=%d\r\n", (long)desc->port, desc->s, ival)); - if (ival > INET_MAX_BUFFER) ival = INET_MAX_BUFFER; - else if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER; + if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER; desc->bufsz = ival; continue; @@ -4594,7 +5234,6 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) if (desc->stype == SOCK_STREAM) { tcp_descriptor* tdesc = (tcp_descriptor*) desc; if (ival < 0) ival = 0; - else if (ival > INET_MAX_BUFFER*2) ival = INET_MAX_BUFFER*2; if (tdesc->low > ival) tdesc->low = ival; tdesc->high = ival; @@ -4605,7 +5244,6 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) if (desc->stype == SOCK_STREAM) { tcp_descriptor* tdesc = (tcp_descriptor*) desc; if (ival < 0) ival = 0; - else if (ival > INET_MAX_BUFFER) ival = INET_MAX_BUFFER; if (tdesc->high < ival) tdesc->high = ival; tdesc->low = ival; @@ -4951,9 +5589,6 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) case INET_LOPT_BUFFER: desc->bufsz = get_int32(curr); curr += 4; - if (desc->bufsz > INET_MAX_BUFFER) - desc->bufsz = INET_MAX_BUFFER; - else if (desc->bufsz < INET_MIN_BUFFER) desc->bufsz = INET_MIN_BUFFER; res = 0; /* This does not affect the kernel buffer size */ @@ -5016,8 +5651,8 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) } case INET_OPT_LINGER: { - CHKLEN(curr, ASSOC_ID_LEN + 2 + 4); - arg.lin.l_onoff = get_int16 (curr); curr += 2; + CHKLEN(curr, 2*4); + arg.lin.l_onoff = get_int32 (curr); curr += 4; arg.lin.l_linger = get_int32 (curr); curr += 4; proto = SOL_SOCKET; @@ -5194,9 +5829,12 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) char *after; # ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_FLAGS int eflags, cflags, hb_enable, hb_disable, - pmtud_enable, pmtud_disable, + pmtud_enable, pmtud_disable; +# ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY + int sackdelay_enable, sackdelay_disable; # endif +# endif CHKLEN(curr, ASSOC_ID_LEN); arg.pap.spp_assoc_id = GET_ASSOC_ID(curr); curr += ASSOC_ID_LEN; @@ -5245,12 +5883,15 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) if (pmtud_enable) cflags |= SPP_PMTUD_ENABLE; if (pmtud_disable) cflags |= SPP_PMTUD_DISABLE; +# ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY + /* The followings are missing in FreeBSD 7.1 */ sackdelay_enable =eflags& SCTP_FLAG_SACDELAY_ENABLE; sackdelay_disable=eflags& SCTP_FLAG_SACDELAY_DISABLE; if (sackdelay_enable && sackdelay_disable) return -1; if (sackdelay_enable) cflags |= SPP_SACKDELAY_ENABLE; if (sackdelay_disable) cflags |= SPP_SACKDELAY_DISABLE; +# endif arg.pap.spp_flags = cflags; # endif @@ -5388,7 +6029,7 @@ static int inet_fill_opts(inet_descriptor* desc, #define PLACE_FOR(Size,Ptr) \ do { \ int need = dest_used + (Size); \ - if (need > INET_MAX_BUFFER) { \ + if (need > INET_MAX_OPT_BUFFER) { \ RETURN_ERROR(); \ } \ if (need > dest_allocated) { \ @@ -5612,7 +6253,7 @@ static int inet_fill_opts(inet_descriptor* desc, buf += 4; data_provided = (int) *buf++; arg_sz = get_int32(buf); - if (arg_sz > INET_MAX_BUFFER) { + if (arg_sz > INET_MAX_OPT_BUFFER) { RETURN_ERROR(); } buf += 4; @@ -5627,8 +6268,8 @@ static int inet_fill_opts(inet_descriptor* desc, buf += arg_sz; len -= arg_sz; } - if (sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz) == - SOCKET_ERROR) { + if (IS_SOCKET_ERROR(sock_getopt(desc->s,proto,type, + arg_ptr,&arg_sz))) { TRUNCATE_TO(0,ptr); continue; } @@ -5645,7 +6286,7 @@ static int inet_fill_opts(inet_descriptor* desc, RETURN_ERROR(); } /* We have 5 bytes allocated to ptr */ - if (sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz) == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(sock_getopt(desc->s,proto,type,arg_ptr,&arg_sz))) { TRUNCATE_TO(0,ptr); continue; } @@ -5726,7 +6367,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen, "miscalculated buffer size"); \ } \ need = (Index) + (N); \ - if (need > INET_MAX_BUFFER/sizeof(ErlDrvTermData)) { \ + if (need > INET_MAX_OPT_BUFFER/sizeof(ErlDrvTermData)) {\ RETURN_ERROR((Spec), -ENOMEM); \ } \ if (need > spec_allocated) { \ @@ -6151,13 +6792,15 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen, if (ap.spp_flags & SPP_PMTUD_DISABLE) { i = LOAD_ATOM (spec, i, am_pmtud_disable); n++; } - +# ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY + /* SPP_SACKDELAY_* not in FreeBSD 7.1 */ if (ap.spp_flags & SPP_SACKDELAY_ENABLE) { i = LOAD_ATOM (spec, i, am_sackdelay_enable); n++; } if (ap.spp_flags & SPP_SACKDELAY_DISABLE) { i = LOAD_ATOM (spec, i, am_sackdelay_disable); n++; } # endif +# endif PLACE_FOR(spec, i, LOAD_NIL_CNT + LOAD_LIST_CNT + 2*LOAD_TUPLE_CNT); @@ -6177,6 +6820,10 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen, struct sctp_sndrcvinfo sri; unsigned int sz = sizeof(sri); + if (buflen < ASSOC_ID_LEN) RETURN_ERROR(spec, -EINVAL); + sri.sinfo_assoc_id = GET_ASSOC_ID(buf); + buf += ASSOC_ID_LEN; + buflen -= ASSOC_ID_LEN; if (sock_getopt(desc->s, IPPROTO_SCTP, SCTP_DEFAULT_SEND_PARAM, &sri, &sz) < 0) continue; /* Fill in the response: */ @@ -6573,7 +7220,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, } } DEBUGF(("inet_ctl(%ld): GETSTAT\r\n", (long) desc->port)); - if (dstlen > INET_MAX_BUFFER) /* sanity check */ + if (dstlen > INET_MAX_OPT_BUFFER) /* sanity check */ return 0; if (dstlen > rsize) { if ((dst = (char*) ALLOC(dstlen)) == NULL) @@ -6589,7 +7236,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, char* dst; int dstlen = 1 /* Reply code */ + len*5; DEBUGF(("inet_ctl(%ld): INET_REQ_SUBSCRIBE\r\n", (long) desc->port)); - if (dstlen > INET_MAX_BUFFER) /* sanity check */ + if (dstlen > INET_MAX_OPT_BUFFER) /* sanity check */ return 0; if (dstlen > rsize) { if ((dst = (char*) ALLOC(dstlen)) == NULL) @@ -6624,6 +7271,13 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, return inet_ctl_getiflist(desc, rbuf, rsize); } + case INET_REQ_GETIFADDRS: { + DEBUGF(("inet_ctl(%ld): GETIFADDRS\r\n", (long)desc->port)); + if (!IS_OPEN(desc)) + return ctl_xerror(EXBADPORT, rbuf, rsize); + return inet_ctl_getifaddrs(desc, rbuf, rsize); + } + case INET_REQ_IFGET: { DEBUGF(("inet_ctl(%ld): IFGET\r\n", (long)desc->port)); if (!IS_OPEN(desc)) @@ -6711,7 +7365,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, if (len != 0) return ctl_error(EINVAL, rbuf, rsize); - if (sock_hostname(tbuf, MAXHOSTNAMELEN) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_hostname(tbuf, MAXHOSTNAMELEN))) return ctl_error(sock_errno(), rbuf, rsize); return ctl_reply(INET_REP_OK, tbuf, strlen(tbuf), rbuf, rsize); } @@ -6728,7 +7382,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, return ctl_error(ENOTCONN, rbuf, rsize); if ((ptr = desc->peer_ptr) == NULL) { ptr = &peer; - if (sock_peer(desc->s, (struct sockaddr*)ptr,&sz) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_peer(desc->s, (struct sockaddr*)ptr,&sz))) return ctl_error(sock_errno(), rbuf, rsize); } if (inet_get_address(desc->sfamily, tbuf, ptr, &sz) < 0) @@ -6765,7 +7419,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, if ((ptr = desc->name_ptr) == NULL) { ptr = &name; - if (sock_name(desc->s, (struct sockaddr*)ptr, &sz) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_name(desc->s, (struct sockaddr*)ptr, &sz))) return ctl_error(sock_errno(), rbuf, rsize); } if (inet_get_address(desc->sfamily, tbuf, ptr, &sz) < 0) @@ -6804,7 +7458,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, if (inet_set_address(desc->sfamily, &local, buf, &len) == NULL) return ctl_error(EINVAL, rbuf, rsize); - if (sock_bind(desc->s,(struct sockaddr*) &local, len) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_bind(desc->s,(struct sockaddr*) &local, len))) return ctl_error(sock_errno(), rbuf, rsize); desc->state = INET_STATE_BOUND; @@ -6831,13 +7485,13 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, if (len < 2) return ctl_error(EINVAL, rbuf, rsize); - n = buf[0]; buf++; len--; + n = get_int8(buf); buf++; len--; if (n >= len) /* the = sign makes the test inklude next length byte */ return ctl_error(EINVAL, rbuf, rsize); memcpy(namebuf, buf, n); namebuf[n] = '\0'; len -= n; buf += n; - n = buf[0]; buf++; len--; + n = get_int8(buf); buf++; len--; if (n > len) return ctl_error(EINVAL, rbuf, rsize); memcpy(protobuf, buf, n); @@ -6860,7 +7514,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, port = get_int16(buf); port = sock_htons(port); buf += 2; - n = buf[0]; buf++; len -= 3; + n = get_int8(buf); buf++; len -= 3; if (n > len) return ctl_error(EINVAL, rbuf, rsize); memcpy(protobuf, buf, n); @@ -7237,7 +7891,7 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, if (len != 2) return ctl_error(EINVAL, rbuf, rsize); backlog = get_int16(buf); - if (sock_listen(desc->inet.s, backlog) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_listen(desc->inet.s, backlog))) return ctl_error(sock_errno(), rbuf, rsize); desc->inet.state = TCP_STATE_LISTEN; return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); @@ -7271,7 +7925,7 @@ static int tcp_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, code = sock_connect(desc->inet.s, (struct sockaddr*) &desc->inet.remote, len); - if ((code == SOCKET_ERROR) && + if (IS_SOCKET_ERROR(code) && ((sock_errno() == ERRNO_BLOCK) || /* Winsock2 */ (sock_errno() == EINPROGRESS))) { /* Unix & OSE!! */ sock_select(INETP(desc), FD_CONNECT, 1); @@ -7947,7 +8601,7 @@ static int tcp_recv(tcp_descriptor* desc, int request_len) n = sock_recv(desc->inet.s, desc->i_ptr, nread, 0); - if (n == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(n)) { int err = sock_errno(); if (err == ECONNRESET) { DEBUGF((" => detected close (connreset)\r\n")); @@ -8449,8 +9103,8 @@ static int tcp_sendv(tcp_descriptor* desc, ErlIOVec* ev) (long)desc->inet.port, desc->inet.s, h_len, len)); if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) { n = 0; - } else if (sock_sendv(desc->inet.s, ev->iov, vsize, &n, 0) - == SOCKET_ERROR) { + } else if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s, ev->iov, + vsize, &n, 0))) { if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) { int err = sock_errno(); DEBUGF(("tcp_sendv(%ld): s=%d, " @@ -8543,7 +9197,7 @@ static int tcp_send(tcp_descriptor* desc, char* ptr, int len) if (desc->tcp_add_flags & TCP_ADDF_DELAY_SEND) { sock_send(desc->inet.s, buf, 0, 0); n = 0; - } else if (sock_sendv(desc->inet.s,iov,2,&n,0) == SOCKET_ERROR) { + } else if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s,iov,2,&n,0))) { if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) { int err = sock_errno(); DEBUGF(("tcp_send(%ld): s=%d,sock_sendv(size=2) errno = %d\r\n", @@ -8616,7 +9270,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event) int code = sock_peer(desc->inet.s, (struct sockaddr*) &desc->inet.remote, &sz); - if (code == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(code)) { desc->inet.state = TCP_STATE_BOUND; /* restore state */ ret = async_error(INETP(desc), sock_errno()); goto done; @@ -8657,7 +9311,7 @@ static int tcp_inet_output(tcp_descriptor* desc, HANDLE event) vsize = vsize > MAX_VSIZE ? MAX_VSIZE : vsize; DEBUGF(("tcp_inet_output(%ld): s=%d, About to send %d items\r\n", (long)desc->inet.port, desc->inet.s, vsize)); - if (sock_sendv(desc->inet.s, iov, vsize, &n, 0)==SOCKET_ERROR) { + if (IS_SOCKET_ERROR(sock_sendv(desc->inet.s, iov, vsize, &n, 0))) { if ((sock_errno() != ERRNO_BLOCK) && (sock_errno() != EINTR)) { DEBUGF(("tcp_inet_output(%ld): sock_sendv(%d) errno = %d\r\n", (long)desc->inet.port, vsize, sock_errno())); @@ -8926,7 +9580,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, sock_select(desc, FD_CONNECT, 1); code = sock_connect(desc->s, &remote.sa, len); - if ((code == SOCKET_ERROR) && (sock_errno() == EINPROGRESS)) { + if (IS_SOCKET_ERROR(code) && (sock_errno() == EINPROGRESS)) { /* XXX: Unix only -- WinSock would have a different cond! */ desc->state = SCTP_STATE_CONNECTING; if (timeout != INET_INFINITY) @@ -8966,7 +9620,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, code = sock_connect(desc->s, (struct sockaddr*) &desc->remote, len); - if (code == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(code)) { sock_connect(desc->s, (struct sockaddr*) NULL, 0); desc->state &= ~INET_F_ACTIVE; return ctl_error(sock_errno(), rbuf, rsize); @@ -9000,7 +9654,7 @@ static int packet_inet_ctl(ErlDrvData e, unsigned int cmd, char* buf, int len, return ctl_error(EINVAL, rbuf, rsize); flag = get_int8(buf); - if (sock_listen(desc->s, flag) == SOCKET_ERROR) + if (IS_SOCKET_ERROR(sock_listen(desc->s, flag))) return ctl_error(sock_errno(), rbuf, rsize); desc->state = SCTP_STATE_LISTEN; /* XXX: not used? */ @@ -9205,7 +9859,7 @@ static void packet_inet_command(ErlDrvData e, char* buf, int len) check_result_code: /* "code" analysis is the same for both SCTP and UDP cases above: */ #endif - if (code == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(code)) { int err = sock_errno(); inet_reply_error(desc, err); } @@ -9304,7 +9958,7 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event) check_result: #endif /* Analyse the result: */ - if (n == SOCKET_ERROR + if (IS_SOCKET_ERROR(n) #ifdef HAVE_SCTP || (short_recv = (IS_SCTP(desc) && !(mhdr.msg_flags & MSG_EOR))) /* NB: here we check for EOR not being set -- this is an error as @@ -9317,11 +9971,13 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event) if (err != ERRNO_BLOCK) { if (!desc->active) { #ifdef HAVE_SCTP - if (short_recv) + if (short_recv) { async_error_am(desc, am_short_recv); - else -#else + } else { async_error(desc, err); + } +#else + async_error(desc, err); #endif driver_cancel_timer(desc->port); sock_select(desc,FD_READ,0); @@ -9419,7 +10075,7 @@ static int packet_inet_output(udp_descriptor* udesc, HANDLE event) int code = sock_peer(desc->s, (struct sockaddr*) &desc->remote, &sz); - if (code == SOCKET_ERROR) { + if (IS_SOCKET_ERROR(code)) { desc->state = PACKET_STATE_BOUND; /* restore state */ ret = async_error(desc, sock_errno()); goto done; @@ -9860,23 +10516,26 @@ int erts_sock_connect(erts_sock_t socket, byte *ip_addr, int len, Uint16 port) if (!inet_set_address(AF_INET, &addr, buf, &blen)) return 0; - if (SOCKET_ERROR == sock_connect(s, + if (IS_SOCKET_ERROR(sock_connect(s, (struct sockaddr *) &addr, - sizeof(struct sockaddr_in))) + sizeof(struct sockaddr_in)))) return 0; return 1; } Sint erts_sock_send(erts_sock_t socket, const void *buf, Sint len) { - return (Sint) sock_send((SOCKET) socket, buf, (size_t) len, 0); + Sint result = (Sint) sock_send((SOCKET) socket, buf, (size_t) len, 0); + if (IS_SOCKET_ERROR(result)) + return SOCKET_ERROR; + return result; } int erts_sock_gethostname(char *buf, int bufsz) { - if (sock_hostname(buf, bufsz) == SOCKET_ERROR) - return -1; + if (IS_SOCKET_ERROR(sock_hostname(buf, bufsz))) + return SOCKET_ERROR; return 0; } diff --git a/erts/emulator/drivers/common/ram_file_drv.c b/erts/emulator/drivers/common/ram_file_drv.c index 4a39a156e6..abedcc933a 100644 --- a/erts/emulator/drivers/common/ram_file_drv.c +++ b/erts/emulator/drivers/common/ram_file_drv.c @@ -35,6 +35,7 @@ #define RAM_FILE_TRUNCATE 14 #define RAM_FILE_PREAD 17 #define RAM_FILE_PWRITE 18 +#define RAM_FILE_FDATASYNC 19 /* other operations */ #define RAM_FILE_GET 30 @@ -45,6 +46,8 @@ #define RAM_FILE_UUENCODE 35 /* uuencode file */ #define RAM_FILE_UUDECODE 36 /* uudecode file */ #define RAM_FILE_SIZE 37 /* get file size */ +#define RAM_FILE_ADVISE 38 /* predeclare the access + * pattern for file data */ /* possible new operations include: DES_ENCRYPT DES_DECRYPT @@ -558,6 +561,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count) numeric_reply(f, 0); /* 0 is not used */ break; + case RAM_FILE_FDATASYNC: + if (f->flags == 0) + error_reply(f, EBADF); + else + reply(f, 1, 0); + break; + case RAM_FILE_FSYNC: if (f->flags == 0) error_reply(f, EBADF); @@ -685,6 +695,13 @@ static void rfile_command(ErlDrvData e, char* buf, int count) case RAM_FILE_UUDECODE: /* uudecode file */ ram_file_uudecode(f); break; + + case RAM_FILE_ADVISE: + if (f->flags == 0) + error_reply(f, EBADF); + else + reply(f, 1, 0); + break; } /* * Ignore anything else -- let the caller hang. diff --git a/erts/emulator/drivers/unix/mem_drv.c b/erts/emulator/drivers/unix/mem_drv.c deleted file mode 100644 index 1417ca1121..0000000000 --- a/erts/emulator/drivers/unix/mem_drv.c +++ /dev/null @@ -1,145 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* Purpose: Access to elib memory statistics */ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#include "sys.h" -#include "erl_driver.h" -#include "elib_stat.h" - -#define MAP_BUF_SIZE 1000 /* Max map size */ -#define HISTO_BUF_SIZE 100 /* Max histogram buckets */ - -static ErlDrvData mem_start(ErlDrvPort); -static int mem_init(void); -static void mem_stop(ErlDrvData); -static void mem_command(ErlDrvData, char*, int); - -const struct driver_entry mem_driver_entry = { - mem_init, - mem_start, - mem_stop, - mem_command, - NULL, - NULL, - "mem_drv" -}; - -static int mem_init(void) -{ - return 0; -} - -static ErlDrvData mem_start(ErlDrvPort port, char* buf) -{ - return (ErlDrvData)port; -} - -static void mem_stop(ErlDrvData port) -{ -} - -void putint32(p, v) -byte* p; int v; -{ - p[0] = (v >> 24) & 0xff; - p[1] = (v >> 16) & 0xff; - p[2] = (v >> 8) & 0xff; - p[3] = (v) & 0xff; -} - -int getint16(p) -byte* p; -{ - return (p[0] << 8) | p[1]; -} - -/* -** Command: -** m L1 L0 -> a heap map of length L1*256 + L0 is returned -** s -> X3 X2 X1 X0 Y3 Y2 Y1 Y0 Z3 Z2 Z1 Z0 -** X == Total heap size bytes -** Y == Total free bytes -** Z == Size of largest free block in bytes -** -** h L1 L0 B0 -> Generate a logarithm historgram base B with L buckets -** l L1 L0 S0 -> Generate a linear histogram with step S with L buckets -*/ -unsigned char outbuf[HISTO_BUF_SIZE*2*4]; - -static void mem_command(ErlDrvData port, char* buf, int count) -{ - if ((count == 1) && buf[0] == 's') { - struct elib_stat info; - char v[3*4]; - - elib_stat(&info); - - putint32(v, info.mem_total*4); - putint32(v+4, info.mem_free*4); - putint32(v+8, info.max_free*4); - driver_output((ErlDrvPort)port, v, 12); - return; - } - else if ((count == 3) && buf[0] == 'm') { - char w[MAP_BUF_SIZE]; - int n = getint16(buf+1); - - if (n > MAP_BUF_SIZE) - n = MAP_BUF_SIZE; - elib_heap_map(w, n); - driver_output((ErlDrvPort)port, w, n); - return; - } - else if ((count == 4) && (buf[0] == 'h' || buf[0] == 'l')) { - unsigned long vf[HISTO_BUF_SIZE]; - unsigned long va[HISTO_BUF_SIZE]; - int n = getint16(buf+1); - int base = (unsigned char) buf[3]; - - if (n >= HISTO_BUF_SIZE) - n = HISTO_BUF_SIZE; - if (buf[0] == 'l') - base = -base; - if (elib_histo(vf, va, n, base) < 0) { - driver_failure((ErlDrvPort)port, -1); - return; - } - else { - char* p = outbuf; - int i; - - for (i = 0; i < n; i++) { - putint32(p, vf[i]); - p += 4; - } - for (i = 0; i < n; i++) { - putint32(p, va[i]); - p += 4; - } - driver_output((ErlDrvPort)port, outbuf, n*8); - } - return; - } - driver_failure((ErlDrvPort)port, -1); -} diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 1d094ee613..4b3934657c 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -98,7 +98,7 @@ extern STATUS copy(char *, char *); #define EF_SAFE_REALLOC(P, S) ef_safe_realloc((P), (S)) #define EF_FREE(P) do { if((P)) driver_free((P)); } while(0) -extern void erl_exit(int n, char *fmt, _DOTS_); +void erl_exit(int n, char *fmt, ...); static void *ef_safe_alloc(Uint s) { @@ -127,7 +127,7 @@ static void *ef_safe_realloc(void *op, Uint s) (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0'))) #ifdef VXWORKS -static FUNCTION(int, vxworks_to_posix, (int vx_errno)); +static int vxworks_to_posix(int vx_errno); #endif /* @@ -146,7 +146,7 @@ static FUNCTION(int, vxworks_to_posix, (int vx_errno)); #define CHECK_PATHLEN(X,Y) /* Nothing */ #endif -static FUNCTION(int, check_error, (int result, Efile_error* errInfo)); +static int check_error(int result, Efile_error* errInfo); static int check_error(int result, Efile_error *errInfo) @@ -587,7 +587,8 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ open directory.*/ char* buffer, /* Pointer to buffer for one filename. */ - size_t size) /* Size of buffer. */ + size_t *size) /* in-out Size of buffer, length + of name. */ { DIR *dp; /* Pointer to directory structure. */ struct dirent* dirp; /* Pointer to directory entry. */ @@ -619,7 +620,8 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ if (IS_DOT_OR_DOTDOT(dirp->d_name)) continue; buffer[0] = '\0'; - strncat(buffer, dirp->d_name, size-1); + strncat(buffer, dirp->d_name, (*size)-1); + *size = strlen(dirp->d_name); return 1; } } @@ -706,6 +708,9 @@ efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ #endif } + if (flags & EFILE_MODE_EXCL) { + mode |= O_EXCL; + } #ifdef VXWORKS if (*name != '/') { @@ -774,6 +779,17 @@ efile_closefile(int fd) } int +efile_fdatasync(Efile_error *errInfo, /* Where to return error codes. */ + int fd) /* File descriptor for file to sync data. */ +{ +#ifdef HAVE_FDATASYNC + return check_error(fdatasync(fd), errInfo); +#else + return efile_fsync(errInfo, fd); +#endif +} + +int efile_fsync(Efile_error *errInfo, /* Where to return error codes. */ int fd) /* File descriptor for file to sync. */ { @@ -1437,3 +1453,14 @@ efile_symlink(Efile_error* errInfo, char* old, char* new) return check_error(symlink(old, new), errInfo); #endif } + +int +efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, + Sint64 length, int advise) +{ +#ifdef HAVE_POSIX_FADVISE + return check_error(posix_fadvise(fd, offset, length, advise), errInfo); +#else + return check_error(0, errInfo); +#endif +} diff --git a/erts/emulator/drivers/win32/mem_drv.c b/erts/emulator/drivers/win32/mem_drv.c deleted file mode 100644 index fa7c46eca8..0000000000 --- a/erts/emulator/drivers/win32/mem_drv.c +++ /dev/null @@ -1,141 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ -/* Purpose: Access to elib memory statistics */ - -#include "sys.h" -#include "erl_driver.h" -#include "elib_stat.h" - -#define MAP_BUF_SIZE 1000 /* Max map size */ -#define HISTO_BUF_SIZE 100 /* Max histogram buckets */ - -static ErlDrvData mem_start(ErlDrvPort, char*); -static int mem_init(void); -static void mem_stop(ErlDrvData); -static void mem_command(ErlDrvData); - -ErlDrvEntry mem_driver_entry = { - mem_init, - mem_start, - mem_stop, - mem_command, - NULL, - NULL, - "mem_drv" -}; - -static int mem_init(void) -{ - return 0; -} - -static ErlDrvData mem_start(ErlDrvPort port, char* buf) -{ - return (ErlDrvData)port; -} - -static void mem_stop(ErlDrvData port) -{ -} - -void putint32(p, v) -byte* p; int v; -{ - p[0] = (v >> 24) & 0xff; - p[1] = (v >> 16) & 0xff; - p[2] = (v >> 8) & 0xff; - p[3] = (v) & 0xff; -} - -int getint16(p) -byte* p; -{ - return (p[0] << 8) | p[1]; -} - -/* -** Command: -** m L1 L0 -> a heap map of length L1*256 + L0 is returned -** s -> X3 X2 X1 X0 Y3 Y2 Y1 Y0 Z3 Z2 Z1 Z0 -** X == Total heap size bytes -** Y == Total free bytes -** Z == Size of largest free block in bytes -** -** h L1 L0 B0 -> Generate a logarithm histogram base B with L buckets -** l L1 L0 S0 -> Generate a linear histogram with step S with L buckets -*/ -unsigned char outbuf[HISTO_BUF_SIZE*2*4]; - -static void mem_command(ErlDrvData port, char* buf, int count) -{ - if ((count == 1) && buf[0] == 's') { - struct elib_stat info; - char v[3*4]; - - elib_stat(&info); - - putint32(v, info.mem_total*4); - putint32(v+4, info.mem_free*4); - putint32(v+8, info.max_free*4); - driver_output((ErlDrvPort)port, v, 12); - return; - } - else if ((count == 3) && buf[0] == 'm') { - char w[MAP_BUF_SIZE]; - int n = getint16(buf+1); - - if (n > MAP_BUF_SIZE) - n = MAP_BUF_SIZE; - elib_heap_map(w, n); - driver_output((ErlDrvPort)port, w, n); - return; - } - else if ((count == 4) && (buf[0] == 'h' || buf[0] == 'l')) { - unsigned long vf[HISTO_BUF_SIZE]; - unsigned long va[HISTO_BUF_SIZE]; - int n = getint16(buf+1); - int base = (unsigned char) buf[3]; - - if (n >= HISTO_BUF_SIZE) - n = HISTO_BUF_SIZE; - if (buf[0] == 'l') - base = -base; - if (elib_histo(vf, va, n, base) < 0) { - driver_failure((ErlDrvPort)port, -1); - return; - } - else { - char* p = outbuf; - int i; - - for (i = 0; i < n; i++) { - putint32(p, vf[i]); - p += 4; - } - for (i = 0; i < n; i++) { - putint32(p, va[i]); - p += 4; - } - driver_output((ErlDrvPort)port, outbuf, n*8); - } - return; - } - driver_failure((ErlDrvPort)port, -1); -} - diff --git a/erts/emulator/drivers/win32/win_con.c b/erts/emulator/drivers/win32/win_con.c index 2202ca655f..14f7941643 100644 --- a/erts/emulator/drivers/win32/win_con.c +++ b/erts/emulator/drivers/win32/win_con.c @@ -704,6 +704,18 @@ FrameWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam) } write_inbuf(&c, 1); return 0; + case WM_MOUSEWHEEL: + { + int delta = GET_WHEEL_DELTA_WPARAM(wParam); + if (delta < 0) { + PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK, + (iVscrollPos + 5)),0); + } else { + WORD pos = ((iVscrollPos - 5) < 0) ? 0 : (iVscrollPos - 5); + PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK,pos),0); + } + return 0; + } case WM_CHAR: c = (TCHAR)wParam; write_inbuf(&c,1); diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 89aaad31da..101853736a 100644..100755 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -23,20 +23,20 @@ #include <windows.h> #include "sys.h" #include <ctype.h> - +#include <wchar.h> #include "erl_efile.h" /* * Microsoft-specific function to map a WIN32 error code to a Posix errno. */ -#define ISSLASH(a) ((a) == '\\' || (a) == '/') +#define ISSLASH(a) ((a) == L'\\' || (a) == L'/') #define ISDIR(st) (((st).st_mode&S_IFMT) == S_IFDIR) #define ISREG(st) (((st).st_mode&S_IFMT) == S_IFREG) #define IS_DOT_OR_DOTDOT(s) \ - (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0'))) + ((s)[0] == L'.' && ((s)[1] == L'\0' || ((s)[1] == L'.' && (s)[2] == L'\0'))) #ifndef INVALID_FILE_ATTRIBUTES #define INVALID_FILE_ATTRIBUTES ((DWORD) 0xFFFFFFFF) @@ -44,9 +44,9 @@ static int check_error(int result, Efile_error* errInfo); static int set_error(Efile_error* errInfo); -static int IsRootUNCName(const char* path); -static int extract_root(char* name); -static unsigned short dos_to_posix_mode(int attr, const char *name); +static int is_root_unc_name(const WCHAR *path); +static int extract_root(WCHAR *name); +static unsigned short dos_to_posix_mode(int attr, const WCHAR *name); static int errno_map(DWORD last_error) { @@ -196,27 +196,26 @@ win_writev(Efile_error* errInfo, int -efile_mkdir(errInfo, name) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to create. */ +efile_mkdir(Efile_error* errInfo, /* Where to return error codes. */ + char* name) /* Name of directory to create. */ { - return check_error(mkdir(name), errInfo); + return check_error(_wmkdir((WCHAR *) name), errInfo); } int -efile_rmdir(errInfo, name) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to delete. */ +efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ + char* name) /* Name of directory to delete. */ { OSVERSIONINFO os; DWORD attr; + WCHAR *wname = (WCHAR *) name; - if (RemoveDirectory(name) != FALSE) { + if (RemoveDirectoryW(wname) != FALSE) { return 1; } errno = errno_map(GetLastError()); if (errno == EACCES) { - attr = GetFileAttributes(name); + attr = GetFileAttributesW(wname); if (attr != (DWORD) -1) { if ((attr & FILE_ATTRIBUTE_DIRECTORY) == 0) { /* @@ -238,21 +237,21 @@ char* name; /* Name of directory to delete. */ GetVersionEx(&os); if (os.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) { HANDLE handle; - WIN32_FIND_DATA data; - char buffer[2*MAX_PATH]; + WIN32_FIND_DATAW data; + WCHAR buffer[2*MAX_PATH]; int len; - len = strlen(name); - strcpy(buffer, name); - if (buffer[0] && buffer[len-1] != '\\' && buffer[len-1] != '/') { - strcat(buffer, "\\"); + len = wcslen(wname); + wcscpy(buffer, wname); + if (buffer[0] && buffer[len-1] != L'\\' && buffer[len-1] != L'/') { + wcscat(buffer, L"\\"); } - strcat(buffer, "*.*"); - handle = FindFirstFile(buffer, &data); + wcscat(buffer, L"*.*"); + handle = FindFirstFileW(buffer, &data); if (handle != INVALID_HANDLE_VALUE) { while (1) { - if ((strcmp(data.cFileName, ".") != 0) - && (strcmp(data.cFileName, "..") != 0)) { + if ((wcscmp(data.cFileName, L".") != 0) + && (wcscmp(data.cFileName, L"..") != 0)) { /* * Found something in this directory. */ @@ -260,7 +259,7 @@ char* name; /* Name of directory to delete. */ errno = EEXIST; break; } - if (FindNextFile(handle, &data) == FALSE) { + if (FindNextFileW(handle, &data) == FALSE) { break; } } @@ -284,19 +283,19 @@ char* name; /* Name of directory to delete. */ } int -efile_delete_file(errInfo, name) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of file to delete. */ +efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ + char* name) /* Name of file to delete. */ { DWORD attr; + WCHAR *wname = (WCHAR *) name; - if (DeleteFile(name) != FALSE) { + if (DeleteFileW(wname) != FALSE) { return 1; } errno = errno_map(GetLastError()); if (errno == EACCES) { - attr = GetFileAttributes(name); + attr = GetFileAttributesW(wname); if (attr != (DWORD) -1) { if (attr & FILE_ATTRIBUTE_DIRECTORY) { /* @@ -308,7 +307,7 @@ char* name; /* Name of file to delete. */ } } } else if (errno == ENOENT) { - attr = GetFileAttributes(name); + attr = GetFileAttributesW(wname); if (attr != (DWORD) -1) { if (attr & FILE_ATTRIBUTE_DIRECTORY) { /* @@ -362,20 +361,21 @@ char* name; /* Name of file to delete. */ */ int -efile_rename(errInfo, src, dst) -Efile_error* errInfo; /* Where to return error codes. */ -char* src; /* Original name. */ -char* dst; /* New name. */ +efile_rename(Efile_error* errInfo, /* Where to return error codes. */ + char* src, /* Original name. */ + char* dst) /* New name. */ { DWORD srcAttr, dstAttr; + WCHAR *wsrc = (WCHAR *) src; + WCHAR *wdst = (WCHAR *) dst; - if (MoveFile(src, dst) != FALSE) { + if (MoveFileW(wsrc, wdst) != FALSE) { return 1; } errno = errno_map(GetLastError()); - srcAttr = GetFileAttributes(src); - dstAttr = GetFileAttributes(dst); + srcAttr = GetFileAttributesW(wsrc); + dstAttr = GetFileAttributesW(wdst); if (srcAttr == (DWORD) -1) { srcAttr = 0; } @@ -390,22 +390,22 @@ char* dst; /* New name. */ if (errno == EACCES) { decode: if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) { - char srcPath[MAX_PATH], dstPath[MAX_PATH]; - char *srcRest, *dstRest; + WCHAR srcPath[MAX_PATH], dstPath[MAX_PATH]; + WCHAR *srcRest, *dstRest; int size; - size = GetFullPathName(src, sizeof(srcPath), srcPath, &srcRest); - if ((size == 0) || (size > sizeof(srcPath))) { + size = GetFullPathNameW(wsrc, MAX_PATH, srcPath, &srcRest); + if ((size == 0) || (size > MAX_PATH)) { return check_error(-1, errInfo); } - size = GetFullPathName(dst, sizeof(dstPath), dstPath, &dstRest); - if ((size == 0) || (size > sizeof(dstPath))) { + size = GetFullPathNameW(wdst, MAX_PATH, dstPath, &dstRest); + if ((size == 0) || (size > MAX_PATH)) { return check_error(-1, errInfo); } if (srcRest == NULL) { - srcRest = srcPath + strlen(srcPath); + srcRest = srcPath + wcslen(srcPath); } - if (strnicmp(srcPath, dstPath, srcRest - srcPath) == 0) { + if (_wcsnicmp(srcPath, dstPath, srcRest - srcPath) == 0) { /* * Trying to move a directory into itself. */ @@ -420,14 +420,14 @@ char* dst; /* New name. */ } (void) extract_root(dstPath); - if (dstPath[0] == '\0') { + if (dstPath[0] == L'\0') { /* * The filename was invalid. (Don't know why, * but play it safe.) */ errno = EINVAL; } - if (stricmp(srcPath, dstPath) != 0) { + if (_wcsicmp(srcPath, dstPath) != 0) { /* * If src is a directory and dst filesystem != src * filesystem, errno should be EXDEV. It is very @@ -463,14 +463,14 @@ char* dst; /* New name. */ * fails, it's because it wasn't empty. */ - if (RemoveDirectory(dst)) { + if (RemoveDirectoryW(wdst)) { /* * Now that that empty directory is gone, we can try * renaming again. If that fails, we'll put this empty * directory back, for completeness. */ - if (MoveFile(src, dst) != FALSE) { + if (MoveFileW(wsrc, wdst) != FALSE) { return 1; } @@ -480,8 +480,8 @@ char* dst; /* New name. */ */ errno = errno_map(GetLastError()); - CreateDirectory(dst, NULL); - SetFileAttributes(dst, dstAttr); + CreateDirectoryW(wdst, NULL); + SetFileAttributesW(wdst, dstAttr); if (errno == EACCES) { /* * Decode the EACCES to a more meaningful error. @@ -506,17 +506,17 @@ char* dst; /* New name. */ * put temp file back to old name. */ - char tempName[MAX_PATH]; + WCHAR tempName[MAX_PATH]; int result, size; - char *rest; + WCHAR *rest; - size = GetFullPathName(dst, sizeof(tempName), tempName, &rest); - if ((size == 0) || (size > sizeof(tempName)) || (rest == NULL)) { + size = GetFullPathNameW(wdst, MAX_PATH, tempName, &rest); + if ((size == 0) || (size > MAX_PATH) || (rest == NULL)) { return check_error(-1, errInfo); } - *rest = '\0'; + *rest = L'\0'; result = -1; - if (GetTempFileName(tempName, "erlr", 0, tempName) != 0) { + if (GetTempFileNameW(tempName, L"erlr", 0, tempName) != 0) { /* * Strictly speaking, need the following DeleteFile and * MoveFile to be joined as an atomic operation so no @@ -524,15 +524,15 @@ char* dst; /* New name. */ * same temp file. */ - DeleteFile(tempName); - if (MoveFile(dst, tempName) != FALSE) { - if (MoveFile(src, dst) != FALSE) { - SetFileAttributes(tempName, FILE_ATTRIBUTE_NORMAL); - DeleteFile(tempName); + DeleteFileW(tempName); + if (MoveFileW(wdst, tempName) != FALSE) { + if (MoveFileW(wsrc, wdst) != FALSE) { + SetFileAttributesW(tempName, FILE_ATTRIBUTE_NORMAL); + DeleteFileW(tempName); return 1; } else { - DeleteFile(dst); - MoveFile(tempName, dst); + DeleteFileW(wdst); + MoveFileW(tempName, wdst); } } @@ -558,11 +558,10 @@ char* dst; /* New name. */ } int -efile_chdir(errInfo, name) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to make current. */ +efile_chdir(Efile_error* errInfo, /* Where to return error codes. */ + char* name) /* Name of directory to make current. */ { - int success = check_error(chdir(name), errInfo); + int success = check_error(_wchdir((WCHAR *) name), errInfo); if (!success && errInfo->posix_errno == EINVAL) /* POSIXification of errno */ errInfo->posix_errno = ENOENT; @@ -570,59 +569,65 @@ char* name; /* Name of directory to make current. */ } int -efile_getdcwd(errInfo, drive, buffer, size) -Efile_error* errInfo; /* Where to return error codes. */ -int drive; /* 0 - current, 1 - A, 2 - B etc. */ -char* buffer; /* Where to return the current directory. */ -size_t size; /* Size of buffer. */ +efile_getdcwd(Efile_error* errInfo, /* Where to return error codes. */ + int drive, /* 0 - current, 1 - A, 2 - B etc. */ + char* buffer, /* Where to return the current directory. */ + size_t size) /* Size of buffer. */ { - if (_getdcwd(drive, buffer, size) == NULL) + WCHAR *wbuffer = (WCHAR *) buffer; + size_t wbuffer_size = size / 2; + if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL) return check_error(-1, errInfo); - for ( ; *buffer; buffer++) - if (*buffer == '\\') - *buffer = '/'; + for ( ; *wbuffer; wbuffer++) + if (*wbuffer == L'\\') + *wbuffer = L'/'; return 1; } int -efile_readdir(errInfo, name, dir_handle, buffer, size) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to open. */ -EFILE_DIR_HANDLE* dir_handle; /* Directory handle of open directory. */ -char* buffer; /* Pointer to buffer for one filename. */ -size_t size; /* Size of buffer. */ +efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ + char* name, /* Name of directory to list */ + EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */ + char* buffer, /* Buffer to put one filename in */ + size_t *size) /* in-out size of buffer/size of filename excluding zero + termination in bytes*/ { HANDLE dir; /* Handle to directory. */ - char wildcard[MAX_PATH]; /* Wildcard to search for. */ - WIN32_FIND_DATA findData; /* Data found by FindFirstFile() or FindNext(). */ + WCHAR wildcard[MAX_PATH]; /* Wildcard to search for. */ + WIN32_FIND_DATAW findData; /* Data found by FindFirstFile() or FindNext(). */ + /* Alignment is not honored, this works on x86 because of alignment fixup by processor. + Not perfect, but faster than alinging by hand (really) */ + WCHAR *wname = (WCHAR *) name; + WCHAR *wbuffer = (WCHAR *) buffer; /* * First time we must setup everything. */ if (*dir_handle == NULL) { - int length = strlen(name); - char* s; + int length = wcslen(wname); + WCHAR* s; if (length+3 >= MAX_PATH) { errno = ENAMETOOLONG; return check_error(-1, errInfo); } - strcpy(wildcard, name); + wcscpy(wildcard, wname); s = wildcard+length-1; - if (*s != '/' && *s != '\\') - *++s = '\\'; - *++s = '*'; - *++s = '\0'; - DEBUGF(("Reading %s\n", wildcard)); - dir = FindFirstFile(wildcard, &findData); + if (*s != L'/' && *s != L'\\') + *++s = L'\\'; + *++s = L'*'; + *++s = L'\0'; + DEBUGF(("Reading %ws\n", wildcard)); + dir = FindFirstFileW(wildcard, &findData); if (dir == INVALID_HANDLE_VALUE) return set_error(errInfo); *dir_handle = (EFILE_DIR_HANDLE) dir; if (!IS_DOT_OR_DOTDOT(findData.cFileName)) { - strcpy(buffer, findData.cFileName); + wcscpy(wbuffer, findData.cFileName); + *size = wcslen(wbuffer)*2; return 1; } } @@ -635,10 +640,11 @@ size_t size; /* Size of buffer. */ dir = (HANDLE) *dir_handle; for (;;) { - if (FindNextFile(dir, &findData)) { + if (FindNextFileW(dir, &findData)) { if (IS_DOT_OR_DOTDOT(findData.cFileName)) continue; - strcpy(buffer, findData.cFileName); + wcscpy(wbuffer, findData.cFileName); + *size = wcslen(wbuffer)*2; return 1; } @@ -655,17 +661,17 @@ size_t size; /* Size of buffer. */ } int -efile_openfile(errInfo, name, flags, pfd, pSize) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to open. */ -int flags; /* Flags to use for opening. */ -int* pfd; /* Where to store the file descriptor. */ -Sint64* pSize; /* Where to store the size of the file. */ +efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ + char* name, /* Name of directory to open. */ + int flags, /* Flags to use for opening. */ + int* pfd, /* Where to store the file descriptor. */ + Sint64* pSize) /* Where to store the size of the file. */ { BY_HANDLE_FILE_INFORMATION fileInfo; /* File information from a handle. */ HANDLE fd; /* Handle to open file. */ DWORD access; /* Access mode: GENERIC_READ, GENERIC_WRITE. */ DWORD crFlags; + WCHAR *wname = (WCHAR *) name; switch (flags & (EFILE_MODE_READ|EFILE_MODE_WRITE)) { case EFILE_MODE_READ: @@ -689,7 +695,11 @@ Sint64* pSize; /* Where to store the size of the file. */ if (flags & EFILE_MODE_APPEND) { crFlags = OPEN_ALWAYS; } - fd = CreateFile(name, access, FILE_SHARE_READ | FILE_SHARE_WRITE, + if (flags & EFILE_MODE_EXCL) { + crFlags = CREATE_NEW; + } + fd = CreateFileW(wname, access, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, crFlags, FILE_ATTRIBUTE_NORMAL, NULL); /* @@ -707,7 +717,7 @@ Sint64* pSize; /* Where to store the size of the file. */ * to EISDIR. */ if (errInfo->posix_errno && - (attr = GetFileAttributes(name)) != INVALID_FILE_ATTRIBUTES && + (attr = GetFileAttributesW(wname)) != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_DIRECTORY)) { errInfo->posix_errno = EISDIR; } @@ -731,9 +741,10 @@ Sint64* pSize; /* Where to store the size of the file. */ int efile_may_openfile(Efile_error* errInfo, char *name) { + WCHAR *wname = (WCHAR *) name; DWORD attr; - if ((attr = GetFileAttributes(name)) == INVALID_FILE_ATTRIBUTES) { + if ((attr = GetFileAttributesW(wname)) == INVALID_FILE_ATTRIBUTES) { return check_error(-1, errInfo); } @@ -742,18 +753,6 @@ efile_may_openfile(Efile_error* errInfo, char *name) { return check_error(-1, errInfo); } return 1; -#if 0 - struct stat statbuf; - - if (stat(name, &statbuf)) { - return check_error(-1, errInfo); - } - if (ISDIR(statbuf)) { - errno = EISDIR; - return check_error(-1, errInfo); - } - return 1; -#endif } void @@ -764,6 +763,15 @@ int fd; /* File descriptor for file to close. */ } int +efile_fdatasync(errInfo, fd) +Efile_error* errInfo; /* Where to return error codes. */ +int fd; /* File descriptor for file to sync. */ +{ + /* Not available in Windows, just call regular fsync */ + return efile_fsync(errInfo, fd); +} + +int efile_fsync(errInfo, fd) Efile_error* errInfo; /* Where to return error codes. */ int fd; /* File descriptor for file to sync. */ @@ -779,16 +787,17 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, char* orig_name, int info_for_link) { HANDLE findhandle; /* Handle returned by FindFirstFile(). */ - WIN32_FIND_DATA findbuf; /* Data return by FindFirstFile(). */ - char name[_MAX_PATH]; + WIN32_FIND_DATAW findbuf; /* Data return by FindFirstFile(). */ + WCHAR name[_MAX_PATH]; int name_len; - char* path; - char pathbuf[_MAX_PATH]; + WCHAR *path; + WCHAR pathbuf[_MAX_PATH]; int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ + WCHAR *worig_name = (WCHAR *) orig_name; /* Don't allow wildcards to be interpreted by system */ - if (strpbrk(orig_name, "?*")) { + if (wcspbrk(worig_name, L"?*")) { enoent: errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; @@ -800,25 +809,25 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, * slash, because it causes FindFirstFile() to fail on Win95. */ - if ((name_len = strlen(orig_name)) >= _MAX_PATH) { + if ((name_len = wcslen(worig_name)) >= _MAX_PATH) { goto enoent; } else { - strcpy(name, orig_name); + wcscpy(name, worig_name); if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != ':') { - name[name_len-1] = '\0'; + name[name_len-2] != L':') { + name[name_len-1] = L'\0'; } } /* Try to get disk from name. If none, get current disk. */ - if (name[1] != ':') { + if (name[1] != L':') { drive = 0; - if (GetCurrentDirectory(sizeof(pathbuf), pathbuf) && - pathbuf[1] == ':') { - drive = tolower(pathbuf[0]) - 'a' + 1; + if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) && + pathbuf[1] == L':') { + drive = towlower(pathbuf[0]) - L'a' + 1; } - } else if (*name && name[2] == '\0') { + } else if (*name && name[2] == L'\0') { /* * X: and nothing more is an error. */ @@ -826,15 +835,15 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; } else - drive = tolower(*name) - 'a' + 1; + drive = towlower(*name) - L'a' + 1; - findhandle = FindFirstFile(name, &findbuf); + findhandle = FindFirstFileW(name, &findbuf); if (findhandle == INVALID_HANDLE_VALUE) { - if (!(strpbrk(name, "./\\") && - (path = _fullpath(pathbuf, name, _MAX_PATH)) && + if (!(wcspbrk(name, L"./\\") && + (path = _wfullpath(pathbuf, name, _MAX_PATH)) && /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ - ((strlen(path) == 3) || IsRootUNCName(path)) && - (GetDriveType(path) > 1) ) ) { + ((wcslen(path) == 3) || is_root_unc_name(path)) && + (GetDriveTypeW(path) > 1) ) ) { errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; @@ -847,8 +856,9 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, findbuf.dwFileAttributes = FILE_ATTRIBUTE_DIRECTORY; findbuf.nFileSizeHigh = 0; findbuf.nFileSizeLow = 0; - findbuf.cFileName[0] = '\0'; + findbuf.cFileName[0] = L'\0'; + pInfo->links = 1; pInfo->modifyTime.year = 1980; pInfo->modifyTime.month = 1; pInfo->modifyTime.day = 1; @@ -861,6 +871,35 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, SYSTEMTIME SystemTime; FILETIME LocalFTime; + /*first check if we are a symlink */ + if (!info_for_link && (findbuf.dwFileAttributes & + FILE_ATTRIBUTE_REPARSE_POINT)){ + /* + * given that we know this is a symlink, + we should be able to find its target */ + WCHAR target_name[_MAX_PATH]; + if (efile_readlink(errInfo, (char *) name, + (char *) target_name,256) == 1) { + FindClose(findhandle); + return efile_fileinfo(errInfo, pInfo, + (char *) target_name, info_for_link); + } + } + + /* number of links: */ + { + HANDLE handle; /* Handle returned by CreateFile() */ + BY_HANDLE_FILE_INFORMATION fileInfo; /* from CreateFile() */ + if (handle = CreateFileW(name, GENERIC_READ, 0,NULL, + OPEN_EXISTING, 0, NULL)) { + GetFileInformationByHandle(handle, &fileInfo); + pInfo->links = fileInfo.nNumberOfLinks; + CloseHandle(handle); + } else { + pInfo->links = 1; + } + } + #define GET_TIME(dst, src) \ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \ !FileTimeToSystemTime(&LocalFTime, &SystemTime)) { \ @@ -895,7 +934,10 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \ pInfo->size_low = findbuf.nFileSizeLow; pInfo->size_high = findbuf.nFileSizeHigh; - if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + if (info_for_link && (findbuf.dwFileAttributes & + FILE_ATTRIBUTE_REPARSE_POINT)) + pInfo->type = FT_SYMLINK; + else if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) pInfo->type = FT_DIRECTORY; else pInfo->type = FT_REGULAR; @@ -906,7 +948,6 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \ pInfo->access = FA_READ|FA_WRITE; pInfo->mode = dos_to_posix_mode(findbuf.dwFileAttributes, name); - pInfo->links = 1; pInfo->major_device = drive; pInfo->minor_device = 0; pInfo->inode = 0; @@ -917,10 +958,9 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \ } int -efile_write_info(errInfo, pInfo, name) -Efile_error* errInfo; -Efile_info* pInfo; -char* name; +efile_write_info(Efile_error* errInfo, + Efile_info* pInfo, + char* name) { SYSTEMTIME timebuf; FILETIME LocalFileTime; @@ -934,12 +974,13 @@ char* name; DWORD attr; DWORD tempAttr; BOOL modifyTime = FALSE; + WCHAR *wname = (WCHAR *) name; /* * Get the attributes for the file. */ - tempAttr = attr = GetFileAttributes((LPTSTR)name); + tempAttr = attr = GetFileAttributesW(wname); if (attr == 0xffffffff) { return set_error(errInfo); } @@ -975,8 +1016,8 @@ char* name; } \ } - MKTIME(ModifyFileTime, pInfo->accessTime, mtime); - MKTIME(AccessFileTime, pInfo->modifyTime, atime); + MKTIME(ModifyFileTime, pInfo->modifyTime, mtime); + MKTIME(AccessFileTime, pInfo->accessTime, atime); MKTIME(CreationFileTime, pInfo->cTime, ctime); #undef MKTIME @@ -993,12 +1034,12 @@ char* name; if (tempAttr & FILE_ATTRIBUTE_READONLY) { tempAttr &= ~FILE_ATTRIBUTE_READONLY; - if (!SetFileAttributes((LPTSTR) name, tempAttr)) { + if (!SetFileAttributesW(wname, tempAttr)) { return set_error(errInfo); } } - fd = CreateFile(name, GENERIC_READ|GENERIC_WRITE, + fd = CreateFileW(wname, GENERIC_READ|GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (fd != INVALID_HANDLE_VALUE) { @@ -1016,7 +1057,7 @@ char* name; */ if (tempAttr != attr) { - if (!SetFileAttributes((LPTSTR) name, attr)) { + if (!SetFileAttributesW(wname, attr)) { return set_error(errInfo); } } @@ -1069,12 +1110,17 @@ char* buf; /* Buffer to write. */ size_t count; /* Number of bytes to write. */ { DWORD written; /* Bytes written in last operation. */ + OVERLAPPED overlapped; + OVERLAPPED* pOverlapped = NULL; if (flags & EFILE_MODE_APPEND) { - (void) SetFilePointer((HANDLE) fd, 0, NULL, FILE_END); + memset(&overlapped, 0, sizeof(overlapped)); + overlapped.Offset = 0xffffffff; + overlapped.OffsetHigh = 0xffffffff; + pOverlapped = &overlapped; } while (count > 0) { - if (!WriteFile((HANDLE) fd, buf, count, &written, NULL)) + if (!WriteFile((HANDLE) fd, buf, count, &written, pOverlapped)) return set_error(errInfo); buf += written; count -= written; @@ -1094,11 +1140,16 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ size_t size) /* Number of bytes to write */ { int cnt; /* Buffers so far written */ + OVERLAPPED overlapped; + OVERLAPPED* pOverlapped = NULL; ASSERT(iovcnt >= 0); if (flags & EFILE_MODE_APPEND) { - (void) SetFilePointer((HANDLE) fd, 0, NULL, FILE_END); + memset(&overlapped, 0, sizeof(overlapped)); + overlapped.Offset = 0xffffffff; + overlapped.OffsetHigh = 0xffffffff; + pOverlapped = &overlapped; } for (cnt = 0; cnt < iovcnt; cnt++) { if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { @@ -1110,7 +1161,7 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ iov[cnt].iov_base + p, iov[cnt].iov_len - p, &w, - NULL)) + pOverlapped)) return set_error(errInfo); } } @@ -1182,7 +1233,7 @@ int flags; /* - * IsRootUNCName - returns TRUE if the argument is a UNC name specifying + * is_root_unc_name - returns TRUE if the argument is a UNC name specifying * a root share. That is, if it is of the form \\server\share\. * This routine will also return true if the argument is of the * form \\server\share (no trailing slash) but Win32 currently @@ -1192,16 +1243,16 @@ int flags; */ static int -IsRootUNCName(const char* path) +is_root_unc_name(const WCHAR *path) { /* * If a root UNC name, path will start with 2 (but not 3) slashes */ - if ((strlen(path) >= 5) /* minimum string is "//x/y" */ + if ((wcslen(path) >= 5) /* minimum string is "//x/y" */ && ISSLASH(path[0]) && ISSLASH(path[1])) { - const char * p = path + 2 ; + const WCHAR *p = path + 2; /* * find the slash between the server name and share name @@ -1244,19 +1295,19 @@ IsRootUNCName(const char* path) */ static int -extract_root(char* name) +extract_root(WCHAR* name) { - int len = strlen(name); + int len = wcslen(name); - if (isalpha(name[0]) && name[1] == ':' && ISSLASH(name[2])) { - int c = name[3]; - name[3] = '\0'; - return c == '\0'; + if (iswalpha(name[0]) && name[1] == L':' && ISSLASH(name[2])) { + WCHAR c = name[3]; + name[3] = L'\0'; + return c == L'\0'; } else if (len < 5 || !ISSLASH(name[0]) || !ISSLASH(name[1])) { goto error; } else { /* Try to find the end of the UNC name. */ - char* p; - int c; + WCHAR* p; + WCHAR c; /* * Find the slash between the server name and share name. @@ -1265,7 +1316,7 @@ extract_root(char* name) for (p = name + 2; *p; p++) if (ISSLASH(*p)) break; - if (*p == '\0') + if (*p == L'\0') goto error; /* @@ -1276,24 +1327,24 @@ extract_root(char* name) if (ISSLASH(*p)) break; c = *p; - *p = '\0'; - return c == '\0' || p[1] == '\0'; + *p = L'\0'; + return c == L'\0' || p[1] == L'\0'; } error: - *name = '\0'; + *name = L'\0'; return 1; } static unsigned short -dos_to_posix_mode(int attr, const char *name) +dos_to_posix_mode(int attr, const WCHAR *name) { register unsigned short uxmode; unsigned dosmode; - register const char *p; + register const WCHAR *p; dosmode = attr & 0xff; - if ((p = name)[1] == ':') + if ((p = name)[1] == L':') p += 2; /* check to see if this is a directory - note we must make a special @@ -1302,7 +1353,7 @@ dos_to_posix_mode(int attr, const char *name) uxmode = (unsigned short) (((ISSLASH(*p) && !p[1]) || (dosmode & FILE_ATTRIBUTE_DIRECTORY) || - *p == '\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG); + *p == L'\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG); /* If attribute byte does not have read-only bit, it is read-write */ @@ -1311,11 +1362,11 @@ dos_to_posix_mode(int attr, const char *name) /* see if file appears to be executable - check extension of name */ - if (p = strrchr(name, '.')) { - if (!stricmp(p, ".exe") || - !stricmp(p, ".cmd") || - !stricmp(p, ".bat") || - !stricmp(p, ".com")) + if (p = wcsrchr(name, L'.')) { + if (!_wcsicmp(p, L".exe") || + !_wcsicmp(p, L".cmd") || + !_wcsicmp(p, L".bat") || + !_wcsicmp(p, L".com")) uxmode |= _S_IEXEC; } @@ -1330,6 +1381,60 @@ dos_to_posix_mode(int attr, const char *name) int efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) { + /* + * load dll and see if we have CreateSymbolicLink at runtime: + * (Vista only) + */ + HINSTANCE hModule = NULL; + WCHAR *wname = (WCHAR *) name; + WCHAR *wbuffer = (WCHAR *) buffer; + if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { + typedef DWORD (WINAPI * GETFINALPATHNAMEBYHANDLEPTR)( + HANDLE hFile, + LPCWSTR lpFilePath, + DWORD cchFilePath, + DWORD dwFlags); + + GETFINALPATHNAMEBYHANDLEPTR pGetFinalPathNameByHandle = + (GETFINALPATHNAMEBYHANDLEPTR)GetProcAddress(hModule, "GetFinalPathNameByHandleW"); + + if (pGetFinalPathNameByHandle == NULL) { + FreeLibrary(hModule); + } else { + /* first check if file is a symlink; {error, einval} otherwise */ + DWORD fileAttributes = GetFileAttributesW(wname); + if ((fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { + BOOLEAN success = 0; + HANDLE h = CreateFileW(wname, GENERIC_READ, 0,NULL, OPEN_EXISTING, 0, NULL); + int len; + if(h != INVALID_HANDLE_VALUE) { + success = pGetFinalPathNameByHandle(h, wbuffer, size,0); + /* GetFinalPathNameByHandle prepends path with "\\?\": */ + len = wcslen(wbuffer); + wmemmove(wbuffer,wbuffer+4,len-3); + if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' && + wbuffer[0] <= L'Z') { + wbuffer[0] = wbuffer[0] + L'a' - L'A'; + } + + for ( ; *wbuffer; wbuffer++) + if (*wbuffer == L'\\') + *wbuffer = L'/'; + CloseHandle(h); + } + FreeLibrary(hModule); + if (success) { + return 1; + } else { + return set_error(errInfo); + } + } else { + FreeLibrary(hModule); + errno = EINVAL; + return check_error(-1, errInfo); + } + } + } errno = ENOTSUP; return check_error(-1, errInfo); } @@ -1338,17 +1443,20 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) int efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) { - WIN32_FIND_DATA wfd; + WIN32_FIND_DATAW wfd; HANDLE fh; - char name[_MAX_PATH]; + WCHAR name[_MAX_PATH+1]; int name_len; - char* path; - char pathbuf[_MAX_PATH]; + WCHAR* path; + WCHAR pathbuf[_MAX_PATH+1]; /* Unclear weather GetCurrentDirectory will access one char after + _MAX_PATH */ + WCHAR *worig_name = (WCHAR *) orig_name; + WCHAR *wbuffer = (WCHAR *) buffer; int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ /* Don't allow wildcards to be interpreted by system */ - if (strpbrk(orig_name, "?*")) { + if (wcspbrk(worig_name, L"?*")) { enoent: errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; @@ -1360,67 +1468,114 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) * slash, because it causes FindFirstFile() to fail on Win95. */ - if ((name_len = strlen(orig_name)) >= _MAX_PATH) { + if ((name_len = wcslen(worig_name)) >= _MAX_PATH) { goto enoent; } else { - strcpy(name, orig_name); + wcscpy(name, worig_name); if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != ':') { - name[name_len-1] = '\0'; + name[name_len-2] != L':') { + name[name_len-1] = L'\0'; } } /* Try to get disk from name. If none, get current disk. */ - if (name[1] != ':') { + if (name[1] != L':') { drive = 0; - if (GetCurrentDirectory(sizeof(pathbuf), pathbuf) && - pathbuf[1] == ':') { - drive = tolower(pathbuf[0]) - 'a' + 1; + if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) && + pathbuf[1] == L':') { + drive = towlower(pathbuf[0]) - L'a' + 1; } - } else if (*name && name[2] == '\0') { + } else if (*name && name[2] == L'\0') { /* * X: and nothing more is an error. */ goto enoent; } else { - drive = tolower(*name) - 'a' + 1; + drive = towlower(*name) - L'a' + 1; } - fh = FindFirstFile(name,&wfd); + fh = FindFirstFileW(name,&wfd); if (fh == INVALID_HANDLE_VALUE) { - if (!(strpbrk(name, "./\\") && - (path = _fullpath(pathbuf, name, _MAX_PATH)) && + if (!(wcspbrk(name, L"./\\") && + (path = _wfullpath(pathbuf, name, _MAX_PATH)) && /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ - ((strlen(path) == 3) || IsRootUNCName(path)) && - (GetDriveType(path) > 1) ) ) { + ((wcslen(path) == 3) || is_root_unc_name(path)) && + (GetDriveTypeW(path) > 1) ) ) { errno = errno_map(GetLastError()); return check_error(-1, errInfo); } /* * Root directories (such as C:\ or \\server\share\ are fabricated. */ - strcpy(buffer,name); + wcscpy(wbuffer,name); return 1; } - strcpy(buffer,wfd.cAlternateFileName); - if (!*buffer) { - strcpy(buffer,wfd.cFileName); + wcscpy(wbuffer,wfd.cAlternateFileName); + if (!*wbuffer) { + wcscpy(wbuffer,wfd.cFileName); } - + FindClose(fh); return 1; } + int efile_link(Efile_error* errInfo, char* old, char* new) { - errno = ENOTSUP; - return check_error(-1, errInfo); + WCHAR *wold = (WCHAR *) old; + WCHAR *wnew = (WCHAR *) new; + if(!CreateHardLinkW(wnew, wold, NULL)) { + return set_error(errInfo); + } + return 1; } int efile_symlink(Efile_error* errInfo, char* old, char* new) { + /* + * Load dll and see if we have CreateSymbolicLink at runtime: + * (Vista only) + */ + HINSTANCE hModule = NULL; + WCHAR *wold = (WCHAR *) old; + WCHAR *wnew = (WCHAR *) new; + if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { + typedef BOOLEAN (WINAPI * CREATESYMBOLICLINKFUNCPTR) ( + LPCWSTR lpSymlinkFileName, + LPCWSTR lpTargetFileName, + DWORD dwFlags); + + CREATESYMBOLICLINKFUNCPTR pCreateSymbolicLink = + (CREATESYMBOLICLINKFUNCPTR) GetProcAddress(hModule, + "CreateSymbolicLinkW"); + /* A for MBCS, W for UNICODE... char* above implies 'W'! */ + if (pCreateSymbolicLink != NULL) { + DWORD attr = GetFileAttributesW(wold); + int flag = (attr != INVALID_FILE_ATTRIBUTES && + attr & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0; + /* SYMBOLIC_LINK_FLAG_DIRECTORY = 1 */ + BOOLEAN success = pCreateSymbolicLink(wnew, wold, flag); + FreeLibrary(hModule); + + if (success) { + return 1; + } else { + return set_error(errInfo); + } + } else + FreeLibrary(hModule); + } errno = ENOTSUP; return check_error(-1, errInfo); } + +int +efile_fadvise(Efile_error* errInfo, int fd, Sint64 offset, + Sint64 length, int advise) +{ + /* posix_fadvise is not available on Windows, do nothing */ + errno = ERROR_SUCCESS; + return check_error(0, errInfo); +} diff --git a/erts/emulator/hipe/hipe_amd64_glue.S b/erts/emulator/hipe/hipe_amd64_glue.S index 83b7b0397b..3376487292 100644 --- a/erts/emulator/hipe/hipe_amd64_glue.S +++ b/erts/emulator/hipe/hipe_amd64_glue.S @@ -346,6 +346,8 @@ nbif_3_gc_after_bif: subq $(16-8), %rsp movq P, %rdi movq %rax, %rsi + xorl %edx, %edx # Pass NULL in regs + xorl %ecx, %ecx # Pass 0 in arity call CSYM(erts_gc_after_bif_call) addq $(16-8), %rsp movl $0, P_NARITY(P) # Note: narity is a 32-bit field @@ -400,7 +402,7 @@ nbif_3_simple_exception: * - the native heap/stack/reds registers are saved in P */ .handle_trap: - movq %rax, P_NARITY(P) + movl %eax, P_NARITY(P) # Note: narity is a 32-bit field movl $HIPE_MODE_SWITCH_RES_TRAP, %eax jmp .nosave_exit diff --git a/erts/emulator/hipe/hipe_arm_glue.S b/erts/emulator/hipe/hipe_arm_glue.S index 5d626a5f69..2bce01954e 100644 --- a/erts/emulator/hipe/hipe_arm_glue.S +++ b/erts/emulator/hipe/hipe_arm_glue.S @@ -311,6 +311,8 @@ nbif_3_gc_after_bif: str TEMP_LR, [P, #P_NRA] str NSP, [P, #P_NSP] mov TEMP_LR, lr + mov r3, #0 /* Pass 0 in arity */ + mov r2, #0 /* Pass NULL in regs */ mov r1, r0 mov r0, P bl erts_gc_after_bif_call diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index b0abfd2310..2a877d8ace 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -440,9 +440,12 @@ BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) align != sizeof(long) && align != sizeof(double))) BIF_ERROR(BIF_P, BADARG); nrbytes = unsigned_val(BIF_ARG_2); + if (nrbytes == 0) + BIF_RET(make_small(0)); block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes); if ((unsigned long)block & (align-1)) - fprintf(stderr, "Yikes! erts_alloc() returned misaligned address %p\r\n", block); + fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n", + __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align); BIF_RET(address_to_term(block, BIF_P)); } diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c index 5188950e17..8f43811537 100644 --- a/erts/emulator/hipe/hipe_bif1.c +++ b/erts/emulator/hipe/hipe_bif1.c @@ -876,22 +876,44 @@ BIF_RETTYPE hipe_bifs_misc_timer_clear_0(BIF_ALIST_0) * + The fallback, which is the same as {X,_} = runtime(statistics). */ +static double fallback_get_hrvtime(void) +{ + unsigned long ms_user; + + elapsed_time_both(&ms_user, NULL, NULL, NULL); + return (double)ms_user; +} + #if USE_PERFCTR #include "hipe_perfctr.h" -static int hrvtime_is_open; -#define hrvtime_is_started() hrvtime_is_open +static int hrvtime_started; /* 0: closed, +1: perfctr, -1: fallback */ +#define hrvtime_is_started() (hrvtime_started != 0) static void start_hrvtime(void) { if (hipe_perfctr_hrvtime_open() >= 0) - hrvtime_is_open = 1; + hrvtime_started = 1; + else + hrvtime_started = -1; } -#define get_hrvtime() hipe_perfctr_hrvtime_get() -#define stop_hrvtime() hipe_perfctr_hrvtime_close() +static void stop_hrvtime(void) +{ + if (hrvtime_started > 0) + hipe_perfctr_hrvtime_close(); + hrvtime_started = 0; +} -#else +static double get_hrvtime(void) +{ + if (hrvtime_started > 0) + return hipe_perfctr_hrvtime_get(); + else + return fallback_get_hrvtime(); +} + +#else /* !USE_PERFCTR */ /* * Fallback, if nothing better exists. @@ -902,15 +924,9 @@ static void start_hrvtime(void) #define hrvtime_is_started() 1 #define start_hrvtime() do{}while(0) #define stop_hrvtime() do{}while(0) +#define get_hrvtime() fallback_get_hrvtime() -static double get_hrvtime(void) -{ - unsigned long ms_user; - elapsed_time_both(&ms_user, NULL, NULL, NULL); - return (double)ms_user; -} - -#endif /* hrvtime support */ +#endif /* !USE_PERFCTR */ BIF_RETTYPE hipe_bifs_get_hrvtime_0(BIF_ALIST_0) { @@ -918,11 +934,8 @@ BIF_RETTYPE hipe_bifs_get_hrvtime_0(BIF_ALIST_0) Eterm res; FloatDef f; - if (!hrvtime_is_started()) { + if (!hrvtime_is_started()) start_hrvtime(); - if (!hrvtime_is_started()) - BIF_RET(NIL); /* arity 0 BIFs may not fail */ - } f.fd = get_hrvtime(); hp = HAlloc(BIF_P, FLOAT_SIZE_OBJECT); res = make_float(hp); diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index e57e293547..6c9e1d9ba7 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* $Id$ @@ -86,7 +86,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) if (is_boxed(gval)) { Eterm *ptr = boxed_val(gval); Eterm val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *nsp_i = val; } else if (in_area(ptr, src, src_size) || @@ -96,7 +96,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) } else if (is_list(gval)) { Eterm *ptr = list_val(gval); Eterm val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (in_area(ptr, src, src_size) || in_area(ptr, oh, oh_size)) { @@ -193,7 +193,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) if (is_boxed(gval)) { Eterm *ptr = boxed_val(gval); Eterm val = *ptr; - if (IS_MOVED(val)) { + if (IS_MOVED_BOXED(val)) { ASSERT(is_boxed(val)); *nsp_i = val; } else if (in_area(ptr, heap, mature_size)) { @@ -205,7 +205,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) } else if (is_list(gval)) { Eterm *ptr = list_val(gval); Eterm val = *ptr; - if (is_non_value(val)) { + if (IS_MOVED_CONS(val)) { *nsp_i = ptr[1]; } else if (in_area(ptr, heap, mature_size)) { MOVE_CONS(ptr, val, old_htop, nsp_i); diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index a77aec7919..900dfc5a8a 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -261,7 +261,7 @@ static const struct literal { /* Field offsets in a process struct */ { "P_HP", offsetof(struct process, htop) }, { "P_HP_LIMIT", offsetof(struct process, stop) }, - { "P_OFF_HEAP_MSO", offsetof(struct process, off_heap.mso) }, + { "P_OFF_HEAP_FIRST", offsetof(struct process, off_heap.first) }, { "P_MBUF", offsetof(struct process, mbuf) }, { "P_ID", offsetof(struct process, id) }, { "P_FLAGS", offsetof(struct process, flags) }, @@ -456,7 +456,7 @@ static const struct rts_param { } rts_params[] = { { 1, "P_OFF_HEAP_FUNS", #if !defined(HYBRID) - 1, offsetof(struct process, off_heap.funs) + 1, offsetof(struct process, off_heap.first) #endif }, diff --git a/erts/emulator/hipe/hipe_ppc_glue.S b/erts/emulator/hipe/hipe_ppc_glue.S index 97b07353f9..c010f4f047 100644 --- a/erts/emulator/hipe/hipe_ppc_glue.S +++ b/erts/emulator/hipe/hipe_ppc_glue.S @@ -476,6 +476,8 @@ CSYM(nbif_3_gc_after_bif): STORE TEMP_LR, P_NRA(P) STORE NSP, P_NSP(P) mflr TEMP_LR + li r6, 0 /* Pass 0 in arity */ + li r5, 0 /* Pass NULL in regs */ mr r4, r3 mr r3, P bl CSYM(erts_gc_after_bif_call) @@ -539,7 +541,7 @@ CSYM(nbif_3_simple_exception): .handle_trap: li r3, HIPE_MODE_SWITCH_RES_TRAP STORE NSP, P_NSP(P) - STORE r4, P_NARITY(P) + stw r4, P_NARITY(P) /* Note: narity is a 32-bit field */ STORE TEMP_LR, P_NRA(P) b .nosave_exit diff --git a/erts/emulator/hipe/hipe_sparc_glue.S b/erts/emulator/hipe/hipe_sparc_glue.S index d1af5c43f5..73cefd4896 100644 --- a/erts/emulator/hipe/hipe_sparc_glue.S +++ b/erts/emulator/hipe/hipe_sparc_glue.S @@ -333,6 +333,8 @@ nbif_3_gc_after_bif: st TEMP_RA, [P+P_NRA] st NSP, [P+P_NSP] mov RA, TEMP_RA + mov 0, %o3 /* Pass 0 in arity */ + mov 0, %o2 /* Pass NULL in regs */ mov %o0, %o1 call erts_gc_after_bif_call mov P, %o0 /* delay slot */ diff --git a/erts/emulator/hipe/hipe_x86_glue.S b/erts/emulator/hipe/hipe_x86_glue.S index 2f7dff39f5..43392111fe 100644 --- a/erts/emulator/hipe/hipe_x86_glue.S +++ b/erts/emulator/hipe/hipe_x86_glue.S @@ -320,11 +320,13 @@ nbif_3_gc_after_bif: .align 4 .gc_after_bif: movl %edx, P_NARITY(P) - subl $(16-4), %esp + subl $(32-4), %esp movl P, (%esp) movl %eax, 4(%esp) + movl $0, 8(%esp) # Pass NULL in regs + movl $0, 12(%esp) # Pass 0 in arity call CSYM(erts_gc_after_bif_call) - addl $(16-4), %esp + addl $(32-4), %esp movl $0, P_NARITY(P) ret diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c index a4fff4ce31..0c61e7bf96 100644 --- a/erts/emulator/hipe/hipe_x86_signal.c +++ b/erts/emulator/hipe/hipe_x86_signal.c @@ -195,7 +195,7 @@ static void do_init(void) #define INIT() do { if (!init_done()) do_init(); } while (0) #endif /* __DARWIN__ */ -#if !defined(__GLIBC__) && !defined(__DARWIN__) +#if !defined(__GLIBC__) && !defined(__DARWIN__) && !defined(__NetBSD__) /* * Assume Solaris/x86 2.8. * There is a number of sigaction() procedures in libc: @@ -231,6 +231,7 @@ static void do_init(void) #define INIT() do { if (!init_done()) do_init(); } while (0) #endif /* not glibc or darwin */ +#if !defined(__NetBSD__) /* * This is our wrapper for sigaction(). sigaction() can be called before * hipe_signal_init() has been executed, especially when threads support @@ -253,7 +254,7 @@ static int my_sigaction(int signum, const struct sigaction *act, struct sigactio } return __next_sigaction(signum, act, oldact); } - +#endif /* * This overrides the C library's core sigaction() procedure, catching * all its internal calls. @@ -268,7 +269,7 @@ int __SIGACTION(int signum, const struct sigaction *act, struct sigaction *oldac /* * This catches the application's own sigaction() calls. */ -#if !defined(__DARWIN__) +#if !defined(__DARWIN__) && !defined(__NetBSD__) int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact) { return my_sigaction(signum, act, oldact); @@ -326,7 +327,9 @@ void hipe_signal_init(void) struct sigaction sa; int i; +#ifndef __NetBSD__ INIT(); +#endif hipe_sigaltstack_init(); diff --git a/erts/emulator/internal_doc/dec.dat b/erts/emulator/internal_doc/dec.dat new file mode 100644 index 0000000000..771ef51baa --- /dev/null +++ b/erts/emulator/internal_doc/dec.dat @@ -0,0 +1,942 @@ +{[59],894}. +{[96],8175}. +{[180],8189}. +{[183],903}. +{[198],1236}. +{[230],1237}. +{[399],1240}. +{[415],1256}. +{[439],1248}. +{[601],1241}. +{[629],1257}. +{[658],1249}. +{[697],884}. +{[768],832}. +{[768,65],192}. +{[768,69],200}. +{[768,73],204}. +{[768,79],210}. +{[768,85],217}. +{[768,87],7808}. +{[768,89],7922}. +{[768,97],224}. +{[768,101],232}. +{[768,105],236}. +{[768,111],242}. +{[768,117],249}. +{[768,119],7809}. +{[768,121],7923}. +{[768,168],8173}. +{[768,770,65],7846}. +{[768,770,69],7872}. +{[768,770,79],7890}. +{[768,770,97],7847}. +{[768,770,101],7873}. +{[768,770,111],7891}. +{[768,772,69],7700}. +{[768,772,79],7760}. +{[768,772,101],7701}. +{[768,772,111],7761}. +{[768,774,65],7856}. +{[768,774,97],7857}. +{[768,776,85],475}. +{[768,776,117],476}. +{[768,776,953],8146}. +{[768,776,965],8162}. +{[768,787,837,913],8074}. +{[768,787,837,919],8090}. +{[768,787,837,937],8106}. +{[768,787,837,945],8066}. +{[768,787,837,951],8082}. +{[768,787,837,969],8098}. +{[768,787,913],7946}. +{[768,787,917],7962}. +{[768,787,919],7978}. +{[768,787,921],7994}. +{[768,787,927],8010}. +{[768,787,937],8042}. +{[768,787,945],7938}. +{[768,787,949],7954}. +{[768,787,951],7970}. +{[768,787,953],7986}. +{[768,787,959],8002}. +{[768,787,965],8018}. +{[768,787,969],8034}. +{[768,788,837,913],8075}. +{[768,788,837,919],8091}. +{[768,788,837,937],8107}. +{[768,788,837,945],8067}. +{[768,788,837,951],8083}. +{[768,788,837,969],8099}. +{[768,788,913],7947}. +{[768,788,917],7963}. +{[768,788,919],7979}. +{[768,788,921],7995}. +{[768,788,927],8011}. +{[768,788,933],8027}. +{[768,788,937],8043}. +{[768,788,945],7939}. +{[768,788,949],7955}. +{[768,788,951],7971}. +{[768,788,953],7987}. +{[768,788,959],8003}. +{[768,788,965],8019}. +{[768,788,969],8035}. +{[768,795,79],7900}. +{[768,795,85],7914}. +{[768,795,111],7901}. +{[768,795,117],7915}. +{[768,837,945],8114}. +{[768,837,951],8130}. +{[768,837,969],8178}. +{[768,913],8122}. +{[768,917],8136}. +{[768,919],8138}. +{[768,921],8154}. +{[768,927],8184}. +{[768,933],8170}. +{[768,937],8186}. +{[768,945],8048}. +{[768,949],8050}. +{[768,951],8052}. +{[768,953],8054}. +{[768,959],8056}. +{[768,965],8058}. +{[768,969],8060}. +{[768,8127],8141}. +{[768,8190],8157}. +{[769],833}. +{[769,65],193}. +{[769,67],262}. +{[769,69],201}. +{[769,71],500}. +{[769,73],205}. +{[769,75],7728}. +{[769,76],313}. +{[769,77],7742}. +{[769,78],323}. +{[769,79],211}. +{[769,80],7764}. +{[769,82],340}. +{[769,83],346}. +{[769,85],218}. +{[769,87],7810}. +{[769,89],221}. +{[769,90],377}. +{[769,97],225}. +{[769,99],263}. +{[769,101],233}. +{[769,103],501}. +{[769,105],237}. +{[769,107],7729}. +{[769,108],314}. +{[769,109],7743}. +{[769,110],324}. +{[769,111],243}. +{[769,112],7765}. +{[769,114],341}. +{[769,115],347}. +{[769,117],250}. +{[769,119],7811}. +{[769,121],253}. +{[769,122],378}. +{[769,168],8174}. +{[769,198],508}. +{[769,216],510}. +{[769,230],509}. +{[769,248],511}. +{[769,770,65],7844}. +{[769,770,69],7870}. +{[769,770,79],7888}. +{[769,770,97],7845}. +{[769,770,101],7871}. +{[769,770,111],7889}. +{[769,771,79],7756}. +{[769,771,85],7800}. +{[769,771,111],7757}. +{[769,771,117],7801}. +{[769,772,69],7702}. +{[769,772,79],7762}. +{[769,772,101],7703}. +{[769,772,111],7763}. +{[769,774,65],7854}. +{[769,774,97],7855}. +{[769,776,73],7726}. +{[769,776,85],471}. +{[769,776,105],7727}. +{[769,776,117],472}. +{[769,776,953],8147}. +{[769,776,965],8163}. +{[769,778,65],506}. +{[769,778,97],507}. +{[769,787,837,913],8076}. +{[769,787,837,919],8092}. +{[769,787,837,937],8108}. +{[769,787,837,945],8068}. +{[769,787,837,951],8084}. +{[769,787,837,969],8100}. +{[769,787,913],7948}. +{[769,787,917],7964}. +{[769,787,919],7980}. +{[769,787,921],7996}. +{[769,787,927],8012}. +{[769,787,937],8044}. +{[769,787,945],7940}. +{[769,787,949],7956}. +{[769,787,951],7972}. +{[769,787,953],7988}. +{[769,787,959],8004}. +{[769,787,965],8020}. +{[769,787,969],8036}. +{[769,788,837,913],8077}. +{[769,788,837,919],8093}. +{[769,788,837,937],8109}. +{[769,788,837,945],8069}. +{[769,788,837,951],8085}. +{[769,788,837,969],8101}. +{[769,788,913],7949}. +{[769,788,917],7965}. +{[769,788,919],7981}. +{[769,788,921],7997}. +{[769,788,927],8013}. +{[769,788,933],8029}. +{[769,788,937],8045}. +{[769,788,945],7941}. +{[769,788,949],7957}. +{[769,788,951],7973}. +{[769,788,953],7989}. +{[769,788,959],8005}. +{[769,788,965],8021}. +{[769,788,969],8037}. +{[769,795,79],7898}. +{[769,795,85],7912}. +{[769,795,111],7899}. +{[769,795,117],7913}. +{[769,807,67],7688}. +{[769,807,99],7689}. +{[769,837,945],8116}. +{[769,837,951],8132}. +{[769,837,959],8180}. +{[769,913],8123}. +{[769,917],8137}. +{[769,919],8139}. +{[769,921],8155}. +{[769,927],8185}. +{[769,933],8171}. +{[769,937],8187}. +{[769,945],8049}. +{[769,949],8051}. +{[769,951],8053}. +{[769,953],8055}. +{[769,959],8057}. +{[769,965],8059}. +{[769,969],8061}. +{[769,1043],1027}. +{[769,1050],1036}. +{[769,1075],1107}. +{[769,1082],1116}. +{[769,8127],8142}. +{[769,8190],8158}. +{[770,65],194}. +{[770,67],264}. +{[770,69],202}. +{[770,71],284}. +{[770,72],292}. +{[770,73],206}. +{[770,74],308}. +{[770,79],212}. +{[770,83],348}. +{[770,85],219}. +{[770,87],372}. +{[770,89],374}. +{[770,90],7824}. +{[770,97],226}. +{[770,99],265}. +{[770,101],234}. +{[770,103],285}. +{[770,104],293}. +{[770,105],238}. +{[770,106],309}. +{[770,111],244}. +{[770,115],349}. +{[770,117],251}. +{[770,119],373}. +{[770,121],375}. +{[770,122],7825}. +{[770,803,65],7852}. +{[770,803,69],7878}. +{[770,803,79],7896}. +{[770,803,97],7853}. +{[770,803,101],7879}. +{[770,803,111],7897}. +{[771,65],195}. +{[771,69],7868}. +{[771,73],296}. +{[771,78],209}. +{[771,79],213}. +{[771,85],360}. +{[771,86],7804}. +{[771,89],7928}. +{[771,97],227}. +{[771,101],7869}. +{[771,105],297}. +{[771,110],241}. +{[771,111],245}. +{[771,117],361}. +{[771,118],7805}. +{[771,121],7929}. +{[771,770,65],7850}. +{[771,770,69],7876}. +{[771,770,79],7894}. +{[771,770,97],7851}. +{[771,770,101],7877}. +{[771,770,111],7895}. +{[771,774,65],7860}. +{[771,774,97],7861}. +{[771,795,79],7904}. +{[771,795,85],7918}. +{[771,795,111],7905}. +{[771,795,117],7919}. +{[772,65],256}. +{[772,69],274}. +{[772,71],7712}. +{[772,73],298}. +{[772,79],332}. +{[772,85],362}. +{[772,97],257}. +{[772,101],275}. +{[772,103],7713}. +{[772,105],299}. +{[772,111],333}. +{[772,117],363}. +{[772,198],482}. +{[772,230],483}. +{[772,775,65],480}. +{[772,775,97],481}. +{[772,776,65],478}. +{[772,776,85],469}. +{[772,776,97],479}. +{[772,776,117],470}. +{[772,803,76],7736}. +{[772,803,82],7772}. +{[772,803,108],7737}. +{[772,803,114],7773}. +{[772,808,79],492}. +{[772,808,111],493}. +{[772,913],8121}. +{[772,921],8153}. +{[772,933],8169}. +{[772,945],8113}. +{[772,953],8145}. +{[772,965],8161}. +{[772,1048],1250}. +{[772,1059],1262}. +{[772,1080],1251}. +{[772,1091],1263}. +{[774,65],258}. +{[774,69],276}. +{[774,71],286}. +{[774,73],300}. +{[774,79],334}. +{[774,85],364}. +{[774,97],259}. +{[774,101],277}. +{[774,103],287}. +{[774,105],301}. +{[774,111],335}. +{[774,117],365}. +{[774,803,65],7862}. +{[774,803,97],7863}. +{[774,807,69],7708}. +{[774,807,101],7709}. +{[774,913],8120}. +{[774,921],8152}. +{[774,933],8168}. +{[774,945],8112}. +{[774,953],8144}. +{[774,965],8160}. +{[774,1040],1232}. +{[774,1045],1238}. +{[774,1046],1217}. +{[774,1048],1049}. +{[774,1059],1038}. +{[774,1072],1233}. +{[774,1077],1239}. +{[774,1078],1218}. +{[774,1080],1081}. +{[774,1091],1118}. +{[775,66],7682}. +{[775,67],266}. +{[775,68],7690}. +{[775,69],278}. +{[775,70],7710}. +{[775,71],288}. +{[775,72],7714}. +{[775,73],304}. +{[775,77],7744}. +{[775,78],7748}. +{[775,80],7766}. +{[775,82],7768}. +{[775,83],7776}. +{[775,84],7786}. +{[775,87],7814}. +{[775,88],7818}. +{[775,89],7822}. +{[775,90],379}. +{[775,98],7683}. +{[775,99],267}. +{[775,100],7691}. +{[775,101],279}. +{[775,102],7711}. +{[775,103],289}. +{[775,104],7715}. +{[775,109],7745}. +{[775,110],7749}. +{[775,112],7767}. +{[775,114],7769}. +{[775,115],7777}. +{[775,116],7787}. +{[775,119],7815}. +{[775,120],7819}. +{[775,121],7823}. +{[775,122],380}. +{[775,383],7835}. +{[775,769,83],7780}. +{[775,769,115],7781}. +{[775,774],784}. +{[775,780,83],7782}. +{[775,780,115],7783}. +{[775,803,83],7784}. +{[775,803,115],7785}. +{[776,65],196}. +{[776,69],203}. +{[776,72],7718}. +{[776,73],207}. +{[776,79],214}. +{[776,85],220}. +{[776,87],7812}. +{[776,88],7820}. +{[776,89],376}. +{[776,97],228}. +{[776,101],235}. +{[776,104],7719}. +{[776,105],239}. +{[776,111],246}. +{[776,116],7831}. +{[776,117],252}. +{[776,119],7813}. +{[776,120],7821}. +{[776,121],255}. +{[776,399],1242}. +{[776,415],1258}. +{[776,601],1243}. +{[776,629],1259}. +{[776,771,79],7758}. +{[776,771,111],7759}. +{[776,772,85],7802}. +{[776,772,117],7803}. +{[776,921],938}. +{[776,933],939}. +{[776,953],970}. +{[776,965],971}. +{[776,978],980}. +{[776,1030],1031}. +{[776,1040],1234}. +{[776,1045],1025}. +{[776,1046],1244}. +{[776,1047],1246}. +{[776,1048],1252}. +{[776,1054],1254}. +{[776,1059],1264}. +{[776,1063],1268}. +{[776,1067],1272}. +{[776,1072],1235}. +{[776,1077],1105}. +{[776,1078],1245}. +{[776,1079],1247}. +{[776,1080],1253}. +{[776,1086],1255}. +{[776,1091],1265}. +{[776,1095],1269}. +{[776,1099],1273}. +{[776,1110],1111}. +{[777,65],7842}. +{[777,69],7866}. +{[777,73],7880}. +{[777,79],7886}. +{[777,85],7910}. +{[777,89],7926}. +{[777,97],7843}. +{[777,101],7867}. +{[777,105],7881}. +{[777,111],7887}. +{[777,117],7911}. +{[777,121],7927}. +{[777,770,65],7848}. +{[777,770,69],7874}. +{[777,770,79],7892}. +{[777,770,97],7849}. +{[777,770,101],7875}. +{[777,770,111],7893}. +{[777,774,65],7858}. +{[777,774,97],7859}. +{[777,795,79],7902}. +{[777,795,85],7916}. +{[777,795,111],7903}. +{[777,795,117],7917}. +{[778,65],197}. +{[778,85],366}. +{[778,97],229}. +{[778,117],367}. +{[778,119],7832}. +{[778,121],7833}. +{[779,79],336}. +{[779,85],368}. +{[779,111],337}. +{[779,117],369}. +{[779,1059],1266}. +{[779,1091],1267}. +{[780,65],461}. +{[780,67],268}. +{[780,68],270}. +{[780,69],282}. +{[780,71],486}. +{[780,73],463}. +{[780,75],488}. +{[780,76],317}. +{[780,78],327}. +{[780,79],465}. +{[780,82],344}. +{[780,83],352}. +{[780,84],356}. +{[780,85],467}. +{[780,90],381}. +{[780,97],462}. +{[780,99],269}. +{[780,100],271}. +{[780,101],283}. +{[780,103],487}. +{[780,105],464}. +{[780,106],496}. +{[780,107],489}. +{[780,108],318}. +{[780,110],328}. +{[780,111],466}. +{[780,114],345}. +{[780,115],353}. +{[780,116],357}. +{[780,117],468}. +{[780,122],382}. +{[780,439],494}. +{[780,658],495}. +{[780,776,85],473}. +{[780,776,117],474}. +{[781,168],901}. +{[781,776],836}. +{[781,776,953],912}. +{[781,776,965],944}. +{[781,913],902}. +{[781,917],904}. +{[781,919],905}. +{[781,921],906}. +{[781,927],908}. +{[781,933],910}. +{[781,937],911}. +{[781,945],940}. +{[781,949],941}. +{[781,951],942}. +{[781,953],943}. +{[781,959],972}. +{[781,965],973}. +{[781,969],974}. +{[781,978],979}. +{[783,65],512}. +{[783,69],516}. +{[783,73],520}. +{[783,79],524}. +{[783,82],528}. +{[783,85],532}. +{[783,97],513}. +{[783,101],517}. +{[783,105],521}. +{[783,111],525}. +{[783,114],529}. +{[783,117],533}. +{[783,1140],1142}. +{[783,1141],1143}. +{[785,65],514}. +{[785,69],518}. +{[785,73],522}. +{[785,79],526}. +{[785,82],530}. +{[785,85],534}. +{[785,97],515}. +{[785,101],519}. +{[785,105],523}. +{[785,111],527}. +{[785,114],531}. +{[785,117],535}. +{[787],835}. +{[787,837,913],8072}. +{[787,837,919],8088}. +{[787,837,937],8104}. +{[787,837,945],8064}. +{[787,837,951],8080}. +{[787,837,969],8096}. +{[787,913],7944}. +{[787,917],7960}. +{[787,919],7976}. +{[787,921],7992}. +{[787,927],8008}. +{[787,937],8040}. +{[787,945],7936}. +{[787,949],7952}. +{[787,951],7968}. +{[787,953],7984}. +{[787,959],8000}. +{[787,961],8164}. +{[787,965],8016}. +{[787,969],8032}. +{[788,837,913],8073}. +{[788,837,919],8089}. +{[788,837,937],8105}. +{[788,837,945],8065}. +{[788,837,951],8081}. +{[788,837,969],8097}. +{[788,913],7945}. +{[788,917],7961}. +{[788,919],7977}. +{[788,921],7993}. +{[788,927],8009}. +{[788,929],8172}. +{[788,933],8025}. +{[788,937],8041}. +{[788,945],7937}. +{[788,949],7953}. +{[788,951],7969}. +{[788,953],7985}. +{[788,959],8001}. +{[788,961],8165}. +{[788,965],8017}. +{[788,969],8033}. +{[795,79],416}. +{[795,85],431}. +{[795,111],417}. +{[795,117],432}. +{[803,65],7840}. +{[803,66],7684}. +{[803,68],7692}. +{[803,69],7864}. +{[803,72],7716}. +{[803,73],7882}. +{[803,75],7730}. +{[803,76],7734}. +{[803,77],7746}. +{[803,78],7750}. +{[803,79],7884}. +{[803,82],7770}. +{[803,83],7778}. +{[803,84],7788}. +{[803,85],7908}. +{[803,86],7806}. +{[803,87],7816}. +{[803,89],7924}. +{[803,90],7826}. +{[803,97],7841}. +{[803,98],7685}. +{[803,100],7693}. +{[803,101],7865}. +{[803,104],7717}. +{[803,105],7883}. +{[803,107],7731}. +{[803,108],7735}. +{[803,109],7747}. +{[803,110],7751}. +{[803,111],7885}. +{[803,114],7771}. +{[803,115],7779}. +{[803,116],7789}. +{[803,117],7909}. +{[803,118],7807}. +{[803,119],7817}. +{[803,121],7925}. +{[803,122],7827}. +{[803,795,79],7906}. +{[803,795,85],7920}. +{[803,795,111],7907}. +{[803,795,117],7921}. +{[804,85],7794}. +{[804,117],7795}. +{[805,65],7680}. +{[805,97],7681}. +{[807,67],199}. +{[807,68],7696}. +{[807,71],290}. +{[807,72],7720}. +{[807,75],310}. +{[807,76],315}. +{[807,78],325}. +{[807,82],342}. +{[807,83],350}. +{[807,84],354}. +{[807,99],231}. +{[807,100],7697}. +{[807,103],291}. +{[807,104],7721}. +{[807,107],311}. +{[807,108],316}. +{[807,110],326}. +{[807,114],343}. +{[807,115],351}. +{[807,116],355}. +{[808,65],260}. +{[808,69],280}. +{[808,73],302}. +{[808,79],490}. +{[808,85],370}. +{[808,97],261}. +{[808,101],281}. +{[808,105],303}. +{[808,111],491}. +{[808,117],371}. +{[813,68],7698}. +{[813,69],7704}. +{[813,76],7740}. +{[813,78],7754}. +{[813,84],7792}. +{[813,85],7798}. +{[813,100],7699}. +{[813,101],7705}. +{[813,108],7741}. +{[813,110],7755}. +{[813,116],7793}. +{[813,117],7799}. +{[814,72],7722}. +{[814,104],7723}. +{[816,69],7706}. +{[816,73],7724}. +{[816,85],7796}. +{[816,101],7707}. +{[816,105],7725}. +{[816,117],7797}. +{[817,66],7686}. +{[817,68],7694}. +{[817,75],7732}. +{[817,76],7738}. +{[817,78],7752}. +{[817,82],7774}. +{[817,84],7790}. +{[817,90],7828}. +{[817,98],7687}. +{[817,100],7695}. +{[817,104],7830}. +{[817,107],7733}. +{[817,108],7739}. +{[817,110],7753}. +{[817,114],7775}. +{[817,116],7791}. +{[817,122],7829}. +{[834,168],8129}. +{[834,776,953],8151}. +{[834,776,965],8167}. +{[834,787,837,913],8078}. +{[834,787,837,919],8094}. +{[834,787,837,937],8110}. +{[834,787,837,945],8070}. +{[834,787,837,951],8086}. +{[834,787,837,969],8102}. +{[834,787,913],7950}. +{[834,787,919],7982}. +{[834,787,921],7998}. +{[834,787,937],8046}. +{[834,787,945],7942}. +{[834,787,951],7974}. +{[834,787,953],7990}. +{[834,787,965],8022}. +{[834,787,969],8038}. +{[834,788,837,913],8079}. +{[834,788,837,919],8095}. +{[834,788,837,937],8111}. +{[834,788,837,945],8071}. +{[834,788,837,951],8087}. +{[834,788,837,969],8103}. +{[834,788,913],7951}. +{[834,788,919],7983}. +{[834,788,921],7999}. +{[834,788,933],8031}. +{[834,788,937],8047}. +{[834,788,945],7943}. +{[834,788,951],7975}. +{[834,788,953],7991}. +{[834,788,965],8023}. +{[834,788,969],8039}. +{[834,837,945],8119}. +{[834,837,951],8135}. +{[834,837,969],8183}. +{[834,945],8118}. +{[834,951],8134}. +{[834,953],8150}. +{[834,965],8166}. +{[834,969],8182}. +{[834,8127],8143}. +{[834,8190],8159}. +{[837,913],8124}. +{[837,919],8140}. +{[837,937],8188}. +{[837,945],8115}. +{[837,951],8131}. +{[837,969],8179}. +{[953],8126}. +{[1463,1488],64302}. +{[1463,1522],64287}. +{[1464,1488],64303}. +{[1465,1493],64331}. +{[1468,1488],64304}. +{[1468,1489],64305}. +{[1468,1490],64306}. +{[1468,1491],64307}. +{[1468,1492],64308}. +{[1468,1493],64309}. +{[1468,1494],64310}. +{[1468,1496],64312}. +{[1468,1497],64313}. +{[1468,1498],64314}. +{[1468,1499],64315}. +{[1468,1500],64316}. +{[1468,1502],64318}. +{[1468,1504],64320}. +{[1468,1505],64321}. +{[1468,1507],64323}. +{[1468,1508],64324}. +{[1468,1510],64326}. +{[1468,1511],64327}. +{[1468,1512],64328}. +{[1468,1513],64329}. +{[1468,1514],64330}. +{[1471,1489],64332}. +{[1471,1499],64333}. +{[1471,1508],64334}. +{[1473,1468,1513],64300}. +{[1473,1513],64298}. +{[1474,1468,1513],64301}. +{[1474,1513],64299}. +{[2364,2325],2392}. +{[2364,2326],2393}. +{[2364,2327],2394}. +{[2364,2332],2395}. +{[2364,2337],2396}. +{[2364,2338],2397}. +{[2364,2344],2345}. +{[2364,2347],2398}. +{[2364,2351],2399}. +{[2364,2352],2353}. +{[2364,2355],2356}. +{[2492,2465],2524}. +{[2492,2466],2525}. +{[2492,2476],2480}. +{[2492,2479],2527}. +{[2494,2503],2507}. +{[2519,2503],2508}. +{[2620,2582],2649}. +{[2620,2583],2650}. +{[2620,2588],2651}. +{[2620,2593],2652}. +{[2620,2603],2654}. +{[2876,2849],2908}. +{[2876,2850],2909}. +{[2876,2863],2911}. +{[2878,2887],2891}. +{[2902,2887],2888}. +{[2903,2887],2892}. +{[3006,3014],3018}. +{[3006,3015],3019}. +{[3031,2962],2964}. +{[3031,3014],3020}. +{[3158,3142],3144}. +{[3266,3270],3274}. +{[3285,3263],3264}. +{[3285,3266,3270],3275}. +{[3285,3270],3271}. +{[3286,3270],3272}. +{[3390,3398],3402}. +{[3390,3399],3403}. +{[3415,3398],3404}. +{[3634,3661],3635}. +{[3762,3789],3763}. +{[3953,3954],3955}. +{[3953,3956],3957}. +{[3953,3968],3969}. +{[3953,3968,4018],3959}. +{[3953,3968,4019],3961}. +{[3968,4018],3958}. +{[3968,4019],3960}. +{[4021,3904],3945}. +{[4021,3984],4025}. +{[4023,3906],3907}. +{[4023,3916],3917}. +{[4023,3921],3922}. +{[4023,3926],3927}. +{[4023,3931],3932}. +{[4023,3986],3987}. +{[4023,3996],3997}. +{[4023,4001],4002}. +{[4023,4006],4007}. +{[4023,4011],4012}. +{[12441,12358],12436}. +{[12441,12363],12364}. +{[12441,12365],12366}. +{[12441,12367],12368}. +{[12441,12369],12370}. +{[12441,12371],12372}. +{[12441,12373],12374}. +{[12441,12375],12376}. +{[12441,12377],12378}. +{[12441,12379],12380}. +{[12441,12381],12382}. +{[12441,12383],12384}. +{[12441,12385],12386}. +{[12441,12388],12389}. +{[12441,12390],12391}. +{[12441,12392],12393}. +{[12441,12399],12400}. +{[12441,12402],12403}. +{[12441,12405],12406}. +{[12441,12408],12409}. +{[12441,12411],12412}. +{[12441,12445],12446}. +{[12441,12454],12532}. +{[12441,12459],12460}. +{[12441,12461],12462}. +{[12441,12463],12464}. +{[12441,12465],12466}. +{[12441,12467],12468}. +{[12441,12469],12470}. +{[12441,12471],12472}. +{[12441,12473],12474}. +{[12441,12475],12476}. +{[12441,12477],12478}. +{[12441,12479],12480}. +{[12441,12481],12482}. +{[12441,12484],12485}. +{[12441,12486],12487}. +{[12441,12488],12489}. +{[12441,12495],12496}. +{[12441,12498],12499}. +{[12441,12501],12502}. +{[12441,12504],12505}. +{[12441,12507],12508}. +{[12441,12527],12535}. +{[12441,12528],12536}. +{[12441,12529],12537}. +{[12441,12530],12538}. +{[12441,12541],12542}. +{[12442,12399],12401}. +{[12442,12402],12404}. +{[12442,12405],12407}. +{[12442,12408],12410}. +{[12442,12411],12413}. +{[12442,12495],12497}. +{[12442,12498],12500}. +{[12442,12501],12503}. +{[12442,12504],12506}. +{[12442,12507],12509}. diff --git a/erts/emulator/internal_doc/dec.erl b/erts/emulator/internal_doc/dec.erl new file mode 100644 index 0000000000..0315f2a52d --- /dev/null +++ b/erts/emulator/internal_doc/dec.erl @@ -0,0 +1,237 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2000-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% This program is used to generate a header file with data for +%% normalizing denormalized unicode. + +%% The C header is generated from a text file containing tuples in the +%% following format: +%% {RevList,Translation} +%% Where 'RevList' is a reversed list of the denormalized repressentation of +%% the character 'Translation'. An example would be the swedish character +%% '�', which would be represented in the file as: +%% {[776,111],246}, as the denormalized representation of codepoint 246 +%% is [111,776] (i.e an 'o' followed by the "double dot accent character 776), +%% while '�' instead is represented as {[776,97],228}, as the denormalized +%% form would be [97,776] (same accent but an 'a' instead). +%% The datafile is generated from the table on Apple's developer connection +%% http://developer.apple.com/library/mac/#technotes/tn/tn1150table.html +%% The generating is done whenever new data is present (i.e. dec.dat has +%% to be changed) and not for every build. The product (the C header) is copied +%% to $ERL_TOP/erts/beam after generation and checked in. +%% The program and the data file is included for reference. + +-module(dec). + +-compile(export_all). + +-define(HASH_SIZE_FACTOR,2). +-define(BIG_PREFIX_SIZE,392). + +-define(INPUT_FILE_NAME,"dec.dat"). +-define(OUTPUT_FILE_NAME,"erl_unicode_normalize.h"). + +read(FName) -> + {ok,L} = file:consult(FName), + [{A,B} || {A,B} <- L, + length(A) > 1% , hd(A) < 769 + ]. + +dec() -> + L = read(?INPUT_FILE_NAME), + G = group(L), + {ok,Out} = file:open(?OUTPUT_FILE_NAME,[write]), + io:format + (Out, + "/*~n" + "* %CopyrightBegin%~n" + "*~n" + "* Copyright Ericsson AB 1999-2010. All Rights Reserved.~n" + "*~n" + "* The contents of this file are subject to the Erlang Public License,~n" + "* Version 1.1, (the \"License\"); you may not use this file except in~n" + "* compliance with the License. You should have received a copy of the~n" + "* Erlang Public License along with this software. If not, it can be~n" + "* retrieved online at http://www.erlang.org/.~n" + "*~n" + "* Software distributed under the License is distributed on an " + "\"AS IS\"~n" + "* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See~n" + "* the License for the specific language governing rights and " + "limitations~n" + "* under the License.~n" + "*~n" + "* %CopyrightEnd%~n" + "*/~n" + "/*~n" + "* This file is automatically generated by ~p.erl, " + "do not edit manually~n" + "*/~n", + [?MODULE]), + + io:format(Out, + "#define HASH_SIZE_FACTOR ~w~n" + "typedef struct _compose_entry {~n" + " Uint16 c;~n" + " Uint16 res;~n" + " Uint16 num_subs;~n" + " struct _compose_entry *subs;~n" + " int *hash;~n" + "} CompEntry;~n~n" + "static int compose_tab_size = ~p;~n", + [?HASH_SIZE_FACTOR,length(G)]), + d(Out,G,[],0), + PreTab = tuple_to_list(make_prefix_table(G,erlang:make_tuple(102,0))), + dump_prefixes(Out,PreTab), +%% Using this cuts down on the searching in the +%% actual implementation, but wastes memory with little real gain.. +%% LL = lists:flatten([PartList || {PartList,_} <- L]), +%% BigPreTab = tuple_to_list( +%% make_big_prefixes(LL, +%% erlang:make_tuple(?BIG_PREFIX_SIZE,0))), +%% dump_big_prefixes(Out,BigPreTab), + file:close(Out), + ok. + + + +d(Out,List,D,C) -> + d_sub(Out,List,D,C), + d_top_hash(Out,List,D,C), + d_top(Out,List,D,C). +d_sub(_Out,[],_D,_C) -> + ok; +d_sub(Out,[{_CP,[],_Res}|T],D,C) -> + d_sub(Out,T,D,C+1); +d_sub(Out,[{_CP,Subs,_Res0}|T],D,C) -> + d(Out,Subs,[C|D],0), + d_sub(Out,T,D,C+1). +d_top(Out,L,D,C) -> + io:format(Out,"static CompEntry ~s[] = {~n",[format_depth(D)]), + d_top_1(Out,L,D,C), + io:format(Out,"}; /* ~s */ ~n",[format_depth(D)]). + +d_top_1(_Out,[],_D,_C) -> + ok; +d_top_1(Out,[{CP,[],Res}|T],D,C) -> + io:format(Out, + "{~w, ~w, 0, NULL, NULL}",[CP,Res]), + if + T =:= [] -> + io:format(Out,"~n",[]); + true -> + io:format(Out,",~n",[]) + end, + d_top_1(Out,T,D,C+1); +d_top_1(Out,[{CP,Subs,_Res}|T],D,C) -> + io:format(Out, + "{~w, 0, ~w, ~s, ~s}",[CP,length(Subs), + format_depth([C|D]), + "hash_"++format_depth([C|D])]), + if + T =:= [] -> + io:format(Out,"~n",[]); + true -> + io:format(Out,",~n",[]) + end, + d_top_1(Out,T,D,C+1). + + +d_top_hash(Out,List,D,_C) -> + HSize = length(List)*?HASH_SIZE_FACTOR, + io:format(Out,"static int ~s[~p] = ~n",["hash_"++format_depth(D),HSize]), + Tup = d_top_hash_1(List,0,erlang:make_tuple(HSize,-1),HSize), + io:format(Out,"~p; /* ~s */ ~n",[Tup,"hash_"++format_depth(D)]). + +d_top_hash_1([],_,Hash,_HSize) -> + Hash; +d_top_hash_1([{CP,_,_}|T],Index,Hash,HSize) -> + Bucket = hash_search(Hash,HSize,CP rem HSize), + d_top_hash_1(T,Index+1,erlang:setelement(Bucket+1,Hash,Index),HSize). + +hash_search(Hash,_HSize,Bucket) when element(Bucket+1,Hash) =:= -1 -> + Bucket; +hash_search(Hash,HSize,Bucket) -> + hash_search(Hash,HSize,(Bucket + 1) rem HSize). + +format_depth(D) -> + lists:reverse(tl(lists:reverse(lists:flatten(["compose_tab_",[ integer_to_list(X) ++ "_" || X <- lists:reverse(D) ]])))). + + + + +make_prefix_table([],Table) -> + Table; +make_prefix_table([{C,_,_}|T],Table) when C =< 4023 -> + Index = (C div 32) + 1 - 24, + Pos = C rem 32, + X = element(Index,Table), + Y = X bor (1 bsl Pos), + NewTab = setelement(Index,Table,Y), + make_prefix_table(T,NewTab); +make_prefix_table([_|T],Tab) -> + make_prefix_table(T,Tab). + +dump_prefixes(Out,L) -> + io:format(Out,"#define COMP_CANDIDATE_MAP_OFFSET 24~n",[]), + io:format(Out,"static Uint32 comp_candidate_map[] = {~n",[]), + dump_prefixes_1(Out,L). +dump_prefixes_1(Out,[H]) -> + io:format(Out," 0x~8.16.0BU~n",[H]), + io:format(Out,"};~n",[]); +dump_prefixes_1(Out,[H|T]) -> + io:format(Out," 0x~8.16.0BU,~n",[H]), + dump_prefixes_1(Out,T). + +%% make_big_prefixes([],Table) -> +%% Table; +%% make_big_prefixes([C|T],Table) -> +%% Index = (C div 32) + 1, +%% Pos = C rem 32, +%% X = element(Index,Table), +%% Y = X bor (1 bsl Pos), +%% NewTab = setelement(Index,Table,Y), +%% make_big_prefixes(T,NewTab). + +%% dump_big_prefixes(Out,L) -> +%% io:format(Out,"#define BIG_COMP_CANDIDATE_SIZE ~w~n", [?BIG_PREFIX_SIZE]), +%% io:format(Out,"static Uint32 big_comp_candidate_map[] = {~n",[]), +%% dump_prefixes_1(Out,L). + +pick([],_,Acc) -> + {lists:reverse(Acc),[]}; +pick([{[H|TT],N}|T],H,Acc) -> + pick(T,H,[{TT,N}|Acc]); +pick([{[H|_],_}|_]=L,M,Acc) when H =/= M -> + {lists:reverse(Acc),L}. + + +group([]) -> + []; +group([{[H],N}|T]) -> + {Part,Rest} = pick(T,H,[]), + [{H,group(Part),N}| group(Rest)]; +group([{[H|_],_}|_]=L) -> + {Part,Rest} = pick(L,H,[]), + [{H,group(Part),0}| group(Rest)]. + + + + + diff --git a/erts/emulator/obsolete/driver.h b/erts/emulator/obsolete/driver.h deleted file mode 100644 index 708fe68e1a..0000000000 --- a/erts/emulator/obsolete/driver.h +++ /dev/null @@ -1,263 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ -/* - * OLD, OBSOLETE include file for erlang driver writers. - * New drivers should use erl_driver.h instead. - */ - -#ifndef __DRIVER_H__ -#define __DRIVER_H__ - -#include <stdlib.h> -#include "driver_int.h" - -#undef _ANSI_ARGS_ -#undef CONST - -#if ((defined(__STDC__) || defined(SABER)) && !defined(NO_PROTOTYPE)) || defined(__cplusplus) || defined(USE_PROTOTYPE) -# define _USING_PROTOTYPES_ 1 -# define _ANSI_ARGS_(x) x -# define CONST const -#else -# define _ANSI_ARGS_(x) () -# define CONST -#endif - -#ifdef __cplusplus -# define EXTERN extern "C" -#else -# define EXTERN extern -#endif - -/* Values for mode arg to driver_select() */ - -#define DO_READ (1 << 0) -#define DO_WRITE (1 << 1) - -/* Flags for set_port_control_flags() */ -#define PORT_CONTROL_FLAG_BINARY 1 -#define PORT_CONTROL_FLAG_HEAVY 2 - -/* This macro is used to name a dynamic driver's init function in */ -/* a way that doesn't lead to conflicts. This is crucial when using */ -/* operating systems that has one namespace for all symbols */ -/* (e.g. VxWorks). Example: if you have an dynamic driver C source */ -/* file named echo_drv.c, you use the macro like this: */ -/* int DRIVER_INIT(echo_drv)(void *handle) */ -#if defined(VXWORKS) -# define DRIVER_INIT(DRIVER_NAME) DRIVER_NAME ## _init -#elif defined(__WIN32__) -# define DRIVER_INIT(DRIVER_NAME) __declspec(dllexport) driver_init -#else -# define DRIVER_INIT(DRIVER_NAME) driver_init -#endif - -typedef int (*F_PTR)(); /* a function pointer */ -typedef long (*L_PTR)(); /* pointer to a function returning long */ - -extern int null_func(); - -/* This structure MUST match Binary in global.h exactly!!! */ -typedef struct driver_binary { - int orig_size; /* total length of binary */ - char orig_bytes[1]; /* the data (char instead of byte!) */ -} DriverBinary; - -typedef struct { - int vsize; /* length of vectors */ - int size; /* total size in bytes */ - SysIOVec* iov; - DriverBinary** binv; -} ErlIOVec; - -/* - * OLD, OBSOLETE driver entry structure. - */ - -typedef struct driver_entry { - F_PTR init; /* called at system start up (no args) */ - L_PTR start; /* called when some one does an open_port - args: port, command (nul-terminated), - additional/alternate args for fd/vanilla/spawn driver. - return value -1 means failure, other - is saved and passed to the other funcs */ - F_PTR stop; /* called when port is closed, and when the - emulator is halted - arg: start_return */ - F_PTR output; /* called when we have output from erlang to the port - args: start_return, buf, buflen */ - F_PTR ready_input; /* called when we have input from one of the driver's - file descriptors - args: start_return, fd */ - F_PTR ready_output; /* called when output is possible to one of the driver's - file descriptors - args: start_return, fd */ - char *driver_name; /* name supplied as {driver,Name,Args} to open_port */ - - F_PTR finish; /* called before unloading (DYNAMIC DRIVERS ONLY) */ - void *handle; /* file handle (DYNAMIC DRIVERS ONLY) */ - F_PTR control; /* "ioctl" for drivers (invoked by port_command/3) */ - F_PTR timeout; /* Reserved */ - F_PTR outputv; /* Reserved */ - F_PTR ready_async; /* Completion routine for driver_async */ - F_PTR padding1[3]; /* pad to match size of modern driver struct */ - int padding2[4]; /* more pad */ - F_PTR padding3[3]; /* even more padding */ -} DriverEntry; - - -/* These are the kernel functions available for driver writers */ - -EXTERN int driver_select _ANSI_ARGS_((int,int,int,int)); - -EXTERN int driver_output _ANSI_ARGS_((int, char*, int)); -EXTERN int driver_output2 _ANSI_ARGS_((int, char*, int, char*, int)); -EXTERN int driver_output_binary _ANSI_ARGS_((int, char*, int, - DriverBinary*, int, int)); -EXTERN int driver_outputv _ANSI_ARGS_((int, char*,int,ErlIOVec*,int)); - -EXTERN int driver_vec_to_buf _ANSI_ARGS_((ErlIOVec*, char*, int)); - -EXTERN int driver_set_timer _ANSI_ARGS_((int, unsigned long)); -EXTERN int driver_cancel_timer _ANSI_ARGS_((int)); - -/* - * The following functions are used to initiate a close of a port - * from a driver. - */ -EXTERN int driver_failure_eof _ANSI_ARGS_((int)); -EXTERN int driver_failure_atom _ANSI_ARGS_((int, char *)); -EXTERN int driver_failure_posix _ANSI_ARGS_((int, int)); -EXTERN int driver_failure _ANSI_ARGS_((int, int)); -EXTERN int driver_exit _ANSI_ARGS_ ((int, int)); - -EXTERN char* erl_errno_id _ANSI_ARGS_((int error)); -EXTERN void set_busy_port _ANSI_ARGS_((int, int)); -EXTERN void add_driver_entry _ANSI_ARGS_((DriverEntry *)); -EXTERN int remove_driver_entry _ANSI_ARGS_((DriverEntry *)); -EXTERN void set_port_control_flags _ANSI_ARGS_((int, int)); - -/* Binary interface */ -/* NOTE: DO NOT overwrite a binary with new data (if the data is delivered); -** since the binary is a shared object it MUST be written once. -*/ - -EXTERN DriverBinary* driver_alloc_binary _ANSI_ARGS_((int)); -EXTERN DriverBinary* driver_realloc_binary _ANSI_ARGS_((DriverBinary*, int)); -EXTERN void driver_free_binary _ANSI_ARGS_((DriverBinary*)); - - -/* Queue interface */ -EXTERN int driver_enqv _ANSI_ARGS_((int, ErlIOVec*, int)); -EXTERN int driver_pushqv _ANSI_ARGS_((int, ErlIOVec*, int)); -EXTERN int driver_deq _ANSI_ARGS_((int, int)); -EXTERN SysIOVec* driver_peekq _ANSI_ARGS_((int, int*)); -EXTERN int driver_sizeq _ANSI_ARGS_((int)); -EXTERN int driver_enq_bin _ANSI_ARGS_((int, DriverBinary*, int, int)); -EXTERN int driver_enq _ANSI_ARGS_((int, char*, int)); -EXTERN int driver_pushq_bin _ANSI_ARGS_((int, DriverBinary*, int, int)); -EXTERN int driver_pushq _ANSI_ARGS_((int, char*, int)); - -/* Memory management */ -EXTERN void *driver_alloc _ANSI_ARGS_((size_t)); -EXTERN void *driver_realloc _ANSI_ARGS_((void*, size_t)); -EXTERN void driver_free _ANSI_ARGS_((void*)); - -/* Shared / dynamic link libraries */ -EXTERN void *driver_dl_open _ANSI_ARGS_((char *)); -EXTERN void *driver_dl_sym _ANSI_ARGS_((void *, char *)); -EXTERN int driver_dl_close _ANSI_ARGS_((void *)); -EXTERN char *driver_dl_error _ANSI_ARGS_((void)); - -/* Async IO functions */ -EXTERN long driver_async _ANSI_ARGS_((int, - unsigned int*, - void (*)(void*), - void *, - void (*)(void*))); -EXTERN int driver_async_cancel _ANSI_ARGS_((long)); - -EXTERN int driver_lock_driver _ANSI_ARGS_((int)); - -/* Threads */ -typedef void* erl_mutex_t; -typedef void* erl_cond_t; -typedef void* erl_thread_t; - -EXTERN erl_mutex_t erts_mutex_create _ANSI_ARGS_((void)); -EXTERN int erts_mutex_destroy _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_lock _ANSI_ARGS_((erl_mutex_t)); -EXTERN int erts_mutex_unlock _ANSI_ARGS_((erl_mutex_t)); - -EXTERN erl_cond_t erts_cond_create _ANSI_ARGS_((void)); -EXTERN int erts_cond_destroy _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_signal _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_broadcast _ANSI_ARGS_((erl_cond_t)); -EXTERN int erts_cond_wait _ANSI_ARGS_((erl_cond_t, erl_mutex_t)); -EXTERN int erts_cond_timedwait _ANSI_ARGS_((erl_cond_t, erl_mutex_t, long)); - -EXTERN int erts_thread_create _ANSI_ARGS_((erl_thread_t*, - void* (*func)(void*), - void* arg, - int detached)); -EXTERN erl_thread_t erts_thread_self _ANSI_ARGS_((void)); -EXTERN void erts_thread_exit _ANSI_ARGS_((void*)); -EXTERN int erts_thread_join _ANSI_ARGS_((erl_thread_t, void**)); -EXTERN int erts_thread_kill _ANSI_ARGS_((erl_thread_t)); - - -typedef unsigned long DriverTermData; - -#define TERM_DATA(x) ((DriverTermData) (x)) - -/* Possible types to send from driver Argument type */ -#define ERL_DRV_NIL ((DriverTermData) 1) /* None */ -#define ERL_DRV_ATOM ((DriverTermData) 2) /* driver_mk_atom(string) */ -#define ERL_DRV_INT ((DriverTermData) 3) /* int */ -#define ERL_DRV_PORT ((DriverTermData) 4) /* driver_mk_port(ix) */ -#define ERL_DRV_BINARY ((DriverTermData) 5) /* ErlDriverBinary*, int */ -#define ERL_DRV_STRING ((DriverTermData) 6) /* char*, int */ -#define ERL_DRV_TUPLE ((DriverTermData) 7) /* int */ -#define ERL_DRV_LIST ((DriverTermData) 8) /* int */ -#define ERL_DRV_STRING_CONS ((DriverTermData) 9) /* char*, int */ -#define ERL_DRV_PID ((DriverTermData) 10) /* driver_connected,... */ - -/* DriverTermData is the type to use for casts when building - * terms that should be sent to connected process, - * for instance a tuple on the form {tcp, Port, [Tag|Binary]} - * - * DriverTermData spec[] = { - * ERL_DRV_ATOM, driver_mk_atom("tcp"), - * ERL_DRV_PORT, driver_mk_port(drv->ix), - * ERL_DRV_INT, REPLY_TAG, - * ERL_DRV_BIN, 50, TERM_DATA(buffer), - * ERL_DRV_LIST, 2, - * ERL_DRV_TUPLE, 3, - * } - * - */ - -EXTERN DriverTermData driver_mk_atom _ANSI_ARGS_ ((char*)); -EXTERN DriverTermData driver_mk_port _ANSI_ARGS_ ((int)); -EXTERN DriverTermData driver_connected _ANSI_ARGS_((int)); -EXTERN DriverTermData driver_caller _ANSI_ARGS_((int)); - -EXTERN int driver_output_term _ANSI_ARGS_((int, DriverTermData *, int)); -EXTERN int driver_send_term _ANSI_ARGS_((int, DriverTermData, DriverTermData *, int)); - -#endif - - diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 1c4c37b01a..010d60eb63 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -35,6 +35,7 @@ #include "global.h" #include "erl_threads.h" #include "erl_mtrace.h" +#include "erl_time.h" #include "big.h" #if HAVE_ERTS_MSEG @@ -271,7 +272,7 @@ schedule_cache_check(void) #endif { cache_check_timer.active = 0; - erl_set_timer(&cache_check_timer, + erts_set_timer(&cache_check_timer, check_cache, NULL, NULL, @@ -382,6 +383,14 @@ mseg_recreate(void *old_seg, Uint old_size, Uint new_size) new_seg = (void *) pmremap((void *) old_seg, (size_t) old_size, (size_t) new_size); +#elif defined(__NetBSD__) + new_seg = (void *) mremap((void *) old_seg, + (size_t) old_size, + NULL, + (size_t) new_size, + 0); + if (new_seg == (void *) MAP_FAILED) + new_seg = NULL; #else new_seg = (void *) mremap((void *) old_seg, (size_t) old_size, diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 5cca33d7eb..4d0ca97889 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -124,19 +124,24 @@ erts_smp_mtx_unlock(&(PS)->mtx) #define ERTS_POLLSET_SET_POLLED_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->polled, (long) 1)) + ((int) erts_smp_atomic_xchg(&(PS)->polled, (erts_aint_t) 1)) #define ERTS_POLLSET_UNSET_POLLED(PS) \ - erts_smp_atomic_set(&(PS)->polled, (long) 0) + erts_smp_atomic_set(&(PS)->polled, (erts_aint_t) 0) #define ERTS_POLLSET_IS_POLLED(PS) \ ((int) erts_smp_atomic_read(&(PS)->polled)) -#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->woken, (long) 1)) -#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ - erts_smp_atomic_set(&(PS)->woken, (long) 1) -#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ - erts_smp_atomic_set(&(PS)->woken, (long) 0) -#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ +#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) set_poller_woken_chk((PS)) +#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ +do { \ + ERTS_THR_MEMORY_BARRIER; \ + erts_smp_atomic_set(&(PS)->woken, (erts_aint_t) 1); \ +} while (0) +#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ +do { \ + erts_smp_atomic_set(&(PS)->woken, (erts_aint_t) 0); \ + ERTS_THR_MEMORY_BARRIER; \ +} while (0) +#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ ((int) erts_smp_atomic_read(&(PS)->woken)) #else @@ -174,9 +179,9 @@ #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic_set(&(PS)->have_update_requests, (long) 1) + erts_smp_atomic_set(&(PS)->have_update_requests, (erts_aint_t) 1) #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic_set(&(PS)->have_update_requests, (long) 0) + erts_smp_atomic_set(&(PS)->have_update_requests, (erts_aint_t) 0) #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \ ((int) erts_smp_atomic_read(&(PS)->have_update_requests)) #else @@ -194,13 +199,18 @@ #else -#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->interrupt, (long) 0)) -#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 0) -#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 1) -#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ +#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS)) +#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ +do { \ + erts_smp_atomic_set(&(PS)->interrupt, (erts_aint_t) 0); \ + ERTS_THR_MEMORY_BARRIER; \ +} while (0) +#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ +do { \ + ERTS_THR_MEMORY_BARRIER; \ + erts_smp_atomic_set(&(PS)->interrupt, (erts_aint_t) 1); \ +} while (0) +#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ ((int) erts_smp_atomic_read(&(PS)->interrupt)) #endif @@ -276,7 +286,7 @@ struct ErtsPollSet_ { ErtsPollSet next; int internal_fd_limit; ErtsFdStatus *fds_status; - int no_of_user_fds; + erts_smp_atomic_t no_of_user_fds; int fds_status_len; #if ERTS_POLL_USE_KERNEL_POLL int kp_fd; @@ -336,16 +346,30 @@ struct ErtsPollSet_ { #endif }; -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP) - static ERTS_INLINE int unset_interrupted_chk(ErtsPollSet ps) { + int res; +#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP) /* This operation isn't atomic, but we have no need at all for an atomic operation here... */ - int res = ps->interrupt; + res = ps->interrupt; ps->interrupt = 0; +#else + res = (int) erts_smp_atomic_xchg(&ps->interrupt, (erts_aint_t) 0); + ERTS_THR_MEMORY_BARRIER; +#endif return res; + +} + +#ifdef ERTS_SMP + +static ERTS_INLINE int +set_poller_woken_chk(ErtsPollSet ps) +{ + ERTS_THR_MEMORY_BARRIER; + return (int) erts_smp_atomic_xchg(&ps->woken, (erts_aint_t) 1); } #endif @@ -828,7 +852,7 @@ write_batch_buf(ErtsPollSet ps, ErtsPollBatchBuf *bbp) ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; ASSERT(ps->fds_status[fd].used_events); ps->fds_status[fd].used_events = 0; - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); update_fallback_pollset(ps, fd); ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); break; @@ -878,11 +902,11 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) events = ERTS_POLL_EV_E2N(ps->fds_status[fd].events); if (!events) { buf[buf_len].events = POLLREMOVE; - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { buf[buf_len].events = events; - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); } else { if ((ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_RST) @@ -972,12 +996,12 @@ batch_update_pollset(ErtsPollSet ps, int fd, ErtsPollBatchBuf *bbp) } if (used_events) { if (!events) { - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); } } else { if (events) - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); } ASSERT((events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); ASSERT((used_events & ~(ERTS_POLL_EV_IN|ERTS_POLL_EV_OUT)) == 0); @@ -1051,7 +1075,7 @@ update_pollset(ErtsPollSet ps, int fd) epe.data.fd = epe_templ.data.fd; res = epoll_ctl(ps->kp_fd, EPOLL_CTL_DEL, fd, &epe); } while (res != 0 && errno == EINTR); - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); ps->fds_status[fd].used_events = 0; } @@ -1059,11 +1083,11 @@ update_pollset(ErtsPollSet ps, int fd) /* A note on EPOLL_CTL_DEL: linux kernel versions before 2.6.9 need a non-NULL event pointer even though it is ignored... */ op = EPOLL_CTL_DEL; - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); } else if (!ps->fds_status[fd].used_events) { op = EPOLL_CTL_ADD; - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); } else { op = EPOLL_CTL_MOD; @@ -1113,7 +1137,7 @@ update_pollset(ErtsPollSet ps, int fd) /* Fall through ... */ case EPOLL_CTL_ADD: { ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_USEFLBCK; - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); #if ERTS_POLL_USE_CONCURRENT_UPDATE if (!*update_fallback) { *update_fallback = 1; @@ -1201,7 +1225,7 @@ static int update_pollset(ErtsPollSet ps, int fd) #if ERTS_POLL_USE_FALLBACK ASSERT(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK); #endif - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); last_pix = --ps->no_poll_fds; if (pix != last_pix) { /* Move last pix to this pix */ @@ -1228,7 +1252,7 @@ static int update_pollset(ErtsPollSet ps, int fd) ASSERT(!(ps->fds_status[fd].flags & ERTS_POLL_FD_FLG_INFLBCK) || fd == ps->kp_fd); #endif - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); ps->fds_status[fd].pix = pix = ps->no_poll_fds++; if (pix >= ps->poll_fds_len) grow_poll_fds(ps, pix); @@ -1279,7 +1303,7 @@ static int update_pollset(ErtsPollSet ps, int fd) if (!ps->fds_status[fd].used_events) { ASSERT(events); - ps->no_of_user_fds++; + erts_smp_atomic_inc(&ps->no_of_user_fds); #if ERTS_POLL_USE_FALLBACK ps->no_select_fds++; ps->fds_status[fd].flags |= ERTS_POLL_FD_FLG_INFLBCK; @@ -1287,7 +1311,7 @@ static int update_pollset(ErtsPollSet ps, int fd) } else if (!events) { ASSERT(ps->fds_status[fd].used_events); - ps->no_of_user_fds--; + erts_smp_atomic_dec(&ps->no_of_user_fds); ps->fds_status[fd].events = events; #if ERTS_POLL_USE_FALLBACK ps->no_select_fds--; @@ -1888,12 +1912,13 @@ static ERTS_INLINE int check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) { ASSERT(!*ps_locked); - if (ps->no_of_user_fds == 0 && tv->tv_usec == 0 && tv->tv_sec == 0) { + if (erts_smp_atomic_read(&ps->no_of_user_fds) == 0 + && tv->tv_usec == 0 && tv->tv_sec == 0) { /* Nothing to poll and zero timeout; done... */ return 0; } else { - long timeout = tv->tv_sec*1000 + tv->tv_usec/1000; + erts_aint_t timeout = tv->tv_sec*1000 + tv->tv_usec/1000; ASSERT(timeout >= 0); erts_smp_atomic_set(&ps->timeout, timeout); #if ERTS_POLL_USE_FALLBACK @@ -1926,7 +1951,7 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) * the maximum number of file descriptors in the poll set. */ struct dvpoll poll_res; - int nfds = ps->no_of_user_fds; + int nfds = (int) erts_smp_atomic_read(&ps->no_of_user_fds); #ifdef ERTS_SMP nfds++; /* Wakeup pipe */ #endif @@ -2087,7 +2112,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, #endif done: - erts_smp_atomic_set(&ps->timeout, LONG_MAX); + erts_smp_atomic_set(&ps->timeout, ERTS_AINT_T_MAX); #ifdef ERTS_POLL_DEBUG_PRINT erts_printf("Leaving %s = erts_poll_wait()\n", res == 0 ? "0" : erl_errno_id(res)); @@ -2125,10 +2150,12 @@ ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set) * is not guaranteed that it will timeout before 'msec' milli seconds. */ void -ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, int set, long msec) +ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, + int set, + long msec) { if (set) { - if (erts_smp_atomic_read(&ps->timeout) > msec) { + if (erts_smp_atomic_read(&ps->timeout) > (erts_aint_t) msec) { ERTS_POLLSET_SET_INTERRUPTED(ps); #if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP) wake_poller(ps); @@ -2204,7 +2231,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->internal_fd_limit = 0; ps->fds_status = NULL; ps->fds_status_len = 0; - ps->no_of_user_fds = 0; + erts_smp_atomic_init(&ps->no_of_user_fds, 0); #if ERTS_POLL_USE_KERNEL_POLL ps->kp_fd = -1; #if ERTS_POLL_USE_EPOLL @@ -2290,7 +2317,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #else erts_smp_atomic_init(&ps->interrupt, 0); #endif - erts_smp_atomic_init(&ps->timeout, LONG_MAX); + erts_smp_atomic_init(&ps->timeout, ERTS_AINT_T_MAX); #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS erts_smp_atomic_init(&ps->no_avoided_wakeups, 0); erts_smp_atomic_init(&ps->no_avoided_interrupts, 0); @@ -2302,7 +2329,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #if ERTS_POLL_USE_FALLBACK ps->fallback_used = 0; #endif - ps->no_of_user_fds = 0; /* Don't count wakeup pipe and fallback fd */ + erts_smp_atomic_set(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */ erts_smp_spin_lock(&pollsets_lock); ps->next = pollsets; @@ -2405,6 +2432,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) while (urqbp) { size += sizeof(ErtsPollSetUpdateRequestsBlock); pending_updates += urqbp->len; + urqbp = urqbp->next; } } #endif @@ -2447,7 +2475,7 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) pip->memory_size = size; - pip->poll_set_size = ps->no_of_user_fds; + pip->poll_set_size = (int) erts_smp_atomic_read(&ps->no_of_user_fds); #ifdef ERTS_SMP pip->poll_set_size++; /* Wakeup pipe */ #endif diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c new file mode 100644 index 0000000000..461e763f03 --- /dev/null +++ b/erts/emulator/sys/common/erl_sys_common_misc.c @@ -0,0 +1,107 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + + +/* + * Darwin needs conversion! + * http://developer.apple.com/library/mac/#qa/qa2001/qa1235.html + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "global.h" + +#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) +#define __DARWIN__ 1 +#endif + +#if !defined(__WIN32__) +#include <locale.h> +#if !defined(HAVE_SETLOCALE) || !defined(HAVE_NL_LANGINFO) || !defined(HAVE_LANGINFO_H) +#define PRIMITIVE_UTF8_CHECK 1 +#else +#include <langinfo.h> +#endif +#endif + +/* Written once and only once */ + +static int filename_encoding = ERL_FILENAME_UNKNOWN; +#if defined(__WIN32__) || defined(__DARWIN__) +static int user_filename_encoding = ERL_FILENAME_UTF8; /* Default unicode on windows */ +#else +static int user_filename_encoding = ERL_FILENAME_LATIN1; +#endif +void erts_set_user_requested_filename_encoding(int encoding) +{ + user_filename_encoding = encoding; +} + +int erts_get_user_requested_filename_encoding(void) +{ + return user_filename_encoding; +} + +void erts_init_sys_common_misc(void) +{ +#if defined(__WIN32__) + /* win_efile will totally fail if this is not set. */ + filename_encoding = ERL_FILENAME_WIN_WCHAR; +#else + if (user_filename_encoding != ERL_FILENAME_UNKNOWN) { + filename_encoding = user_filename_encoding; + } else { + char *l; + filename_encoding = ERL_FILENAME_LATIN1; +# ifdef PRIMITIVE_UTF8_CHECK + setlocale(LC_CTYPE, ""); /* Set international environment, + ignore result */ + if (((l = getenv("LC_ALL")) && *l) || + ((l = getenv("LC_CTYPE")) && *l) || + ((l = getenv("LANG")) && *l)) { + if (strstr(l, "UTF-8")) { + filename_encoding = ERL_FILENAME_UTF8; + } + } + +# else + l = setlocale(LC_CTYPE, ""); /* Set international environment */ + if (l != NULL) { + if (strcmp(nl_langinfo(CODESET), "UTF-8") == 0) { + filename_encoding = ERL_FILENAME_UTF8; + } + } +# endif + } +# if defined(__DARWIN__) + if (filename_encoding == ERL_FILENAME_UTF8) { + filename_encoding = ERL_FILENAME_UTF8_MAC; + } +# endif +#endif +} + +int erts_get_native_filename_encoding(void) +{ + return filename_encoding; +} diff --git a/erts/emulator/sys/unix/erl9_start.c b/erts/emulator/sys/unix/erl9_start.c deleted file mode 100644 index 578062d7e2..0000000000 --- a/erts/emulator/sys/unix/erl9_start.c +++ /dev/null @@ -1,130 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif -#include "sys.h" -#include "erl_vm.h" -#include "global.h" -#include <stdio.h> -#include <string.h> -#include <stdlib.h> - -/* - * XXX This is a temporary dummy to make sys.c happy until we'll rewrite it. - */ -unsigned preloaded_size_ring0 = 1; -unsigned char preloaded_ring0[1] = {0}; - -Preload pre_loaded[] = { - {"ring0", 1, preloaded_ring0}, - {0, 0, 0} -}; - -int -main(int argc, char** argv) -{ - char sbuf[1024]; - struct { - void* p; - int sz; - } bins[2]; - int bin_num = 0; - FILE* fp; - char* progname = argv[0]; - char* eq; - - argv++, argc--; - - if (argc > 0 && argv[0][0] == '-') { - argv++, argc--; - } - if (argc < 1) { - abort(); - } - if ((fp = fopen(argv[0], "r")) == NULL) { - abort(); - } - - /* Needs to be called before any memory allocation */ - erts_short_init(); - - while (fgets(sbuf, sizeof sbuf, fp)) { - if (sbuf[0] == '#') { - continue; /* Comment */ - } else if (sbuf[0] == 'e' && strncmp("exec", sbuf, 4) == 0) { - continue; /* Comment ;-) */ - } else if ((eq = strchr(sbuf, '=')) != NULL) { - char* val; - char* p = strchr(sbuf, '\n'); - if (p) { - *p = '\0'; - } - *eq = '\0'; - val = erts_read_env(sbuf); - if (val == NULL) { - *eq = '='; - erts_sys_putenv(sbuf, eq - &sbuf[0]); - } - erts_free_read_env(val); - } else if (sbuf[0] == ':' && '0' <= sbuf[1] && sbuf[1] <= '9') { - int load_size = atoi(sbuf+1); - void* bin; - - bin = malloc(load_size); - if (fread(bin, 1, load_size, fp) != load_size) { - abort(); - } - bins[bin_num].p = bin; - bins[bin_num].sz = load_size; - bin_num++; - } else if (strcmp(sbuf, "--end--\n") == 0) { - int rval; - Eterm mod = NIL; - char *val; - - fclose(fp); - - if (bin_num != 2) { - abort(); - } - - val = erts_read_env("ERLBREAKHANDLER"); - if (val) { - init_break_handler(); - } - erts_free_read_env(val); - - if ((rval = erts_load_module(NULL, 0, NIL, &mod, bins[0].p, bins[0].sz)) < 0) { - fprintf(stderr, "%s: Load of initial module failed: %d\n", - progname, rval); - abort(); - } - erts_first_process(mod, bins[1].p, bins[1].sz, argc, argv); - free(bins[0].p); - free(bins[1].p); - process_main(); - abort(); - } else { - fprintf(stderr, "%s: bad line: %s\n", progname, sbuf); - abort(); - } - } - abort(); -} diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index 2d5ef882f6..824678a0bb 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -329,11 +329,4 @@ extern int exit_async(void); #define ERTS_EXIT_AFTER_DUMP _exit -#ifdef ERTS_TIMER_THREAD -struct erts_iwait; /* opaque for clients */ -extern struct erts_iwait *erts_iwait_init(void); -extern void erts_iwait_wait(struct erts_iwait *iwait, struct timeval *delay); -extern void erts_iwait_interrupt(struct erts_iwait *iwait); -#endif /* ERTS_TIMER_THREAD */ - #endif /* #ifndef _ERL_UNIX_SYS_H */ diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 50b208848f..bfc04faa45 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -75,6 +75,7 @@ static erts_smp_rwmtx_t environ_rwmtx; #include "erl_sys_driver.h" #include "erl_check_io.h" +#include "erl_cpu_topology.h" #ifndef DISABLE_VFORK #define DISABLE_VFORK 0 @@ -123,8 +124,6 @@ static ErtsSysReportExit *report_exit_transit_list; extern int check_async_ready(void); extern int driver_interrupt(int, int); -/*EXTERN_FUNCTION(void, increment_time, (int));*/ -/*EXTERN_FUNCTION(int, next_time, (_VOID_));*/ extern void do_break(void); extern void erl_sys_args(int*, char**); @@ -221,10 +220,10 @@ static struct fd_data { } *fd_data; /* indexed by fd */ /* static FUNCTION(int, write_fill, (int, char*, int)); unused? */ -static FUNCTION(void, note_child_death, (int, int)); +static void note_child_death(int, int); #if CHLDWTHR -static FUNCTION(void *, child_waiter, (void *)); +static void* child_waiter(void *); #endif /********************* General functions ****************************/ @@ -238,9 +237,9 @@ static int max_files = -1; #ifdef ERTS_SMP erts_smp_atomic_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 1) + erts_smp_atomic_set(&erts_break_requested, (erts_aint_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 0) + erts_smp_atomic_set(&erts_break_requested, (erts_aint_t) 0) #else volatile int erts_break_requested = 0; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) @@ -384,18 +383,6 @@ MALLOC_USE_HASH(1); #endif #ifdef USE_THREADS -static void *ethr_internal_alloc(size_t size) -{ - return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size); -} -static void *ethr_internal_realloc(void *ptr, size_t size) -{ - return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size); -} -static void ethr_internal_free(void *ptr) -{ - erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr); -} #ifdef ERTS_THR_HAVE_SIG_FUNCS /* @@ -413,7 +400,7 @@ typedef struct { #ifdef ERTS_THR_HAVE_SIG_FUNCS sigset_t saved_sigmask; #endif - int unbind_child; + int sched_bind_data; } erts_thr_create_data_t; /* @@ -424,15 +411,13 @@ static void * thr_create_prepare(void) { erts_thr_create_data_t *tcdp; - ErtsSchedulerData *esdp; tcdp = erts_alloc(ERTS_ALC_T_TMP, sizeof(erts_thr_create_data_t)); #ifdef ERTS_THR_HAVE_SIG_FUNCS erts_thr_sigmask(SIG_BLOCK, &thr_create_sigmask, &tcdp->saved_sigmask); #endif - esdp = erts_get_scheduler_data(); - tcdp->unbind_child = esdp && erts_is_scheduler_bound(esdp); + tcdp->sched_bind_data = erts_sched_bind_atthrcreate_prepare(); return (void *) tcdp; } @@ -444,6 +429,8 @@ thr_create_cleanup(void *vtcdp) { erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp; + erts_sched_bind_atthrcreate_parent(tcdp->sched_bind_data); + #ifdef ERTS_THR_HAVE_SIG_FUNCS /* Restore signalmask... */ erts_thr_sigmask(SIG_SETMASK, &tcdp->saved_sigmask, NULL); @@ -457,6 +444,10 @@ thr_create_prepare_child(void *vtcdp) { erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp; +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_thread_setup(); +#endif + #ifndef NO_FPE_SIGNALS /* * We do not want fp exeptions in other threads than the @@ -466,12 +457,7 @@ thr_create_prepare_child(void *vtcdp) erts_thread_disable_fpe(); #endif - if (tcdp->unbind_child) { - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - erts_unbind_from_cpu(erts_cpuinfo); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - } - + erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data); } #endif /* #ifdef USE_THREADS */ @@ -484,9 +470,6 @@ erts_sys_pre_init(void) #ifdef USE_THREADS { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; - eid.alloc = ethr_internal_alloc; - eid.realloc = ethr_internal_realloc; - eid.free = ethr_internal_free; eid.thread_create_child_func = thr_create_prepare_child; /* Before creation in parent */ @@ -534,13 +517,14 @@ erts_sys_pre_init(void) #endif #endif /* USE_THREADS */ erts_smp_atomic_init(&sys_misc_mem_sz, 0); - erts_smp_rwmtx_init(&environ_rwmtx, "environ"); } void erl_sys_init(void) { + erts_smp_rwmtx_init(&environ_rwmtx, "environ"); #if !DISABLE_VFORK + { int res; char bindir[MAXPATHLEN]; size_t bindirsz = sizeof(bindir); @@ -570,6 +554,7 @@ erl_sys_init(void) bindir, DIR_SEPARATOR_CHAR, CHILD_SETUP_PROG_NAME); + } #endif #ifdef USE_SETLINEBUF @@ -1324,10 +1309,18 @@ static char **build_unix_environment(char *block) } } - for (j = 0; j < i; j++) { + for (j = 0; j < i; ) { size_t last = strlen(cpp[j])-1; if (cpp[j][last] == '=' && strchr(cpp[j], '=') == cpp[j]+last) { cpp[j] = cpp[--len]; + if (len < i) { + i--; + } else { + j++; + } + } + else { + j++; } } @@ -1464,9 +1457,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op CHLD_STAT_LOCK; - unbind = erts_is_scheduler_bound(NULL); - if (unbind) - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + unbind = erts_sched_bind_atfork_prepare(); #if !DISABLE_VFORK /* See fork/vfork discussion before this function. */ @@ -1479,7 +1470,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op if (pid == 0) { /* The child! Setup child... */ - if (unbind && erts_unbind_from_cpu(erts_cpuinfo) != 0) + if (erts_sched_bind_atfork_child(unbind) != 0) goto child_error; /* OBSERVE! @@ -1580,8 +1571,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op cs_argv[CS_ARGV_PROGNAME_IX] = child_setup_prog; cs_argv[CS_ARGV_WD_IX] = opts->wd ? opts->wd : "."; - cs_argv[CS_ARGV_UNBIND_IX] - = (unbind ? erts_get_unbind_from_cpu_str(erts_cpuinfo) : "false"); + cs_argv[CS_ARGV_UNBIND_IX] = erts_sched_bind_atvfork_child(unbind); cs_argv[CS_ARGV_FD_CR_IX] = fd_close_range; for (i = 0; i < CS_ARGV_NO_OF_DUP2_OPS; i++) cs_argv[CS_ARGV_DUP2_OP_IX(i)] = &dup2_op[i][0]; @@ -1630,8 +1620,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op } #endif - if (unbind) - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + erts_sched_bind_atfork_parent(unbind); if (pid == -1) { saved_errno = errno; @@ -2563,7 +2552,6 @@ extern Preload pre_loaded[]; void erts_sys_alloc_init(void) { - elib_ensure_initialized(); } void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz) @@ -3121,226 +3109,3 @@ erl_sys_args(int* argc, char** argv) } *argc = j; } - -#ifdef ERTS_TIMER_THREAD - -/* - * Interruptible-wait facility: low-level synchronisation state - * and methods that are implementation dependent. - * - * Constraint: Every implementation must define 'struct erts_iwait' - * with a field 'erts_smp_atomic_t state;'. - */ - -/* values for struct erts_iwait's state field */ -#define IWAIT_WAITING 0 -#define IWAIT_AWAKE 1 -#define IWAIT_INTERRUPT 2 - -#if 0 /* XXX: needs feature test in erts/configure.in */ - -/* - * This is an implementation of the interruptible wait facility on - * top of Linux-specific futexes. - */ -#include <asm/unistd.h> -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 -static int sys_futex(void *futex, int op, int val, const struct timespec *timeout) -{ - return syscall(__NR_futex, futex, op, val, timeout); -} - -struct erts_iwait { - erts_smp_atomic_t state; /* &state.counter is our futex */ -}; - -static void iwait_lowlevel_init(struct erts_iwait *iwait) { /* empty */ } - -static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay) -{ - struct timespec timeout; - int res; - - timeout.tv_sec = delay->tv_sec; - timeout.tv_nsec = delay->tv_usec * 1000; - res = sys_futex((void*)&iwait->state.counter, FUTEX_WAIT, IWAIT_WAITING, &timeout); - if (res < 0 && errno != ETIMEDOUT && errno != EWOULDBLOCK && errno != EINTR) - perror("FUTEX_WAIT"); -} - -static void iwait_lowlevel_interrupt(struct erts_iwait *iwait) -{ - int res = sys_futex((void*)&iwait->state.counter, FUTEX_WAKE, 1, NULL); - if (res < 0) - perror("FUTEX_WAKE"); -} - -#else /* using poll() or select() */ - -/* - * This is an implementation of the interruptible wait facility on - * top of pipe(), poll() or select(), read(), and write(). - */ -struct erts_iwait { - erts_smp_atomic_t state; - int read_fd; /* wait polls and reads this fd */ - int write_fd; /* interrupt writes this fd */ -}; - -static void iwait_lowlevel_init(struct erts_iwait *iwait) -{ - int fds[2]; - - if (pipe(fds) < 0) { - perror("pipe()"); - exit(1); - } - iwait->read_fd = fds[0]; - iwait->write_fd = fds[1]; -} - -#if defined(ERTS_USE_POLL) - -#include <sys/poll.h> -#define PERROR_POLL "poll()" - -static int iwait_lowlevel_poll(int read_fd, struct timeval *delay) -{ - struct pollfd pollfd; - int timeout; - - pollfd.fd = read_fd; - pollfd.events = POLLIN; - pollfd.revents = 0; - timeout = delay->tv_sec * 1000 + delay->tv_usec / 1000; - return poll(&pollfd, 1, timeout); -} - -#else /* !ERTS_USE_POLL */ - -#include <sys/select.h> -#define PERROR_POLL "select()" - -static int iwait_lowlevel_poll(int read_fd, struct timeval *delay) -{ - fd_set readfds; - - FD_ZERO(&readfds); - FD_SET(read_fd, &readfds); - return select(read_fd + 1, &readfds, NULL, NULL, delay); -} - -#endif /* !ERTS_USE_POLL */ - -static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay) -{ - int res; - char buf[64]; - - res = iwait_lowlevel_poll(iwait->read_fd, delay); - if (res > 0) - (void)read(iwait->read_fd, buf, sizeof buf); - else if (res < 0 && errno != EINTR) - perror(PERROR_POLL); -} - -static void iwait_lowlevel_interrupt(struct erts_iwait *iwait) -{ - int res = write(iwait->write_fd, "!", 1); - if (res < 0) - perror("write()"); -} - -#endif /* using poll() or select() */ - -#if 0 /* not using poll() or select() */ -/* - * This is an implementation of the interruptible wait facility on - * top of pthread_cond_timedwait(). This has two problems: - * 1. pthread_cond_timedwait() requires an absolute time point, - * so the relative delay must be converted to absolute time. - * Worse, this breaks if the machine's time is adjusted while - * we're preparing to wait. - * 2. Each cond operation requires additional mutex lock/unlock operations. - * - * Problem 2 is probably not too bad on Linux (they'll just become - * relatively cheap futex operations), but problem 1 is the real killer. - * Only use this implementation if no better alternatives are available! - */ -struct erts_iwait { - erts_smp_atomic_t state; - pthread_cond_t cond; - pthread_mutex_t mutex; -}; - -static void iwait_lowlevel_init(struct erts_iwait *iwait) -{ - iwait->cond = (pthread_cond_t) PTHREAD_COND_INITIALIZER; - iwait->mutex = (pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER; -} - -static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay) -{ - struct timeval tmp; - struct timespec timeout; - - /* Due to pthread_cond_timedwait()'s use of absolute - time, this must be the real gettimeofday(), _not_ - the "smoothed" one beam/erl_time_sup.c implements. */ - gettimeofday(&tmp, NULL); - - tmp.tv_sec += delay->tv_sec; - tmp.tv_usec += delay->tv_usec; - if (tmp.tv_usec >= 1000*1000) { - tmp.tv_usec -= 1000*1000; - tmp.tv_sec += 1; - } - timeout.tv_sec = tmp.tv_sec; - timeout.tv_nsec = tmp.tv_usec * 1000; - pthread_mutex_lock(&iwait->mutex); - pthread_cond_timedwait(&iwait->cond, &iwait->mutex, &timeout); - pthread_mutex_unlock(&iwait->mutex); -} - -static void iwait_lowlevel_interrupt(struct erts_iwait *iwait) -{ - pthread_mutex_lock(&iwait->mutex); - pthread_cond_signal(&iwait->cond); - pthread_mutex_unlock(&iwait->mutex); -} - -#endif /* not using POLL */ - -/* - * Interruptible-wait facility. This is just a wrapper around the - * low-level synchronisation code, where we maintain our logical - * state in order to suppress some state transitions. - */ - -struct erts_iwait *erts_iwait_init(void) -{ - struct erts_iwait *iwait = malloc(sizeof *iwait); - if (!iwait) { - perror("malloc"); - exit(1); - } - iwait_lowlevel_init(iwait); - erts_smp_atomic_init(&iwait->state, IWAIT_AWAKE); - return iwait; -} - -void erts_iwait_wait(struct erts_iwait *iwait, struct timeval *delay) -{ - if (erts_smp_atomic_xchg(&iwait->state, IWAIT_WAITING) != IWAIT_INTERRUPT) - iwait_lowlevel_wait(iwait, delay); - erts_smp_atomic_set(&iwait->state, IWAIT_AWAKE); -} - -void erts_iwait_interrupt(struct erts_iwait *iwait) -{ - if (erts_smp_atomic_xchg(&iwait->state, IWAIT_INTERRUPT) == IWAIT_WAITING) - iwait_lowlevel_interrupt(iwait); -} - -#endif /* ERTS_TIMER_THREAD */ diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c index c59c99f65e..6e9376b0f3 100644 --- a/erts/emulator/sys/unix/sys_float.c +++ b/erts/emulator/sys/unix/sys_float.c @@ -476,7 +476,7 @@ static int mask_fpe(void) #endif -#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__))) || (defined(__OpenBSD__) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__)) +#if (defined(__linux__) && (defined(__i386__) || defined(__x86_64__) || defined(__sparc__) || defined(__powerpc__))) || (defined(__DARWIN__) && (defined(__i386__) || defined(__x86_64__) || defined(__ppc__))) || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__))) || ((defined(__NetBSD__) || defined(__OpenBSD__)) && defined(__x86_64__)) || (defined(__sun__) && defined(__x86_64__)) #if defined(__linux__) && defined(__i386__) #if !defined(X86_FXSR_MAGIC) @@ -519,6 +519,10 @@ static int mask_fpe(void) #define mc_pc(mc) ((mc)->mc_rip) #elif defined(__FreeBSD__) && defined(__i386__) #define mc_pc(mc) ((mc)->mc_eip) +#elif defined(__NetBSD__) && defined(__x86_64__) +#define mc_pc(mc) ((mc)->__gregs[_REG_RIP]) +#elif defined(__NetBSD__) && defined(__i386__) +#define mc_pc(mc) ((mc)->__gregs[_REG_EIP]) #elif defined(__OpenBSD__) && defined(__x86_64__) #define mc_pc(mc) ((mc)->sc_rip) #elif defined(__sun__) && defined(__x86_64__) @@ -610,6 +614,23 @@ static void fpe_sig_action(int sig, siginfo_t *si, void *puc) struct env87 *env87 = &savefpu->sv_87.sv_env; env87->en_sw &= ~0xFF; } +#elif defined(__NetBSD__) && defined(__x86_64__) + mcontext_t *mc = &uc->uc_mcontext; + struct fxsave64 *fxsave = (struct fxsave64 *)&mc->__fpregs; + pc = mc_pc(mc); + fxsave->fx_mxcsr = 0x1F80; + fxsave->fx_fsw &= ~0xFF; +#elif defined(__NetBSD__) && defined(__i386__) + mcontext_t *mc = &uc->uc_mcontext; + pc = mc_pc(mc); + if (uc->uc_flags & _UC_FXSAVE) { + struct envxmm *envxmm = (struct envxmm *)&mc->__fpregs; + envxmm->en_mxcsr = 0x1F80; + envxmm->en_sw &= ~0xFF; + } else { + struct env87 *env87 = (struct env87 *)&mc->__fpregs; + env87->en_sw &= ~0xFF; + } #elif defined(__OpenBSD__) && defined(__x86_64__) struct fxsave64 *fxsave = uc->sc_fpstate; pc = mc_pc(uc); @@ -799,8 +820,17 @@ sys_chars_to_double(char* buf, double* fp) } #ifdef NO_FPE_SIGNALS - if (errno == ERANGE && (*fp == 0.0 || *fp == HUGE_VAL || *fp == -HUGE_VAL)) { - return -1; + if (errno == ERANGE) { + if (*fp == HUGE_VAL || *fp == -HUGE_VAL) { + /* overflow, should give error */ + return -1; + } else if (t == s && *fp == 0.0) { + /* This should give 0.0 - OTP-7178 */ + errno = 0; + + } else if (*fp == 0.0) { + return -1; + } } #endif return 0; @@ -810,7 +840,9 @@ int matherr(struct exception *exc) { #if !defined(NO_FPE_SIGNALS) - set_current_fp_exception((unsigned long)__builtin_return_address(0)); + volatile unsigned long *fpexnp = erts_get_current_fp_exception(); + if (fpexnp != NULL) + *fpexnp = (unsigned long)__builtin_return_address(0); #endif return 1; } diff --git a/erts/emulator/sys/vxworks/sys.c b/erts/emulator/sys/vxworks/sys.c index 411b4b37cf..c6e7b65f32 100644 --- a/erts/emulator/sys/vxworks/sys.c +++ b/erts/emulator/sys/vxworks/sys.c @@ -85,7 +85,7 @@ EXTERN_FUNCTION(void, erl_exit, (int n, char*, _DOTS_)); EXTERN_FUNCTION(void, erl_error, (char*, va_list)); EXTERN_FUNCTION(int, driver_interrupt, (int, int)); EXTERN_FUNCTION(void, increment_time, (int)); -EXTERN_FUNCTION(int, next_time, (_VOID_)); +EXTERN_FUNCTION(int, erts_next_time, (_VOID_)); EXTERN_FUNCTION(void, set_reclaim_free_function, (FreeFunction)); EXTERN_FUNCTION(int, erl_mem_info_get, (MEM_PART_STATS *)); EXTERN_FUNCTION(void, erl_crash_dump, (char* file, int line, char* fmt, ...)); diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index d816cc2c07..d84ae2ede2 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ #ifdef HAVE_CONFIG_H @@ -297,31 +297,58 @@ struct ErtsPollSet_ { #define ERTS_POLLSET_UNLOCK(PS) \ erts_smp_mtx_unlock(&(PS)->mtx) #define ERTS_POLLSET_SET_POLLED_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->polled, (long) 1)) + ((int) erts_smp_atomic_xchg(&(PS)->polled, (erts_aint_t) 1)) #define ERTS_POLLSET_SET_POLLED(PS) \ - erts_smp_atomic_set(&(PS)->polled, (long) 1) + erts_smp_atomic_set(&(PS)->polled, (erts_aint_t) 1) #define ERTS_POLLSET_UNSET_POLLED(PS) \ - erts_smp_atomic_set(&(PS)->polled, (long) 0) + erts_smp_atomic_set(&(PS)->polled, (erts_aint_t) 0) #define ERTS_POLLSET_IS_POLLED(PS) \ ((int) erts_smp_atomic_read(&(PS)->polled)) -#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->woken, (long) 1)) -#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ - erts_smp_atomic_set(&(PS)->woken, (long) 1) -#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ - erts_smp_atomic_set(&(PS)->woken, (long) 0) -#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ + +#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) set_poller_woken_chk((PS)) +#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ +do { \ + ERTS_THR_MEMORY_BARRIER; \ + erts_smp_atomic_set(&(PS)->woken, (erts_aint_t) 1); \ +} while (0) +#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ +do { \ + erts_smp_atomic_set(&(PS)->woken, (erts_aint_t) 0); \ + ERTS_THR_MEMORY_BARRIER; \ +} while (0) +#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ ((int) erts_smp_atomic_read(&(PS)->woken)) -#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->interrupt, (long) 0)) -#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 0) -#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 1) -#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ +#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS)) +#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ +do { \ + erts_smp_atomic_set(&(PS)->interrupt, (erts_aint_t) 0); \ + ERTS_THR_MEMORY_BARRIER; \ +} while (0) +#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ +do { \ + ERTS_THR_MEMORY_BARRIER; \ + erts_smp_atomic_set(&(PS)->interrupt, (erts_aint_t) 1); \ +} while (0) +#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ ((int) erts_smp_atomic_read(&(PS)->interrupt)) +static ERTS_INLINE int +unset_interrupted_chk(ErtsPollSet ps) +{ + int res = (int) erts_smp_atomic_xchg(&ps->interrupt, (erts_aint_t) 0); + ERTS_THR_MEMORY_BARRIER; + return res; + +} + +static ERTS_INLINE int +set_poller_woken_chk(ErtsPollSet ps) +{ + ERTS_THR_MEMORY_BARRIER; + return (int) erts_smp_atomic_xchg(&ps->woken, (erts_aint_t) 1); +} + #else #define ERTS_POLLSET_LOCK(PS) @@ -386,9 +413,9 @@ struct ErtsPollSet_ { #ifdef ERTS_SMP extern erts_smp_atomic_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 1) + erts_smp_atomic_set(&erts_break_requested, (erts_aint_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 0) + erts_smp_atomic_set(&erts_break_requested, (erts_aint_t) 0) #else extern volatile int erts_break_requested; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) @@ -959,7 +986,7 @@ void erts_poll_interrupt_timed(ErtsPollSet ps, HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,msec)); #ifdef ERTS_SMP if (set) { - if (erts_smp_atomic_read(&ps->timeout) > msec) { + if (erts_smp_atomic_read(&ps->timeout) > (erts_aint_t) msec) { ERTS_POLLSET_SET_INTERRUPTED(ps); wake_poller(ps); } @@ -1201,7 +1228,7 @@ int erts_poll_wait(ErtsPollSet ps, erts_mtx_unlock(&w->mtx); } done: - erts_smp_atomic_set(&ps->timeout, LONG_MAX); + erts_smp_atomic_set(&ps->timeout, ERTS_AINT_T_MAX); *len = num; ERTS_POLLSET_UNLOCK(ps); HARDTRACEF(("Out erts_poll_wait")); @@ -1287,7 +1314,7 @@ ErtsPollSet erts_poll_create_pollset(void) erts_smp_mtx_init(&ps->mtx, "pollset"); erts_smp_atomic_init(&ps->interrupt, 0); #endif - erts_smp_atomic_init(&ps->timeout, LONG_MAX); + erts_smp_atomic_init(&ps->timeout, ERTS_AINT_T_MAX); HARDTRACEF(("Out erts_poll_create_pollset")); return ps; diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h index 4949998abc..1347eead91 100644 --- a/erts/emulator/sys/win32/erl_win_dyn_driver.h +++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h @@ -87,15 +87,15 @@ WDD_TYPEDEF(unsigned long, erts_alc_test, (unsigned long, unsigned long, unsigned long, unsigned long)); -WDD_TYPEDEF(long, driver_binary_get_refc, (ErlDrvBinary *dbp)); -WDD_TYPEDEF(long, driver_binary_inc_refc, (ErlDrvBinary *dbp)); -WDD_TYPEDEF(long, driver_binary_dec_refc, (ErlDrvBinary *dbp)); +WDD_TYPEDEF(ErlDrvSInt, driver_binary_get_refc, (ErlDrvBinary *dbp)); +WDD_TYPEDEF(ErlDrvSInt, driver_binary_inc_refc, (ErlDrvBinary *dbp)); +WDD_TYPEDEF(ErlDrvSInt, driver_binary_dec_refc, (ErlDrvBinary *dbp)); WDD_TYPEDEF(ErlDrvPDL, driver_pdl_create, (ErlDrvPort)); WDD_TYPEDEF(void, driver_pdl_lock, (ErlDrvPDL)); WDD_TYPEDEF(void, driver_pdl_unlock, (ErlDrvPDL)); -WDD_TYPEDEF(long, driver_pdl_get_refc, (ErlDrvPDL)); -WDD_TYPEDEF(long, driver_pdl_inc_refc, (ErlDrvPDL)); -WDD_TYPEDEF(long, driver_pdl_dec_refc, (ErlDrvPDL)); +WDD_TYPEDEF(ErlDrvSInt, driver_pdl_get_refc, (ErlDrvPDL)); +WDD_TYPEDEF(ErlDrvSInt, driver_pdl_inc_refc, (ErlDrvPDL)); +WDD_TYPEDEF(ErlDrvSInt, driver_pdl_dec_refc, (ErlDrvPDL)); WDD_TYPEDEF(void, driver_system_info, (ErlDrvSysInfo *, size_t)); WDD_TYPEDEF(int, driver_get_now, (ErlDrvNowData *)); WDD_TYPEDEF(int, driver_monitor_process, (ErlDrvPort port, diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 46dee826f0..37041ed987 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -31,12 +31,13 @@ #include "global.h" #include "erl_threads.h" #include "../../drivers/win32/win_con.h" +#include "erl_cpu_topology.h" void erts_sys_init_float(void); void erl_start(int, char**); -void erl_exit(int n, char*, _DOTS_); +void erl_exit(int n, char*, ...); void erl_error(char*, va_list); void erl_crash_dump(char*, int, char*, ...); @@ -66,14 +67,17 @@ static void async_read_file(struct async_io* aio, LPVOID buf, DWORD numToRead); static int async_write_file(struct async_io* aio, LPVOID buf, DWORD numToWrite); static int get_overlapped_result(struct async_io* aio, LPDWORD pBytesRead, BOOL wait); -static FUNCTION(BOOL, CreateChildProcess, (char *, HANDLE, HANDLE, - HANDLE, LPHANDLE, BOOL, - LPVOID, LPTSTR, unsigned, - char **, int *)); +static BOOL create_child_process(char *, HANDLE, HANDLE, + HANDLE, LPHANDLE, BOOL, + LPVOID, LPTSTR, unsigned, + char **, int *); static int create_pipe(LPHANDLE, LPHANDLE, BOOL, BOOL); -static int ApplicationType(const char* originalName, char fullPath[MAX_PATH], +static int application_type(const char* originalName, char fullPath[MAX_PATH], BOOL search_in_path, BOOL handle_quotes, int *error_return); +static int application_type_w(const char* originalName, WCHAR fullPath[MAX_PATH], + BOOL search_in_path, BOOL handle_quotes, + int *error_return); HANDLE erts_service_event; @@ -86,15 +90,19 @@ static erts_smp_atomic_t pipe_creation_counter; static erts_smp_mtx_t sys_driver_data_lock; -/* Results from ApplicationType is one of */ +/* Results from application_type(_w) is one of */ #define APPL_NONE 0 #define APPL_DOS 1 #define APPL_WIN3X 2 #define APPL_WIN32 3 -static FUNCTION(int, driver_write, (long, HANDLE, byte*, int)); +static int driver_write(long, HANDLE, byte*, int); static void common_stop(int); static int create_file_thread(struct async_io* aio, int mode); +#ifdef ERTS_SMP +static void close_active_handle(ErlDrvPort, HANDLE handle); +static DWORD WINAPI threaded_handle_closer(LPVOID param); +#endif static DWORD WINAPI threaded_reader(LPVOID param); static DWORD WINAPI threaded_writer(LPVOID param); static DWORD WINAPI threaded_exiter(LPVOID param); @@ -132,6 +140,13 @@ static BOOL win_console = FALSE; static OSVERSIONINFO int_os_version; /* Version information for Win32. */ +/*#define USE_CANCELIOEX + Disabled the use of CancelIoEx as its been seen to cause problem with some + drivers. Not sure what to blame; faulty drivers or some form of invalid use. +*/ +#if defined(ERTS_SMP) && defined(USE_CANCELIOEX) +static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED); +#endif /* This is the system's main function (which may or may not be called "main") - do general system-dependent initialization @@ -676,25 +691,50 @@ release_driver_data(DriverData* dp) erts_smp_mtx_lock(&sys_driver_data_lock); #ifdef ERTS_SMP - /* This is a workaround for the fact that CancelIo cant cancel - requests issued by another thread and that we still cant use - CancelIoEx as that's only availabele in Vista etc. */ - if(dp->in.async_io_active && dp->in.fd != INVALID_HANDLE_VALUE) { - CloseHandle(dp->in.fd); - dp->in.fd = INVALID_HANDLE_VALUE; - DEBUGF(("Waiting for the in event thingie")); - WaitForSingleObject(dp->in.ov.hEvent,INFINITE); - DEBUGF(("...done\n")); - } - if(dp->out.async_io_active && dp->out.fd != INVALID_HANDLE_VALUE) { - CloseHandle(dp->out.fd); - dp->out.fd = INVALID_HANDLE_VALUE; - DEBUGF(("Waiting for the out event thingie")); - WaitForSingleObject(dp->out.ov.hEvent,INFINITE); - DEBUGF(("...done\n")); +#ifdef USE_CANCELIOEX + if (fpCancelIoEx != NULL) { + if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { + (*fpCancelIoEx)(dp->in.fd, NULL); + } + if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) { + (*fpCancelIoEx)(dp->out.fd, NULL); + } + } + else +#endif + { + /* This is a workaround for the fact that CancelIo cant cancel + requests issued by another thread and that we cant use + CancelIoEx as that's only available in Vista etc. + R14: Avoid scheduler deadlock by only wait for 10ms, and then spawn + a thread that will keep waiting in in order to close handles. */ + HANDLE handles[2]; + int i = 0; + int timeout = 10; + if(dp->in.async_io_active && dp->in.fd != INVALID_HANDLE_VALUE) { + CloseHandle(dp->in.fd); + dp->in.fd = INVALID_HANDLE_VALUE; + DEBUGF(("Waiting for the in event thingie")); + if (WaitForSingleObject(dp->in.ov.hEvent,timeout) == WAIT_TIMEOUT) { + close_active_handle(dp->port_num, dp->in.ov.hEvent); + dp->in.ov.hEvent = NULL; + timeout = 0; + } + DEBUGF(("...done\n")); + } + if(dp->out.async_io_active && dp->out.fd != INVALID_HANDLE_VALUE) { + CloseHandle(dp->out.fd); + dp->out.fd = INVALID_HANDLE_VALUE; + DEBUGF(("Waiting for the out event thingie")); + if (WaitForSingleObject(dp->out.ov.hEvent,timeout) == WAIT_TIMEOUT) { + close_active_handle(dp->port_num, dp->out.ov.hEvent); + dp->out.ov.hEvent = NULL; + } + DEBUGF(("...done\n")); + } } #else - if (dp->out.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { + if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { CancelIo(dp->in.fd); } if (dp->out.thread == (HANDLE) -1 && dp->out.fd != INVALID_HANDLE_VALUE) { @@ -737,6 +777,88 @@ release_driver_data(DriverData* dp) erts_smp_mtx_unlock(&sys_driver_data_lock); } +#ifdef ERTS_SMP + +struct handles_to_be_closed { + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + unsigned cnt; +}; +static struct handles_to_be_closed* htbc_curr = NULL; +CRITICAL_SECTION htbc_lock; + +static void close_active_handle(ErlDrvPort port_num, HANDLE handle) +{ + struct handles_to_be_closed* htbc; + int i; + EnterCriticalSection(&htbc_lock); + htbc = htbc_curr; + if (htbc == NULL || htbc->cnt >= MAXIMUM_WAIT_OBJECTS) { + DWORD tid; + HANDLE thread; + + htbc = (struct handles_to_be_closed*) erts_alloc(ERTS_ALC_T_DRV_TAB, + sizeof(*htbc)); + htbc->handles[0] = CreateAutoEvent(FALSE); + htbc->cnt = 1; + thread = (HANDLE *) _beginthreadex(NULL, 0, threaded_handle_closer, htbc, 0, &tid); + CloseHandle(thread); + } + htbc->handles[htbc->cnt++] = handle; + driver_select(port_num, (ErlDrvEvent)handle, ERL_DRV_USE_NO_CALLBACK, 0); + SetEvent(htbc->handles[0]); + htbc_curr = htbc; + LeaveCriticalSection(&htbc_lock); +} + +static DWORD WINAPI +threaded_handle_closer(LPVOID param) +{ + struct handles_to_be_closed* htbc = (struct handles_to_be_closed*) param; + unsigned ix; + DWORD res; + DEBUGF(("threaded_handle_closer %p started\r\n", htbc)); + EnterCriticalSection(&htbc_lock); + for (;;) { + { + HANDLE* handles = htbc->handles; + unsigned cnt = htbc->cnt; + DWORD timeout = (htbc == htbc_curr) ? INFINITE : 10*1000; + + LeaveCriticalSection(&htbc_lock); + DEBUGF(("threaded_handle_closer %p waiting for %d handles\r\n", htbc, cnt)); + res = WaitForMultipleObjects(cnt, handles, FALSE, timeout); + } + EnterCriticalSection(&htbc_lock); + switch (res) { + case WAIT_OBJECT_0: + case WAIT_TIMEOUT: + break; /* got some more handles to wait for maybe */ + default: + ix = res - WAIT_OBJECT_0; + if (ix > 0 && ix < htbc->cnt) { + CloseHandle(htbc->handles[ix]); + htbc->handles[ix] = htbc->handles[--htbc->cnt]; + } + } + if (htbc != htbc_curr) { + if (htbc->cnt == 1) { /* no real handles left */ + break; + } + /* The thread with most free slots will be "current" */ + if (htbc->cnt < htbc_curr->cnt) { + htbc_curr = htbc; + DEBUGF(("threaded_handle_closer %p made current\r\n", htbc)); + } + } + } + LeaveCriticalSection(&htbc_lock); + CloseHandle(htbc->handles[0]); + erts_free(ERTS_ALC_T_DRV_TAB, htbc); + DEBUGF(("threaded_handle_closer %p terminating\r\n", htbc)); + return 0; +} +#endif /* ERTS_SMP */ + /* * Stores input and output file descriptors in the DriverData structure, * and calls driver_select(). @@ -1026,12 +1148,18 @@ static int spawn_init() { int i; - +#if defined(ERTS_SMP) && defined(USE_CANCELIOEX) + HMODULE module = GetModuleHandle("kernel32"); + fpCancelIoEx = (BOOL (WINAPI *)(HANDLE,LPOVERLAPPED)) + ((module != NULL) ? GetProcAddress(module,"CancelIoEx") : NULL); + DEBUGF(("fpCancelIoEx = %p\r\n", fpCancelIoEx)); +#endif driver_data = (struct driver_data *) erts_alloc(ERTS_ALC_T_DRV_TAB, max_files * sizeof(struct driver_data)); erts_smp_atomic_add(&sys_misc_mem_sz, max_files*sizeof(struct driver_data)); for (i = 0; i < max_files; i++) driver_data[i].port_num = PORT_FREE; + return 0; } @@ -1110,8 +1238,10 @@ spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts) */ DEBUGF(("Spawning \"%s\"\n", name)); - envir = win_build_environment(envir); - ok = CreateChildProcess(name, + envir = win_build_environment(envir); /* Still an ansi environment, could be + converted to unicode for spawn_executable, but + that is not done (yet) */ + ok = create_child_process(name, hChildStdin, hChildStdout, hChildStderr, @@ -1190,7 +1320,7 @@ create_file_thread(AsyncIo* aio, int mode) } /* - * A helper function used by CreateChildProcess(). + * A helper function used by create_child_process(). * Parses a command line with arguments and returns the length of the * first part containing the program name. * Example: input = "\"Program Files\"\\erl arg1 arg2" @@ -1231,24 +1361,25 @@ int parse_command(char* cmd){ return i; } -BOOL need_quotes(char *str) +static BOOL need_quotes(WCHAR *str) { int in_quote = 0; int backslashed = 0; int naked_space = 0; - while (*str != '\0') { + + while (*str != L'\0') { switch (*str) { - case '\\' : + case L'\\' : backslashed = !backslashed; break; - case '"': + case L'"': if (backslashed) { backslashed=0; } else { in_quote = !in_quote; } break; - case ' ': + case L' ': backslashed = 0; if (!(backslashed || in_quote)) { naked_space++; @@ -1267,7 +1398,7 @@ BOOL need_quotes(char *str) /* *---------------------------------------------------------------------- * - * CreateChildProcess -- + * create_child_process -- * * Create a child process that has pipes as its * standard input, output, and error. The child process runs @@ -1292,7 +1423,7 @@ BOOL need_quotes(char *str) */ static BOOL -CreateChildProcess +create_child_process ( char *origcmd, /* Command line for child process (including * name of executable). Or whole executable if st is @@ -1311,14 +1442,12 @@ CreateChildProcess ) { PROCESS_INFORMATION piProcInfo = {0}; - STARTUPINFO siStartInfo = {0}; BOOL ok = FALSE; int applType; /* Not to be changed for different types of executables */ int staticCreateFlags = GetPriorityClass(GetCurrentProcess()); int createFlags = DETACHED_PROCESS; char *newcmdline = NULL; - char execPath[MAX_PATH]; int cmdlength; char* thecommand; LPTSTR appname = NULL; @@ -1326,14 +1455,17 @@ CreateChildProcess *errno_return = -1; - siStartInfo.cb = sizeof(STARTUPINFO); - siStartInfo.dwFlags = STARTF_USESTDHANDLES; - siStartInfo.hStdInput = hStdin; - siStartInfo.hStdOutput = hStdout; - siStartInfo.hStdError = hStderr; - if (st != ERTS_SPAWN_EXECUTABLE) { + STARTUPINFO siStartInfo = {0}; + char execPath[MAX_PATH]; + + siStartInfo.cb = sizeof(STARTUPINFO); + siStartInfo.dwFlags = STARTF_USESTDHANDLES; + siStartInfo.hStdInput = hStdin; + siStartInfo.hStdOutput = hStdout; + siStartInfo.hStdError = hStderr; + /* * Parse out the program name from the command line (it can be quoted and * contain spaces). @@ -1345,9 +1477,9 @@ CreateChildProcess thecommand[cmdlength] = '\0'; DEBUGF(("spawn command: %s\n", thecommand)); - applType = ApplicationType(thecommand, execPath, TRUE, + applType = application_type(thecommand, execPath, TRUE, TRUE, errno_return); - DEBUGF(("ApplicationType returned for (%s) is %d\n", thecommand, applType)); + DEBUGF(("application_type returned for (%s) is %d\n", thecommand, applType)); erts_free(ERTS_ALC_T_TMP, (void *) thecommand); if (applType == APPL_NONE) { erts_free(ERTS_ALC_T_TMP,newcmdline); @@ -1376,126 +1508,147 @@ CreateChildProcess strcat(newcmdline, execPath); strcat(newcmdline, origcmd+cmdlength); - } else { /* ERTS_SPAWN_EXECUTABLE */ + DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags)); + ok = CreateProcessA(appname, + newcmdline, + NULL, + NULL, + TRUE, + createFlags | staticCreateFlags, + env, + wd, + &siStartInfo, + &piProcInfo); + + } else { /* ERTS_SPAWN_EXECUTABLE, filename and args are in unicode ({utf16,little}) */ int run_cmd = 0; - applType = ApplicationType(origcmd, execPath, FALSE, FALSE, - errno_return); + STARTUPINFOW siStartInfo = {0}; + WCHAR execPath[MAX_PATH]; + + + siStartInfo.cb = sizeof(STARTUPINFOW); + siStartInfo.dwFlags = STARTF_USESTDHANDLES; + siStartInfo.hStdInput = hStdin; + siStartInfo.hStdOutput = hStdout; + siStartInfo.hStdError = hStderr; + + applType = application_type_w(origcmd, (char *) execPath, FALSE, FALSE, + errno_return); if (applType == APPL_NONE) { return FALSE; } if (applType == APPL_DOS) { - /* - * See comment above - */ + /* + * See comment above + */ - siStartInfo.wShowWindow = SW_HIDE; - siStartInfo.dwFlags |= STARTF_USESHOWWINDOW; - createFlags = CREATE_NEW_CONSOLE; - run_cmd = 1; + siStartInfo.wShowWindow = SW_HIDE; + siStartInfo.dwFlags |= STARTF_USESHOWWINDOW; + createFlags = CREATE_NEW_CONSOLE; + run_cmd = 1; } else if (hide) { - DEBUGF(("hiding window\n")); - siStartInfo.wShowWindow = SW_HIDE; - siStartInfo.dwFlags |= STARTF_USESHOWWINDOW; - createFlags = 0; + DEBUGF(("hiding window\n")); + siStartInfo.wShowWindow = SW_HIDE; + siStartInfo.dwFlags |= STARTF_USESHOWWINDOW; + createFlags = 0; } if (run_cmd) { - char cmdPath[MAX_PATH]; + WCHAR cmdPath[MAX_PATH]; int cmdType; - cmdType = ApplicationType("cmd.exe", cmdPath, TRUE, FALSE, errno_return); + cmdType = application_type_w((char *) L"cmd.exe", (char *) cmdPath, TRUE, FALSE, errno_return); if (cmdType == APPL_NONE || cmdType == APPL_DOS) { return FALSE; } - appname = (char *) erts_alloc(ERTS_ALC_T_TMP, strlen(cmdPath)+1); - strcpy(appname,cmdPath); + appname = (char *) erts_alloc(ERTS_ALC_T_TMP, (wcslen(cmdPath)+1)*sizeof(WCHAR)); + wcscpy((WCHAR *) appname,cmdPath); } else { - appname = (char *) erts_alloc(ERTS_ALC_T_TMP, strlen(execPath)+1); - strcpy(appname,execPath); + appname = (char *) erts_alloc(ERTS_ALC_T_TMP, (wcslen(execPath)+1)*sizeof(WCHAR)); + wcscpy((WCHAR *) appname, execPath); } - if (argv == NULL) { + if (argv == NULL) { BOOL orig_need_q = need_quotes(execPath); - char *ptr; - int ocl = strlen(execPath); + WCHAR *ptr; + int ocl = wcslen(execPath); if (run_cmd) { newcmdline = (char *) erts_alloc(ERTS_ALC_T_TMP, - ocl + ((orig_need_q) ? 3 : 1) - + 11); - memcpy(newcmdline,"cmd.exe /c ",11); - ptr = newcmdline + 11; + (ocl + ((orig_need_q) ? 3 : 1) + + 11)*sizeof(WCHAR)); + memcpy(newcmdline,L"cmd.exe /c ",11*sizeof(WCHAR)); + ptr = (WCHAR *) (newcmdline + (11*sizeof(WCHAR))); } else { newcmdline = (char *) erts_alloc(ERTS_ALC_T_TMP, - ocl + ((orig_need_q) ? 3 : 1)); - ptr = newcmdline; + (ocl + ((orig_need_q) ? 3 : 1))*sizeof(WCHAR)); + ptr = (WCHAR *) newcmdline; } if (orig_need_q) { - *ptr++ = '"'; + *ptr++ = L'"'; } - memcpy(ptr,execPath,ocl); + memcpy(ptr,execPath,ocl*sizeof(WCHAR)); ptr += ocl; if (orig_need_q) { - *ptr++ = '"'; + *ptr++ = L'"'; } - *ptr = '\0'; + *ptr = L'\0'; } else { int sum = 1; /* '\0' */ - char **ar = argv; - char *n; + WCHAR **ar = (WCHAR **) argv; + WCHAR *n; char *save_arg0 = NULL; if (argv[0] == erts_default_arg0 || run_cmd) { save_arg0 = argv[0]; - argv[0] = execPath; + argv[0] = (char *) execPath; } if (run_cmd) { sum += 11; /* cmd.exe /c */ } while (*ar != NULL) { - sum += strlen(*ar); + sum += wcslen(*ar); if (need_quotes(*ar)) { sum += 2; /* quotes */ } sum++; /* space */ ++ar; } - ar = argv; - newcmdline = erts_alloc(ERTS_ALC_T_TMP, sum); - n = newcmdline; + ar = (WCHAR **) argv; + newcmdline = erts_alloc(ERTS_ALC_T_TMP, sum*sizeof(WCHAR)); + n = (WCHAR *) newcmdline; if (run_cmd) { - memcpy(n,"cmd.exe /c ",11); + memcpy(n,L"cmd.exe /c ",11*sizeof(WCHAR)); n += 11; } while (*ar != NULL) { int q = need_quotes(*ar); - sum = strlen(*ar); + sum = wcslen(*ar); if (q) { - *n++ = '"'; + *n++ = L'"'; } - memcpy(n,*ar,sum); + memcpy(n,*ar,sum*sizeof(WCHAR)); n += sum; if (q) { - *n++ = '"'; + *n++ = L'"'; } - *n++ = ' '; + *n++ = L' '; ++ar; } - ASSERT(n > newcmdline); - *(n-1) = '\0'; + *(n-1) = L'\0'; if (save_arg0 != NULL) { argv[0] = save_arg0; } } - } - DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags)); - ok = CreateProcess(appname, - newcmdline, - NULL, - NULL, - TRUE, - createFlags | staticCreateFlags, - env, - wd, - &siStartInfo, - &piProcInfo); - + DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags)); + ok = CreateProcessW((WCHAR *) appname, + (WCHAR *) newcmdline, + NULL, + NULL, + TRUE, + createFlags | staticCreateFlags, + env, + (WCHAR *) wd, + &siStartInfo, + &piProcInfo); + + } /* end SPAWN_EXECUTABLE */ if (newcmdline != NULL) { erts_free(ERTS_ALC_T_TMP,newcmdline); } @@ -1614,7 +1767,7 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o -static int ApplicationType +static int application_type ( const char *originalName, /* Name of the application to find. */ char fullPath[MAX_PATH], /* Filled with complete path to @@ -1768,6 +1921,146 @@ static int ApplicationType return applType; } +static int application_type_w (const char *originalName, /* Name of the application to find. */ + WCHAR wfullpath[MAX_PATH],/* Filled with complete path to + * application. */ + BOOL search_in_path, /* If we should search the system wide path */ + BOOL handle_quotes, /* If we should handle quotes around executable */ + int *error_return) /* A place to put an error code */ +{ + int applType, i; + HANDLE hFile; + WCHAR *ext, *rest; + char buf[2]; + DWORD read; + IMAGE_DOS_HEADER header; + static WCHAR extensions[][5] = {L"", L".com", L".exe", L".bat"}; + int is_quoted; + int len; + WCHAR *wname = (WCHAR *) originalName; + WCHAR xfullpath[MAX_PATH]; + + len = wcslen(wname); + is_quoted = handle_quotes && len > 0 && wname[0] == L'"' && + wname[len-1] == L'"'; + + applType = APPL_NONE; + *error_return = ENOENT; + for (i = 0; i < (int) (sizeof(extensions) / sizeof(extensions[0])); i++) { + if(is_quoted) { + lstrcpynW(xfullpath, wname+1, MAX_PATH - 7); /* Cannot start using StringCchCopy yet, we support + older platforms */ + len = wcslen(xfullpath); + if(len > 0) { + xfullpath[len-1] = L'\0'; + } + } else { + lstrcpynW(xfullpath, wname, MAX_PATH - 5); + } + wcscat(xfullpath, extensions[i]); + /* It seems that the Unicode version does not allow in and out parameter to overlap. */ + SearchPathW((search_in_path) ? NULL : L".", xfullpath, NULL, MAX_PATH, wfullpath, &rest); + + /* + * Ignore matches on directories or data files, return if identified + * a known type. + */ + + if (GetFileAttributesW(wfullpath) & FILE_ATTRIBUTE_DIRECTORY) { + continue; + } + + ext = wcsrchr(wfullpath, L'.'); + if ((ext != NULL) && (_wcsicmp(ext, L".bat") == 0)) { + *error_return = EACCES; + applType = APPL_DOS; + break; + } + + hFile = CreateFileW(wfullpath, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) { + continue; + } + + *error_return = EACCES; /* If considered an error, + it's an access error */ + header.e_magic = 0; + ReadFile(hFile, (void *) &header, sizeof(header), &read, NULL); + if (header.e_magic != IMAGE_DOS_SIGNATURE) { + /* + * Doesn't have the magic number for relocatable executables. If + * filename ends with .com, assume it's a DOS application anyhow. + * Note that we didn't make this assumption at first, because some + * supposed .com files are really 32-bit executables with all the + * magic numbers and everything. + */ + + CloseHandle(hFile); + if ((ext != NULL) && (_wcsicmp(ext, L".com") == 0)) { + applType = APPL_DOS; + break; + } + continue; + } + if (header.e_lfarlc != sizeof(header)) { + /* + * All Windows 3.X and Win32 and some DOS programs have this value + * set here. If it doesn't, assume that since it already had the + * other magic number it was a DOS application. + */ + + CloseHandle(hFile); + applType = APPL_DOS; + break; + } + + /* + * The DWORD at header.e_lfanew points to yet another magic number. + */ + + buf[0] = '\0'; + SetFilePointer(hFile, header.e_lfanew, NULL, FILE_BEGIN); + ReadFile(hFile, (void *) buf, 2, &read, NULL); + CloseHandle(hFile); + + if ((buf[0] == 'L') && (buf[1] == 'E')) { + applType = APPL_DOS; + } else if ((buf[0] == 'N') && (buf[1] == 'E')) { + applType = APPL_WIN3X; + } else if ((buf[0] == 'P') && (buf[1] == 'E')) { + applType = APPL_WIN32; + } else { + continue; + } + break; + } + + if (applType == APPL_NONE) { + return APPL_NONE; + } + + if ((applType == APPL_DOS) || (applType == APPL_WIN3X)) { + /* + * Replace long path name of executable with short path name for + * 16-bit applications. Otherwise the application may not be able + * to correctly parse its own command line to separate off the + * application name from the arguments. + */ + + GetShortPathNameW(wfullpath, wfullpath, MAX_PATH); + } + if (is_quoted) { + /* restore quotes on quoted program name */ + len = wcslen(wfullpath); + memmove(wfullpath+1,wfullpath,len*sizeof(WCHAR)); + wfullpath[0]=L'"'; + wfullpath[len+1]=L'"'; + wfullpath[len+2]=L'\0'; + } + return applType; +} + /* * Thread function used to emulate overlapped reading. */ @@ -2545,7 +2838,6 @@ erts_sys_main_thread(void) void erts_sys_alloc_init(void) { - elib_ensure_initialized(); } void *erts_sys_alloc(ErtsAlcType_t t, void *x, Uint sz) @@ -2892,19 +3184,51 @@ check_supported_os_version(void) } #ifdef USE_THREADS -static void *ethr_internal_alloc(size_t size) + +typedef struct { + int sched_bind_data; +} erts_thr_create_data_t; + +/* + * thr_create_prepare() is called in parent thread before thread creation. + * Returned value is passed as argument to thr_create_cleanup(). + */ +static void * +thr_create_prepare(void) { - return erts_alloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, (Uint) size); + erts_thr_create_data_t *tcdp; + + tcdp = erts_alloc(ERTS_ALC_T_TMP, sizeof(erts_thr_create_data_t)); + tcdp->sched_bind_data = erts_sched_bind_atthrcreate_prepare(); + + return (void *) tcdp; } -static void *ethr_internal_realloc(void *ptr, size_t size) + + +/* thr_create_cleanup() is called in parent thread after thread creation. */ +static void +thr_create_cleanup(void *vtcdp) { - return erts_realloc_fnf(ERTS_ALC_T_ETHR_INTERNAL, ptr, (Uint) size); + erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp; + + erts_sched_bind_atthrcreate_parent(tcdp->sched_bind_data); + + erts_free(ERTS_ALC_T_TMP, tcdp); } -static void ethr_internal_free(void *ptr) + +static void +thr_create_prepare_child(void *vtcdp) { - erts_free(ERTS_ALC_T_ETHR_INTERNAL, ptr); + erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp; + +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_thread_setup(); +#endif /* ERTS_ENABLE_LOCK_COUNT */ + + erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data); } -#endif + +#endif /* USE_THREADS */ void erts_sys_pre_init(void) @@ -2915,9 +3239,13 @@ erts_sys_pre_init(void) #ifdef USE_THREADS { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; - eid.alloc = ethr_internal_alloc; - eid.realloc = ethr_internal_realloc; - eid.free = ethr_internal_free; + + eid.thread_create_child_func = thr_create_prepare_child; + /* Before creation in parent */ + eid.thread_create_prepare_func = thr_create_prepare; + /* After creation in parent */ + eid.thread_create_parent_func = thr_create_cleanup, + erts_thr_init(&eid); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init(); @@ -2925,15 +3253,8 @@ erts_sys_pre_init(void) } #endif erts_smp_atomic_init(&sys_misc_mem_sz, 0); - erts_sys_env_init(); } -/* - * the last two only used for standalone erlang - * they should are used by sae_main in beam dll to - * enable standalone execution via erl_api-routines - */ - void noinherit_std_handle(DWORD type) { HANDLE h = GetStdHandle(type); @@ -2947,6 +3268,8 @@ void erl_sys_init(void) { HANDLE handle; + erts_sys_env_init(); + noinherit_std_handle(STD_OUTPUT_HANDLE); noinherit_std_handle(STD_INPUT_HANDLE); noinherit_std_handle(STD_ERROR_HANDLE); @@ -2956,6 +3279,7 @@ void erl_sys_init(void) #ifdef ERTS_SMP erts_smp_tsd_key_create(&win32_errstr_key); + InitializeCriticalSection(&htbc_lock); #endif erts_smp_atomic_init(&pipe_creation_counter,0); /* diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c index ac4be3f316..02c8433a10 100644 --- a/erts/emulator/sys/win32/sys_env.c +++ b/erts/emulator/sys/win32/sys_env.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2002-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -145,15 +145,17 @@ merge_environment(char *old, char *add) for(j = 0; a_arg[j] != NULL; ++j){ char **tmp; char *current = a_arg[j]; + char *eq_p = strchr(current,'='); + int unset = (eq_p!=NULL && eq_p[1]=='\0'); if ((tmp = find_arg(c_arg, current)) != NULL) { - if (current[strlen(current)-1] != '=') { + if (!unset) { *tmp = current; } else { *tmp = c_arg[--i]; c_arg[i] = NULL; } - } else if (current[strlen(current)-1] != '=') { + } else if (!unset) { c_arg[i++] = current; c_arg[i] = NULL; } diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c index d2449a1bdb..262f84babc 100644 --- a/erts/emulator/sys/win32/sys_interrupt.c +++ b/erts/emulator/sys/win32/sys_interrupt.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. + * Copyright Ericsson AB 1997-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,9 +33,9 @@ #ifdef ERTS_SMP erts_smp_atomic_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 1) + erts_smp_atomic_set(&erts_break_requested, (erts_aint_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 0) + erts_smp_atomic_set(&erts_break_requested, (erts_aint_t) 0) #else volatile int erts_break_requested = 0; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index df2faf1450..7259e1b84d 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -75,14 +75,15 @@ MODULES= \ node_container_SUITE \ nofrag_SUITE \ num_bif_SUITE \ - obsolete_SUITE \ op_SUITE \ port_SUITE \ port_bif_SUITE \ process_SUITE \ pseudoknot_SUITE \ + receive_SUITE \ ref_SUITE \ register_SUITE \ + mtx_SUITE \ save_calls_SUITE \ send_term_SUITE \ sensitive_SUITE \ @@ -100,6 +101,7 @@ MODULES= \ trace_local_SUITE \ trace_meta_SUITE \ trace_call_count_SUITE \ + trace_call_time_SUITE \ scheduler_SUITE \ old_scheduler_SUITE \ z_SUITE \ @@ -117,7 +119,8 @@ NO_OPT= bs_bincomp \ bs_match_int \ bs_match_tail \ bs_match_misc \ - bs_utf + bs_utf \ + guard NO_OPT_MODULES= $(NO_OPT:%=%_no_opt_SUITE) diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl index 228ff15341..32ac07cb2d 100644 --- a/erts/emulator/test/beam_SUITE.erl +++ b/erts/emulator/test/beam_SUITE.erl @@ -20,7 +20,8 @@ -module(beam_SUITE). -export([all/1, packed_registers/1, apply_last/1, apply_last_bif/1, - buildo_mucho/1, heap_sizes/1, big_lists/1, fconv/1]). + buildo_mucho/1, heap_sizes/1, big_lists/1, fconv/1, + select_val/1]). -export([applied/2]). @@ -28,7 +29,7 @@ all(suite) -> [packed_registers, apply_last, apply_last_bif, buildo_mucho, - heap_sizes, big_lists]. + heap_sizes, big_lists, select_val]. %% Verify that apply(M, F, A) is really tail recursive. @@ -302,3 +303,19 @@ do_fconv(nil, Float) when is_float(Float) -> Float + []; do_fconv(tuple_literal, Float) when is_float(Float) -> Float + {a,b}. + +select_val(Config) when is_list(Config) -> + ?line zero = do_select_val(0), + ?line big = do_select_val(1 bsl 64), + ?line integer = do_select_val(42), + ok. + +do_select_val(X) -> + case X of + 0 -> + zero; + 1 bsl 64 -> + big; + Int when is_integer(Int) -> + integer + end. diff --git a/erts/emulator/test/beam_literals_SUITE.erl b/erts/emulator/test/beam_literals_SUITE.erl index 75841adbfc..1eda939cf8 100644 --- a/erts/emulator/test/beam_literals_SUITE.erl +++ b/erts/emulator/test/beam_literals_SUITE.erl @@ -23,7 +23,8 @@ matching_bigs/1, matching_more_bigs/1, matching_bigs_and_smalls/1, badmatch/1, case_clause/1, receiving/1, literal_type_tests/1, - put_list/1, fconv/1, literal_case_expression/1]). + put_list/1, fconv/1, literal_case_expression/1, + increment/1]). -include("test_server.hrl"). @@ -32,7 +33,7 @@ all(suite) -> matching_bigs, matching_more_bigs, matching_bigs_and_smalls, badmatch, case_clause, receiving, literal_type_tests, - put_list, fconv, literal_case_expression]. + put_list, fconv, literal_case_expression, increment]. putting(doc) -> "Test creating lists and tuples containing big number literals."; putting(Config) when is_list(Config) -> @@ -48,6 +49,7 @@ matching_bigs(doc) -> "Test matching of a few big number literals (in Beam," matching_bigs(Config) when is_list(Config) -> a = matching1(3972907842873739), b = matching1(-389789298378939783333333333333333333784), + other = matching1(3141699999999999999999999999999999999), other = matching1(42). matching_smalls(doc) -> "Test matching small numbers (both positive and negative)."; @@ -405,14 +407,51 @@ fconv_2(F) when is_float(F) -> literal_case_expression(Config) when is_list(Config) -> ?line DataDir = ?config(data_dir, Config), ?line Src = filename:join(DataDir, "literal_case_expression"), - ?line {ok,literal_case_expression=Mod,Code} = compile:file(Src, [from_asm,binary]), + ?line {ok,literal_case_expression=Mod,Code} = + compile:file(Src, [from_asm,binary]), ?line {module,Mod} = code:load_binary(Mod, Src, Code), ?line ok = Mod:x(), ?line ok = Mod:y(), + ?line ok = Mod:zi1(), + ?line ok = Mod:zi2(), + ?line ok = Mod:za1(), + ?line ok = Mod:za2(), ?line true = code:delete(Mod), ?line code:purge(Mod), ok. +%% Test the i_increment instruction. +increment(Config) when is_list(Config) -> + %% In the 32-bit emulator, Neg32 can be represented as a small, + %% but -Neg32 cannot. Therefore the i_increment instruction must + %% not be used in the subtraction that follows (since i_increment + %% cannot handle a bignum literal). + Neg32 = -(1 bsl 27), + Big32 = id(1 bsl 32), + Result32 = (1 bsl 32) + (1 bsl 27), + ?line Result32 = Big32 + (1 bsl 27), + ?line Result32 = Big32 - Neg32, + + %% Same thing, but for the 64-bit emulator. + Neg64 = -(1 bsl 59), + Big64 = id(1 bsl 64), + Result64 = (1 bsl 64) + (1 bsl 59), + ?line Result64 = Big64 + (1 bsl 59), + ?line Result64 = Big64 - Neg64, + + %% Test error handling for the i_increment instruction. + Bad = id(bad), + ?line {'EXIT',{badarith,_}} = (catch Bad + 42), + + %% Small operands, but a big result. + Res32 = 1 bsl 27, + Small32 = id(Res32-1), + ?line Res32 = Small32 + 1, + Res64 = 1 bsl 59, + Small64 = id(Res64-1), + ?line Res64 = Small64 + 1, + ok. + %% Help functions. chksum(Term) -> diff --git a/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S b/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S index c0ffe9ab53..bfdfc079dc 100644 --- a/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S +++ b/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S @@ -1,10 +1,11 @@ {module, literal_case_expression}. %% version = 0 -{exports, [{module_info,0},{module_info,1},{x,0},{y,0}]}. +{exports, [{module_info,0},{module_info,1},{x,0},{y,0}, + {zi1,0},{zi2,0},{za1,0},{za2,0}]}. {attributes, []}. -{labels, 15}. +{labels, 32}. {function, x, 0, 2}. @@ -52,6 +53,81 @@ {label,10}. {case_end,{float,34.0000}}. +{function, zi1, 0, 16}. + {label,15}. + {func_info,{atom,literal_case_expression},{atom,zi1},0}. + {label,16}. + {test,is_integer,{f,19},[{integer,42}]}. + {select_val,{integer,42}, + {f,18}, + {list,[{integer,42}, + {f,17}, + {integer,1000}, + {f,18}]}}. + {label,17}. + {move,{atom,ok},{x,0}}. + return. + {label,18}. + {move,{atom,error},{x,0}}. + return. + {label,19}. + {case_end,{integer,42}}. + +{function, zi2, 0, 16}. + {label,20}. + {func_info,{atom,literal_case_expression},{atom,zi2},0}. + {label,21}. + {test,is_integer,{f,23},[{integer,42}]}. + {select_val,{integer,42}, + {f,23}, + {list,[{integer,42}, + {f,22}, + {integer,1000}, + {f,23}]}}. + {label,22}. + {move,{atom,ok},{x,0}}. + return. + {label,23}. + {move,{atom,error},{x,0}}. + return. + +{function, za1, 0, 25}. + {label,24}. + {func_info,{atom,literal_case_expression},{atom,za1},0}. + {label,25}. + {test,is_atom,{f,28},[{atom,x}]}. + {select_val,{atom,x}, + {f,27}, + {list,[{atom,a}, + {f,27}, + {atom,x}, + {f,26}]}}. + {label,26}. + {move,{atom,ok},{x,0}}. + return. + {label,27}. + {move,{atom,error},{x,0}}. + return. + {label,28}. + {case_end,{atom,x}}. + +{function, za2, 0, 30}. + {label,29}. + {func_info,{atom,literal_case_expression},{atom,za2},0}. + {label,30}. + {test,is_atom,{f,32},[{atom,x}]}. + {select_val,{atom,x}, + {f,32}, + {list,[{atom,a}, + {f,32}, + {atom,x}, + {f,31}]}}. + {label,31}. + {move,{atom,ok},{x,0}}. + return. + {label,32}. + {move,{atom,error},{x,0}}. + return. {function, module_info, 0, 12}. {label,11}. diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index cfbc5dfe81..b4ef0e6d5a 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2005-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -22,12 +22,13 @@ -include("test_server.hrl"). -export([all/1,init_per_testcase/2,fin_per_testcase/2, + types/1, t_list_to_existing_atom/1,os_env/1,otp_7526/1, binary_to_atom/1,binary_to_existing_atom/1, atom_to_binary/1,min_max/1]). all(suite) -> - [t_list_to_existing_atom,os_env,otp_7526, + [types,t_list_to_existing_atom,os_env,otp_7526, atom_to_binary,binary_to_atom,binary_to_existing_atom, min_max]. @@ -39,6 +40,73 @@ fin_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). +types(Config) when is_list(Config) -> + c:l(erl_bif_types), + case erlang:function_exported(erl_bif_types, module_info, 0) of + false -> + %% Fail cleanly. + ?line ?t:fail("erl_bif_types not compiled"); + true -> + types_1() + end. + +types_1() -> + ?line List0 = erlang:system_info(snifs), + + %% Ignore missing type information for hipe BIFs. + ?line List = [MFA || {M,_,_}=MFA <- List0, M =/= hipe_bifs], + + case [MFA || MFA <- List, not known_types(MFA)] of + [] -> + types_2(List); + BadTypes -> + io:put_chars("No type information:\n"), + io:format("~p\n", [lists:sort(BadTypes)]), + ?line ?t:fail({length(BadTypes),bifs_without_types}) + end. + +types_2(List) -> + BadArity = [MFA || {M,F,A}=MFA <- List, + begin + Types = erl_bif_types:arg_types(M, F, A), + length(Types) =/= A + end], + case BadArity of + [] -> + types_3(List); + [_|_] -> + io:put_chars("Bifs with bad arity\n"), + io:format("~p\n", [BadArity]), + ?line ?t:fail({length(BadArity),bad_arity}) + end. + +types_3(List) -> + BadSmokeTest = [MFA || {M,F,A}=MFA <- List, + begin + try erl_bif_types:type(M, F, A) of + Type -> + %% Test that type is returned. + not erl_types:is_erl_type(Type) + catch + Class:Error -> + io:format("~p: ~p ~p\n", + [MFA,Class,Error]), + true + end + end], + case BadSmokeTest of + [] -> + ok; + [_|_] -> + io:put_chars("Bifs with failing calls to erlang_bif_types:type/3 " + "(or with bogus return values):\n"), + io:format("~p\n", [BadSmokeTest]), + ?line ?t:fail({length(BadSmokeTest),bad_smoke_test}) + end. + +known_types({M,F,A}) -> + erl_bif_types:is_known(M, F, A). + t_list_to_existing_atom(Config) when is_list(Config) -> ?line all = list_to_existing_atom("all"), ?line ?MODULE = list_to_existing_atom(?MODULE_STRING), @@ -308,6 +376,18 @@ min_max(Config) when is_list(Config) -> ?line 42.0 = erlang:min(42.0, 42), ?line 42.0 = erlang:max(42.0, 42), + %% And now (R14) they are also autoimported! + ?line a = min(id(a), a), + ?line a = min(id(a), b), + ?line a = min(id(b), a), + ?line b = min(id(b), b), + ?line a = max(id(a), a), + ?line b = max(id(a), b), + ?line b = max(id(b), a), + ?line b = max(id(b), b), + + ?line 42.0 = min(42.0, 42), + ?line 42.0 = max(42.0, 42), ok. diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 7ecc31aa29..77d2579848 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -438,11 +438,11 @@ terms(Config) when is_list(Config) -> ok end, Term = binary_to_term(Bin), - Term = erlang:binary_to_term(Bin, [safe]), + Term = binary_to_term(Bin, [safe]), Unaligned = make_unaligned_sub_binary(Bin), Term = binary_to_term(Unaligned), - Term = erlang:binary_to_term(Unaligned, []), - Term = erlang:binary_to_term(Bin, [safe]), + Term = binary_to_term(Unaligned, []), + Term = binary_to_term(Bin, [safe]), BinC = erlang:term_to_binary(Term, [compressed]), Term = binary_to_term(BinC), true = size(BinC) =< size(Bin), @@ -542,7 +542,7 @@ bad_bin_to_term(BadBin) -> {'EXIT',{badarg,_}} = (catch binary_to_term(BadBin)). bad_bin_to_term(BadBin,Opts) -> - {'EXIT',{badarg,_}} = (catch erlang:binary_to_term(BadBin,Opts)). + {'EXIT',{badarg,_}} = (catch binary_to_term(BadBin,Opts)). safe_binary_to_term2(doc) -> "Test safety options for binary_to_term/2"; safe_binary_to_term2(Config) when is_list(Config) -> @@ -553,7 +553,7 @@ safe_binary_to_term2(Config) when is_list(Config) -> BadRef = <<131,114,0,3,BadHostAtom/binary,0,<<0,0,0,255>>/binary, Empty/binary,Empty/binary>>, ?line bad_bin_to_term(BadRef, [safe]), % good ref, with a bad atom - ?line fullsweep_after = erlang:binary_to_term(<<131,100,0,15,"fullsweep_after">>, [safe]), % should be a good atom + ?line fullsweep_after = binary_to_term(<<131,100,0,15,"fullsweep_after">>, [safe]), % should be a good atom BadExtFun = <<131,113,100,0,4,98,108,117,101,100,0,4,109,111,111,110,97,3>>, ?line bad_bin_to_term(BadExtFun, [safe]), ok. diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index 3d9b51d278..138a19a626 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -27,7 +27,7 @@ mem_leak/1, coerce_to_float/1, bjorn/1, huge_float_field/1, huge_binary/1, system_limit/1, badarg/1, copy_writable_binary/1, kostis/1, dynamic/1, bs_add/1, - otp_7422/1]). + otp_7422/1, zero_width/1]). -include("test_server.hrl"). @@ -36,7 +36,7 @@ all(suite) -> not_used, in_guard, mem_leak, coerce_to_float, bjorn, huge_float_field, huge_binary, system_limit, badarg, copy_writable_binary, kostis, dynamic, bs_add, - otp_7422]. + otp_7422, zero_width]. big(1) -> 57285702734876389752897683. @@ -786,5 +786,20 @@ otp_7422_bin(N) when N < 512 -> end), otp_7422_bin(N+1); otp_7422_bin(_) -> ok. + +zero_width(Config) when is_list(Config) -> + ?line Z = id(0), + Small = id(42), + Big = id(1 bsl 128), + ?line <<>> = <<Small:Z>>, + ?line <<>> = <<Small:0>>, + ?line <<>> = <<Big:Z>>, + ?line <<>> = <<Big:0>>, + + ?line {'EXIT',{badarg,_}} = (catch <<not_a_number:0>>), + ?line {'EXIT',{badarg,_}} = (catch <<(id(not_a_number)):Z>>), + ?line {'EXIT',{badarg,_}} = (catch <<(id(not_a_number)):0>>), + + ok. id(I) -> I. diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl index 9b16170293..7350aef4ec 100644 --- a/erts/emulator/test/busy_port_SUITE.erl +++ b/erts/emulator/test/busy_port_SUITE.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% @@ -182,7 +182,7 @@ system_monitor(Config) when is_list(Config) -> ?line Master ! {Owner, {command, "u"}}, ?line {Busy,beta} = rec(Void), ?line Void = rec(Void), - ?line NewMonitor = erlang:system_monitor(OldMonitor), + ?line _NewMonitor = erlang:system_monitor(OldMonitor), ?line OldMonitor = erlang:system_monitor(), ?line OldMonitor = erlang:system_monitor(OldMonitor), %% @@ -361,7 +361,6 @@ soft_busy_driver(Config) when is_list(Config) -> hs_test(Config, false). hs_test(Config, HardBusy) when is_list(Config) -> - ?line Me = self(), ?line DrvName = case HardBusy of true -> 'hard_busy_drv'; false -> 'soft_busy_drv' @@ -479,7 +478,7 @@ hs_busy_pcmd(Prt, Opts, StartFun, EndFun) -> Tester ! {self(), doing_port_command}, Start = os:timestamp(), Res = try {return, - erlang:port_command(Prt, [], Opts)} + port_command(Prt, [], Opts)} catch Exception:Error -> {Exception, Error} end, End = os:timestamp(), diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 33351a3cc9..5707de30d3 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -320,6 +320,9 @@ make_stub(Config) when is_list(Config) -> (catch code:make_stub_module(my_code_test, bit_sized_binary(Code), {[],[]})), + ?line {'EXIT',{badarg,_}} = + (catch code:make_stub_module(my_code_test_with_wrong_name, + Code, {[],[]})), ok. make_stub_many_funs(Config) when is_list(Config) -> diff --git a/erts/emulator/test/decode_packet_SUITE.erl b/erts/emulator/test/decode_packet_SUITE.erl index 6cde286871..d9e961be2f 100644 --- a/erts/emulator/test/decode_packet_SUITE.erl +++ b/erts/emulator/test/decode_packet_SUITE.erl @@ -304,6 +304,10 @@ http(Config) when is_list(Config) -> {ok, {http_request, 'GET', ResB, {1,1}}, Rest} = decode_pkt(http_bin,Bin) end, lists:foreach(UriF, http_uri_variants()), + + %% Response with empty phrase + ?line {ok,{http_response,{1,1},200,[]},<<>>} = decode_pkt(http, <<"HTTP/1.1 200\r\n">>, []), + ?line {ok,{http_response,{1,1},200,<<>>},<<>>} = decode_pkt(http_bin, <<"HTTP/1.1 200\r\n">>, []), ok. http_with_bin(http) -> diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 8f48d8a992..f26455e6da 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -1,23 +1,24 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% -module(distribution_SUITE). +-compile(r12). %% Tests distribution and the tcp driver. @@ -26,6 +27,7 @@ -export([all/1, ping/1, bulk_send/1, bulk_send_small/1, bulk_send_big/1, + bulk_send_bigbig/1, local_send/1, local_send_small/1, local_send_big/1, local_send_legal/1, link_to_busy/1, exit_to_busy/1, lost_exit/1, link_to_dead/1, link_to_dead_new_node/1, @@ -37,6 +39,7 @@ atom_roundtrip/1, atom_roundtrip_r12b/1, contended_atom_cache_entry/1, + bad_dist_structure/1, bad_dist_ext/1, bad_dist_ext_receive/1, bad_dist_ext_process_info/1, @@ -49,7 +52,8 @@ -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0, roundtrip/1, bounce/1, do_dist_auto_connect/1, inet_rpc_server/1, dist_parallel_sender/3, dist_parallel_receiver/0, - dist_evil_parallel_receiver/0]). + dist_evil_parallel_receiver/0, + sendersender/4, sendersender2/4]). all(suite) -> [ ping, bulk_send, local_send, link_to_busy, exit_to_busy, @@ -58,6 +62,7 @@ all(suite) -> [ stop_dist, trap_bif, dist_auto_connect, dist_parallel_send, atom_roundtrip, atom_roundtrip_r12b, contended_atom_cache_entry, + bad_dist_structure, bad_dist_ext ]. @@ -120,7 +125,7 @@ bulk_send(doc) -> "the time. This tests that a process that is suspended on a ", "busy port will eventually be resumed."]; bulk_send(suite) -> - [bulk_send_small, bulk_send_big]. + [bulk_send_small, bulk_send_big, bulk_send_bigbig]. bulk_send_small(Config) when is_list(Config) -> ?line bulk_send(64, 32). @@ -128,6 +133,9 @@ bulk_send_small(Config) when is_list(Config) -> bulk_send_big(Config) when is_list(Config) -> ?line bulk_send(32, 64). +bulk_send_bigbig(Config) when is_list(Config) -> + ?line bulk_sendsend(32*5, 4). + bulk_send(Terms, BinSize) -> ?line Dog = test_server:timetrap(test_server:seconds(30)), @@ -144,6 +152,53 @@ bulk_send(Terms, BinSize) -> ?line test_server:timetrap_cancel(Dog), {comment, integer_to_list(trunc(Size/1024/Elapsed+0.5)) ++ " K/s"}. +bulk_sendsend(Terms, BinSize) -> + {Rate1, MonitorCount1} = bulk_sendsend2(Terms, BinSize, 5), + {Rate2, MonitorCount2} = bulk_sendsend2(Terms, BinSize, 995), + Ratio = if MonitorCount2 == 0 -> MonitorCount1 / 1.0; + true -> MonitorCount1 / MonitorCount2 + end, + %% A somewhat arbitrary ratio, but hopefully one that will accomodate + %% a wide range of CPU speeds. + true = (Ratio > 8.0), + {comment, + integer_to_list(Rate1) ++ " K/s, " ++ + integer_to_list(Rate2) ++ " K/s, " ++ + integer_to_list(MonitorCount1) ++ " monitor msgs, " ++ + integer_to_list(MonitorCount2) ++ " monitor msgs, " ++ + float_to_list(Ratio) ++ " monitor ratio"}. + +bulk_sendsend2(Terms, BinSize, BusyBufSize) -> + ?line Dog = test_server:timetrap(test_server:seconds(30)), + + ?line io:format("Sending ~w binaries, each of size ~w K", + [Terms, BinSize]), + ?line {ok, NodeRecv} = start_node(bulk_receiver), + ?line Recv = spawn(NodeRecv, erlang, apply, [fun receiver/2, [0, 0]]), + ?line Bin = list_to_binary(lists:duplicate(BinSize*1024, 253)), + %%?line Size = Terms*size(Bin), + + %% SLF LEFT OFF HERE. + %% When the caller uses small hunks, like 4k via + %% bulk_sendsend(32*5, 4), then (on my laptop at least), we get + %% zero monitor messages. But if we use "+zdbbl 5", then we + %% get a lot of monitor messages. So, if we can count up the + %% total number of monitor messages that we get when running both + %% default busy size and "+zdbbl 5", and if the 5 case gets + %% "many many more" monitor messages, then we know we're working. + + ?line {ok, NodeSend} = start_node(bulk_sender, "+zdbbl " ++ integer_to_list(BusyBufSize)), + ?line _Send = spawn(NodeSend, erlang, apply, [fun sendersender/4, [self(), Recv, Bin, Terms]]), + ?line {Elapsed, {_TermsN, SizeN}, MonitorCount} = + receive {sendersender, BigRes} -> + BigRes + end, + ?line stop_node(NodeRecv), + ?line stop_node(NodeSend), + + ?line test_server:timetrap_cancel(Dog), + {trunc(SizeN/1024/Elapsed+0.5), MonitorCount}. + sender(To, _Bin, 0) -> To ! {done, self()}, receive @@ -154,6 +209,43 @@ sender(To, Bin, Left) -> To ! {term, Bin}, sender(To, Bin, Left-1). +%% Sender process to be run on a slave node + +sendersender(Parent, To, Bin, Left) -> + erlang:system_monitor(self(), [busy_dist_port]), + [spawn(fun() -> sendersender2(To, Bin, Left, false) end) || + _ <- lists:seq(1,1)], + {USec, {Res, MonitorCount}} = + timer:tc(?MODULE, sendersender2, [To, Bin, Left, true]), + Parent ! {sendersender, {USec/1000000, Res, MonitorCount}}. + +sendersender2(To, Bin, Left, SendDone) -> + sendersender3(To, Bin, Left, SendDone, 0). + +sendersender3(To, _Bin, 0, SendDone, MonitorCount) -> + if SendDone -> + To ! {done, self()}; + true -> + ok + end, + receive + {monitor, _Pid, _Type, _Info} -> + sendersender3(To, _Bin, 0, SendDone, MonitorCount + 1) + after 0 -> + if SendDone -> + receive + Any when is_tuple(Any), size(Any) == 2 -> + {Any, MonitorCount} + end; + true -> + exit(normal) + end + end; +sendersender3(To, Bin, Left, SendDone, MonitorCount) -> + To ! {term, Bin}, + %%timer:sleep(50), + sendersender3(To, Bin, Left-1, SendDone, MonitorCount). + %% Receiver process to be run on a slave node. receiver(Terms, Size) -> @@ -432,7 +524,7 @@ sink1() -> lost_exit(doc) -> "Test that EXIT and DOWN messages send to another node are not lost if " - "if the distribution port is busy."; + "the distribution port is busy."; lost_exit(Config) when is_list(Config) -> ?line {ok, Node} = start_node(lost_exit), @@ -1053,8 +1145,7 @@ contended_atom_cache_entry(Config) when is_list(Config) -> ?line {ok, SNode} = start_node(Config), ?line {ok, RNode} = start_node(Config), ?line Success = make_ref(), - ?line Mstr - = spawn_link( + ?line spawn_link( SNode, fun () -> erts_debug:set_internal_state(available_internal_state, @@ -1111,13 +1202,13 @@ contended_atom_cache_entry(Config) when is_list(Config) -> ?line stop_node(RNode), ?line ok. -send_ref_atom(To, Ref, Atom, 0) -> +send_ref_atom(_To, _Ref, _Atom, 0) -> ok; send_ref_atom(To, Ref, Atom, N) -> To ! {Ref, Atom}, send_ref_atom(To, Ref, Atom, N-1). -receive_ref_atom(Ref, Atom, 0) -> +receive_ref_atom(_Ref, _Atom, 0) -> ok; receive_ref_atom(Ref, Atom, N) -> receive @@ -1152,7 +1243,7 @@ unwanted_cixs() -> nodes()). -get_conflicting_atoms(CIX, 0) -> +get_conflicting_atoms(_CIX, 0) -> []; get_conflicting_atoms(CIX, N) -> {A, B, C} = now(), @@ -1166,6 +1257,186 @@ get_conflicting_atoms(CIX, N) -> get_conflicting_atoms(CIX, N) end. +-define(COOKIE, ''). +-define(DOP_LINK, 1). +-define(DOP_SEND, 2). +-define(DOP_EXIT, 3). +-define(DOP_UNLINK, 4). +-define(DOP_REG_SEND, 6). +-define(DOP_GROUP_LEADER, 7). +-define(DOP_EXIT2, 8). + +-define(DOP_SEND_TT, 12). +-define(DOP_EXIT_TT, 13). +-define(DOP_REG_SEND_TT, 16). +-define(DOP_EXIT2_TT, 18). + +-define(DOP_MONITOR_P, 19). +-define(DOP_DEMONITOR_P, 20). +-define(DOP_MONITOR_P_EXIT, 21). + +start_monitor(Offender,P) -> + ?line Parent = self(), + ?line Q = spawn(Offender, + fun () -> + Ref = erlang:monitor(process,P), + Parent ! {self(),ref,Ref}, + receive + just_stay_alive -> ok + end + end), + ?line Ref = receive + {Q,ref,R} -> + R + after 5000 -> + error + end, + io:format("Ref is ~p~n",[Ref]), + ok. +start_link(Offender,P) -> + ?line Parent = self(), + ?line Q = spawn(Offender, + fun () -> + process_flag(trap_exit,true), + link(P), + Parent ! {self(),ref,P}, + receive + just_stay_alive -> ok + end + end), + ?line Ref = receive + {Q,ref,R} -> + R + after 5000 -> + error + end, + io:format("Ref is ~p~n",[Ref]), + ok. + +bad_dist_structure(suite) -> + []; +bad_dist_structure(doc) -> + ["Test dist messages with valid structure (binary to term ok) but malformed" + "control content"]; +bad_dist_structure(Config) when is_list(Config) -> + %process_flag(trap_exit,true), + ODog = ?config(watchdog, Config), + ?t:timetrap_cancel(ODog), + Dog = ?t:timetrap(?t:seconds(15)), + + ?line {ok, Offender} = start_node(bad_dist_structure_offender), + ?line {ok, Victim} = start_node(bad_dist_structure_victim), + ?line start_node_monitors([Offender,Victim]), + ?line Parent = self(), + ?line P = spawn(Victim, + fun () -> + process_flag(trap_exit,true), + Parent ! {self(), started}, + receive check_msgs -> ok end, + bad_dist_struct_check_msgs([one, + two]), + Parent ! {self(), messages_checked}, + receive done -> ok end + end), + ?line receive {P, started} -> ok end, + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line verify_up(Offender, Victim), + ?line true = lists:member(Offender, rpc:call(Victim, erlang, nodes, [])), + ?line start_monitor(Offender,P), + ?line P ! one, + ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal,normal},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_LINK},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_UNLINK,'replace'},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_UNLINK,'replace',make_ref()},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_UNLINK,make_ref(),P},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_UNLINK,normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P,normal},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P,normal},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT,'replace',P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT,make_ref(),normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT_TT,'replace',token,P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT_TT,make_ref(),token,normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT2,'replace',P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT2,make_ref(),normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT2_TT,'replace',token,P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT2_TT,make_ref(),token,normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace'},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace','atomic'},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace',P},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name},2,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name,token},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace',''},2,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',P},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name,{token}},2,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND_TT,'',P},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND_TT,'',name,token},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND,''},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND,'',name},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND,'',P,{token}},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line P ! two, + ?line P ! check_msgs, + ?line receive + {P, messages_checked} -> ok + after 5000 -> + exit(victim_is_dead) + end, + + ?line {message_queue_len, 0} + = rpc:call(Victim, erlang, process_info, [P, message_queue_len]), + + ?line unlink(P), + ?line P ! done, + ?line stop_node(Offender), + ?line stop_node(Victim), + ?t:timetrap_cancel(Dog), + ok. bad_dist_ext(doc) -> []; bad_dist_ext(suite) -> @@ -1393,6 +1664,22 @@ bad_dist_ext_connection_id(Config) when is_list(Config) -> ?line stop_node(Victim). +bad_dist_struct_check_msgs([]) -> + receive + Msg -> + exit({unexpected_message, Msg}) + after 0 -> + ok + end; +bad_dist_struct_check_msgs([M|Ms]) -> + receive + {'EXIT',_,_} = EM -> + io:format("Ignoring exit message: ~p~n",[EM]), + bad_dist_struct_check_msgs([M|Ms]); + Msg -> + M = Msg, + bad_dist_struct_check_msgs(Ms) + end. bad_dist_ext_check_msgs([]) -> receive Msg -> @@ -1407,24 +1694,6 @@ bad_dist_ext_check_msgs([M|Ms]) -> bad_dist_ext_check_msgs(Ms) end. --define(COOKIE, ''). --define(DOP_LINK, 1). --define(DOP_SEND, 2). --define(DOP_EXIT, 3). --define(DOP_UNLINK, 4). --define(DOP_NODE_LINK, 5). --define(DOP_REG_SEND, 6). --define(DOP_GROUP_LEADER, 7). --define(DOP_EXIT2, 8). - --define(DOP_SEND_TT, 12). --define(DOP_EXIT_TT, 13). --define(DOP_REG_SEND_TT, 16). --define(DOP_EXIT2_TT, 18). - --define(DOP_MONITOR_P, 19). --define(DOP_DEMONITOR_P, 20). --define(DOP_MONITOR_P_EXIT, 21). dport_reg_send(Node, Name, Msg) -> DPrt = case dport(Node) of @@ -1456,6 +1725,39 @@ dport_send(To, Msg) -> ?COOKIE, To}), dmsg_ext(Msg)]). +send_bad_structure(Offender,Victim,Bad,WhereToPutSelf) -> + send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,[]). +send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> + Parent = self(), + Done = make_ref(), + spawn(Offender, + fun () -> + Node = node(Victim), + pong = net_adm:ping(Node), + DPrt = dport(Node), + Bad1 = case WhereToPutSelf of + 0 -> + Bad; + N when N > 0 -> + setelement(N,Bad,self()) + end, + DData = [dmsg_hdr(), + dmsg_ext(Bad1)] ++ + case PayLoad of + [] -> []; + _Other -> [dmsg_ext(PayLoad)] + end, + port_command(DPrt, DData), + Parent ! {DData,Done} + end), + receive + {WhatSent,Done} -> + io:format("Offender sent ~p~n",[WhatSent]), + ok + after 5000 -> + exit(unable_to_send) + end. + %% send_bad_msgs(): %% Send a valid distribution header and control message @@ -1539,10 +1841,10 @@ dmsg_bad_hdr() -> 255]. % 255 atom references -dmsg_fake_hdr1() -> - A = <<"fake header atom 1">>, - [131, % Version Magic - $D, 1, 16#8, 0, size(A), A]. % Fake header +%% dmsg_fake_hdr1() -> +%% A = <<"fake header atom 1">>, +%% [131, % Version Magic +%% $D, 1, 16#8, 0, size(A), A]. % Fake header dmsg_fake_hdr2() -> A1 = <<"fake header atom 1">>, @@ -1727,7 +2029,7 @@ flush_node_changes() -> node_monitor_loop(Master) -> receive - {nodeup, Node, InfoList} = Msg -> + {nodeup, Node, _InfoList} = Msg -> Master ! {nodeup, node(), Node}, ?t:format("~p ~p: ~p~n", [node(), erlang:now(), Msg]), node_monitor_loop(Master); @@ -1764,9 +2066,9 @@ verify_no_down(A, B) -> ok end. -verify_down(A, B) -> - receive {nodedown, A, B, _} -> ok end, - receive {nodedown, B, A, _} -> ok end. +%% verify_down(A, B) -> +%% receive {nodedown, A, B, _} -> ok end, +%% receive {nodedown, B, A, _} -> ok end. verify_down(A, ReasonA, B, ReasonB) -> receive @@ -1786,11 +2088,11 @@ from(H, [H | T]) -> T; from(H, [_ | T]) -> from(H, T); from(_, []) -> []. -fun_spawn(Fun) -> - fun_spawn(Fun, []). +%% fun_spawn(Fun) -> +%% fun_spawn(Fun, []). -fun_spawn(Fun, Args) -> - spawn_link(erlang, apply, [Fun, Args]). +%% fun_spawn(Fun, Args) -> +%% spawn_link(erlang, apply, [Fun, Args]). long_or_short() -> diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl index 542c8dffbe..11a7a61586 100644 --- a/erts/emulator/test/erl_link_SUITE.erl +++ b/erts/emulator/test/erl_link_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -1050,7 +1050,6 @@ stop_node(Node) -> -define(DOP_SEND, 2). -define(DOP_EXIT, 3). -define(DOP_UNLINK, 4). --define(DOP_NODE_LINK, 5). -define(DOP_REG_SEND, 6). -define(DOP_GROUP_LEADER, 7). -define(DOP_EXIT2, 8). diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl index e60a999df1..934a1b10a4 100644 --- a/erts/emulator/test/erts_debug_SUITE.erl +++ b/erts/emulator/test/erts_debug_SUITE.erl @@ -21,10 +21,10 @@ -include("test_server.hrl"). -export([all/1,init_per_testcase/2,fin_per_testcase/2, - flat_size/1,flat_size_big/1,df/1]). + flat_size/1,flat_size_big/1,df/1,instructions/1]). all(suite) -> - [flat_size,flat_size_big,df]. + [flat_size,flat_size_big,df,instructions]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(2)), @@ -70,3 +70,8 @@ df(Config) when is_list(Config) -> pps() -> {erlang:ports()}. + +instructions(Config) when is_list(Config) -> + ?line Is = erts_debug:instructions(), + ?line _ = [list_to_atom(I) || I <- Is], + ok. diff --git a/erts/emulator/test/float_SUITE.erl b/erts/emulator/test/float_SUITE.erl index 102e472ea6..99e9457985 100644 --- a/erts/emulator/test/float_SUITE.erl +++ b/erts/emulator/test/float_SUITE.erl @@ -22,7 +22,10 @@ -include("test_server.hrl"). -export([all/1,init_per_testcase/2,fin_per_testcase/2, - fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1,bad_float_unpack/1]). + fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1, + bad_float_unpack/1]). +-export([otp_7178/1]). + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(3)), @@ -33,7 +36,29 @@ fin_per_testcase(_Func, Config) -> ?t:timetrap_cancel(Dog). all(suite) -> - [fpe,fp_drv,fp_drv_thread,denormalized,match,bad_float_unpack]. + [fpe, + fp_drv, + fp_drv_thread, + otp_7178, + denormalized, + match, + bad_float_unpack]. + +%% +%% OTP-7178, list_to_float on very small numbers should give 0.0 +%% instead of exception, i.e. ignore underflow. +%% +otp_7178(suite) -> + []; +otp_7178(doc) -> + ["test that list_to_float on very small numbers give 0.0"]; +otp_7178(Config) when is_list(Config) -> + ?line X = list_to_float("1.0e-325"), + ?line true = (X < 0.00000001) and (X > -0.00000001), + ?line Y = list_to_float("1.0e-325325325"), + ?line true = (Y < 0.00000001) and (Y > -0.00000001), + ?line {'EXIT', {badarg,_}} = (catch list_to_float("1.0e83291083210")), + ok. %% Forces floating point exceptions and tests that subsequent, legal, %% operations are calculated correctly. Original version by Sebastian diff --git a/erts/emulator/test/fun_r12_SUITE.erl b/erts/emulator/test/fun_r12_SUITE.erl index f21299ba67..9262731dcb 100644 --- a/erts/emulator/test/fun_r12_SUITE.erl +++ b/erts/emulator/test/fun_r12_SUITE.erl @@ -18,6 +18,7 @@ %% -module(fun_r12_SUITE). +-compile(r12). -export([all/1,init_per_testcase/2,fin_per_testcase/2,dist_old_release/1]). diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl index 23482a20d7..8fef36dfaf 100644 --- a/erts/emulator/test/guard_SUITE.erl +++ b/erts/emulator/test/guard_SUITE.erl @@ -1,33 +1,34 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% -module(guard_SUITE). -export([all/1, bad_arith/1, bad_tuple/1, test_heap_guards/1, guard_bifs/1, - type_tests/1]). + type_tests/1,guard_bif_binary_part/1]). -include("test_server.hrl"). -export([init/3]). -import(lists, [member/2]). -all(suite) -> [bad_arith, bad_tuple, test_heap_guards, guard_bifs, type_tests]. +all(suite) -> [bad_arith, bad_tuple, test_heap_guards, guard_bifs, + type_tests, guard_bif_binary_part]. bad_arith(doc) -> "Test that a bad arithmetic operation in a guard works correctly."; bad_arith(Config) when is_list(Config) -> @@ -136,6 +137,170 @@ init(Fun, Args, Filler) -> dummy(_) -> ok. +-define(MASK_ERROR(EXPR),mask_error((catch (EXPR)))). +mask_error({'EXIT',{Err,_}}) -> + Err; +mask_error(Else) -> + Else. + +guard_bif_binary_part(doc) -> + ["Test the binary_part/2,3 guard BIF's extensively"]; +guard_bif_binary_part(Config) when is_list(Config) -> + %% Overflow tests that need to be unoptimized + ?line badarg = + ?MASK_ERROR( + binary_part(<<1,2,3>>,{16#FFFFFFFFFFFFFFFF, + -16#7FFFFFFFFFFFFFFF-1})), + ?line badarg = + ?MASK_ERROR( + binary_part(<<1,2,3>>,{16#FFFFFFFFFFFFFFFF, + 16#7FFFFFFFFFFFFFFF})), + F = fun(X) -> + Master = self(), + {Pid,Ref} = spawn_monitor( fun() -> + A = lists:duplicate(X,a), + B = [do_binary_part_guard() | A], + Master ! {self(),hd(B)}, + ok + end), + receive + {Pid,ok} -> + erlang:demonitor(Ref,[flush]), + ok; + Error -> + Error + end + end, + [ ok = F(N) || N <- lists:seq(1,10000) ], + ok. + + +do_binary_part_guard() -> + ?line 1 = bptest(<<1,2,3>>), + ?line 2 = bptest(<<2,1,3>>), + ?line error = bptest(<<1>>), + ?line error = bptest(<<>>), + ?line error = bptest(apa), + ?line 3 = bptest(<<2,3,3>>), + % With one variable (pos) + ?line 1 = bptest(<<1,2,3>>,1), + ?line 2 = bptest(<<2,1,3>>,1), + ?line error = bptest(<<1>>,1), + ?line error = bptest(<<>>,1), + ?line error = bptest(apa,1), + ?line 3 = bptest(<<2,3,3>>,1), + % With one variable (length) + ?line 1 = bptesty(<<1,2,3>>,1), + ?line 2 = bptesty(<<2,1,3>>,1), + ?line error = bptesty(<<1>>,1), + ?line error = bptesty(<<>>,1), + ?line error = bptesty(apa,1), + ?line 3 = bptesty(<<2,3,3>>,2), + % With one variable (whole tuple) + ?line 1 = bptestx(<<1,2,3>>,{1,1}), + ?line 2 = bptestx(<<2,1,3>>,{1,1}), + ?line error = bptestx(<<1>>,{1,1}), + ?line error = bptestx(<<>>,{1,1}), + ?line error = bptestx(apa,{1,1}), + ?line 3 = bptestx(<<2,3,3>>,{1,2}), + % With two variables + ?line 1 = bptest(<<1,2,3>>,1,1), + ?line 2 = bptest(<<2,1,3>>,1,1), + ?line error = bptest(<<1>>,1,1), + ?line error = bptest(<<>>,1,1), + ?line error = bptest(apa,1,1), + ?line 3 = bptest(<<2,3,3>>,1,2), + % Direct (autoimported) call, these will be evaluated by the compiler... + ?line <<2>> = binary_part(<<1,2,3>>,1,1), + ?line <<1>> = binary_part(<<2,1,3>>,1,1), + % Compiler warnings due to constant evaluation expected (3) + ?line badarg = ?MASK_ERROR(binary_part(<<1>>,1,1)), + ?line badarg = ?MASK_ERROR(binary_part(<<>>,1,1)), + ?line badarg = ?MASK_ERROR(binary_part(apa,1,1)), + ?line <<3,3>> = binary_part(<<2,3,3>>,1,2), + % Direct call through apply + ?line <<2>> = apply(erlang,binary_part,[<<1,2,3>>,1,1]), + ?line <<1>> = apply(erlang,binary_part,[<<2,1,3>>,1,1]), + % Compiler warnings due to constant evaluation expected (3) + ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[<<1>>,1,1])), + ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[<<>>,1,1])), + ?line badarg = ?MASK_ERROR(apply(erlang,binary_part,[apa,1,1])), + ?line <<3,3>> = apply(erlang,binary_part,[<<2,3,3>>,1,2]), + % Constant propagation + ?line Bin = <<1,2,3>>, + ?line ok = if + binary_part(Bin,1,1) =:= <<2>> -> + ok; + %% Compiler warning, clause cannot match (expected) + true -> + error + end, + ?line ok = if + binary_part(Bin,{1,1}) =:= <<2>> -> + ok; + %% Compiler warning, clause cannot match (expected) + true -> + error + end, + ok. + + +bptest(B) when length(B) =:= 1337 -> + 1; +bptest(B) when binary_part(B,{1,1}) =:= <<2>> -> + 1; +bptest(B) when erlang:binary_part(B,1,1) =:= <<1>> -> + 2; +bptest(B) when erlang:binary_part(B,{1,2}) =:= <<3,3>> -> + 3; +bptest(_) -> + error. + +bptest(B,A) when length(B) =:= A -> + 1; +bptest(B,A) when binary_part(B,{A,1}) =:= <<2>> -> + 1; +bptest(B,A) when erlang:binary_part(B,A,1) =:= <<1>> -> + 2; +bptest(B,A) when erlang:binary_part(B,{A,2}) =:= <<3,3>> -> + 3; +bptest(_,_) -> + error. + +bptestx(B,A) when length(B) =:= A -> + 1; +bptestx(B,A) when binary_part(B,A) =:= <<2>> -> + 1; +bptestx(B,A) when erlang:binary_part(B,A) =:= <<1>> -> + 2; +bptestx(B,A) when erlang:binary_part(B,A) =:= <<3,3>> -> + 3; +bptestx(_,_) -> + error. + +bptesty(B,A) when length(B) =:= A -> + 1; +bptesty(B,A) when binary_part(B,{1,A}) =:= <<2>> -> + 1; +bptesty(B,A) when erlang:binary_part(B,1,A) =:= <<1>> -> + 2; +bptesty(B,A) when erlang:binary_part(B,{1,A}) =:= <<3,3>> -> + 3; +bptesty(_,_) -> + error. + +bptest(B,A,_C) when length(B) =:= A -> + 1; +bptest(B,A,C) when binary_part(B,{A,C}) =:= <<2>> -> + 1; +bptest(B,A,C) when erlang:binary_part(B,A,C) =:= <<1>> -> + 2; +bptest(B,A,C) when erlang:binary_part(B,{A,C}) =:= <<3,3>> -> + 3; +bptest(_,_,_) -> + error. + + guard_bifs(doc) -> "Test all guard bifs with nasty (but legal arguments)."; guard_bifs(Config) when is_list(Config) -> ?line Big = -237849247829874297658726487367328971246284736473821617265433, diff --git a/erts/emulator/test/hash_SUITE.erl b/erts/emulator/test/hash_SUITE.erl index 85bdb8bff8..f5d1871bfb 100644 --- a/erts/emulator/test/hash_SUITE.erl +++ b/erts/emulator/test/hash_SUITE.erl @@ -480,14 +480,14 @@ otp_5292_test() -> S2 = md5([md5(hash_int(S, E, PH)) || {Start, N, Sz} <- d(), {S, E} <- int(Start, N, Sz)]), ?line Comment = case S1 of - <<43,186,76,102,87,4,110,245,203,177,206,6,130,69,43,99>> -> + <<4,248,208,156,200,131,7,1,173,13,239,173,112,81,16,174>> -> ?line big = erlang:system_info(endian), "Big endian machine"; - <<21,206,139,15,149,28,167,81,98,225,132,254,49,125,174,195>> -> + <<180,28,33,231,239,184,71,125,76,47,227,241,78,184,176,233>> -> ?line little = erlang:system_info(endian), "Little endian machine" end, - ?line <<140,37,79,80,26,242,130,22,20,229,123,240,223,244,43,99>> = S2, + ?line <<124,81,198,121,174,233,19,137,10,83,33,80,226,111,238,99>> = S2, ?line 2 = erlang:hash(1, (1 bsl 27) -1), ?line {'EXIT', _} = (catch erlang:hash(1, (1 bsl 27))), {comment, Comment}. @@ -507,7 +507,7 @@ hash_int(Start, End, F) -> {Start, End, md5(HL)}. md5(T) -> - erlang:md5(term_to_binary(T)). + erlang:md5(term_to_binary(T)). bit_level_binaries() -> ?line [3511317,7022633,14044578,28087749,56173436,112344123,90467083|_] = diff --git a/erts/emulator/test/mtx_SUITE.erl b/erts/emulator/test/mtx_SUITE.erl new file mode 100644 index 0000000000..ae77fe4d89 --- /dev/null +++ b/erts/emulator/test/mtx_SUITE.erl @@ -0,0 +1,473 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Stress tests of rwmutex implementation. +%% +%% Author: Rickard Green +%% +-module(mtx_SUITE). + +%%-define(line_trace,true). + +-include("test_server.hrl"). + +-export([all/1, init_per_suite/1, end_per_suite/1, init_per_testcase/2, fin_per_testcase/2]). + +-export([long_rwlock/1, + hammer_ets_rwlock/1, + hammer_rwlock/1, + hammer_rwlock_check/1, + hammer_tryrwlock/1, + hammer_tryrwlock_check/1, + hammer_sched_long_rwlock/1, + hammer_sched_long_rwlock_check/1, + hammer_sched_long_freqread_rwlock/1, + hammer_sched_long_freqread_rwlock_check/1, + hammer_sched_long_tryrwlock/1, + hammer_sched_long_tryrwlock_check/1, + hammer_sched_long_freqread_tryrwlock/1, + hammer_sched_long_freqread_tryrwlock_check/1, + hammer_sched_rwlock/1, + hammer_sched_rwlock_check/1, + hammer_sched_freqread_rwlock/1, + hammer_sched_freqread_rwlock_check/1, + hammer_sched_tryrwlock/1, + hammer_sched_tryrwlock_check/1, + hammer_sched_freqread_tryrwlock/1, + hammer_sched_freqread_tryrwlock_check/1]). + +init_per_suite(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Lib = filename:join([DataDir, atom_to_list(?MODULE)]), + ok = erlang:load_nif(Lib, none), + Config. + +end_per_suite(Config) when is_list(Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = ?t:timetrap(?t:minutes(15)), + [{watchdog, Dog}|Config]. + +fin_per_testcase(_Func, Config) -> + Dog = ?config(watchdog, Config), + ?t:timetrap_cancel(Dog). + +all(suite) -> + [ + long_rwlock, + hammer_rwlock_check, + hammer_rwlock, + hammer_tryrwlock_check, + hammer_tryrwlock, + hammer_ets_rwlock, + hammer_sched_long_rwlock_check, + hammer_sched_long_rwlock, + hammer_sched_long_freqread_rwlock_check, + hammer_sched_long_freqread_rwlock, + hammer_sched_long_tryrwlock_check, + hammer_sched_long_tryrwlock, + hammer_sched_long_freqread_tryrwlock_check, + hammer_sched_long_freqread_tryrwlock, + hammer_sched_rwlock_check, + hammer_sched_rwlock, + hammer_sched_freqread_rwlock_check, + hammer_sched_freqread_rwlock, + hammer_sched_tryrwlock_check, + hammer_sched_tryrwlock, + hammer_sched_freqread_tryrwlock_check, + hammer_sched_freqread_tryrwlock + ]. + +long_rwlock(Config) when is_list(Config) -> + statistics(runtime), + LLRes = long_rw_test(), + {_, RunTime} = statistics(runtime), + %% A very short run time is expected, since + %% threads in the test mostly wait + ?t:format("RunTime=~p~n", [RunTime]), + ?line true = RunTime < 100, + ?line RunTimeStr = "Run-time during test was "++integer_to_list(RunTime)++" ms.", + case LLRes of + ok -> + {comment, RunTimeStr}; + {comment, Comment} -> + {comment, Comment ++ " " ++ RunTimeStr} + end. + +hammer_rwlock(Config) when is_list(Config) -> + hammer_rw_test(false). + +hammer_rwlock_check(Config) when is_list(Config) -> + hammer_rw_test(true). + +hammer_tryrwlock(Config) when is_list(Config) -> + hammer_tryrw_test(false). + +hammer_tryrwlock_check(Config) when is_list(Config) -> + hammer_tryrw_test(true). + +hammer_sched_rwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, false, true, 0, 0). + +hammer_sched_rwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, true, true, 0, 0). + +hammer_sched_freqread_rwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, false, true, 0, 0). + +hammer_sched_freqread_rwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, true, true, 0, 0). + +hammer_sched_tryrwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, false, false, 0, 100). + +hammer_sched_tryrwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, true, false, 0, 100). + +hammer_sched_freqread_tryrwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, false, false, 0, 100). + +hammer_sched_freqread_tryrwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, true, false, 0, 100). + +hammer_sched_long_rwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, false, true, 100, 0). + +hammer_sched_long_rwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, true, true, 100, 0). + +hammer_sched_long_freqread_rwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, false, true, 100, 0). + +hammer_sched_long_freqread_rwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, true, true, 100, 0). + +hammer_sched_long_tryrwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, false, false, 100, 100). + +hammer_sched_long_tryrwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, true, false, 100, 100). + +hammer_sched_long_freqread_tryrwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, false, false, 100, 100). + +hammer_sched_long_freqread_tryrwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, true, false, 100, 100). + +hammer_sched_rwlock_test(FreqRead, LockCheck, Blocking, WaitLocked, WaitUnlocked) -> + case create_rwlock(FreqRead, LockCheck) of + enotsup -> + {skipped, "Not supported."}; + RWLock -> + Onln = erlang:system_info(schedulers_online), + NWPs = case Onln div 3 of + 1 -> case Onln < 4 of + true -> 1; + false -> 2 + end; + X -> X + end, + NRPs = Onln - NWPs, + NoLockOps = ((((50000000 div Onln) + div case {Blocking, WaitLocked} of + {false, 0} -> 1; + _ -> 10 + end) + div (case WaitLocked == 0 of + true -> 1; + false -> WaitLocked*250 + end)) + div handicap()), + ?t:format("NoLockOps=~p~n", [NoLockOps]), + Sleep = case Blocking of + true -> NoLockOps; + false -> NoLockOps div 10 + end, + WPs = lists:map( + fun (Sched) -> + spawn_opt( + fun () -> + io:format("Writer on scheduler ~p.~n", + [Sched]), + Sched = erlang:system_info(scheduler_id), + receive go -> gone end, + hammer_sched_rwlock_proc(RWLock, + Blocking, + true, + WaitLocked, + WaitUnlocked, + NoLockOps, + Sleep), + Sched = erlang:system_info(scheduler_id) + end, + [link, {scheduler, Sched}]) + end, + lists:seq(1, NWPs)), + RPs = lists:map( + fun (Sched) -> + spawn_opt( + fun () -> + io:format("Reader on scheduler ~p.~n", + [Sched]), + Sched = erlang:system_info(scheduler_id), + receive go -> gone end, + hammer_sched_rwlock_proc(RWLock, + Blocking, + false, + WaitLocked, + WaitUnlocked, + NoLockOps, + Sleep), + Sched = erlang:system_info(scheduler_id) + end, + [link, {scheduler, Sched}]) + end, + lists:seq(NWPs + 1, NWPs + NRPs)), + Procs = WPs ++ RPs, + case {Blocking, WaitLocked} of + {_, 0} -> ok; + {false, _} -> ok; + _ -> statistics(runtime) + end, + lists:foreach(fun (P) -> P ! go end, Procs), + lists:foreach(fun (P) -> + M = erlang:monitor(process, P), + receive + {'DOWN', M, process, P, _} -> + ok + end + end, + Procs), + case {Blocking, WaitLocked} of + {_, 0} -> ok; + {false, _} -> ok; + _ -> + {_, RunTime} = statistics(runtime), + ?t:format("RunTime=~p~n", [RunTime]), + ?line true = RunTime < 500, + {comment, + "Run-time during test was " + ++ integer_to_list(RunTime) + ++ " ms."} + end + end. + +hammer_sched_rwlock_proc(_RWLock, + _Blocking, + _WriteOp, + _WaitLocked, + _WaitUnlocked, + 0, + _Sleep) -> + ok; +hammer_sched_rwlock_proc(RWLock, + Blocking, + WriteOp, + WaitLocked, + WaitUnlocked, + Times, + Sleep) when Times rem Sleep == 0 -> + rwlock_op(RWLock, Blocking, WriteOp, WaitLocked, WaitUnlocked), + hammer_sched_rwlock_proc(RWLock, + Blocking, + WriteOp, + WaitLocked, + WaitUnlocked, + Times - 1, + Sleep); +hammer_sched_rwlock_proc(RWLock, + Blocking, + WriteOp, + WaitLocked, + WaitUnlocked, + Times, + Sleep) -> + rwlock_op(RWLock, Blocking, WriteOp, WaitLocked, 0), + hammer_sched_rwlock_proc(RWLock, + Blocking, + WriteOp, + WaitLocked, + WaitUnlocked, + Times - 1, + Sleep). + +-define(HAMMER_ETS_RWLOCK_REPEAT_TIMES, 1). +-define(HAMMER_ETS_RWLOCK_TSIZE, 500). + +hammer_ets_rwlock(Config) when is_list(Config) -> + {Ops, Procs} = case handicap() of + 1 -> {20000, 500}; + 2 -> {20000, 50}; + 3 -> {2000, 50}; + _ -> {200, 50} + end, + ?t:format("Procs=~p~nOps=~p~n", [Procs, Ops]), + lists:foreach(fun (XOpts) -> + ?t:format("Running with extra opts: ~p", [XOpts]), + hammer_ets_rwlock_test(XOpts, true, 2, Ops, + Procs, false) + end, + [[], + [{read_concurrency, true}], + [{write_concurrency, true}], + [{read_concurrency, true},{write_concurrency, true}]]), + ok. + +%% Aux funcs + +long_rw_test() -> + exit(no_nif_implementation). + +hammer_rw_test(_Arg) -> + exit(no_nif_implementation). + +hammer_tryrw_test(_Arg) -> + exit(no_nif_implementation). + +create_rwlock(_FreqRead, _LockCheck) -> + exit(no_nif_implementation). + +rwlock_op(_RWLock, _Blocking, _WriteOp, _WaitLocked, _WaitUnlocked) -> + exit(no_nif_implementation). + +hammer_ets_rwlock_put_data() -> + put(?MODULE, {"here are some", data, "to store", make_ref()}). + +hammer_ets_rwlock_get_data() -> + get(?MODULE). + +hammer_ets_rwlock_ops(_T, _UW, _N, _C, _SC, 0) -> + ok; +hammer_ets_rwlock_ops(T, UW, N, C, SC, Tot) when N >= ?HAMMER_ETS_RWLOCK_TSIZE -> + hammer_ets_rwlock_ops(T, UW, 0, C, SC, Tot); +hammer_ets_rwlock_ops(T, UW, N, 0, SC, Tot) -> + case UW of + true -> + true = ets:insert(T, {N, Tot, hammer_ets_rwlock_get_data()}); + false -> + [{N, _, _}] = ets:lookup(T, N) + end, + hammer_ets_rwlock_ops(T, UW, N+1, SC, SC, Tot-1); +hammer_ets_rwlock_ops(T, UW, N, C, SC, Tot) -> + case UW of + false -> + true = ets:insert(T, {N, Tot, hammer_ets_rwlock_get_data()}); + true -> + [{N, _, _}] = ets:lookup(T, N) + end, + hammer_ets_rwlock_ops(T, UW, N+1, C-1, SC, Tot-1). + +hammer_ets_rwlock_init(T, N) when N < ?HAMMER_ETS_RWLOCK_TSIZE -> + ets:insert(T, {N, N, N}), + hammer_ets_rwlock_init(T, N+1); +hammer_ets_rwlock_init(_T, _N) -> + ok. + +hammer_ets_rwlock_test(XOpts, UW, C, N, NP, SC) -> + receive after 100 -> ok end, + {TP, TM} = spawn_monitor( + fun () -> + _L = repeat_list( + fun () -> + Caller = self(), + T = fun () -> + Parent = self(), + hammer_ets_rwlock_put_data(), + T=ets:new(x, [public | XOpts]), + hammer_ets_rwlock_init(T, 0), + Ps0 = repeat_list( + fun () -> + spawn_link( + fun () -> + hammer_ets_rwlock_put_data(), + receive go -> ok end, + hammer_ets_rwlock_ops(T, UW, N, C, C, N), + Parent ! {done, self()}, + receive after infinity -> ok end + end) + end, + NP - case SC of + false -> 0; + _ -> 1 + end), + Ps = case SC of + false -> Ps0; + _ -> [spawn_link(fun () -> + hammer_ets_rwlock_put_data(), + receive go -> ok end, + hammer_ets_rwlock_ops(T, UW, N, SC, SC, N), + Parent ! {done, self()}, + receive after infinity -> ok end + end) | Ps0] + end, + Start = now(), + lists:foreach(fun (P) -> P ! go end, Ps), + lists:foreach(fun (P) -> receive {done, P} -> ok end end, Ps), + Stop = now(), + lists:foreach(fun (P) -> + unlink(P), + exit(P, bang), + M = erlang:monitor(process, P), + receive + {'DOWN', M, process, P, _} -> ok + end + end, Ps), + Res = timer:now_diff(Stop, Start)/1000000, + Caller ! {?MODULE, self(), Res} + end, + TP = spawn_link(T), + receive + {?MODULE, TP, Res} -> + Res + end + end, + ?HAMMER_ETS_RWLOCK_REPEAT_TIMES) + end), + receive + {'DOWN', TM, process, TP, _} -> ok + end. + +repeat_list(Fun, N) -> + repeat_list(Fun, N, []). + +repeat_list(_Fun, 0, Acc) -> + Acc; +repeat_list(Fun, N, Acc) -> + repeat_list(Fun, N-1, [Fun()|Acc]). + + +handicap() -> + X0 = case catch (erlang:system_info(logical_processors_available) >= + erlang:system_info(schedulers_online)) of + true -> 1; + _ -> 2 + end, + case erlang:system_info(build_type) of + opt -> + X0; + ReallySlow when ReallySlow == debug; + ReallySlow == valgrind; + ReallySlow == purify -> + X0*3; + _Slow -> + X0*2 + end. + diff --git a/erts/emulator/test/mtx_SUITE_data/Makefile.src b/erts/emulator/test/mtx_SUITE_data/Makefile.src new file mode 100644 index 0000000000..b6c843269c --- /dev/null +++ b/erts/emulator/test/mtx_SUITE_data/Makefile.src @@ -0,0 +1,30 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2010. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +include @erts_lib_include_internal_generated@@[email protected] +include @erts_lib_include_internal_generated@@DS@erts_internal.mk + +NIF_LIBS = mtx_SUITE@dll@ + +SHLIB_EXTRA_CFLAGS = $(ETHR_DEFS) -I@erts_lib_include_internal@ -I@erts_lib_include_internal_generated@ +LIBS = @ERTS_LIBS@ + +all: $(NIF_LIBS) + +@SHLIB_RULES@ diff --git a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c new file mode 100644 index 0000000000..818023211c --- /dev/null +++ b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c @@ -0,0 +1,692 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Stress tests of rwmutex implementation. + * + * Author: Rickard Green + */ + +#include "erl_nif.h" + +#ifdef __WIN32__ +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include <windows.h> +#else +# include "ethread.h" +# include "erl_misc_utils.h" +# include <unistd.h> +#endif + +#include <errno.h> +#include <stdio.h> + +static int +fail(const char *file, int line, const char *function, const char *assertion); + +#undef ASSERT +#define ASSERT(X) ((void) ((X) ? 1 : fail(__FILE__, __LINE__, __func__, #X))) + +#ifdef __WIN32__ +/* + * We cannot access the ethread symbols directly; test + * what we got in the nif api instead... + */ +#define HAVE_FREQREAD_SUPPORT 0 +#define RWMUTEX_T ErlNifRWLock +#define RWMUTEX_CREATE(FR) enif_rwlock_create("dummy") +#define RWMUTEX_DESTROY enif_rwlock_destroy +#define RWMUTEX_WLOCK enif_rwlock_rwlock +#define RWMUTEX_TRYWLOCK enif_rwlock_tryrwlock +#define RWMUTEX_WUNLOCK enif_rwlock_rwunlock +#define RWMUTEX_TRYRLOCK enif_rwlock_tryrlock +#define RWMUTEX_RLOCK enif_rwlock_rlock +#define RWMUTEX_RUNLOCK enif_rwlock_runlock +#define THR_ID ErlNifTid +#define THR_CREATE(A, B, C, D) enif_thread_create("dummy", (A), (B), (C), (D)) +#define THR_JOIN enif_thread_join +#define ATOMIC_T volatile LONG +#define ATOMIC_INIT(VarP, Val) (*(VarP) = (Val)) +#define ATOMIC_SET(VarP, Val) (*(VarP) = (Val)) +#define ATOMIC_READ(VarP) (*(VarP)) +#define ATOMIC_INC InterlockedIncrement +#define ATOMIC_DEC InterlockedDecrement + +#else + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +# define HAVE_FREQREAD_SUPPORT 1 +#else +# define HAVE_FREQREAD_SUPPORT 0 +#endif + +#define RWMUTEX_T ethr_rwmutex +static ethr_rwmutex * +RWMUTEX_CREATE(int freqread) +{ + ethr_rwmutex *rwmtx = enif_alloc(sizeof(ethr_rwmutex)); + ethr_rwmutex_opt rwmtx_opt = ETHR_RWMUTEX_OPT_DEFAULT_INITER; + if (freqread) + rwmtx_opt.type = ETHR_RWMUTEX_TYPE_FREQUENT_READ; + ASSERT(rwmtx); + ASSERT(ethr_rwmutex_init_opt(rwmtx, &rwmtx_opt) == 0); + return rwmtx; +} +static void +RWMUTEX_DESTROY(ethr_rwmutex *rwmtx) +{ + ASSERT(ethr_rwmutex_destroy(rwmtx) == 0); + enif_free(rwmtx); +} +#define RWMUTEX_TRYWLOCK ethr_rwmutex_tryrwlock +#define RWMUTEX_WLOCK ethr_rwmutex_rwlock +#define RWMUTEX_WUNLOCK ethr_rwmutex_rwunlock +#define RWMUTEX_TRYRLOCK ethr_rwmutex_tryrlock +#define RWMUTEX_RLOCK ethr_rwmutex_rlock +#define RWMUTEX_RUNLOCK ethr_rwmutex_runlock +#define THR_ID ethr_tid +#define THR_CREATE ethr_thr_create +#define THR_JOIN ethr_thr_join +#define ATOMIC_T ethr_atomic_t +#define ATOMIC_INIT ethr_atomic_init +#define ATOMIC_SET ethr_atomic_set +#define ATOMIC_READ ethr_atomic_read +#define ATOMIC_INC ethr_atomic_inc +#define ATOMIC_DEC ethr_atomic_dec + +#endif + + +#if !defined(__func__) +# if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# if !defined(__GNUC__) || __GNUC__ < 2 +# define __func__ "[unknown_function]" +# else +# define __func__ __FUNCTION__ +# endif +# endif +#endif + +static void milli_sleep(int ms); +static int get_bool(ErlNifEnv* env, ERL_NIF_TERM term); + +/* + * Long rwlock testcase + */ + +#define LONG_RW_NO_W_THREADS 6 +#define LONG_RW_NO_THREADS 20 +#define LONG_RW_NO_WLOCK_COUNT 100 + +typedef struct { + RWMUTEX_T *rwlock; + ATOMIC_T *is_wlocked; + ATOMIC_T *is_rlocked; + int *stop; + int *count; + int sleep; +} long_rw_t; + +static void * +long_rw_w(void *varg) +{ + long_rw_t *arg = varg; + int stop = 0; + do { + RWMUTEX_WLOCK(arg->rwlock); + ASSERT(!ATOMIC_READ(arg->is_wlocked)); + ATOMIC_SET(arg->is_wlocked, 1); + ASSERT(!ATOMIC_READ(arg->is_rlocked)); + milli_sleep(arg->sleep); + if (++(*arg->count) > LONG_RW_NO_WLOCK_COUNT) + stop = *arg->stop = 1; + ATOMIC_SET(arg->is_wlocked, 0); + ASSERT(!ATOMIC_READ(arg->is_rlocked)); + RWMUTEX_WUNLOCK(arg->rwlock); + } while (!stop); + return NULL; +} + +static void * +long_rw_r(void *varg) +{ + long_rw_t *arg = varg; + int stop; + do { + RWMUTEX_RLOCK(arg->rwlock); + ASSERT(!ATOMIC_READ(arg->is_wlocked)); + ATOMIC_INC(arg->is_rlocked); + milli_sleep(arg->sleep); + stop = *arg->stop; + ATOMIC_DEC(arg->is_rlocked); + ASSERT(!ATOMIC_READ(arg->is_wlocked)); + RWMUTEX_RUNLOCK(arg->rwlock); + } while (!stop); + return NULL; +} + + +static ERL_NIF_TERM long_rw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + int res, freqread, i, count, stop; + ATOMIC_T is_wlocked, is_rlocked; + THR_ID tid[LONG_RW_NO_THREADS]; + long_rw_t arg; + long_rw_t targ[LONG_RW_NO_THREADS]; + + ATOMIC_INIT(&is_wlocked, 0); + ATOMIC_INIT(&is_rlocked, 0); + + freqread = 0; + + arg.is_wlocked = &is_wlocked; + arg.is_rlocked = &is_rlocked; + arg.count = &count; + arg.stop = &stop; + + restart: + + stop = 0; + count = 0; + + arg.rwlock = RWMUTEX_CREATE(freqread); + + ASSERT(arg.rwlock); + + for (i = 0; i < LONG_RW_NO_W_THREADS; i++) { + targ[i] = arg; + targ[i].sleep = 100 + i*10; + ASSERT(THR_CREATE(&tid[i], long_rw_w, &targ[i], NULL) == 0); + } + for (; i < LONG_RW_NO_THREADS; i++) { + targ[i] = arg; + targ[i].sleep = 100; + ASSERT(THR_CREATE(&tid[i], long_rw_r, &targ[i], NULL) == 0); + } + for (i = 0; i < LONG_RW_NO_THREADS; i++) + ASSERT(THR_JOIN(tid[i], NULL) == 0); + + ASSERT(!ATOMIC_READ(arg.is_wlocked)); + ASSERT(!ATOMIC_READ(arg.is_rlocked)); + + RWMUTEX_DESTROY(arg.rwlock); + + if (HAVE_FREQREAD_SUPPORT && !freqread) { + freqread = 1; + goto restart; + } + + if (freqread) + return enif_make_atom(env, "ok"); + else + return enif_make_tuple2(env, + enif_make_atom(env, + "comment"), + enif_make_string(env, + "No frequent read test made.", + ERL_NIF_LATIN1)); +} + +/* + * Hammer rwlock testcase + */ + +#define HAMMER_RW_NO_W_THREADS 6 +#define HAMMER_RW_NO_THREADS 20 +#define HAMMER_RW_NO_WLOCK_COUNT 1000000 + +typedef struct { + RWMUTEX_T *rwlock; + ATOMIC_T is_locked; + int lock_check; + int stop; + int count; +} hammer_rw_t; + +static void * +hammer_rw_w(void *varg) +{ + hammer_rw_t *arg = varg; + int stop = 0; + do { + RWMUTEX_WLOCK(arg->rwlock); + if (arg->lock_check) { + ASSERT(!ATOMIC_READ(&arg->is_locked)); + ATOMIC_SET(&arg->is_locked, -1); + } + if (++arg->count > HAMMER_RW_NO_WLOCK_COUNT) + stop = arg->stop = 1; + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) == -1); + ATOMIC_SET(&arg->is_locked, 0); + } + RWMUTEX_WUNLOCK(arg->rwlock); + } while (!stop); + return NULL; +} + +static void * +hammer_rw_r(void *varg) +{ + hammer_rw_t *arg = varg; + int stop; + do { + RWMUTEX_RLOCK(arg->rwlock); + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) >= 0); + ATOMIC_INC(&arg->is_locked); + } + stop = arg->stop; + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) > 0); + ATOMIC_DEC(&arg->is_locked); + } + RWMUTEX_RUNLOCK(arg->rwlock); + } while (!stop); + return NULL; +} + + +static ERL_NIF_TERM hammer_rw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + hammer_rw_t arg; + char buf[10]; + int res, freqread, i; + THR_ID tid[HAMMER_RW_NO_THREADS]; + + if (argc != 1) + goto badarg; + + arg.lock_check = get_bool(env, argv[0]); + if (arg.lock_check < 0) + goto badarg; + + ATOMIC_INIT(&arg.is_locked, 0); + + freqread = 0; + + restart: + arg.stop = 0; + arg.count = 0; + + arg.rwlock = RWMUTEX_CREATE(freqread); + + ASSERT(arg.rwlock); + + for (i = 0; i < HAMMER_RW_NO_W_THREADS; i++) + ASSERT(THR_CREATE(&tid[i], hammer_rw_w, &arg, NULL) == 0); + for (; i < HAMMER_RW_NO_THREADS; i++) + ASSERT(THR_CREATE(&tid[i], hammer_rw_r, &arg, NULL) == 0); + for (i = 0; i < HAMMER_RW_NO_THREADS; i++) + ASSERT(THR_JOIN(tid[i], NULL) == 0); + + ASSERT(!ATOMIC_READ(&arg.is_locked)); + + RWMUTEX_DESTROY(arg.rwlock); + + if (HAVE_FREQREAD_SUPPORT && !freqread) { + freqread = 1; + goto restart; + } + + if (freqread) + return enif_make_atom(env, "ok"); + else + return enif_make_tuple2(env, + enif_make_atom(env, + "comment"), + enif_make_string(env, + "No frequent read test made.", + ERL_NIF_LATIN1)); + badarg: + return enif_make_badarg(env); +} + +/* + * Hammer try rwlock testcase + */ + +#define HAMMER_TRYRW_NO_W_THREADS 10 +#define HAMMER_TRYRW_NO_THREADS 20 +#define HAMMER_TRYRW_NO_WLOCK_COUNT 10000000 +#define HAMMER_TRYRW_NO_RLOCK_COUNT 10000000 +#define HAMMER_TRYRW_NO_WLOCK_WAIT_COUNT ((10*HAMMER_TRYRW_NO_WLOCK_COUNT)/8) +#define HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT ((10*HAMMER_TRYRW_NO_RLOCK_COUNT)/8) + +typedef struct { + RWMUTEX_T *rwlock; + ATOMIC_T is_locked; + int lock_check; + int w_count; + ATOMIC_T r_count; +} hammer_tryrw_t; + +static void * +hammer_tryrw_w(void *varg) +{ + hammer_tryrw_t *arg = varg; + int stop = 0; + int wait = 0; + do { + while (EBUSY == RWMUTEX_TRYWLOCK(arg->rwlock)); + if (arg->lock_check) { + ASSERT(!ATOMIC_READ(&arg->is_locked)); + ATOMIC_SET(&arg->is_locked, -1); + } + if (++arg->w_count > HAMMER_TRYRW_NO_WLOCK_COUNT) + stop = 1; + else if (arg->w_count > HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT) + wait = 1; + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) == -1); + ATOMIC_SET(&arg->is_locked, 0); + } + RWMUTEX_WUNLOCK(arg->rwlock); + if (wait) + milli_sleep(1); + } while (!stop); + return NULL; +} + +static void * +hammer_tryrw_r(void *varg) +{ + hammer_tryrw_t *arg = varg; + long r_count; + int stop = 0; + int wait = 0; + do { + while (EBUSY == RWMUTEX_TRYRLOCK(arg->rwlock)); + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) >= 0); + ATOMIC_INC(&arg->is_locked); + } + ATOMIC_INC(&arg->r_count); + r_count = ATOMIC_READ(&arg->r_count); + if (r_count > HAMMER_TRYRW_NO_RLOCK_COUNT) + stop = 1; + else if (r_count > HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT) + wait = 1; + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) > 0); + ATOMIC_DEC(&arg->is_locked); + } + RWMUTEX_RUNLOCK(arg->rwlock); + if (wait) + milli_sleep(1); + } while (!stop); + return NULL; +} + + +static ERL_NIF_TERM hammer_tryrw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + hammer_tryrw_t arg; + char buf[10]; + int res, freqread, i; + THR_ID tid[HAMMER_TRYRW_NO_THREADS]; + + if (argc != 1) + goto badarg; + + arg.lock_check = get_bool(env, argv[0]); + if (arg.lock_check < 0) + goto badarg; + + ATOMIC_INIT(&arg.is_locked, 0); + freqread = 0; + + restart: + + arg.w_count = 0; + ATOMIC_INIT(&arg.r_count, 0); + + arg.rwlock = RWMUTEX_CREATE(freqread); + + ASSERT(arg.rwlock); + + for (i = 0; i < HAMMER_TRYRW_NO_W_THREADS; i++) + ASSERT(THR_CREATE(&tid[i], hammer_tryrw_w, &arg, NULL) == 0); + for (; i < HAMMER_TRYRW_NO_THREADS; i++) + ASSERT(THR_CREATE(&tid[i], hammer_tryrw_r, &arg, NULL) == 0); + for (i = 0; i < HAMMER_TRYRW_NO_THREADS; i++) + ASSERT(THR_JOIN(tid[i], NULL) == 0); + + ASSERT(!ATOMIC_READ(&arg.is_locked)); + + RWMUTEX_DESTROY(arg.rwlock); + + if (HAVE_FREQREAD_SUPPORT && !freqread) { + freqread = 1; + goto restart; + } + + if (freqread) + return enif_make_atom(env, "ok"); + else + return enif_make_tuple2(env, + enif_make_atom(env, + "comment"), + enif_make_string(env, + "No frequent read test made.", + ERL_NIF_LATIN1)); + badarg: + return enif_make_badarg(env); +} + +typedef struct { + int lock_check; + ATOMIC_T is_locked; + RWMUTEX_T *rwlock; +} rwlock_resource_t; + +static void +rwlock_destructor(ErlNifEnv* env, void* obj) +{ + rwlock_resource_t *rwlr = obj; + if (rwlr->lock_check) + ASSERT(!ATOMIC_READ(&rwlr->is_locked)); + RWMUTEX_DESTROY(rwlr->rwlock); +} + +/* + * create_rwlock(FreqRead, LockCheck) + */ + +static ERL_NIF_TERM +create_rwlock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + int lock_check, freqread; + ERL_NIF_TERM rwlock_term; + rwlock_resource_t *rwlr; + char buf[100]; + + if (argc != 2) + goto badarg; + + freqread = get_bool(env, argv[0]); + if (freqread < 0) + goto badarg; + + if (!HAVE_FREQREAD_SUPPORT && freqread) + return enif_make_atom(env, "enotsup"); + + lock_check = get_bool(env, argv[1]); + if (lock_check < 0) + goto badarg; + + rwlr = enif_alloc_resource(enif_priv_data(env), sizeof(rwlock_resource_t)); + rwlr->lock_check = lock_check; + ATOMIC_INIT(&rwlr->is_locked, 0); + rwlr->rwlock = RWMUTEX_CREATE(freqread); + rwlock_term = enif_make_resource(env, rwlr); + enif_release_resource(rwlr); + return rwlock_term; + + badarg: + return enif_make_badarg(env); +} + +/* + * rwlock_op(RWLock, Blocking, WriteOp, WaitTime) + */ + +static ERL_NIF_TERM +rwlock_op(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + rwlock_resource_t *rwlr; + int blocking, write, wait_locked, wait_unlocked; + + if (argc != 5) + goto badarg; + + if (!enif_get_resource(env, argv[0], enif_priv_data(env), (void **) &rwlr)) + goto badarg; + + blocking = get_bool(env, argv[1]); + if (blocking < 0) + goto badarg; + + write = get_bool(env, argv[2]); + if (write < 0) + goto badarg; + + if (!enif_get_int(env, argv[3], &wait_locked)) + goto badarg; + if (wait_locked < 0) + goto badarg; + + if (!enif_get_int(env, argv[4], &wait_unlocked)) + goto badarg; + if (wait_unlocked < 0) + goto badarg; + + if (write) { + if (blocking) + RWMUTEX_WLOCK(rwlr->rwlock); + else + while (EBUSY == RWMUTEX_TRYWLOCK(rwlr->rwlock)); + if (rwlr->lock_check) { + ASSERT(!ATOMIC_READ(&rwlr->is_locked)); + ATOMIC_SET(&rwlr->is_locked, -1); + } + } + else { + if (blocking) + RWMUTEX_RLOCK(rwlr->rwlock); + else + while (EBUSY == RWMUTEX_TRYRLOCK(rwlr->rwlock)); + if (rwlr->lock_check) { + ASSERT(ATOMIC_READ(&rwlr->is_locked) >= 0); + ATOMIC_INC(&rwlr->is_locked); + } + } + + if (wait_locked) + milli_sleep(wait_locked); + + if (write) { + if (rwlr->lock_check) { + ASSERT(ATOMIC_READ(&rwlr->is_locked) == -1); + ATOMIC_SET(&rwlr->is_locked, 0); + } + RWMUTEX_WUNLOCK(rwlr->rwlock); + } + else { + if (rwlr->lock_check) { + ASSERT(ATOMIC_READ(&rwlr->is_locked) > 0); + ATOMIC_DEC(&rwlr->is_locked); + } + RWMUTEX_RUNLOCK(rwlr->rwlock); + } + + if (wait_unlocked) + milli_sleep(wait_unlocked); + + return enif_make_atom(env, "ok"); + badarg: + return enif_make_badarg(env); +} + +static int load_nif_lib(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + *priv_data = enif_open_resource_type(env, + NULL, + "rwlock_resource", + rwlock_destructor, + ERL_NIF_RT_CREATE, + NULL); + if (*priv_data) + return 0; + else + return -1; +} + +/* + * 0 -> false + * >0 -> true + * <0 -> error + */ + +static int +get_bool(ErlNifEnv* env, ERL_NIF_TERM term) +{ + int res; + char buf[10]; + + res = enif_get_atom(env, term, buf, sizeof(buf), ERL_NIF_LATIN1); + if (res == 0) + return -1; + if (strcmp("false", buf) == 0) + return 0; + else if (strcmp("true", buf) == 0) + return 1; + else + return -1; +} + +static int +fail(const char *file, int line, const char *function, const char *assertion) +{ + fprintf(stderr, "%s:%d: Assertion failed in %s(): %s\n", + file, line, function, assertion); + abort(); +} + +static void +milli_sleep(int ms) +{ +#ifdef __WIN32__ + Sleep(ms); +#else + while (erts_milli_sleep(ms) != 0); +#endif +} + +static ErlNifFunc nif_funcs[] = { + {"long_rw_test", 0, long_rw_test}, + {"hammer_rw_test", 1, hammer_rw_test}, + {"hammer_tryrw_test", 1, hammer_tryrw_test}, + {"create_rwlock", 2, create_rwlock}, + {"rwlock_op", 5, rwlock_op} +}; + +ERL_NIF_INIT(mtx_SUITE, nif_funcs, load_nif_lib, NULL, NULL, NULL) diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 522caec8f1..42947aa6be 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -20,23 +20,39 @@ -module(nif_SUITE). %%-define(line_trace,true). -%%-define(CHECK(Exp,Got), ?line check(Exp,Got,?LINE)). --define(CHECK(Exp,Got), ?line Exp = Got). +-define(CHECK(Exp,Got), check(Exp,Got,?LINE)). +%%-define(CHECK(Exp,Got), ?line Exp = Got). -include("test_server.hrl"). --export([all/1, fin_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1, +-export([all/1, + %%init_per_testcase/2, + fin_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1, types/1, many_args/1, binaries/1, get_string/1, get_atom/1, api_macros/1, - from_array/1, iolist_as_binary/1, resource/1, resource_takeover/1, - threading/1, neg/1]). + from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, resource_takeover/1, + threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, is_checks/1, + get_length/1, make_atom/1, make_string/1]). -export([many_args_100/100]). + + +%% -export([lib_version/0,call_history/0,hold_nif_mod_priv_data/1,nif_mod_call_history/0, +%% list_seq/1,type_test/0,tuple_2_list/1,is_identical/2,compare/2, +%% clone_bin/1,make_sub_bin/3,string_to_bin/2,atom_to_bin/2,macros/1, +%% tuple_2_list_and_tuple/1,iolist_2_bin/1,get_resource_type/1,alloc_resource/2, +%% make_resource/1,get_resource/2,release_resource/1,last_resource_dtor_call/0, +%% make_new_resource/2,make_new_resource_binary/1,send_list_seq/2,send_new_blob/2, +%% alloc_msgenv/0,clear_msgenv/1,grow_blob/2,send_blob/2,send_blob_thread/3, +%% join_send_thread/1]). + + -define(nif_stub,nif_stub_error(?LINE)). all(suite) -> [basic, reload, upgrade, heap_frag, types, many_args, binaries, get_string, - get_atom, api_macros, from_array, iolist_as_binary, resource, - resource_takeover, threading, neg]. + get_atom, api_macros, from_array, iolist_as_binary, resource, resource_binary, + resource_takeover, threading, send, send2, send3, send_threaded, neg, is_checks, + get_length, make_atom, make_string]. %%init_per_testcase(_Case, Config) -> %% ?line Dog = ?t:timetrap(?t:seconds(60*60*24)), @@ -57,7 +73,7 @@ basic(Config) when is_list(Config) -> ?line true = (lib_version() =/= undefined), ?line [{load,1,1,101},{lib_version,1,2,102}] = call_history(), ?line [] = call_history(), - ?line [?MODULE] = erlang:system_info(taints), + ?line true = lists:member(?MODULE, erlang:system_info(taints)), ok. reload(doc) -> ["Test reload callback in nif lib"]; @@ -91,7 +107,8 @@ reload(Config) when is_list(Config) -> ?line true = erlang:purge_module(nif_mod), ?line [{unload,1,3,103}] = nif_mod_call_history(), - ?line [?MODULE, nif_mod] = erlang:system_info(taints), + ?line true = lists:member(?MODULE, erlang:system_info(taints)), + ?line true = lists:member(nif_mod, erlang:system_info(taints)), ?line verify_tmpmem(TmpMem), ok. @@ -181,7 +198,8 @@ upgrade(Config) when is_list(Config) -> ?line true = erlang:purge_module(nif_mod), ?line [{unload,2,4,204}] = nif_mod_call_history(), - ?line [?MODULE, nif_mod] = erlang:system_info(taints), + ?line true = lists:member(?MODULE, erlang:system_info(taints)), + ?line true = lists:member(nif_mod, erlang:system_info(taints)), ?line verify_tmpmem(TmpMem), ok. @@ -473,12 +491,51 @@ resource_new_do2(Type) -> {{PtrA,BinA}, {ResB,PtrB,BinB}}. resource_neg(TypeA) -> + resource_neg_do(TypeA), + + catch exit(42), % dummy exception to purge saved stacktraces from earlier exception + erlang:garbage_collect(), + ?line {_,_,2} = last_resource_dtor_call(), + ok. + +resource_neg_do(TypeA) -> TypeB = get_resource_type(1), - Aptr = alloc_resource(TypeA, <<"Arnold">>), - Bptr = alloc_resource(TypeB, <<"Bobo">>), - ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeA, Bptr)), - ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeB, Aptr)), + ResA = make_new_resource(TypeA, <<"Arnold">>), + ResB= make_new_resource(TypeB, <<"Bobo">>), + ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeA, ResB)), + ?line {'EXIT',{badarg,_}} = (catch get_resource(TypeB, ResA)), ok. + +resource_binary(doc) -> ["Test enif_make_resource_binary"]; +resource_binary(suite) -> []; +resource_binary(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + ?line {Ptr,Bin} = resource_binary_do(), + erlang:garbage_collect(), + Last = last_resource_dtor_call(), + ?CHECK({Ptr,Bin,1}, Last), + ok. + +resource_binary_do() -> + Bin = <<"Hej Hopp i lingonskogen">>, + ?line {Ptr,ResBin1} = make_new_resource_binary(Bin), + ?line ResBin1 = Bin, + ?line ResInfo = {Ptr,_} = get_resource(binary_resource_type,ResBin1), + + Papa = self(), + Forwarder = spawn_link(fun() -> forwarder(Papa) end), + io:format("sending to forwarder pid=~p\n",[Forwarder]), + Forwarder ! ResBin1, + ResBin2 = receive_any(), + ?line ResBin2 = ResBin1, + ?line ResInfo = get_resource(binary_resource_type,ResBin2), + Forwarder ! terminate, + ?line {Forwarder, 1} = receive_any(), + erlang:garbage_collect(), + ?line ResInfo = get_resource(binary_resource_type,ResBin1), + ?line ResInfo = get_resource(binary_resource_type,ResBin2), + ResInfo. + -define(RT_CREATE,1). -define(RT_TAKEOVER,2). @@ -672,7 +729,8 @@ resource_takeover(Config) when is_list(Config) -> ?line ok = forget_resource(AN4), ?line [] = nif_mod_call_history(), - ?line [?MODULE, nif_mod] = erlang:system_info(taints), + ?line true = lists:member(?MODULE, erlang:system_info(taints)), + ?line true = lists:member(nif_mod, erlang:system_info(taints)), ?line verify_tmpmem(TmpMem), ok. @@ -743,7 +801,282 @@ threading(Config) when is_list(Config) -> ?line ok = tester:load_nif_lib(Config, "tsd"), ?line ok = tester:run(). + +send(doc) -> ["Test NIF message sending"]; +send(Config) when is_list(Config) -> + ensure_lib_loaded(Config), + + N = 1500, + List = lists:seq(1,N), + ?line {ok,1} = send_list_seq(N, self), + ?line {ok,1} = send_list_seq(N, self()), + ?line List = receive_any(), + ?line List = receive_any(), + Papa = self(), + spawn_link(fun() -> ?line {ok,1} = send_list_seq(N, Papa) end), + ?line List = receive_any(), + + ?line {ok, 1, BlobS} = send_new_blob(self(), other_term()), + ?line BlobR = receive_any(), + io:format("Sent ~p\nGot ~p\n", [BlobS, BlobR]), + ?line BlobR = BlobS, + + %% send to dead pid + {DeadPid, DeadMon} = spawn_monitor(fun() -> void end), + ?line {'DOWN', DeadMon, process, DeadPid, normal} = receive_any(), + {ok,0} = send_list_seq(7, DeadPid), + ok. + +send2(doc) -> ["More NIF message sending"]; +send2(Config) when is_list(Config) -> + ensure_lib_loaded(Config), + + send2_do1(fun send_blob_dbg/2), + ok. + +send_threaded(doc) -> ["Send msg from user thread"]; +send_threaded(Config) when is_list(Config) -> + case erlang:system_info(smp_support) of + true -> + send2_do1(fun(ME,To) -> send_blob_thread_dbg(ME,To,join) end), + send2_do1(fun(ME,To) -> send_blob_thread_and_join(ME,To) end), + ok; + false -> + {skipped,"No threaded send on non-SMP"} + end. + + +send2_do1(SendBlobF) -> + io:format("sending to self=~p\n",[self()]), + send2_do2(SendBlobF, self()), + + Papa = self(), + Forwarder = spawn_link(fun() -> forwarder(Papa) end), + io:format("sending to forwarder pid=~p\n",[Forwarder]), + send2_do2(SendBlobF, Forwarder), + Forwarder ! terminate, + ?line {Forwarder, 4} = receive_any(), + ok. + +send2_do2(SendBlobF, To) -> + MsgEnv = alloc_msgenv(), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + ?line {ok,1,Blob0} = SendBlobF(MsgEnv, To), + ?line Blob1 = receive_any(), + ?line Blob1 = Blob0, + clear_msgenv(MsgEnv), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + ?line {ok,1,Blob2} = SendBlobF(MsgEnv, To), + ?line Blob3 = receive_any(), + ?line Blob3 = Blob2, + + clear_msgenv(MsgEnv), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + + clear_msgenv(MsgEnv), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + ?line {ok,1,Blob4} = SendBlobF(MsgEnv, To), + ?line Blob5 = receive_any(), + ?line Blob5 = Blob4, + + clear_msgenv(MsgEnv), + clear_msgenv(MsgEnv), + repeat(50, fun(_) -> grow_blob(MsgEnv,other_term()) end, []), + ?line {ok,1,Blob6} = SendBlobF(MsgEnv, To), + ?line Blob7 = receive_any(), + ?line Blob7 = Blob6, + + ok. + + +send_blob_thread_and_join(MsgEnv, To) -> + ?line {ok,Blob} = send_blob_thread_dbg(MsgEnv, To, no_join), + ?line {ok,SendRes} = join_send_thread(MsgEnv), + {ok,SendRes,Blob}. + +send_blob_dbg(MsgEnv, To) -> + Ret = send_blob(MsgEnv, To), + %%io:format("send_blob to ~p returned ~p\n",[To,Ret]), + Ret. + +send_blob_thread_dbg(MsgEnv, To, Join) -> + Ret = send_blob_thread(MsgEnv, To, Join), + %%io:format("send_blob_thread to ~p Join=~p returned ~p\n",[To,Join,Ret]), + Ret. + + +forwarder(To) -> + forwarder(To, 0). +forwarder(To, N) -> + case receive_any() of + terminate -> + To ! {self(), N}; + Msg -> + To ! Msg, + forwarder(To, N+1) + end. + +other_term() -> + {fun(X,Y) -> X*Y end, make_ref()}. + +send3(doc) -> ["Message sending stress test"]; +send3(Config) when is_list(Config) -> + %% Let a number of processes send random message blobs between each other + %% using enif_send. Kill and spawn new ones randomly to keep a ~constant + %% number of workers running. + Seed = now(), + io:format("seed: ~p\n",[Seed]), + random:seed(Seed), + ets:new(nif_SUITE,[named_table,public]), + ?line true = ets:insert(nif_SUITE,{send3,0,0,0,0}), + timer:send_after(10000, timeout), % Run for 10 seconds + SpawnCnt = send3_controller(0, [], [], 20), + ?line [{_,Rcv,SndOk,SndFail,Balance}] = ets:lookup(nif_SUITE,send3), + io:format("spawns=~p received=~p, sent=~p send-failure=~p balance=~p\n", + [SpawnCnt,Rcv,SndOk,SndFail,Balance]), + ets:delete(nif_SUITE). + +send3_controller(SpawnCnt, [], _, infinity) -> + SpawnCnt; +send3_controller(SpawnCnt0, Mons0, Pids0, Tick) -> + receive + timeout -> + io:format("Timeout. Sending 'halt' to ~p\n",[Pids0]), + lists:foreach(fun(P) -> P ! {halt,self()} end, Pids0), + lists:foreach(fun(P) -> receive {halted,P} -> ok end end, Pids0), + QTot = lists:foldl(fun(P,QSum) -> + {message_queue_len,QLen} = + erlang:process_info(P,message_queue_len), + QSum + QLen + end, 0, Pids0), + io:format("Total queue length ~p\n",[QTot]), + lists:foreach(fun(P) -> P ! die end, Pids0), + send3_controller(SpawnCnt0, Mons0, [], infinity); + {'DOWN', MonRef, process, _Pid, _} -> + Mons1 = lists:delete(MonRef, Mons0), + %%io:format("Got DOWN from ~p. Monitors left: ~p\n",[Pid,Mons1]), + send3_controller(SpawnCnt0, Mons1, Pids0, Tick) + after Tick -> + Max = 20, + N = length(Pids0), + PidN = random:uniform(Max), + %%io:format("N=~p PidN=~p Pids0=~p\n", [N,PidN,Pids0]), + case PidN > N of + true -> + {NewPid,Mon} = spawn_opt(fun send3_proc/0, [link,monitor]), + lists:foreach(fun(P) -> P ! {is_born,NewPid} end, Pids0), + ?line Balance = ets:lookup_element(nif_SUITE,send3,5), + Inject = (Balance =< 0), + case Inject of + true -> ok; + false -> ets:update_element(nif_SUITE,send3,{5,-1}) + end, + NewPid ! {pids,Pids0,Inject}, + send3_controller(SpawnCnt0+1, [Mon|Mons0], [NewPid|Pids0], Tick); + false -> + KillPid = lists:nth(PidN,Pids0), + KillPid ! die, + Pids1 = lists:delete(KillPid, Pids0), + lists:foreach(fun(P) -> P ! {is_dead,KillPid} end, Pids1), + send3_controller(SpawnCnt0, Mons0, Pids1, Tick) + end + end. + +send3_proc() -> + %%io:format("Process ~p spawned\n",[self()]), + send3_proc([self()], {0,0,0}, {1,2,3,4,5}). +send3_proc(Pids0, Counters={Rcv,SndOk,SndFail}, State0) -> + %%io:format("~p: Pids0=~p", [self(), Pids0]), + %%timer:sleep(10), + receive + {pids, Pids1, Inject} -> + %%io:format("~p: got ~p Inject=~p\n", [self(), Pids1, Inject]), + ?line Pids0 = [self()], + Pids2 = [self() | Pids1], + case Inject of + true -> send3_proc_send(Pids2, Counters, State0); + false -> send3_proc(Pids2, Counters, State0) + end; + {is_born, Pid} -> + %%io:format("~p: is_born ~p, got ~p\n", [self(), Pid, Pids0]), + send3_proc([Pid | Pids0], Counters, State0); + {is_dead, Pid} -> + Pids1 = lists:delete(Pid,Pids0), + %%io:format("~p: is_dead ~p, got ~p\n", [self(), Pid, Pids1]), + send3_proc(Pids1, Counters, State0); + {blob, Blob0} -> + %%io:format("~p: blob ~p\n", [self(), Blob0]), + State1 = send3_new_state(State0, Blob0), + send3_proc_send(Pids0, {Rcv+1,SndOk,SndFail}, State1); + die -> + %%io:format("Process ~p terminating, stats = ~p\n",[self(),Counters]), + {message_queue_len,Dropped} = erlang:process_info(self(),message_queue_len), + _R = ets:update_counter(nif_SUITE,send3, + [{2,Rcv},{3,SndOk},{4,SndFail},{5,1-Dropped}]), + %%io:format("~p: dies R=~p\n", [self(), R]), + ok; + {halt,Papa} -> + Papa ! {halted,self()}, + io:format("~p halted\n",[self()]), + receive die -> ok end, + io:format("~p dying\n",[self()]) + end. + +send3_proc_send(Pids, {Rcv,SndOk,SndFail}, State0) -> + To = lists:nth(random:uniform(length(Pids)),Pids), + Blob = send3_make_blob(), + State1 = send3_new_state(State0,Blob), + case send3_send(To, Blob) of + true -> + send3_proc(Pids, {Rcv,SndOk+1,SndFail}, State1); + false -> + send3_proc(Pids, {Rcv,SndOk,SndFail+1}, State1) + end. + + +send3_make_blob() -> + case random:uniform(20)-1 of + 0 -> {term,[]}; + N -> + MsgEnv = alloc_msgenv(), + repeat(N bsr 1, + fun(_) -> grow_blob(MsgEnv,other_term(),random:uniform(1 bsl 20)) + end, void), + case (N band 1) of + 0 -> {term,copy_blob(MsgEnv)}; + 1 -> {msgenv,MsgEnv} + end + end. + +send3_send(Pid, Msg) -> + %% 90% enif_send and 10% normal bang + case random:uniform(10) of + 1 -> send3_send_bang(Pid,Msg); + _ -> send3_send_nif(Pid,Msg) + end. +send3_send_nif(Pid, {term,Blob}) -> + %%io:format("~p send term nif\n",[self()]), + send_term(Pid, {blob, Blob}) =:= 1; +send3_send_nif(Pid, {msgenv,MsgEnv}) -> + %%io:format("~p send blob nif\n",[self()]), + send3_blob(MsgEnv, Pid, blob) =:= 1. + +send3_send_bang(Pid, {term,Blob}) -> + %%io:format("~p send term bang\n",[self()]), + Pid ! {blob, Blob}, + true; +send3_send_bang(Pid, {msgenv,MsgEnv}) -> + %%io:format("~p send blob bang\n",[self()]), + Pid ! {blob, copy_blob(MsgEnv)}, + true. + +send3_new_state(State, Blob) -> + case random:uniform(5+2) of + N when N =< 5-> setelement(N, State, Blob); + _ -> State % Don't store blob + end. + neg(doc) -> ["Negative testing of load_nif"]; neg(Config) when is_list(Config) -> TmpMem = tmpmem(), @@ -759,7 +1092,17 @@ neg(Config) when is_list(Config) -> ?line verify_tmpmem(TmpMem), ?line ok. +is_checks(doc) -> ["Test all enif_is functions"]; +is_checks(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + ?line ok = check_is(hejsan, <<19,98>>, make_ref(), ok, fun() -> ok end, + self(), hd(erlang:ports()), [], [1,9,9,8], + {hejsan, "hejsan", [$h,"ejs",<<"an">>]}). +get_length(doc) -> ["Test all enif_get_length functions"]; +get_length(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + ?line ok = length_test(hejsan, "hejsan", [], [], not_a_list). ensure_lib_loaded(Config) -> ensure_lib_loaded(Config, 1). @@ -773,6 +1116,23 @@ ensure_lib_loaded(Config, Ver) -> ok end. +make_atom(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + An0Atom = an0atom, + An0Atom0 = 'an\000atom\000', + ?line Atoms = make_atoms(), + ?line 7 = size(Atoms), + ?line Atoms = {An0Atom,An0Atom,An0Atom,An0Atom0,An0Atom,An0Atom,An0Atom0}. + +make_string(Config) when is_list(Config) -> + ?line ensure_lib_loaded(Config, 1), + ?line Strings = make_strings(), + ?line 5 = size(Strings), + A0String = "a0string", + A0String0 = [$a,0,$s,$t,$r,$i,$n,$g,0], + AStringWithAccents = [$E,$r,$l,$a,$n,$g,$ ,16#e4,$r,$ ,$e,$t,$t,$ ,$g,$e,$n,$e,$r,$e,$l,$l,$t,$ ,$p,$r,$o,$g,$r,$a,$m,$s,$p,$r,16#e5,$k], + ?line Strings = {A0String,A0String,A0String,A0String0, AStringWithAccents}. + tmpmem() -> case erlang:system_info({allocator,temp_alloc}) of false -> undefined; @@ -821,13 +1181,18 @@ call(Pid,Cmd) -> receive_any() -> receive M -> M end. -%% check(Exp,Got,Line) -> -%% case Got of -%% Exp -> Exp; -%% _ -> -%% io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]), -%% Got -%% end. +repeat(0, _, Arg) -> + Arg; +repeat(N, Fun, Arg0) -> + repeat(N-1, Fun, Fun(Arg0)). + +check(Exp,Got,Line) -> + case Got of + Exp -> Exp; + _ -> + io:format("CHECK at ~p: Expected ~p but got ~p\n",[Line,Exp,Got]), + Got + end. %% The NIFs: @@ -855,6 +1220,23 @@ get_resource(_,_) -> ?nif_stub. release_resource(_) -> ?nif_stub. last_resource_dtor_call() -> ?nif_stub. make_new_resource(_,_) -> ?nif_stub. +check_is(_,_,_,_,_,_,_,_,_,_) -> ?nif_stub. +length_test(_,_,_,_,_) -> ?nif_stub. +make_atoms() -> ?nif_stub. +make_strings() -> ?nif_stub. +make_new_resource_binary(_) -> ?nif_stub. +send_list_seq(_,_) -> ?nif_stub. +send_new_blob(_,_) -> ?nif_stub. +alloc_msgenv() -> ?nif_stub. +clear_msgenv(_) -> ?nif_stub. +grow_blob(_,_) -> ?nif_stub. +grow_blob(_,_,_) -> ?nif_stub. +send_blob(_,_) -> ?nif_stub. +send3_blob(_,_,_) -> ?nif_stub. +send_blob_thread(_,_,_) -> ?nif_stub. +join_send_thread(_) -> ?nif_stub. +copy_blob(_) -> ?nif_stub. +send_term(_,_) -> ?nif_stub. nif_stub_error(Line) -> exit({nif_not_loaded,module,?MODULE,line,Line}). diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index 3ad4f93374..8489124966 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -28,6 +28,12 @@ static int static_cntA; /* zero by default */ static int static_cntB = NIF_SUITE_LIB_VER * 100; +static ERL_NIF_TERM atom_self; +static ERL_NIF_TERM atom_ok; +static ERL_NIF_TERM atom_join; +static ERL_NIF_TERM atom_binary_resource_type; + + typedef struct { int ref_cnt; @@ -38,7 +44,7 @@ typedef struct void add_call(ErlNifEnv* env, PrivData* data, const char* func_name) { - CallInfo* call = enif_alloc(env, sizeof(CallInfo)+strlen(func_name)); + CallInfo* call = enif_alloc(sizeof(CallInfo)+strlen(func_name)); strcpy(call->func_name, func_name); call->lib_ver = NIF_SUITE_LIB_VER; call->next = data->call_history; @@ -49,7 +55,7 @@ void add_call(ErlNifEnv* env, PrivData* data, const char* func_name) call->arg_sz = 0; } -#define ADD_CALL(FUNC_NAME) add_call(env, enif_get_data(env),FUNC_NAME) +#define ADD_CALL(FUNC_NAME) add_call(env, enif_priv_data(env),FUNC_NAME) static void* resource_dtor_last = NULL; static unsigned resource_dtor_last_sz = 0; @@ -60,15 +66,24 @@ static void resource_dtor(ErlNifEnv* env, void* obj) { resource_dtor_last = obj; resource_dtor_cnt++; - resource_dtor_last_sz = enif_sizeof_resource(env, obj); + resource_dtor_last_sz = enif_sizeof_resource(obj); assert(resource_dtor_last_sz <= sizeof(resource_dtor_last_data)); memcpy(resource_dtor_last_data, obj, resource_dtor_last_sz); } +static ErlNifResourceType* msgenv_resource_type; +static void msgenv_dtor(ErlNifEnv* env, void* obj); + +static ErlNifResourceType* binary_resource_type; +static void binary_resource_dtor(ErlNifEnv* env, void* obj); +struct binary_resource { + unsigned char* data; + unsigned size; +}; + static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { - /*ERL_NIF_TERM head, tail;*/ - PrivData* data = enif_alloc(env, sizeof(PrivData)); + PrivData* data = enif_alloc(sizeof(PrivData)); assert(data != NULL); data->ref_cnt = 1; data->call_history = NULL; @@ -76,41 +91,71 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) add_call(env, data, "load"); - /* - head = load_info; - data->rt_cnt = 0; - for (head=load_info; enif_get_list_cell(env,load_info,&head,&tail); - head=tail) { - char buf[20]; - int n = enif_get_string(env,head,buf,sizeof(buf)); - assert(n > 0); - assert(i < sizeof(data->rt_arr)/sizeof(*data->rt_arr)); - data->rt_arr[data->rt_cnt++].t = enif_create_resource_type(env,buf,resource_dtor, - ERL_NIF_RT_CREATE,NULL); - } - assert(enif_is_empty_list(env,head)); - */ - data->rt_arr[0].t = enif_open_resource_type(env,"Gold",resource_dtor, + data->rt_arr[0].t = enif_open_resource_type(env,NULL,"Gold",resource_dtor, ERL_NIF_RT_CREATE,NULL); - data->rt_arr[1].t = enif_open_resource_type(env,"Silver",resource_dtor, + data->rt_arr[1].t = enif_open_resource_type(env,NULL,"Silver",resource_dtor, ERL_NIF_RT_CREATE,NULL); + binary_resource_type = enif_open_resource_type(env,NULL,"nif_SUITE.binary", + binary_resource_dtor, + ERL_NIF_RT_CREATE, NULL); + + msgenv_resource_type = enif_open_resource_type(env,NULL,"nif_SUITE.msgenv", + msgenv_dtor, + ERL_NIF_RT_CREATE, NULL); + + atom_self = enif_make_atom(env,"self"); + atom_ok = enif_make_atom(env,"ok"); + atom_join = enif_make_atom(env,"join"); + atom_binary_resource_type = enif_make_atom(env,"binary_resource_type"); + *priv_data = data; return 0; } +static void resource_takeover(ErlNifEnv* env, PrivData* priv) +{ + ErlNifResourceFlags tried; + ErlNifResourceType* rt; + rt = enif_open_resource_type(env, NULL, "Gold", resource_dtor, + ERL_NIF_RT_TAKEOVER, &tried); + assert(rt == priv->rt_arr[0].t); + assert(tried == ERL_NIF_RT_TAKEOVER); + rt = enif_open_resource_type(env, NULL, "Silver", resource_dtor, + ERL_NIF_RT_TAKEOVER, &tried); + assert(rt == priv->rt_arr[1].t); + assert(tried == ERL_NIF_RT_TAKEOVER); + + rt = enif_open_resource_type(env, NULL, "nif_SUITE.binary", binary_resource_dtor, + ERL_NIF_RT_TAKEOVER, &tried); + assert(rt != NULL); + assert(tried == ERL_NIF_RT_TAKEOVER); + assert(binary_resource_type==NULL || binary_resource_type == rt); + binary_resource_type = rt; + + rt = enif_open_resource_type(env, NULL, "nif_SUITE.msgenv", msgenv_dtor, + ERL_NIF_RT_TAKEOVER, &tried); + assert(rt != NULL); + assert(tried == ERL_NIF_RT_TAKEOVER); + assert(msgenv_resource_type==NULL || msgenv_resource_type == rt); + msgenv_resource_type = rt; +} + static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { - add_call(env, *priv_data, "reload"); + PrivData* priv = (PrivData*) *priv_data; + add_call(env, priv, "reload"); + resource_takeover(env,priv); return 0; } static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { - PrivData* data = *old_priv_data; - add_call(env, data, "upgrade"); - data->ref_cnt++; - *priv_data = *old_priv_data; + PrivData* priv = (PrivData*) *old_priv_data; + add_call(env, priv, "upgrade"); + priv->ref_cnt++; + *priv_data = *old_priv_data; + resource_takeover(env,priv); return 0; } @@ -119,7 +164,10 @@ static void unload(ErlNifEnv* env, void* priv_data) PrivData* data = priv_data; add_call(env, data, "unload"); if (--data->ref_cnt == 0) { - enif_free(env, priv_data); + if (data->nif_mod != NULL) { + NifModPrivData_release(data->nif_mod); + } + enif_free(priv_data); } } @@ -149,28 +197,28 @@ static ERL_NIF_TERM make_call_history(ErlNifEnv* env, CallInfo** headp) enif_make_int(env,call->static_cntB)); list = enif_make_list_cell(env, tpl, list); *headp = call->next; - enif_free(env,call); + enif_free(call); } return list; } static ERL_NIF_TERM call_history(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - PrivData* data = (PrivData*) enif_get_data(env); + PrivData* data = (PrivData*) enif_priv_data(env); return make_call_history(env,&data->call_history); } static ERL_NIF_TERM hold_nif_mod_priv_data(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - PrivData* data = (PrivData*) enif_get_data(env); + PrivData* data = (PrivData*) enif_priv_data(env); unsigned long ptr_as_ulong; if (!enif_get_ulong(env,argv[0],&ptr_as_ulong)) { return enif_make_badarg(env); } - if (data->nif_mod != NULL && --(data->nif_mod->ref_cnt) == 0) { - enif_free(env,data->nif_mod); + if (data->nif_mod != NULL) { + NifModPrivData_release(data->nif_mod); } data->nif_mod = (NifModPrivData*) ptr_as_ulong; return enif_make_int(env,++(data->nif_mod->ref_cnt)); @@ -178,7 +226,7 @@ static ERL_NIF_TERM hold_nif_mod_priv_data(ErlNifEnv* env, int argc, const ERL_N static ERL_NIF_TERM nif_mod_call_history(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - PrivData* data = (PrivData*) enif_get_data(env); + PrivData* data = (PrivData*) enif_priv_data(env); ERL_NIF_TERM ret; if (data->nif_mod == NULL) { return enif_make_string(env,"nif_mod pointer is NULL", ERL_NIF_LATIN1); @@ -248,6 +296,30 @@ static int test_ulong(ErlNifEnv* env, unsigned long i1) return 1; } +static int test_int64(ErlNifEnv* env, ErlNifSInt64 i1) +{ + ErlNifSInt64 i2 = 0; + ERL_NIF_TERM int_term = enif_make_int64(env, i1); + if (!enif_get_int64(env,int_term, &i2) || i1 != i2) { + fprintf(stderr, "test_int64(%ld) ...FAILED i2=%ld\r\n", + (long)i1, (long)i2); + return 0; + } + return 1; +} + +static int test_uint64(ErlNifEnv* env, ErlNifUInt64 i1) +{ + ErlNifUInt64 i2 = 0; + ERL_NIF_TERM int_term = enif_make_uint64(env, i1); + if (!enif_get_uint64(env,int_term, &i2) || i1 != i2) { + fprintf(stderr, "test_ulong(%lu) ...FAILED i2=%lu\r\n", + (unsigned long)i1, (unsigned long)i2); + return 0; + } + return 1; +} + static int test_double(ErlNifEnv* env, double d1) { double d2 = 0; @@ -271,6 +343,8 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ unsigned uint; long slong; unsigned long ulong; + ErlNifSInt64 sint64; + ErlNifUInt64 uint64; double d; ERL_NIF_TERM atom, ref1, ref2; @@ -304,11 +378,25 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ slong -= slong / 3 + 1; } while (slong >= 0); + sint64 = ((ErlNifSInt64)1 << 63); /* INT64_MIN */ + do { + if (!test_int64(env,sint64)) { + goto error; + } + sint64 += ~sint64 / 3 + 1; + } while (sint64 < 0); + sint64 = ((ErlNifUInt64)1 << 63) - 1; /* INT64_MAX */ + do { + if (!test_int64(env,sint64)) { + goto error; + } + sint64 -= sint64 / 3 + 1; + } while (sint64 >= 0); uint = UINT_MAX; for (;;) { if (!test_uint(env,uint)) { - + goto error; } if (uint == 0) break; uint -= uint / 3 + 1; @@ -316,11 +404,19 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ ulong = ULONG_MAX; for (;;) { if (!test_ulong(env,ulong)) { - + goto error; } if (ulong == 0) break; ulong -= ulong / 3 + 1; } + uint64 = (ErlNifUInt64)-1; /* UINT64_MAX */ + for (;;) { + if (!test_uint64(env,uint64)) { + goto error; + } + if (uint64 == 0) break; + uint64 -= uint64 / 3 + 1; + } if (MAX_SMALL < INT_MAX) { /* 32-bit */ for (i=-10 ; i <= 10; i++) { @@ -343,24 +439,31 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ for (i=-10 ; i < 10; i++) { if (!test_long(env,MAX_SMALL+i) || !test_ulong(env,MAX_SMALL+i) || - !test_long(env,MIN_SMALL+i)) { + !test_long(env,MIN_SMALL+i) || + !test_int64(env,MAX_SMALL+i) || !test_uint64(env,MAX_SMALL+i) || + !test_int64(env,MIN_SMALL+i)) { goto error; } + if (MAX_SMALL < INT_MAX) { + if (!test_int(env,MAX_SMALL+i) || !test_uint(env,MAX_SMALL+i) || + !test_int(env,MIN_SMALL+i)) { + goto error; + } + } } - for (d=3.141592e-100 ; d < 1e100 ; d *= 9.97) { if (!test_double(env,d) || !test_double(env,-d)) { goto error; } } - if (!enif_make_existing_atom(env,"nif_SUITE", &atom) - || !enif_is_identical(env,atom,enif_make_atom(env,"nif_SUITE"))) { + if (!enif_make_existing_atom(env,"nif_SUITE", &atom, ERL_NIF_LATIN1) + || !enif_is_identical(atom,enif_make_atom(env,"nif_SUITE"))) { fprintf(stderr, "nif_SUITE not an atom?\r\n"); goto error; } for (i=2; i; i--) { - if (enif_make_existing_atom(env,"nif_SUITE_pink_unicorn", &atom)) { + if (enif_make_existing_atom(env,"nif_SUITE_pink_unicorn", &atom, ERL_NIF_LATIN1)) { fprintf(stderr, "pink unicorn exist?\r\n"); goto error; } @@ -368,7 +471,7 @@ static ERL_NIF_TERM type_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[ ref1 = enif_make_ref(env); ref2 = enif_make_ref(env); if (!enif_is_ref(env,ref1) || !enif_is_ref(env,ref2) - || enif_is_identical(env,ref1,ref2) || enif_compare(env,ref1,ref2)==0) { + || enif_is_identical(ref1,ref2) || enif_compare(ref1,ref2)==0) { fprintf(stderr, "strange refs?\r\n"); goto error; } @@ -398,7 +501,7 @@ static ERL_NIF_TERM is_identical(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar if (argc != 2) { return enif_make_badarg(env); } - return enif_make_atom(env, (enif_is_identical(env,argv[0],argv[1]) ? + return enif_make_atom(env, (enif_is_identical(argv[0],argv[1]) ? "true" : "false")); } @@ -407,7 +510,7 @@ static ERL_NIF_TERM compare(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) if (argc != 2) { return enif_make_badarg(env); } - return enif_make_int(env, enif_compare(env,argv[0],argv[1])); + return enif_make_int(env, enif_compare(argv[0],argv[1])); } static ERL_NIF_TERM many_args_100(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -454,7 +557,7 @@ static ERL_NIF_TERM string_to_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM a unsigned size; int n; if (!enif_get_int(env,argv[1],(int*)&size) - || !enif_alloc_binary(env,size,&obin)) { + || !enif_alloc_binary(size,&obin)) { return enif_make_badarg(env); } n = enif_get_string(env, argv[0], (char*)obin.data, size, ERL_NIF_LATIN1); @@ -468,10 +571,10 @@ static ERL_NIF_TERM atom_to_bin(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg unsigned size; int n; if (!enif_get_int(env,argv[1],(int*)&size) - || !enif_alloc_binary(env,size,&obin)) { + || !enif_alloc_binary(size,&obin)) { return enif_make_badarg(env); } - n = enif_get_atom(env, argv[0], (char*)obin.data, size); + n = enif_get_atom(env, argv[0], (char*)obin.data, size, ERL_NIF_LATIN1); return enif_make_tuple(env, 2, enif_make_int(env,n), enif_make_binary(env,&obin)); } @@ -552,7 +655,7 @@ static ERL_NIF_TERM last_resource_dtor_call(ErlNifEnv* env, int argc, const ERL_ static ERL_NIF_TERM get_resource_type(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - PrivData* data = (PrivData*) enif_get_data(env); + PrivData* data = (PrivData*) enif_priv_data(env); int ix; if (!enif_get_int(env, argv[0], &ix) || ix >= 2) { @@ -568,7 +671,7 @@ static ERL_NIF_TERM alloc_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM union { void* p; long l;} data; if (!enif_get_long(env, argv[0], &type.l) || !enif_inspect_binary(env, argv[1], &data_bin) - || (data.p = enif_alloc_resource(env, type.t, data_bin.size))==NULL) { + || (data.p = enif_alloc_resource(type.t, data_bin.size))==NULL) { return enif_make_badarg(env); } @@ -593,28 +696,65 @@ static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TE ERL_NIF_TERM ret; if (!enif_get_long(env, argv[0], &type.l) || !enif_inspect_binary(env, argv[1], &data_bin) - || (data = enif_alloc_resource(env, type.t, data_bin.size))==NULL) { + || (data = enif_alloc_resource(type.t, data_bin.size))==NULL) { return enif_make_badarg(env); } ret = enif_make_resource(env, data); memcpy(data, data_bin.data, data_bin.size); - enif_release_resource(env, data); + enif_release_resource(data); return ret; } +static ERL_NIF_TERM make_new_resource_binary(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifBinary data_bin; + union { struct binary_resource* p; void* vp; long l;} br; + void* buf; + ERL_NIF_TERM ret; + if (!enif_inspect_binary(env, argv[0], &data_bin) + || (br.vp = enif_alloc_resource(binary_resource_type, + sizeof(struct binary_resource)))==NULL + || (buf = enif_alloc(data_bin.size)) == NULL) { + + return enif_make_badarg(env); + } + memset(br.vp,0xba,sizeof(struct binary_resource)); /* avoid valgrind warning */ + br.p->data = buf; + br.p->size = data_bin.size; + memcpy(br.p->data, data_bin.data, data_bin.size); + ret = enif_make_resource_binary(env, br.vp, br.p->data, br.p->size); + enif_release_resource(br.p); + return enif_make_tuple2(env, enif_make_long(env,br.l), ret); +} + +static void binary_resource_dtor(ErlNifEnv* env, void* obj) +{ + struct binary_resource* br = (struct binary_resource*) obj; + resource_dtor(env,obj); + assert(br->data != NULL); + enif_free(br->data); + br->data = NULL; +} + static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary data_bin; union { ErlNifResourceType* t; long l; } type; union { void* p; long l; } data; - if (!enif_get_long(env, argv[0], &type.l) + type.t = NULL; + if (enif_is_identical(argv[0], atom_binary_resource_type)) { + type.t = binary_resource_type; + } + else { + enif_get_long(env, argv[0], &type.l); + } + if (type.t == NULL || !enif_get_resource(env, argv[1], type.t, &data.p)) { return enif_make_badarg(env); } - - enif_alloc_binary(env, enif_sizeof_resource(env,data.p), &data_bin); + enif_alloc_binary(enif_sizeof_resource(data.p), &data_bin); memcpy(data_bin.data, data.p, data_bin.size); return enif_make_tuple2(env, enif_make_long(env,data.l), enif_make_binary(env, &data_bin)); @@ -626,10 +766,595 @@ static ERL_NIF_TERM release_resource(ErlNifEnv* env, int argc, const ERL_NIF_TER if (!enif_get_long(env, argv[0], &data.l)) { return enif_make_badarg(env); } - enif_release_resource(env, data.p); + enif_release_resource(data.p); return enif_make_atom(env,"ok"); } +/* + * argv[0] an atom + * argv[1] a binary + * argv[2] a ref + * argv[3] 'ok' + * argv[4] a fun + * argv[5] a pid + * argv[6] a port + * argv[7] an empty list + * argv[8] a non-empty list + * argv[9] a tuple + */ +static ERL_NIF_TERM check_is(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM ok_atom = enif_make_atom(env, "ok"); + + if (!enif_is_atom(env, argv[0])) return enif_make_badarg(env); + if (!enif_is_binary(env, argv[1])) return enif_make_badarg(env); + if (!enif_is_ref(env, argv[2])) return enif_make_badarg(env); + if (!enif_is_identical(argv[3], ok_atom)) return enif_make_badarg(env); + if (!enif_is_fun(env, argv[4])) return enif_make_badarg(env); + if (!enif_is_pid(env, argv[5])) return enif_make_badarg(env); + if (!enif_is_port(env, argv[6])) return enif_make_badarg(env); + if (!enif_is_empty_list(env, argv[7])) return enif_make_badarg(env); + if (!enif_is_list(env, argv[7])) return enif_make_badarg(env); + if (!enif_is_list(env, argv[8])) return enif_make_badarg(env); + if (!enif_is_tuple(env, argv[9])) return enif_make_badarg(env); + + return ok_atom; +} + +/* + * argv[0] atom with length of 6 + * argv[1] list with length of 6 + * argv[2] empty list + * argv[3] not an atom + * argv[4] not a list + */ +static ERL_NIF_TERM length_test(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + unsigned len; + + if (!enif_get_atom_length(env, argv[0], &len, ERL_NIF_LATIN1) || len != 6) + return enif_make_badarg(env); + + if (!enif_get_list_length(env, argv[1], &len) || len != 6) + return enif_make_badarg(env); + + if (!enif_get_list_length(env, argv[2], &len) || len != 0) + return enif_make_badarg(env); + + if (enif_get_atom_length(env, argv[3], &len, ERL_NIF_LATIN1)) + return enif_make_badarg(env); + + if (enif_get_list_length(env, argv[4], &len)) + return enif_make_badarg(env); + + return enif_make_atom(env, "ok"); +} + +static ERL_NIF_TERM make_atoms(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ERL_NIF_TERM arr[7]; + ERL_NIF_TERM existingatom0a, existingatom0b; + ERL_NIF_TERM existing0atom0; + const char * const an0atom = "an0atom"; + const char an0atom0[8] = {'a','n','\0','a','t','o','m',0}; + + arr[0] = enif_make_atom(env, "an0atom"); + arr[1] = enif_make_atom_len(env, "an0atom", 7); + arr[2] = enif_make_atom_len(env, an0atom, 7); + arr[3] = enif_make_atom_len(env, an0atom0, 8); + + if (!enif_make_existing_atom(env, "an0atom", &existingatom0a, ERL_NIF_LATIN1)) + return enif_make_atom(env, "error"); + arr[4] = existingatom0a; + + if (!enif_make_existing_atom_len(env, an0atom, 7, &existingatom0b, ERL_NIF_LATIN1)) + return enif_make_atom(env, "error"); + arr[5] = existingatom0b; + + if (!enif_make_existing_atom_len(env, an0atom0, 8, &existing0atom0, ERL_NIF_LATIN1)) + return enif_make_atom(env, "error"); + arr[6] = existing0atom0; + + return enif_make_tuple7(env, + arr[0],arr[1],arr[2],arr[3],arr[4],arr[5],arr[6]); +} + +static ERL_NIF_TERM make_strings(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + const char a0string[8] = {'a','0','s','t','r','i','n','g'}; + const char a0string0[9] = {'a','\0','s','t','r','i','n','g',0}; + const char astringwith8bits[37] = {'E','r','l','a','n','g',' ',0xE4 /* 'ä' */,'r',' ','e','t','t',' ','g','e','n','e','r','e','l','l','t',' ','p','r','o','g','r','a','m','s','p','r', 0xE5 /* 'å' */,'k',0}; + + return enif_make_tuple5(env, + enif_make_string(env, "a0string", ERL_NIF_LATIN1), + enif_make_string_len(env, "a0string", 8, ERL_NIF_LATIN1), + enif_make_string_len(env, a0string, 8, ERL_NIF_LATIN1), + enif_make_string_len(env, a0string0, 9, ERL_NIF_LATIN1), + enif_make_string(env, astringwith8bits, ERL_NIF_LATIN1)); +} +static ERL_NIF_TERM send_list_seq(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifPid to; + ERL_NIF_TERM msg; + ErlNifEnv* msg_env; + int i, res; + + if (!enif_get_int(env, argv[0], &i)) { + return enif_make_badarg(env); + } + if (argv[1] == atom_self) { + enif_self(env, &to); + } + else if (!enif_get_local_pid(env, argv[1], &to)) { + return enif_make_badarg(env); + } + msg_env = enif_alloc_env(); + msg = enif_make_list(msg_env,0); + for ( ; i>0 ; i--) { + msg = enif_make_list_cell(msg_env, enif_make_int(msg_env, i), msg); + } + res = enif_send(env, &to, msg_env, msg); + enif_free_env(msg_env); + return enif_make_tuple2(env, atom_ok, enif_make_int(env,res)); +} + +static void fill(void* dst, unsigned bytes, int seed) +{ + unsigned char* ptr = dst; + int i; + for (i=bytes; i>0; i--) { + *ptr++ = seed; + seed += 7; + } +} + +#define MAKE_TERM_REUSE_LEN 16 +struct make_term_info +{ + ErlNifEnv* caller_env; + ErlNifEnv* dst_env; + ERL_NIF_TERM reuse[MAKE_TERM_REUSE_LEN]; + unsigned reuse_push; + unsigned reuse_pull; + ErlNifResourceType* resource_type; + ERL_NIF_TERM other_term; + ERL_NIF_TERM blob; + ErlNifPid to_pid; + ErlNifTid tid; + ErlNifCond* cond; + ErlNifMutex* mtx; + int send_it; + int send_res; + unsigned n; +}; + + +static void push_term(struct make_term_info* mti, ERL_NIF_TERM term) +{ + unsigned ix = (mti->reuse_push++) % MAKE_TERM_REUSE_LEN; + mti->reuse[ix] = term; + //enif_fprintf(stderr, "push at %u: %T\r\n", ix, term); +} +static ERL_NIF_TERM pull_term(struct make_term_info* mti) +{ + unsigned ix; + if (mti->reuse_pull >= mti->reuse_push && + mti->reuse_push < MAKE_TERM_REUSE_LEN) { + mti->reuse_pull = 0; + if (mti->reuse_push == 0) { + mti->reuse[0] = enif_make_list(mti->dst_env, 0); + } + } + ix = (mti->reuse_pull++) % MAKE_TERM_REUSE_LEN; + //enif_fprintf(stderr, "pull from %u: %T\r\n", ix, mti->reuse[ix]); + return mti->reuse[ix]; +} + +static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res); + +static ERL_NIF_TERM make_term_binary(struct make_term_info* mti, int n) +{ + ErlNifBinary bin; + enif_alloc_binary(100, &bin); + fill(bin.data, bin.size, n); + return enif_make_binary(mti->dst_env, &bin); +} + +static ERL_NIF_TERM make_term_int(struct make_term_info* mti, int n) +{ + int i; + fill(&i, sizeof(i), n); + return enif_make_int(mti->dst_env, i); +} + +static ERL_NIF_TERM make_term_ulong(struct make_term_info* mti, int n) +{ + unsigned long ul; + fill(&ul, sizeof(ul), n); + return enif_make_ulong(mti->dst_env, ul); +} + +static ERL_NIF_TERM make_term_double(struct make_term_info* mti, int n) +{ + double d = 3.141592; + return enif_make_double(mti->dst_env, d); +} +static ERL_NIF_TERM make_term_atom(struct make_term_info* mti, int n) +{ + return enif_make_atom(mti->dst_env, "make_term_n"); +} +static ERL_NIF_TERM make_term_existing_atom(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM res; + int exist = enif_make_existing_atom(mti->dst_env, "nif_SUITE", &res, + ERL_NIF_LATIN1); + assert(exist); + return res; +} +static ERL_NIF_TERM make_term_string(struct make_term_info* mti, int n) +{ + return enif_make_string(mti->dst_env, "Hello!", ERL_NIF_LATIN1); +} +static ERL_NIF_TERM make_term_ref(struct make_term_info* mti, int n) +{ + return enif_make_ref(mti->dst_env); +} +static ERL_NIF_TERM make_term_sub_binary(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM orig; + unsigned char* ptr = enif_make_new_binary(mti->dst_env, 10, &orig); + fill(ptr, 10, n); + return enif_make_sub_binary(mti->dst_env, orig, 3, 5); +} +static ERL_NIF_TERM make_term_uint(struct make_term_info* mti, int n) +{ + unsigned int ui; + fill(&ui, sizeof(ui), n); + return enif_make_uint(mti->dst_env, ui); +} +static ERL_NIF_TERM make_term_long(struct make_term_info* mti, int n) +{ + long l; + fill(&l, sizeof(l), n); + return enif_make_long(mti->dst_env, l); +} +static ERL_NIF_TERM make_term_tuple0(struct make_term_info* mti, int n) +{ + return enif_make_tuple(mti->dst_env, 0); +} +static ERL_NIF_TERM make_term_list0(struct make_term_info* mti, int n) +{ + return enif_make_list(mti->dst_env, 0); +} +static ERL_NIF_TERM make_term_resource(struct make_term_info* mti, int n) +{ + void* resource = enif_alloc_resource(mti->resource_type, 10); + ERL_NIF_TERM term; + fill(resource, 10, n); + term = enif_make_resource(mti->dst_env, resource); + enif_release_resource(resource); + return term; +} +static ERL_NIF_TERM make_term_new_binary(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM res; + unsigned char* ptr = enif_make_new_binary(mti->dst_env,20,&res); + fill(ptr, 20, n); + return res; +} +static ERL_NIF_TERM make_term_caller_pid(struct make_term_info* mti, int n) +{ + ErlNifPid pid; + return enif_make_pid(mti->dst_env, enif_self(mti->caller_env, &pid)); +} + +static ERL_NIF_TERM make_term_tuple(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[3]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + t[2] = pull_term(mti); + return enif_make_tuple3(mti->dst_env, t[0], t[1], t[2]); +} +static ERL_NIF_TERM make_term_list(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[3]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + t[2] = pull_term(mti); + return enif_make_list3(mti->dst_env, t[0], t[1], t[2]); +} +static ERL_NIF_TERM make_term_list_cell(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[2]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + return enif_make_list_cell(mti->dst_env, t[0], t[1]); +} +static ERL_NIF_TERM make_term_tuple_from_array(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[3]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + t[2] = pull_term(mti); + return enif_make_tuple_from_array(mti->dst_env, t, 3); +} +static ERL_NIF_TERM make_term_list_from_array(struct make_term_info* mti, int n) +{ + ERL_NIF_TERM t[3]; + t[0] = pull_term(mti); + t[1] = pull_term(mti); + t[2] = pull_term(mti); + return enif_make_list_from_array(mti->dst_env, t, 3); +} +static ERL_NIF_TERM make_term_garbage(struct make_term_info* mti, int n) +{ + (void) enif_make_string(mti->dst_env, "garbage string", ERL_NIF_LATIN1); + return pull_term(mti); +} +static ERL_NIF_TERM make_term_copy(struct make_term_info* mti, int n) +{ + return enif_make_copy(mti->dst_env, mti->other_term); +} + +typedef ERL_NIF_TERM Make_term_Func(struct make_term_info*, int); +static Make_term_Func* make_funcs[] = { + make_term_binary, + make_term_int, + make_term_ulong, + make_term_double, + make_term_atom, + make_term_existing_atom, + make_term_string, + //make_term_ref, + make_term_sub_binary, + make_term_uint, + make_term_long, + make_term_tuple0, + make_term_list0, + make_term_resource, + make_term_new_binary, + make_term_caller_pid, + make_term_tuple, + make_term_list, + make_term_list_cell, + make_term_tuple_from_array, + make_term_list_from_array, + make_term_garbage, + make_term_copy +}; +static unsigned num_of_make_funcs() +{ + return sizeof(make_funcs)/sizeof(*make_funcs); +} +static int make_term_n(struct make_term_info* mti, int n, ERL_NIF_TERM* res) +{ + if (n < num_of_make_funcs()) { + *res = make_funcs[n](mti, n); + push_term(mti, *res); + return 1; + } + return 0; +} + +static ERL_NIF_TERM make_blob(ErlNifEnv* caller_env, ErlNifEnv* dst_env, + ERL_NIF_TERM other_term) +{ + PrivData* priv = (PrivData*) enif_priv_data(caller_env); + ERL_NIF_TERM term, list; + int n = 0; + struct make_term_info mti; + mti.caller_env = caller_env; + mti.dst_env = dst_env; + mti.reuse_push = 0; + mti.reuse_pull = 0; + mti.resource_type = priv->rt_arr[0].t; + mti.other_term = other_term; + + list = enif_make_list(dst_env, 0); + while (make_term_n(&mti, n++, &term)) { + list = enif_make_list_cell(dst_env, term, list); + } + return list; +} + +static ERL_NIF_TERM send_new_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifPid to; + ERL_NIF_TERM msg, copy; + ErlNifEnv* msg_env; + int res; + + if (!enif_get_local_pid(env, argv[0], &to)) { + return enif_make_badarg(env); + } + msg_env = enif_alloc_env(); + msg = make_blob(env,msg_env, argv[1]); + copy = make_blob(env,env, argv[1]); + res = enif_send(env, &to, msg_env, msg); + enif_free_env(msg_env); + return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy); +} + +static ERL_NIF_TERM alloc_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + PrivData* priv = (PrivData*) enif_priv_data(env); + struct make_term_info* mti; + ERL_NIF_TERM ret; + + mti = (struct make_term_info*) enif_alloc_resource(msgenv_resource_type, + sizeof(*mti)); + mti->caller_env = NULL; + mti->dst_env = enif_alloc_env(); + mti->reuse_push = 0; + mti->reuse_pull = 0; + mti->resource_type = priv->rt_arr[0].t; + mti->other_term = enif_make_list(mti->dst_env, 0); + mti->blob = enif_make_list(mti->dst_env, 0); + mti->mtx = enif_mutex_create("nif_SUITE:mtx"); + mti->cond = enif_cond_create("nif_SUITE:cond"); + mti->send_res = 0xcafebabe; + mti->n = 0; + ret = enif_make_resource(env, mti); + enif_release_resource(mti); + return ret; +} + +static void msgenv_dtor(ErlNifEnv* env, void* obj) +{ + struct make_term_info* mti = (struct make_term_info*) obj; + if (mti->dst_env != NULL) { + enif_free_env(mti->dst_env); + } + enif_mutex_destroy(mti->mtx); + enif_cond_destroy(mti->cond); +} + +static ERL_NIF_TERM clear_msgenv(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { + void* vp; + struct make_term_info* p; + }mti; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) { + return enif_make_badarg(env); + } + enif_clear_env(mti.p->dst_env); + mti.p->reuse_pull = 0; + mti.p->reuse_push = 0; + mti.p->blob = enif_make_list(mti.p->dst_env, 0); + return atom_ok; +} + +static ERL_NIF_TERM grow_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + ERL_NIF_TERM term; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) + || (argc>2 && !enif_get_uint(env,argv[2], &mti.p->n))) { + return enif_make_badarg(env); + } + mti.p->caller_env = env; + mti.p->other_term = argv[1]; + mti.p->n %= num_of_make_funcs(); + make_term_n(mti.p, mti.p->n++, &term); + mti.p->blob = enif_make_list_cell(mti.p->dst_env, term, mti.p->blob); + return atom_ok; +} + +static ERL_NIF_TERM send_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + ErlNifPid to; + ERL_NIF_TERM copy; + int res; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) + || !enif_get_local_pid(env, argv[1], &to)) { + return enif_make_badarg(env); + } + copy = enif_make_copy(env, mti.p->blob); + res = enif_send(env, &to, mti.p->dst_env, mti.p->blob); + return enif_make_tuple3(env, atom_ok, enif_make_int(env,res), copy); +} + +static ERL_NIF_TERM send3_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + ErlNifPid to; + ERL_NIF_TERM copy; + int res; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) + || !enif_get_local_pid(env, argv[1], &to)) { + return enif_make_badarg(env); + } + mti.p->blob = enif_make_tuple2(mti.p->dst_env, + enif_make_copy(mti.p->dst_env, argv[2]), + mti.p->blob); + res = enif_send(env, &to, mti.p->dst_env, mti.p->blob); + return enif_make_int(env,res); +} + +void* threaded_sender(void *arg) +{ + + union { void* vp; struct make_term_info* p; }mti; + mti.vp = arg; + + enif_mutex_lock(mti.p->mtx); + while (!mti.p->send_it) { + enif_cond_wait(mti.p->cond, mti.p->mtx); + } + mti.p->send_it = 0; + enif_mutex_unlock(mti.p->mtx); + mti.p->send_res = enif_send(NULL, &mti.p->to_pid, mti.p->dst_env, mti.p->blob); + return NULL; +} + +static ERL_NIF_TERM send_blob_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + ERL_NIF_TERM copy; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp) + || !enif_get_local_pid(env,argv[1], &mti.p->to_pid)) { + return enif_make_badarg(env); + } + copy = enif_make_copy(env, mti.p->blob); + + mti.p->send_it = enif_is_identical(argv[2],atom_join); + if (enif_thread_create("nif_SUITE:send_from_thread", &mti.p->tid, + threaded_sender, mti.p, NULL) != 0) { + return enif_make_badarg(env); + } + if (enif_is_identical(argv[2],atom_join)) { + int err = enif_thread_join(mti.p->tid, NULL); + assert(err == 0); + return enif_make_tuple3(env, atom_ok, enif_make_int(env, mti.p->send_res), copy); + } + else { + enif_keep_resource(mti.vp); + return enif_make_tuple2(env, atom_ok, copy); + } +} + +static ERL_NIF_TERM join_send_thread(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + int err; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) { + return enif_make_badarg(env); + } + enif_mutex_lock(mti.p->mtx); + mti.p->send_it = 1; + enif_cond_signal(mti.p->cond); + enif_mutex_unlock(mti.p->mtx); + err = enif_thread_join(mti.p->tid, NULL); + assert(err == 0); + enif_release_resource(mti.vp); + return enif_make_tuple2(env, atom_ok, enif_make_int(env, mti.p->send_res)); +} + +static ERL_NIF_TERM copy_blob(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + union { void* vp; struct make_term_info* p; }mti; + if (!enif_get_resource(env, argv[0], msgenv_resource_type, &mti.vp)) { + return enif_make_badarg(env); + } + return enif_make_copy(env, mti.p->blob); +} + +static ERL_NIF_TERM send_term(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifEnv* menv; + ErlNifPid pid; + int ret; + if (!enif_get_local_pid(env, argv[0], &pid)) { + return enif_make_badarg(env); + } + menv = enif_alloc_env(); + ret = enif_send(env, &pid, menv, enif_make_copy(menv, argv[1])); + enif_free_env(menv); + return enif_make_int(env, ret); +} static ErlNifFunc nif_funcs[] = { @@ -656,8 +1381,25 @@ static ErlNifFunc nif_funcs[] = {"get_resource", 2, get_resource}, {"release_resource", 1, release_resource}, {"last_resource_dtor_call", 0, last_resource_dtor_call}, - {"make_new_resource", 2, make_new_resource} - + {"make_new_resource", 2, make_new_resource}, + {"check_is", 10, check_is}, + {"length_test", 5, length_test}, + {"make_atoms", 0, make_atoms}, + {"make_strings", 0, make_strings}, + {"make_new_resource", 2, make_new_resource}, + {"make_new_resource_binary", 1, make_new_resource_binary}, + {"send_list_seq", 2, send_list_seq}, + {"send_new_blob", 2, send_new_blob}, + {"alloc_msgenv", 0, alloc_msgenv}, + {"clear_msgenv", 1, clear_msgenv}, + {"grow_blob", 2, grow_blob}, + {"grow_blob", 3, grow_blob}, + {"send_blob", 2, send_blob}, + {"send3_blob", 3, send3_blob}, + {"send_blob_thread", 3, send_blob_thread}, + {"join_send_thread", 1, join_send_thread}, + {"copy_blob", 1, copy_blob}, + {"send_term", 2, send_term} }; ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload) diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c index 75df9d56d5..e32d10057c 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_mod.c +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c @@ -42,6 +42,11 @@ static ERL_NIF_TERM am_resource_type; static ERL_NIF_TERM am_resource_dtor_A; static ERL_NIF_TERM am_resource_dtor_B; +static NifModPrivData* priv_data(ErlNifEnv* env) +{ + return (NifModPrivData*) enif_priv_data(env); +} + static void init(ErlNifEnv* env) { am_true = enif_make_atom(env, "true"); @@ -54,7 +59,7 @@ static void init(ErlNifEnv* env) static void add_call_with_arg(ErlNifEnv* env, NifModPrivData* data, const char* func_name, const char* arg, int arg_sz) { - CallInfo* call = enif_alloc(env, sizeof(CallInfo)+strlen(func_name) + arg_sz); + CallInfo* call = (CallInfo*)enif_alloc(sizeof(CallInfo)+strlen(func_name) + arg_sz); strcpy(call->func_name, func_name); call->lib_ver = NIF_LIB_VER; call->static_cntA = ++static_cntA; @@ -78,7 +83,7 @@ static void add_call(ErlNifEnv* env, NifModPrivData* data,const char* func_name) add_call_with_arg(env, data, func_name, NULL, 0); } -#define ADD_CALL(FUNC_NAME) add_call(env, enif_priv_data(env),FUNC_NAME) +#define ADD_CALL(FUNC_NAME) add_call(env, priv_data(env),FUNC_NAME) #define STRINGIFY_(X) #X #define STRINGIFY(X) STRINGIFY_(X) @@ -87,56 +92,56 @@ static void resource_dtor_A(ErlNifEnv* env, void* a) { const char dtor_name[] = "resource_dtor_A_v" STRINGIFY(NIF_LIB_VER); - add_call_with_arg(env, enif_priv_data(env), dtor_name, - a, enif_sizeof_resource(env, a)); + add_call_with_arg(env, priv_data(env), dtor_name, (const char*)a, + enif_sizeof_resource(a)); } static void resource_dtor_B(ErlNifEnv* env, void* a) { - const char dtor_name[] = "resource_dtor_B_v" STRINGIFY(NIF_LIB_VER); + const char dtor_name[] = "resource_dtor_B_v" STRINGIFY(NIF_LIB_VER); - add_call_with_arg(env, enif_priv_data(env), dtor_name, - a, enif_sizeof_resource(env, a)); + add_call_with_arg(env, priv_data(env), dtor_name, (const char*)a, + enif_sizeof_resource(a)); } /* {resource_type, Ix|null, ErlNifResourceFlags in, "TypeName", dtor(A|B|null), ErlNifResourceFlags out}*/ static void open_resource_type(ErlNifEnv* env, ERL_NIF_TERM op_tpl) { - NifModPrivData* data = enif_priv_data(env); + NifModPrivData* data = priv_data(env); const ERL_NIF_TERM* arr; int arity; char rt_name[30]; - union { enum ErlNifResourceFlags e; int i; } flags, exp_res, got_res; + union { ErlNifResourceFlags e; int i; } flags, exp_res, got_res; unsigned ix; ErlNifResourceDtor* dtor; ErlNifResourceType* got_ptr; CHECK(enif_get_tuple(env, op_tpl, &arity, &arr)); CHECK(arity == 6); - CHECK(enif_is_identical(env, arr[0], am_resource_type)); + CHECK(enif_is_identical(arr[0], am_resource_type)); CHECK(enif_get_int(env, arr[2], &flags.i)); CHECK(enif_get_string(env, arr[3], rt_name, sizeof(rt_name), ERL_NIF_LATIN1) > 0); CHECK(enif_get_int(env, arr[5], &exp_res.i)); - if (enif_is_identical(env, arr[4], am_null)) { + if (enif_is_identical(arr[4], am_null)) { dtor = NULL; } - else if (enif_is_identical(env, arr[4], am_resource_dtor_A)) { + else if (enif_is_identical(arr[4], am_resource_dtor_A)) { dtor = resource_dtor_A; } else { - CHECK(enif_is_identical(env, arr[4], am_resource_dtor_B)); + CHECK(enif_is_identical(arr[4], am_resource_dtor_B)); dtor = resource_dtor_B; } - got_ptr = enif_open_resource_type(env, rt_name, dtor, + got_ptr = enif_open_resource_type(env, NULL, rt_name, dtor, flags.e, &got_res.e); if (enif_get_uint(env, arr[1], &ix) && ix < RT_MAX && got_ptr != NULL) { data->rt_arr[ix] = got_ptr; } else { - CHECK(enif_is_identical(env, arr[1], am_null)); + CHECK(enif_is_identical(arr[1], am_null)); CHECK(got_ptr == NULL); } CHECK(got_res.e == exp_res.e); @@ -144,7 +149,7 @@ static void open_resource_type(ErlNifEnv* env, ERL_NIF_TERM op_tpl) static void do_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info) { - NifModPrivData* data = enif_priv_data(env); + NifModPrivData* data = priv_data(env); ERL_NIF_TERM head, tail; unsigned ix; for (ix=0; ix<RT_MAX; ix++) { @@ -158,17 +163,18 @@ static void do_load_info(ErlNifEnv* env, ERL_NIF_TERM load_info) CHECK(enif_is_empty_list(env, head)); } -static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int load(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info) { NifModPrivData* data; init(env); - data = enif_alloc(env, sizeof(NifModPrivData)); + data = (NifModPrivData*) enif_alloc(sizeof(NifModPrivData)); CHECK(data != NULL); - *priv_data = data; + *priv = data; data->mtx = enif_mutex_create("nif_mod_priv_data"); data->ref_cnt = 1; data->call_history = NULL; + add_call(env, data, "load"); do_load_info(env, load_info); @@ -176,39 +182,35 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int reload(ErlNifEnv* env, void** priv, ERL_NIF_TERM load_info) { + NifModPrivData* data = (NifModPrivData*) *priv; init(env); - add_call(env, *priv_data, "reload"); + add_call(env, data, "reload"); do_load_info(env, load_info); return 0; } -static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +static int upgrade(ErlNifEnv* env, void** priv, void** old_priv_data, ERL_NIF_TERM load_info) { - NifModPrivData* data = *old_priv_data; + NifModPrivData* data = (NifModPrivData*) *old_priv_data; init(env); add_call(env, data, "upgrade"); data->ref_cnt++; - *priv_data = *old_priv_data; + *priv = *old_priv_data; do_load_info(env, load_info); return 0; } -static void unload(ErlNifEnv* env, void* priv_data) +static void unload(ErlNifEnv* env, void* priv) { - NifModPrivData* data = priv_data; + NifModPrivData* data = (NifModPrivData*) priv; + int is_last; add_call(env, data, "unload"); - enif_mutex_lock(data->mtx); - if (--data->ref_cnt == 0) { - enif_mutex_unlock(data->mtx); - enif_mutex_destroy(data->mtx); - enif_free(env, data); - } - enif_mutex_unlock(data->mtx); + NifModPrivData_release(data); } static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -220,12 +222,12 @@ static ERL_NIF_TERM lib_version(ErlNifEnv* env, int argc, const ERL_NIF_TERM arg static ERL_NIF_TERM get_priv_data_ptr(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ADD_CALL("get_priv_data_ptr"); - return enif_make_ulong(env, (unsigned long)enif_priv_data(env)); + return enif_make_ulong(env, (unsigned long)priv_data(env)); } static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - NifModPrivData* data = (NifModPrivData*) enif_priv_data(env); + NifModPrivData* data = priv_data(env); ErlNifBinary ibin; char* a; ERL_NIF_TERM ret; @@ -234,22 +236,22 @@ static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TE || !enif_inspect_binary(env, argv[1], &ibin)) { return enif_make_badarg(env); } - a = enif_alloc_resource(env, data->rt_arr[ix], ibin.size); + a = (char*) enif_alloc_resource(data->rt_arr[ix], ibin.size); memcpy(a, ibin.data, ibin.size); ret = enif_make_resource(env, a); - enif_release_resource(env, a); + enif_release_resource(a); return ret; } static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - NifModPrivData* data = (NifModPrivData*) enif_priv_data(env); + NifModPrivData* data = priv_data(env); ErlNifBinary obin; unsigned ix; void* a; if (!enif_get_uint(env, argv[0], &ix) || ix >= RT_MAX || !enif_get_resource(env, argv[1], data->rt_arr[ix], &a) - || !enif_alloc_binary(env, enif_sizeof_resource(env, a), &obin)) { + || !enif_alloc_binary(enif_sizeof_resource(a), &obin)) { return enif_make_badarg(env); } memcpy(obin.data, a, obin.size); diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.h b/erts/emulator/test/nif_SUITE_data/nif_mod.h index 0eaf91d6e1..cd0ecf4b54 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_mod.h +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.h @@ -20,3 +20,15 @@ typedef struct ErlNifResourceType* rt_arr[RT_MAX]; }NifModPrivData; +#define NifModPrivData_release(NMPD) \ + do { \ + int is_last; \ + enif_mutex_lock((NMPD)->mtx); \ + is_last = (--(NMPD)->ref_cnt == 0); \ + enif_mutex_unlock((NMPD)->mtx); \ + if (is_last) { \ + enif_mutex_destroy((NMPD)->mtx); \ + enif_free((NMPD)); \ + } \ + }while (0) + diff --git a/erts/emulator/test/obsolete_SUITE.erl b/erts/emulator/test/obsolete_SUITE.erl deleted file mode 100644 index b191f84ee0..0000000000 --- a/erts/emulator/test/obsolete_SUITE.erl +++ /dev/null @@ -1,123 +0,0 @@ -%% -%% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2004-2010. All Rights Reserved. -%% -%% The contents of this file are subject to the Erlang Public License, -%% Version 1.1, (the "License"); you may not use this file except in -%% compliance with the License. You should have received a copy of the -%% Erlang Public License along with this software. If not, it can be -%% retrieved online at http://www.erlang.org/. -%% -%% Software distributed under the License is distributed on an "AS IS" -%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -%% the License for the specific language governing rights and limitations -%% under the License. -%% -%% %CopyrightEnd% -%% - - --module(obsolete_SUITE). --author('[email protected]'). --compile(nowarn_obsolete_guard). - --export([all/1]). - --export([erl_threads/1]). - --include("test_server.hrl"). - --define(DEFAULT_TIMETRAP_SECS, 240). - -all(doc) -> []; -all(suite) -> - case catch erlang:system_info({wordsize,external}) of - 4 -> [erl_threads]; - _ -> {skip, "Only expected to work on true 32-bit architectures"} - end. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%% Testcases %% -%% %% - -erl_threads(suite) -> []; -erl_threads(doc) -> []; -erl_threads(Cfg) -> - ?line case erlang:system_info(threads) of - true -> - ?line drv_case(Cfg, erl_threads); - false -> - ?line {skip, "Emulator not compiled with threads support"} - end. - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%% %% -%% Internal functions %% -%% %% - -drv_case(Config, CaseName) -> - drv_case(Config, CaseName, ""). - -drv_case(Config, CaseName, TimeTrap) when integer(TimeTrap) -> - drv_case(Config, CaseName, "", TimeTrap); -drv_case(Config, CaseName, Command) when list(Command) -> - drv_case(Config, CaseName, Command, ?DEFAULT_TIMETRAP_SECS). - -drv_case(Config, CaseName, TimeTrap, Command) when list(Command), - integer(TimeTrap) -> - drv_case(Config, CaseName, Command, TimeTrap); -drv_case(Config, CaseName, Command, TimeTrap) when list(Config), - atom(CaseName), - list(Command), - integer(TimeTrap) -> - case ?t:os_type() of - {Family, _} when Family == unix; Family == win32 -> - ?line run_drv_case(Config, CaseName, Command, TimeTrap); - SkipOs -> - ?line {skipped, - lists:flatten(["Not run on " - | io_lib:format("~p",[SkipOs])])} - end. - -run_drv_case(Config, CaseName, Command, TimeTrap) -> - ?line Dog = test_server:timetrap(test_server:seconds(TimeTrap)), - ?line DataDir = ?config(data_dir,Config), - case erl_ddll:load_driver(DataDir, CaseName) of - ok -> ok; - {error, Error} -> - io:format("~s\n", [erl_ddll:format_error(Error)]), - ?line ?t:fail() - end, - ?line Port = open_port({spawn, atom_to_list(CaseName)}, []), - ?line true = is_port(Port), - ?line Port ! {self(), {command, Command}}, - ?line Result = receive_drv_result(Port, CaseName), - ?line Port ! {self(), close}, - ?line receive - {Port, closed} -> - ok - end, - ?line ok = erl_ddll:unload_driver(CaseName), - ?line test_server:timetrap_cancel(Dog), - ?line Result. - -receive_drv_result(Port, CaseName) -> - ?line receive - {print, Port, CaseName, Str} -> - ?line ?t:format("~s", [Str]), - ?line receive_drv_result(Port, CaseName); - {'EXIT', Port, Error} -> - ?line ?t:fail(Error); - {'EXIT', error, Error} -> - ?line ?t:fail(Error); - {failed, Port, CaseName, Comment} -> - ?line ?t:fail(Comment); - {skipped, Port, CaseName, Comment} -> - ?line {skipped, Comment}; - {succeeded, Port, CaseName, ""} -> - ?line succeeded; - {succeeded, Port, CaseName, Comment} -> - ?line {comment, Comment} - end. diff --git a/erts/emulator/test/obsolete_SUITE_data/Makefile.src b/erts/emulator/test/obsolete_SUITE_data/Makefile.src deleted file mode 100644 index d8e2b861c0..0000000000 --- a/erts/emulator/test/obsolete_SUITE_data/Makefile.src +++ /dev/null @@ -1,33 +0,0 @@ -# ``The contents of this file are subject to the Erlang Public License, -# Version 1.1, (the "License"); you may not use this file except in -# compliance with the License. You should have received a copy of the -# Erlang Public License along with this software. If not, it can be -# retrieved via the world wide web at http://www.erlang.org/. -# -# Software distributed under the License is distributed on an "AS IS" -# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See -# the License for the specific language governing rights and limitations -# under the License. -# -# The Initial Developer of the Original Code is Ericsson Utvecklings AB. -# Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings -# AB. All Rights Reserved.'' -# -# $Id$ -# - -TEST_DRVS = erl_threads@dll@ -CC = @CC@ -LD = @LD@ -CFLAGS = @SHLIB_CFLAGS@ -I@erl_include@ @DEFS@ -SHLIB_EXTRA_LDLIBS = testcase_driver@obj@ - -all: $(TEST_DRVS) - -@SHLIB_RULES@ - -testcase_driver@obj@: testcase_driver.c testcase_driver.h -$(TEST_DRVS): testcase_driver@obj@ - - - diff --git a/erts/emulator/test/obsolete_SUITE_data/erl_threads.c b/erts/emulator/test/obsolete_SUITE_data/erl_threads.c deleted file mode 100644 index 27a5163121..0000000000 --- a/erts/emulator/test/obsolete_SUITE_data/erl_threads.c +++ /dev/null @@ -1,302 +0,0 @@ -/* ``The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved via the world wide web at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * The Initial Developer of the Original Code is Ericsson Utvecklings AB. - * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings - * AB. All Rights Reserved.'' - * - * $Id$ - */ - -#include "testcase_driver.h" - -#ifndef __WIN32__ - -#define NO_OF_THREADS 2 - -#include <unistd.h> -#include <errno.h> - -static int die; -static int cw_passed; -static int res_tf0; -static int res_tf1; -static erl_mutex_t mtx; -static erl_cond_t cnd; -static erl_thread_t tid[NO_OF_THREADS]; -static int need_join[NO_OF_THREADS]; - -typedef struct { - int n; -} thr_arg_t; - - -static void *tf0(void *vta) -{ - int r; - - if (((thr_arg_t *) vta)->n != 0) - goto fail; - - r = erts_mutex_lock(mtx); - if (r != 0) { - erts_mutex_unlock(mtx); - goto fail; - } - - r = erts_cond_wait(cnd, mtx); - if (r != 0 || die) { - erts_mutex_unlock(mtx); - goto fail; - } - - cw_passed++; - - r = erts_cond_wait(cnd, mtx); - if (r != 0 || die) { - erts_mutex_unlock(mtx); - goto fail; - } - - cw_passed++; - - r = erts_mutex_unlock(mtx); - if (r != 0) - goto fail; - - res_tf0 = 0; - - return (void *) &res_tf0; - - fail: - return NULL; -} - - -static void *tf1(void *vta) -{ - int r; - - if (((thr_arg_t *) vta)->n != 1) - goto fail; - - r = erts_mutex_lock(mtx); - if (r != 0) { - erts_mutex_unlock(mtx); - goto fail; - } - - r = erts_cond_wait(cnd, mtx); - if (r != 0 || die) { - erts_mutex_unlock(mtx); - goto fail; - } - - cw_passed++; - - r = erts_cond_wait(cnd, mtx); - if (r != 0 || die) { - erts_mutex_unlock(mtx); - goto fail; - } - - cw_passed++; - - r = erts_mutex_unlock(mtx); - if (r != 0) - goto fail; - - res_tf1 = 1; - - erts_thread_exit((void *) &res_tf1); - - res_tf1 = 4711; - - fail: - return NULL; -} - -#endif /* #ifndef __WIN32__ */ - -void -testcase_run(TestCaseState_t *tcs) -{ -#ifdef __WIN32__ - testcase_skipped(tcs, "Nothing to test; not supported on windows."); -#else - int i, r; - void *tres[NO_OF_THREADS]; - thr_arg_t ta[NO_OF_THREADS]; - erl_thread_t t1; - - die = 0; - cw_passed = 0; - - for (i = 0; i < NO_OF_THREADS; i++) - need_join[i] = 0; - - res_tf0 = 17; - res_tf1 = 17; - - cnd = mtx = NULL; - - /* Create mutex and cond */ - mtx = erts_mutex_create(); - ASSERT(tcs, mtx); - cnd = erts_cond_create(); - ASSERT(tcs, cnd); - - /* Create the threads */ - ta[0].n = 0; - r = erts_thread_create(&tid[0], tf0, (void *) &ta[0], 0); - ASSERT(tcs, r == 0); - need_join[0] = 1; - - ta[1].n = 1; - r = erts_thread_create(&tid[1], tf1, (void *) &ta[1], 0); - ASSERT(tcs, r == 0); - need_join[1] = 1; - - /* Make sure the threads waits on cond wait */ - sleep(1); - - r = erts_mutex_lock(mtx); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - ASSERT_CLNUP(tcs, cw_passed == 0, (void) erts_mutex_unlock(mtx)); - - - /* Let one thread pass one cond wait */ - r = erts_cond_signal(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - r = erts_mutex_unlock(mtx); - ASSERT(tcs, r == 0); - - sleep(1); - - r = erts_mutex_lock(mtx); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - ASSERT_CLNUP(tcs, cw_passed == 1, (void) erts_mutex_unlock(mtx)); - - - /* Let both threads pass one cond wait */ - r = erts_cond_broadcast(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - r = erts_mutex_unlock(mtx); - ASSERT(tcs, r == 0); - - sleep(1); - - r = erts_mutex_lock(mtx); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - ASSERT_CLNUP(tcs, cw_passed == 3, (void) erts_mutex_unlock(mtx)); - - - /* Let the thread that only have passed one cond wait pass the other one */ - r = erts_cond_signal(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - r = erts_mutex_unlock(mtx); - ASSERT(tcs, r == 0); - - sleep(1); - - r = erts_mutex_lock(mtx); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - ASSERT_CLNUP(tcs, cw_passed == 4, (void) erts_mutex_unlock(mtx)); - - /* Both threads should have passed both cond waits and exited; - join them and check returned values */ - - r = erts_thread_join(tid[0], &tres[0]); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - need_join[0] = 0; - - ASSERT_CLNUP(tcs, tres[0] == &res_tf0, (void) erts_mutex_unlock(mtx)); - ASSERT_CLNUP(tcs, res_tf0 == 0, (void) erts_mutex_unlock(mtx)); - - r = erts_thread_join(tid[1], &tres[1]); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - need_join[1] = 0; - - ASSERT_CLNUP(tcs, tres[1] == &res_tf1, (void) erts_mutex_unlock(mtx)); - ASSERT_CLNUP(tcs, res_tf1 == 1, (void) erts_mutex_unlock(mtx)); - - /* Test signaling when noone waits */ - - r = erts_cond_signal(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - /* Test broadcasting when noone waits */ - - r = erts_cond_broadcast(cnd); - ASSERT_CLNUP(tcs, r == 0, (void) erts_mutex_unlock(mtx)); - - /* erts_cond_timedwait() not supported anymore */ - r = erts_cond_timedwait(cnd, mtx, 1000); - ASSERT_CLNUP(tcs, r != 0, (void) erts_mutex_unlock(mtx)); - ASSERT_CLNUP(tcs, - strcmp(erl_errno_id(r), "enotsup") == 0, - (void) erts_mutex_unlock(mtx)); - - r = erts_mutex_unlock(mtx); - ASSERT(tcs, r == 0); - - r = erts_mutex_destroy(mtx); - ASSERT(tcs, r == 0); - mtx = NULL; - - r = erts_cond_destroy(cnd); - ASSERT(tcs, r == 0); - cnd = NULL; - - /* ... */ - t1 = erts_thread_self(); - - if (cw_passed == 4711) { - /* We don't want to execute this just check that the - symbol/symbols is/are defined */ - erts_thread_kill(t1); - } - -#endif /* #ifndef __WIN32__ */ -} - -char * -testcase_name(void) -{ - return "erl_threads"; -} - -void -testcase_cleanup(TestCaseState_t *tcs) -{ - int i; - for (i = 0; i < NO_OF_THREADS; i++) { - if (need_join[i]) { - erts_mutex_lock(mtx); - die = 1; - erts_cond_broadcast(cnd); - erts_mutex_unlock(mtx); - erts_thread_join(tid[1], NULL); - } - } - if (mtx) - erts_mutex_destroy(mtx); - if (cnd) - erts_cond_destroy(cnd); -} - diff --git a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c b/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c deleted file mode 100644 index 99d5adb041..0000000000 --- a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.c +++ /dev/null @@ -1,262 +0,0 @@ -/* ``The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved via the world wide web at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * The Initial Developer of the Original Code is Ericsson Utvecklings AB. - * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings - * AB. All Rights Reserved.'' - * - * $Id$ - */ - -#include "testcase_driver.h" -#include <stdio.h> -#include <stdlib.h> -#include <stdarg.h> -#include <setjmp.h> -#include <string.h> - -#ifdef __WIN32__ -#undef HAVE_VSNPRINTF -#define HAVE_VSNPRINTF 1 -#define vsnprintf _vsnprintf -#endif - -#ifndef HAVE_VSNPRINTF -#define HAVE_VSNPRINTF 0 -#endif - -#define COMMENT_BUF_SZ 4096 - -#define TESTCASE_FAILED 0 -#define TESTCASE_SKIPPED 1 -#define TESTCASE_SUCCEEDED 2 - -typedef struct { - TestCaseState_t visible; - int port; - int result; - jmp_buf done_jmp_buf; - char *comment; - char comment_buf[COMMENT_BUF_SZ]; -} InternalTestCaseState_t; - -long testcase_drv_start(int port, char *command); -int testcase_drv_stop(long drv_data); -int testcase_drv_run(long drv_data, char *buf, int len); - -static DriverEntry testcase_drv_entry = { - NULL, - testcase_drv_start, - testcase_drv_stop, - testcase_drv_run -}; - - -int DRIVER_INIT(testcase_drv)(void *arg) -{ - testcase_drv_entry.driver_name = testcase_name(); - return (int) &testcase_drv_entry; -} - -long -testcase_drv_start(int port, char *command) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) - driver_alloc(sizeof(InternalTestCaseState_t)); - if (!itcs) { - return -1; - } - - itcs->visible.testcase_name = testcase_name(); - itcs->visible.extra = NULL; - itcs->port = port; - itcs->result = TESTCASE_FAILED; - itcs->comment = ""; - - return (long) itcs; -} - -int -testcase_drv_stop(long drv_data) -{ - testcase_cleanup((TestCaseState_t *) drv_data); - driver_free((void *) drv_data); - return 0; -} - -int -testcase_drv_run(long drv_data, char *buf, int len) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) drv_data; - DriverTermData result_atom; - DriverTermData msg[12]; - - itcs->visible.command = buf; - itcs->visible.command_len = len; - - if (setjmp(itcs->done_jmp_buf) == 0) { - testcase_run((TestCaseState_t *) itcs); - itcs->result = TESTCASE_SUCCEEDED; - } - - switch (itcs->result) { - case TESTCASE_SUCCEEDED: - result_atom = driver_mk_atom("succeeded"); - break; - case TESTCASE_SKIPPED: - result_atom = driver_mk_atom("skipped"); - break; - case TESTCASE_FAILED: - default: - result_atom = driver_mk_atom("failed"); - break; - } - - msg[0] = ERL_DRV_ATOM; - msg[1] = (DriverTermData) result_atom; - - msg[2] = ERL_DRV_PORT; - msg[3] = driver_mk_port(itcs->port); - - msg[4] = ERL_DRV_ATOM; - msg[5] = driver_mk_atom(itcs->visible.testcase_name); - - msg[6] = ERL_DRV_STRING; - msg[7] = (DriverTermData) itcs->comment; - msg[8] = (DriverTermData) strlen(itcs->comment); - - msg[9] = ERL_DRV_TUPLE; - msg[10] = (DriverTermData) 4; - - driver_output_term(itcs->port, msg, 11); - return 0; -} - -int -testcase_assertion_failed(TestCaseState_t *tcs, - char *file, int line, char *assertion) -{ - testcase_failed(tcs, "%s:%d: Assertion failed: \"%s\"", - file, line, assertion); - return 0; -} - -void -testcase_printf(TestCaseState_t *tcs, char *frmt, ...) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; - DriverTermData msg[12]; - va_list va; - va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif - va_end(va); - - msg[0] = ERL_DRV_ATOM; - msg[1] = (DriverTermData) driver_mk_atom("print"); - - msg[2] = ERL_DRV_PORT; - msg[3] = driver_mk_port(itcs->port); - - msg[4] = ERL_DRV_ATOM; - msg[5] = driver_mk_atom(itcs->visible.testcase_name); - - msg[6] = ERL_DRV_STRING; - msg[7] = (DriverTermData) itcs->comment_buf; - msg[8] = (DriverTermData) strlen(itcs->comment_buf); - - msg[9] = ERL_DRV_TUPLE; - msg[10] = (DriverTermData) 4; - - driver_output_term(itcs->port, msg, 11); -} - - -void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; - va_list va; - va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif - va_end(va); - - itcs->result = TESTCASE_SUCCEEDED; - itcs->comment = itcs->comment_buf; - - longjmp(itcs->done_jmp_buf, 1); -} - -void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; - va_list va; - va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif - va_end(va); - - itcs->result = TESTCASE_SKIPPED; - itcs->comment = itcs->comment_buf; - - longjmp(itcs->done_jmp_buf, 1); -} - -void testcase_failed(TestCaseState_t *tcs, char *frmt, ...) -{ - InternalTestCaseState_t *itcs = (InternalTestCaseState_t *) tcs; - char buf[10]; - size_t bufsz = sizeof(buf); - va_list va; - va_start(va, frmt); -#if HAVE_VSNPRINTF - vsnprintf(itcs->comment_buf, COMMENT_BUF_SZ, frmt, va); -#else - vsprintf(itcs->comment_buf, frmt, va); -#endif - va_end(va); - - itcs->result = TESTCASE_FAILED; - itcs->comment = itcs->comment_buf; - - if (erl_drv_getenv("ERL_ABORT_ON_FAILURE", buf, &bufsz) == 0 - && strcmp("true", buf) == 0) { - fprintf(stderr, "Testcase \"%s\" failed: %s\n", - itcs->visible.testcase_name, itcs->comment); - abort(); - } - - longjmp(itcs->done_jmp_buf, 1); -} - -void *testcase_alloc(size_t size) -{ - return driver_alloc(size); -} - -void *testcase_realloc(void *ptr, size_t size) -{ - return driver_realloc(ptr, size); -} - -void testcase_free(void *ptr) -{ - driver_free(ptr); -} diff --git a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h b/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h deleted file mode 100644 index 3d85ca6df0..0000000000 --- a/erts/emulator/test/obsolete_SUITE_data/testcase_driver.h +++ /dev/null @@ -1,57 +0,0 @@ -/* ``The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved via the world wide web at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * The Initial Developer of the Original Code is Ericsson Utvecklings AB. - * Portions created by Ericsson are Copyright 1999, Ericsson Utvecklings - * AB. All Rights Reserved.'' - * - * $Id$ - */ - -#ifndef TESTCASE_DRIVER_H__ -#define TESTCASE_DRIVER_H__ - -#include "obsolete/driver.h" -#include <stdlib.h> - -typedef struct { - char *testcase_name; - char *command; - int command_len; - void *extra; -} TestCaseState_t; - -#define ASSERT_CLNUP(TCS, B, CLN) \ -do { \ - if (!(B)) { \ - CLN; \ - testcase_assertion_failed((TCS), __FILE__, __LINE__, #B); \ - } \ -} while (0) - -#define ASSERT(TCS, B) ASSERT_CLNUP(TCS, B, (void) 0) - -void testcase_printf(TestCaseState_t *tcs, char *frmt, ...); -void testcase_succeeded(TestCaseState_t *tcs, char *frmt, ...); -void testcase_skipped(TestCaseState_t *tcs, char *frmt, ...); -void testcase_failed(TestCaseState_t *tcs, char *frmt, ...); -int testcase_assertion_failed(TestCaseState_t *tcs, char *file, int line, - char *assertion); -void *testcase_alloc(size_t size); -void *testcase_realloc(void *ptr, size_t size); -void testcase_free(void *ptr); - - -char *testcase_name(void); -void testcase_run(TestCaseState_t *tcs); -void testcase_cleanup(TestCaseState_t *tcs); - -#endif diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index eb69bf917b..7fe532abd0 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -88,7 +88,7 @@ otp_3906/1, otp_4389/1, win_massive/1, win_massive_client/1, mix_up_ports/1, otp_5112/1, otp_5119/1, otp_6224/1, exit_status_multi_scheduling_block/1, ports/1, - spawn_driver/1,spawn_executable/1, + spawn_driver/1, spawn_executable/1, close_deaf_port/1, unregister_name/1]). -export([]). @@ -113,7 +113,7 @@ all(suite) -> otp_3906, otp_4389, win_massive, mix_up_ports, otp_5112, otp_5119, exit_status_multi_scheduling_block, - ports, spawn_driver, spawn_executable, + ports, spawn_driver, spawn_executable, close_deaf_port, unregister_name ]. @@ -878,13 +878,20 @@ env2(Config) -> "nisse" = os:getenv(Long) end), - + ?line env_slave(Temp, [{"must_define_something","some_value"}, - {"certainly_not_existing",false}, + {"certainly_not_existing",false}, {"ends_with_equal", "value="}, {Long,false}, {"glurf","a glorfy string"}]), + %% A lot of non existing variables (mingled with existing) + NotExistingList = [{lists:flatten(io_lib:format("V~p_not_existing",[X])),false} + || X <- lists:seq(1,150)], + ExistingList = [{lists:flatten(io_lib:format("V~p_existing",[X])),"a_value"} + || X <- lists:seq(1,150)], + ?line env_slave(Temp, lists:sort(ExistingList ++ NotExistingList)), + ?line test_server:timetrap_cancel(Dog), ok. @@ -2293,3 +2300,37 @@ load_driver(Dir, Driver) -> io:format("~s\n", [erl_ddll:format_error(Error)]), Res end. + + +close_deaf_port(doc) -> ["Send data to port program that does not read it, then close port." + "Primary targeting Windows to test threaded_handle_closer in sys.c"]; +close_deaf_port(suite) -> []; +close_deaf_port(Config) when is_list(Config) -> + ?line Dog = test_server:timetrap(test_server:seconds(100)), + ?line DataDir = ?config(data_dir, Config), + ?line DeadPort = os:find_executable("dead_port", DataDir), + ?line Port = open_port({spawn,DeadPort++" 60"},[]), + ?line erlang:port_command(Port,"Hello, can you hear me!?!?"), + ?line port_close(Port), + + Res = close_deaf_port_1(0, DeadPort), + io:format("Waiting for OS procs to terminate...\n"), + receive after 5*1000 -> ok end, + ?line test_server:timetrap_cancel(Dog), + Res. + +close_deaf_port_1(1000, _) -> + ok; +close_deaf_port_1(N, Cmd) -> + Timeout = integer_to_list(random:uniform(5*1000)), + ?line try open_port({spawn_executable,Cmd},[{args,[Timeout]}]) of + Port -> + ?line erlang:port_command(Port,"Hello, can you hear me!?!?"), + ?line port_close(Port), + close_deaf_port_1(N+1, Cmd) + catch + _:eagain -> + {comment, "Could not spawn more than " ++ integer_to_list(N) ++ " OS processes."} + end. + + diff --git a/erts/emulator/test/port_SUITE_data/Makefile.src b/erts/emulator/test/port_SUITE_data/Makefile.src index d97b37c9ae..ff822ae720 100644 --- a/erts/emulator/test/port_SUITE_data/Makefile.src +++ b/erts/emulator/test/port_SUITE_data/Makefile.src @@ -3,7 +3,7 @@ LD = @LD@ CFLAGS = @CFLAGS@ -I@erl_include@ @DEFS@ CROSSLDFLAGS = @CROSSLDFLAGS@ -PROGS = port_test@exe@ echo_args@exe@ +PROGS = port_test@exe@ echo_args@exe@ dead_port@exe@ DRIVERS = echo_drv@dll@ exit_drv@dll@ failure_drv@dll@ all: $(PROGS) $(DRIVERS) port_test.@EMULATOR@ diff --git a/erts/emulator/test/port_SUITE_data/dead_port.c b/erts/emulator/test/port_SUITE_data/dead_port.c new file mode 100644 index 0000000000..68e96fbf14 --- /dev/null +++ b/erts/emulator/test/port_SUITE_data/dead_port.c @@ -0,0 +1,102 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +#ifdef VXWORKS +#include <vxWorks.h> +#include <taskVarLib.h> +#include <taskLib.h> +#include <sysLib.h> +#include <string.h> +#include <ioLib.h> +#endif + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <errno.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#ifndef __WIN32__ +#include <unistd.h> + +#ifdef VXWORKS +#include "reclaim.h" +#include <sys/times.h> +#else +#include <sys/time.h> +#endif + +#define O_BINARY 0 +#define _setmode(fd, mode) +#endif + +#ifdef __WIN32__ +#include "windows.h" +#include "winbase.h" +#endif + + +#ifdef VXWORKS +#define MAIN(argc, argv) port_test(argc, argv) +#else +#define MAIN(argc, argv) main(argc, argv) +#endif + + +extern int errno; + +static void delay(unsigned ms); + + +MAIN(argc, argv) +int argc; +char *argv[]; +{ + int x; + if (argc < 2) { + fprintf(stderr,"Usage %s <milliseconds>\n",argv[0]); + return 1; + } + if ((x = atoi(argv[1])) <= 0) { + fprintf(stderr,"Usage %s <milliseconds>\n",argv[0]); + return 1; + } + delay(x); + return 0; +} + +static void +delay(unsigned ms) +{ +#ifdef VXWORKS + taskDelay((sysClkRateGet() * ms) / 1000); +#else +#ifdef __WIN32__ + Sleep(ms); +#else + struct timeval t; + t.tv_sec = ms/1000; + t.tv_usec = (ms % 1000) * 1000; + + select(0, NULL, NULL, NULL, &t); +#endif +#endif +} diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl new file mode 100644 index 0000000000..40ebf2bd21 --- /dev/null +++ b/erts/emulator/test/receive_SUITE.erl @@ -0,0 +1,113 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(receive_SUITE). + +%% Tests receive after. + +-include("test_server.hrl"). + +-export([all/1, + call_with_huge_message_queue/1,receive_in_between/1]). + +-export([init_per_testcase/2,fin_per_testcase/2]). + +all(suite) -> + [call_with_huge_message_queue,receive_in_between]. + +init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> + Dog=?t:timetrap(?t:minutes(3)), + [{watchdog, Dog}|Config]. + +fin_per_testcase(_Func, Config) -> + Dog=?config(watchdog, Config), + ?t:timetrap_cancel(Dog). + +call_with_huge_message_queue(Config) when is_list(Config) -> + ?line Pid = spawn_link(fun echo_loop/0), + + ?line {Time,ok} = tc(fun() -> calls(10, Pid) end), + + ?line [self() ! {msg,N} || N <- lists:seq(1, 500000)], + erlang:garbage_collect(), + ?line {NewTime,ok} = tc(fun() -> calls(10, Pid) end), + io:format("Time for empty message queue: ~p", [Time]), + io:format("Time for huge message queue: ~p", [NewTime]), + + case (NewTime+1) / (Time+1) of + Q when Q < 10 -> + ok; + Q -> + io:format("Q = ~p", [Q]), + ?line ?t:fail() + end, + ok. + +calls(0, _) -> ok; +calls(N, Pid) -> + {ok,{ultimate_answer,42}} = call(Pid, {ultimate_answer,42}), + calls(N-1, Pid). + +call(Pid, Msg) -> + Mref = erlang:monitor(process, Pid), + Pid ! {Mref,{self(),Msg}}, + receive + {Mref, Reply} -> + erlang:demonitor(Mref, [flush]), + {ok, Reply}; + {'DOWN', Mref, _, _, Reason} -> + exit(Reason) + end. + +receive_in_between(Config) when is_list(Config) -> + ?line Pid = spawn_link(fun echo_loop/0), + ?line [{ok,{a,b}} = call2(Pid, {a,b}) || _ <- lists:seq(1, 100000)], + ok. + +call2(Pid, Msg) -> + self() ! dummy, + Mref = erlang:monitor(process, Pid), + Pid ! {Mref,{self(),Msg}}, + receive_one(), + receive + {Mref,Reply} -> + erlang:demonitor(Mref, [flush]), + {ok,Reply}; + {'DOWN',Mref,_,_,Reason} -> + exit(Reason) + end. + +receive_one() -> + receive + dummy -> ok + end. + +%%% +%%% Common helpers. +%%% + +echo_loop() -> + receive + {Ref,{Pid,Msg}} -> + Pid ! {Ref,Msg}, + echo_loop() + end. + +tc(Fun) -> + timer:tc(erlang, apply, [Fun,[]]). diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index c9101b77c2..06442bfad6 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -33,7 +33,7 @@ -include("test_server.hrl"). %-compile(export_all). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/1, init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]). -export([equal/1, few_low/1, @@ -47,9 +47,11 @@ scheduler_bind/1, scheduler_bind_types/1, cpu_topology/1, + update_cpu_info/1, sct_cmd/1, sbt_cmd/1, - scheduler_suspend/1]). + scheduler_suspend/1, + reader_groups/1]). -define(DEFAULT_TIMEOUT, ?t:minutes(10)). @@ -67,7 +69,8 @@ all(suite) -> equal_with_high_max, bound_process, scheduler_bind, - scheduler_suspend]. + scheduler_suspend, + reader_groups]. init_per_testcase(Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), @@ -81,6 +84,10 @@ fin_per_testcase(_Case, Config) when is_list(Config) -> ?t:timetrap_cancel(Dog), ok. +end_per_suite(Config) -> + catch erts_debug:set_internal_state(available_internal_state, false), + Config. + -define(ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED, (2000*2000)). -define(DEFAULT_TEST_REDS_PER_SCHED, 200000000). @@ -243,6 +250,7 @@ bound_loop(NS, N, M, Sched) -> scheduler_bind(suite) -> [scheduler_bind_types, cpu_topology, + update_cpu_info, sct_cmd, sbt_cmd]. @@ -766,6 +774,137 @@ cpu_topology_cmdline_test(Config, Topology, Cmd) -> ?line stop_node(Node), ?line ok. +update_cpu_info(Config) when is_list(Config) -> + ?line OldOnline = erlang:system_info(schedulers_online), + ?line OldAff = get_affinity_mask(), + ?line ?t:format("START - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n", + [OldAff, OldOnline, erlang:system_info(scheduler_bindings)]), + ?line case {erlang:system_info(logical_processors_available), OldAff} of + {Avail, _} when Avail == unknown; OldAff == unknown -> + %% Nothing much to test; just a smoke test + case erlang:system_info(update_cpu_info) of + unchanged -> ?line ok; + changed -> ?line ok + end; + _ -> + try + ?line adjust_schedulers_online(), + case erlang:system_info(schedulers_online) of + 1 -> + %% Nothing much to test; just a smoke test + ?line ok; + Onln0 -> + %% unset least significant bit + ?line Aff = (OldAff band (OldAff - 1)), + ?line set_affinity_mask(Aff), + ?line Onln1 = Onln0 - 1, + ?line case adjust_schedulers_online() of + {Onln0, Onln1} -> + ?line Onln1 = erlang:system_info(schedulers_online), + ?line receive after 500 -> ok end, + ?line ?t:format("TEST - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n", + [Aff, Onln1, erlang:system_info(scheduler_bindings)]), + ?line unchanged = adjust_schedulers_online(), + ?line ok; + Fail -> + ?line ?t:fail(Fail) + end + end + after + set_affinity_mask(OldAff), + adjust_schedulers_online(), + erlang:system_flag(schedulers_online, OldOnline), + receive after 500 -> ok end, + ?t:format("END - Affinity mask: ~p - Schedulers online: ~p - Scheduler bindings: ~p~n", + [get_affinity_mask(), + erlang:system_info(schedulers_online), + erlang:system_info(scheduler_bindings)]) + end + end. + +adjust_schedulers_online() -> + case erlang:system_info(update_cpu_info) of + unchanged -> + unchanged; + changed -> + Avail = erlang:system_info(logical_processors_available), + {erlang:system_flag(schedulers_online, Avail), Avail} + end. + +read_affinity(Data) -> + Exp = "pid " ++ os:getpid() ++ "'s current affinity mask", + case string:tokens(Data, ":") of + [Exp, DirtyAffinityStr] -> + AffinityStr = string:strip(string:strip(DirtyAffinityStr, + both, $ ), + both, $\n), + case catch erlang:list_to_integer(AffinityStr, 16) of + Affinity when is_integer(Affinity) -> + Affinity; + _ -> + bad + end; + _ -> + bad + end. + +get_affinity_mask(Port, Status, Affinity) when Status == unknown; + Affinity == unknown -> + receive + {Port,{data, Data}} -> + get_affinity_mask(Port, Status, read_affinity(Data)); + {Port,{exit_status,S}} -> + get_affinity_mask(Port, S, Affinity) + end; +get_affinity_mask(Port, Status, bad) -> + unknown; +get_affinity_mask(Port, Status, Affinity) -> + Affinity. + +get_affinity_mask() -> + case ?t:os_type() of + {unix, linux} -> + case catch open_port({spawn, "taskset -p " ++ os:getpid()}, + [exit_status]) of + Port when is_port(Port) -> + get_affinity_mask(Port, unknown, unknown); + _ -> + unknown + end; + _ -> + unknown + end. + +set_affinity_mask(Port, unknown) -> + receive + {Port,{data, _}} -> + set_affinity_mask(Port, unknown); + {Port,{exit_status,Status}} -> + set_affinity_mask(Port, Status) + end; +set_affinity_mask(Port, Status) -> + receive + {Port,{data, _}} -> + set_affinity_mask(Port, unknown) + after 0 -> + Status + end. + +set_affinity_mask(Mask) -> + Cmd = lists:flatten(["taskset -p ", + io_lib:format("~.16b", [Mask]), + " ", + os:getpid()]), + case catch open_port({spawn, Cmd}, [exit_status]) of + Port when is_port(Port) -> + case set_affinity_mask(Port, unknown) of + 0 -> ok; + _ -> exit(failed_to_set_affinity) + end; + _ -> + exit(failed_to_set_affinity) + end. + sct_cmd(Config) when is_list(Config) -> ?line Topology = ?TOPOLOGY_A_TERM, ?line OldRelFlags = clear_erl_rel_flags(), @@ -902,7 +1041,8 @@ scheduler_suspend_test(Config, Schedulers) -> ?line [SState] = mcall(Node, [fun () -> erlang:system_info(schedulers_state) end]), - ?line {Sched, _, _} = SState, + ?line ?t:format("SState=~p~n", [SState]), + ?line {Sched, SchedOnln, _SchedAvail} = SState, ?line true = is_integer(Sched), ?line [ok] = mcall(Node, [fun () -> sst0_loop(300) end]), ?line [ok] = mcall(Node, [fun () -> sst1_loop(300) end]), @@ -914,6 +1054,14 @@ scheduler_suspend_test(Config, Schedulers) -> fun () -> sst2_loop(200) end, fun () -> sst3_loop(Sched, 200) end]), ?line [SState] = mcall(Node, [fun () -> + case Sched == SchedOnln of + false -> + Sched = erlang:system_flag( + schedulers_online, + SchedOnln); + true -> + ok + end, erlang:system_info(schedulers_state) end]), ?line stop_node(Node), @@ -956,12 +1104,361 @@ sst3_loop(S, N) -> erlang:system_flag(schedulers_online, 1), erlang:system_flag(schedulers_online, S), sst3_loop(S, N-1). + +reader_groups(Config) when is_list(Config) -> + %% White box testing. These results are correct, but other results + %% could be too... + + %% The actual tilepro64 topology + CPUT0 = [{processor,[{node,[{core,{logical,0}}, + {core,{logical,1}}, + {core,{logical,2}}, + {core,{logical,8}}, + {core,{logical,9}}, + {core,{logical,10}}, + {core,{logical,11}}, + {core,{logical,16}}, + {core,{logical,17}}, + {core,{logical,18}}, + {core,{logical,19}}, + {core,{logical,24}}, + {core,{logical,25}}, + {core,{logical,27}}, + {core,{logical,29}}]}, + {node,[{core,{logical,3}}, + {core,{logical,4}}, + {core,{logical,5}}, + {core,{logical,6}}, + {core,{logical,7}}, + {core,{logical,12}}, + {core,{logical,13}}, + {core,{logical,14}}, + {core,{logical,15}}, + {core,{logical,20}}, + {core,{logical,21}}, + {core,{logical,22}}, + {core,{logical,23}}, + {core,{logical,28}}, + {core,{logical,30}}]}, + {node,[{core,{logical,31}}, + {core,{logical,36}}, + {core,{logical,37}}, + {core,{logical,38}}, + {core,{logical,44}}, + {core,{logical,45}}, + {core,{logical,46}}, + {core,{logical,47}}, + {core,{logical,51}}, + {core,{logical,52}}, + {core,{logical,53}}, + {core,{logical,54}}, + {core,{logical,55}}, + {core,{logical,60}}, + {core,{logical,61}}]}, + {node,[{core,{logical,26}}, + {core,{logical,32}}, + {core,{logical,33}}, + {core,{logical,34}}, + {core,{logical,35}}, + {core,{logical,39}}, + {core,{logical,40}}, + {core,{logical,41}}, + {core,{logical,42}}, + {core,{logical,43}}, + {core,{logical,48}}, + {core,{logical,49}}, + {core,{logical,50}}, + {core,{logical,58}}]}]}], + + ?line [{0,1},{1,1},{2,1},{3,3},{4,3},{5,3},{6,3},{7,3},{8,1},{9,1},{10,1}, + {11,1},{12,3},{13,3},{14,4},{15,4},{16,2},{17,2},{18,2},{19,2}, + {20,4},{21,4},{22,4},{23,4},{24,2},{25,2},{26,7},{27,2},{28,4}, + {29,2},{30,4},{31,5},{32,7},{33,7},{34,7},{35,7},{36,5},{37,5}, + {38,5},{39,7},{40,7},{41,8},{42,8},{43,8},{44,5},{45,5},{46,5}, + {47,6},{48,8},{49,8},{50,8},{51,6},{52,6},{53,6},{54,6},{55,6}, + {58,8},{60,6},{61,6}] + = reader_groups_map(CPUT0, 8), + + CPUT1 = [n([p([c([t(l(0)),t(l(1)),t(l(2)),t(l(3))]), + c([t(l(4)),t(l(5)),t(l(6)),t(l(7))]), + c([t(l(8)),t(l(9)),t(l(10)),t(l(11))]), + c([t(l(12)),t(l(13)),t(l(14)),t(l(15))])]), + p([c([t(l(16)),t(l(17)),t(l(18)),t(l(19))]), + c([t(l(20)),t(l(21)),t(l(22)),t(l(23))]), + c([t(l(24)),t(l(25)),t(l(26)),t(l(27))]), + c([t(l(28)),t(l(29)),t(l(30)),t(l(31))])])]), + n([p([c([t(l(32)),t(l(33)),t(l(34)),t(l(35))]), + c([t(l(36)),t(l(37)),t(l(38)),t(l(39))]), + c([t(l(40)),t(l(41)),t(l(42)),t(l(43))]), + c([t(l(44)),t(l(45)),t(l(46)),t(l(47))])]), + p([c([t(l(48)),t(l(49)),t(l(50)),t(l(51))]), + c([t(l(52)),t(l(53)),t(l(54)),t(l(55))]), + c([t(l(56)),t(l(57)),t(l(58)),t(l(59))]), + c([t(l(60)),t(l(61)),t(l(62)),t(l(63))])])]), + n([p([c([t(l(64)),t(l(65)),t(l(66)),t(l(67))]), + c([t(l(68)),t(l(69)),t(l(70)),t(l(71))]), + c([t(l(72)),t(l(73)),t(l(74)),t(l(75))]), + c([t(l(76)),t(l(77)),t(l(78)),t(l(79))])]), + p([c([t(l(80)),t(l(81)),t(l(82)),t(l(83))]), + c([t(l(84)),t(l(85)),t(l(86)),t(l(87))]), + c([t(l(88)),t(l(89)),t(l(90)),t(l(91))]), + c([t(l(92)),t(l(93)),t(l(94)),t(l(95))])])]), + n([p([c([t(l(96)),t(l(97)),t(l(98)),t(l(99))]), + c([t(l(100)),t(l(101)),t(l(102)),t(l(103))]), + c([t(l(104)),t(l(105)),t(l(106)),t(l(107))]), + c([t(l(108)),t(l(109)),t(l(110)),t(l(111))])]), + p([c([t(l(112)),t(l(113)),t(l(114)),t(l(115))]), + c([t(l(116)),t(l(117)),t(l(118)),t(l(119))]), + c([t(l(120)),t(l(121)),t(l(122)),t(l(123))]), + c([t(l(124)),t(l(125)),t(l(126)),t(l(127))])])])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3}, + {10,3},{11,3},{12,4},{13,4},{14,4},{15,4},{16,5},{17,5},{18,5}, + {19,5},{20,6},{21,6},{22,6},{23,6},{24,7},{25,7},{26,7},{27,7}, + {28,8},{29,8},{30,8},{31,8},{32,9},{33,9},{34,9},{35,9},{36,10}, + {37,10},{38,10},{39,10},{40,11},{41,11},{42,11},{43,11},{44,12}, + {45,12},{46,12},{47,12},{48,13},{49,13},{50,13},{51,13},{52,14}, + {53,14},{54,14},{55,14},{56,15},{57,15},{58,15},{59,15},{60,16}, + {61,16},{62,16},{63,16},{64,17},{65,17},{66,17},{67,17},{68,18}, + {69,18},{70,18},{71,18},{72,19},{73,19},{74,19},{75,19},{76,20}, + {77,20},{78,20},{79,20},{80,21},{81,21},{82,21},{83,21},{84,22}, + {85,22},{86,22},{87,22},{88,23},{89,23},{90,23},{91,23},{92,24}, + {93,24},{94,24},{95,24},{96,25},{97,25},{98,25},{99,25},{100,26}, + {101,26},{102,26},{103,26},{104,27},{105,27},{106,27},{107,27}, + {108,28},{109,28},{110,28},{111,28},{112,29},{113,29},{114,29}, + {115,29},{116,30},{117,30},{118,30},{119,30},{120,31},{121,31}, + {122,31},{123,31},{124,32},{125,32},{126,32},{127,32}] + = reader_groups_map(CPUT1, 128), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1}, + {11,1},{12,1},{13,1},{14,1},{15,1},{16,1},{17,1},{18,1},{19,1}, + {20,1},{21,1},{22,1},{23,1},{24,1},{25,1},{26,1},{27,1},{28,1}, + {29,1},{30,1},{31,1},{32,1},{33,1},{34,1},{35,1},{36,1},{37,1}, + {38,1},{39,1},{40,1},{41,1},{42,1},{43,1},{44,1},{45,1},{46,1}, + {47,1},{48,1},{49,1},{50,1},{51,1},{52,1},{53,1},{54,1},{55,1}, + {56,1},{57,1},{58,1},{59,1},{60,1},{61,1},{62,1},{63,1},{64,2}, + {65,2},{66,2},{67,2},{68,2},{69,2},{70,2},{71,2},{72,2},{73,2}, + {74,2},{75,2},{76,2},{77,2},{78,2},{79,2},{80,2},{81,2},{82,2}, + {83,2},{84,2},{85,2},{86,2},{87,2},{88,2},{89,2},{90,2},{91,2}, + {92,2},{93,2},{94,2},{95,2},{96,2},{97,2},{98,2},{99,2},{100,2}, + {101,2},{102,2},{103,2},{104,2},{105,2},{106,2},{107,2},{108,2}, + {109,2},{110,2},{111,2},{112,2},{113,2},{114,2},{115,2},{116,2}, + {117,2},{118,2},{119,2},{120,2},{121,2},{122,2},{123,2},{124,2}, + {125,2},{126,2},{127,2}] + = reader_groups_map(CPUT1, 2), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,2},{5,2},{6,2},{7,2},{8,3},{9,3},{10,3}, + {11,3},{12,3},{13,3},{14,3},{15,3},{16,4},{17,4},{18,4},{19,4}, + {20,4},{21,4},{22,4},{23,4},{24,5},{25,5},{26,5},{27,5},{28,5}, + {29,5},{30,5},{31,5},{32,6},{33,6},{34,6},{35,6},{36,6},{37,6}, + {38,6},{39,6},{40,7},{41,7},{42,7},{43,7},{44,7},{45,7},{46,7}, + {47,7},{48,8},{49,8},{50,8},{51,8},{52,8},{53,8},{54,8},{55,8}, + {56,9},{57,9},{58,9},{59,9},{60,9},{61,9},{62,9},{63,9},{64,10}, + {65,10},{66,10},{67,10},{68,10},{69,10},{70,10},{71,10},{72,11}, + {73,11},{74,11},{75,11},{76,11},{77,11},{78,11},{79,11},{80,12}, + {81,12},{82,12},{83,12},{84,12},{85,12},{86,12},{87,12},{88,13}, + {89,13},{90,13},{91,13},{92,13},{93,13},{94,13},{95,13},{96,14}, + {97,14},{98,14},{99,14},{100,14},{101,14},{102,14},{103,14}, + {104,15},{105,15},{106,15},{107,15},{108,15},{109,15},{110,15}, + {111,15},{112,16},{113,16},{114,16},{115,16},{116,16},{117,16}, + {118,16},{119,16},{120,17},{121,17},{122,17},{123,17},{124,17}, + {125,17},{126,17},{127,17}] + = reader_groups_map(CPUT1, 17), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1},{5,1},{6,1},{7,1},{8,1},{9,1},{10,1}, + {11,1},{12,1},{13,1},{14,1},{15,1},{16,2},{17,2},{18,2},{19,2}, + {20,2},{21,2},{22,2},{23,2},{24,2},{25,2},{26,2},{27,2},{28,2}, + {29,2},{30,2},{31,2},{32,3},{33,3},{34,3},{35,3},{36,3},{37,3}, + {38,3},{39,3},{40,3},{41,3},{42,3},{43,3},{44,3},{45,3},{46,3}, + {47,3},{48,4},{49,4},{50,4},{51,4},{52,4},{53,4},{54,4},{55,4}, + {56,4},{57,4},{58,4},{59,4},{60,4},{61,4},{62,4},{63,4},{64,5}, + {65,5},{66,5},{67,5},{68,5},{69,5},{70,5},{71,5},{72,5},{73,5}, + {74,5},{75,5},{76,5},{77,5},{78,5},{79,5},{80,6},{81,6},{82,6}, + {83,6},{84,6},{85,6},{86,6},{87,6},{88,6},{89,6},{90,6},{91,6}, + {92,6},{93,6},{94,6},{95,6},{96,7},{97,7},{98,7},{99,7},{100,7}, + {101,7},{102,7},{103,7},{104,7},{105,7},{106,7},{107,7},{108,7}, + {109,7},{110,7},{111,7},{112,7},{113,7},{114,7},{115,7},{116,7}, + {117,7},{118,7},{119,7},{120,7},{121,7},{122,7},{123,7},{124,7}, + {125,7},{126,7},{127,7}] + = reader_groups_map(CPUT1, 7), + + ?line CPUT2 = [p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))]), + p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]), + p([t(l(10))]), + p([c(l(11)),c(l(12)),c(l(13))]), + p([c(l(14)),c(l(15))])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2},{6,2},{7,2},{8,2},{9,2}, + {10,3}, + {11,4},{12,4},{13,4}, + {14,5},{15,5}] = reader_groups_map(CPUT2, 5), + + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,5},{13,5}, + {14,6},{15,6}] = reader_groups_map(CPUT2, 6), + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,6},{13,6}, + {14,7},{15,7}] = reader_groups_map(CPUT2, 7), + + ?line [{0,1},{1,1},{2,2},{3,2},{4,2}, + {5,3},{6,3},{7,3},{8,3},{9,3}, + {10,4}, + {11,5},{12,6},{13,6}, + {14,7},{15,8}] = reader_groups_map(CPUT2, 8), + + ?line [{0,1},{1,2},{2,2},{3,3},{4,3}, + {5,4},{6,4},{7,4},{8,4},{9,4}, + {10,5}, + {11,6},{12,7},{13,7}, + {14,8},{15,9}] = reader_groups_map(CPUT2, 9), + + ?line [{0,1},{1,2},{2,2},{3,3},{4,3}, + {5,4},{6,4},{7,4},{8,4},{9,4}, + {10,5}, + {11,6},{12,7},{13,8}, + {14,9},{15,10}] = reader_groups_map(CPUT2, 10), + + ?line [{0,1},{1,2},{2,3},{3,4},{4,4}, + {5,5},{6,5},{7,5},{8,5},{9,5}, + {10,6}, + {11,7},{12,8},{13,9}, + {14,10},{15,11}] = reader_groups_map(CPUT2, 11), + + ?line [{0,1},{1,2},{2,3},{3,4},{4,5}, + {5,6},{6,6},{7,6},{8,6},{9,6}, + {10,7}, + {11,8},{12,9},{13,10}, + {14,11},{15,12}] = reader_groups_map(CPUT2, 100), + + CPUT3 = [p([t(l(5)),t(l(6)),t(l(7)),t(l(8)),t(l(9))]), + p([t(l(10))]), + p([c(l(11)),c(l(12)),c(l(13))]), + p([c(l(14)),c(l(15))]), + p([c(l(0)),c(l(1)),c(l(2)),c(l(3)),c(l(4))])], + + ?line [{0,5},{1,5},{2,6},{3,6},{4,6}, + {5,1},{6,1},{7,1},{8,1},{9,1}, + {10,2},{11,3},{12,3},{13,3}, + {14,4},{15,4}] = reader_groups_map(CPUT3, 6), + + CPUT4 = [p([t(l(0)),t(l(1)),t(l(2)),t(l(3)),t(l(4))]), + p([t(l(5))]), + p([c(l(6)),c(l(7)),c(l(8))]), + p([c(l(9)),c(l(10))]), + p([c(l(11)),c(l(12)),c(l(13)),c(l(14)),c(l(15))])], + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2}, + {6,3},{7,3},{8,3}, + {9,4},{10,4}, + {11,5},{12,5},{13,6},{14,6},{15,6}] = reader_groups_map(CPUT4, 6), + + ?line [{0,1},{1,1},{2,1},{3,1},{4,1}, + {5,2}, + {6,3},{7,4},{8,4}, + {9,5},{10,5}, + {11,6},{12,6},{13,7},{14,7},{15,7}] = reader_groups_map(CPUT4, 7), + + ?line [{0,1},{65535,2}] = reader_groups_map([c(l(0)),c(l(65535))], 10), + + ?line ok. +reader_groups_map(CPUT, Groups) -> + Old = erlang:system_info({cpu_topology, defined}), + erlang:system_flag(cpu_topology, CPUT), + enable_internal_state(), + Res = erts_debug:get_internal_state({reader_groups_map, Groups}), + erlang:system_flag(cpu_topology, Old), + lists:sort(Res). + %% %% Utils %% +tilera_cpu_topology() -> + [{processor,[{node,[{core,{logical,0}}, + {core,{logical,1}}, + {core,{logical,2}}, + {core,{logical,8}}, + {core,{logical,9}}, + {core,{logical,10}}, + {core,{logical,11}}, + {core,{logical,16}}, + {core,{logical,17}}, + {core,{logical,18}}, + {core,{logical,19}}, + {core,{logical,24}}, + {core,{logical,25}}, + {core,{logical,27}}, + {core,{logical,29}}]}, + {node,[{core,{logical,3}}, + {core,{logical,4}}, + {core,{logical,5}}, + {core,{logical,6}}, + {core,{logical,7}}, + {core,{logical,12}}, + {core,{logical,13}}, + {core,{logical,14}}, + {core,{logical,15}}, + {core,{logical,20}}, + {core,{logical,21}}, + {core,{logical,22}}, + {core,{logical,23}}, + {core,{logical,28}}, + {core,{logical,30}}]}, + {node,[{core,{logical,31}}, + {core,{logical,36}}, + {core,{logical,37}}, + {core,{logical,38}}, + {core,{logical,44}}, + {core,{logical,45}}, + {core,{logical,46}}, + {core,{logical,47}}, + {core,{logical,51}}, + {core,{logical,52}}, + {core,{logical,53}}, + {core,{logical,54}}, + {core,{logical,55}}, + {core,{logical,60}}, + {core,{logical,61}}]}, + {node,[{core,{logical,26}}, + {core,{logical,32}}, + {core,{logical,33}}, + {core,{logical,34}}, + {core,{logical,35}}, + {core,{logical,39}}, + {core,{logical,40}}, + {core,{logical,41}}, + {core,{logical,42}}, + {core,{logical,43}}, + {core,{logical,48}}, + {core,{logical,49}}, + {core,{logical,50}}, + {core,{logical,58}}]}]}]. + +l(Id) -> + {logical, Id}. + +t(X) -> + {thread, X}. + +c(X) -> + {core, X}. + +p(X) -> + {processor, X}. + +n(X) -> + {node, X}. + mcall(Node, Funs) -> Parent = self(), Refs = lists:map(fun (Fun) -> diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl index 489adbd660..819aa34886 100644 --- a/erts/emulator/test/send_term_SUITE.erl +++ b/erts/emulator/test/send_term_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2009. All Rights Reserved. +%% Copyright Ericsson AB 2005-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -61,7 +61,7 @@ basic(Config) when is_list(Config) -> ?line ExpectExt2Term = term(P, 5), %% ERL_DRV_INT, ERL_DRV_UINT - ?line case erlang:system_info(wordsize) of + ?line case erlang:system_info({wordsize, external}) of 4 -> ?line {-1, 4294967295} = term(P, 6); 8 -> @@ -76,40 +76,43 @@ basic(Config) when is_list(Config) -> ?line ExpectedBinTup = term(P, 7), %% single terms - ?line [] = term(P, 8), % ERL_DRV_NIL - ?line '' = term(P, 9), % ERL_DRV_ATOM - ?line an_atom = term(P, 10), % ERL_DRV_ATOM - ?line -4711 = term(P, 11), % ERL_DRV_INT - ?line 4711 = term(P, 12), % ERL_DRV_UINT - ?line P = term(P, 13), % ERL_DRV_PORT - ?line <<>> = term(P, 14), % ERL_DRV_BINARY - ?line <<"hejsan">> = term(P, 15), % ERL_DRV_BINARY - ?line <<>> = term(P, 16), % ERL_DRV_BUF2BINARY - ?line <<>> = term(P, 17), % ERL_DRV_BUF2BINARY - ?line <<"hoppsan">> = term(P, 18), % ERL_DRV_BUF2BINARY - ?line "" = term(P, 19), % ERL_DRV_STRING - ?line "" = term(P, 20), % ERL_DRV_STRING - ?line "hippsan" = term(P, 21), % ERL_DRV_STRING - ?line {} = term(P, 22), % ERL_DRV_TUPLE - ?line [] = term(P, 23), % ERL_DRV_LIST - ?line Self = term(P, 24), % ERL_DRV_PID - ?line [] = term(P, 25), % ERL_DRV_STRING_CONS - ?line AFloat = term(P, 26), % ERL_DRV_FLOAT + Singles = [{[], 8}, % ERL_DRV_NIL + {'', 9}, % ERL_DRV_ATOM + {an_atom, 10}, % ERL_DRV_ATOM + {-4711, 11}, % ERL_DRV_INT + {4711, 12}, % ERL_DRV_UINT + {P, 13}, % ERL_DRV_PORT + {<<>>, 14}, % ERL_DRV_BINARY + {<<"hejsan">>, 15}, % ERL_DRV_BINARY + {<<>>, 16}, % ERL_DRV_BUF2BINARY + {<<>>, 17}, % ERL_DRV_BUF2BINARY + {<<"hoppsan">>, 18}, % ERL_DRV_BUF2BINARY + {"", 19}, % ERL_DRV_STRING + {"", 20}, % ERL_DRV_STRING + {"hippsan", 21}, % ERL_DRV_STRING + {{}, 22}, % ERL_DRV_TUPLE + {[], 23}, % ERL_DRV_LIST + {Self, 24}, % ERL_DRV_PID + {[], 25}, % ERL_DRV_STRING_CONS + {[], 27}, % ERL_DRV_EXT2TERM + {18446744073709551615, 28}, % ERL_DRV_UINT64 + {20233590931456, 29}, % ERL_DRV_UINT64 + {4711, 30}, % ERL_DRV_UINT64 + {0, 31}, % ERL_DRV_UINT64 + {9223372036854775807, 32}, % ERL_DRV_INT64 + {20233590931456, 33}, % ERL_DRV_INT64 + {4711, 34}, % ERL_DRV_INT64 + {0, 35}, % ERL_DRV_INT64 + {-1, 36}, % ERL_DRV_INT64 + {-4711, 37}, % ERL_DRV_INT64 + {-20233590931456, 38}, % ERL_DRV_INT64 + {-9223372036854775808, 39}], % ERL_DRV_INT64 + ?line {Terms, Ops} = lists:unzip(Singles), + ?line Terms = term(P,Ops), + + AFloat = term(P, 26), % ERL_DRV_FLOAT ?line true = AFloat < 0.001, ?line true = AFloat > -0.001, - ?line [] = term(P, 27), % ERL_DRV_EXT2TERM - ?line 18446744073709551615 = term(P, 28), % ERL_DRV_UINT64 - ?line 20233590931456 = term(P, 29), % ERL_DRV_UINT64 - ?line 4711 = term(P, 30), % ERL_DRV_UINT64 - ?line 0 = term(P, 31), % ERL_DRV_UINT64 - ?line 9223372036854775807 = term(P, 32), % ERL_DRV_INT64 - ?line 20233590931456 = term(P, 33), % ERL_DRV_INT64 - ?line 4711 = term(P, 34), % ERL_DRV_INT64 - ?line 0 = term(P, 35), % ERL_DRV_INT64 - ?line -1 = term(P, 36), % ERL_DRV_INT64 - ?line -4711 = term(P, 37), % ERL_DRV_INT64 - ?line -20233590931456 = term(P, 38), % ERL_DRV_INT64 - ?line -9223372036854775808 = term(P, 39), % ERL_DRV_INT64 %% Failure cases. ?line [] = term(P, 127), diff --git a/erts/emulator/test/send_term_SUITE_data/send_term_drv.c b/erts/emulator/test/send_term_SUITE_data/send_term_drv.c index 6638de0560..165cce2e9d 100644 --- a/erts/emulator/test/send_term_SUITE_data/send_term_drv.c +++ b/erts/emulator/test/send_term_SUITE_data/send_term_drv.c @@ -17,6 +17,7 @@ */ #include "erl_driver.h" +#include <stdio.h> #include <errno.h> #include <string.h> @@ -65,12 +66,21 @@ static void fail_term(ErlDrvTermData* msg, int len, int line); static void send_term_drv_run(ErlDrvData port, char *buf, int count) { - ErlDrvTermData msg[1024]; - - switch (*buf) { + char buf7[1024]; + ErlDrvTermData spec[1024]; + ErlDrvTermData* msg = spec; + ErlDrvBinary* bins[15]; + int bin_ix = 0; + ErlDrvSInt64 s64[15]; + int s64_ix = 0; + ErlDrvUInt64 u64[15]; + int u64_ix = 0; + int i = 0; + + for (i=0; i<count; i++) switch (buf[i]) { case 0: msg[0] = ERL_DRV_NIL; - output_term(msg, 1); + msg += 1; break; case 1: /* Most term types inside a tuple. */ @@ -102,7 +112,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[22] = driver_connected(erlang_port); msg[23] = ERL_DRV_TUPLE; msg[24] = (ErlDrvTermData) 7; - output_term(msg, 25); + msg += 25; } break; @@ -117,7 +127,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[i] = ERL_DRV_NIL; msg[i+1] = ERL_DRV_LIST; msg[i+2] = (ErlDrvTermData) 201; - output_term(msg, i+3); + msg += i+3; } break; @@ -126,7 +136,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) ErlDrvBinary* bin; int i; - bin = driver_alloc_binary(256); + bin = bins[bin_ix++] = driver_alloc_binary(256); for (i = 0; i < 256; i++) { bin->orig_bytes[i] = i; } @@ -140,8 +150,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[7] = (ErlDrvTermData) 23; msg[8] = ERL_DRV_TUPLE; msg[9] = (ErlDrvTermData) 2; - output_term(msg, 10); - driver_free_binary(bin); + msg += 10; } break; @@ -152,11 +161,11 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[3] = driver_caller(erlang_port); msg[4] = ERL_DRV_TUPLE; msg[5] = (ErlDrvTermData) 2; - output_term(msg, 6); + msg += 6; break; case 5: - output_term(msg, make_ext_term_list(msg, 0)); + msg += make_ext_term_list(msg, 0); break; case 6: @@ -166,94 +175,91 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[3] = ~((ErlDrvTermData) 0); msg[4] = ERL_DRV_TUPLE; msg[5] = (ErlDrvTermData) 2; - output_term(msg, 6); + msg += 6; break; case 7: { int len = 0; - char buf[1024]; - memset(buf, 17, sizeof(buf)); + memset(buf7, 17, sizeof(buf7)); /* empty heap binary */ msg[len++] = ERL_DRV_BUF2BINARY; msg[len++] = (ErlDrvTermData) NULL; /* NULL is ok if size == 0 */ msg[len++] = (ErlDrvTermData) 0; /* empty heap binary again */ msg[len++] = ERL_DRV_BUF2BINARY; - msg[len++] = (ErlDrvTermData) &buf[0]; /* ptr is ok if size == 0 */ + msg[len++] = (ErlDrvTermData) buf7; /* ptr is ok if size == 0 */ msg[len++] = (ErlDrvTermData) 0; /* heap binary */ msg[len++] = ERL_DRV_BUF2BINARY; - msg[len++] = (ErlDrvTermData) &buf[0]; + msg[len++] = (ErlDrvTermData) buf7; msg[len++] = (ErlDrvTermData) 17; /* off heap binary */ msg[len++] = ERL_DRV_BUF2BINARY; - msg[len++] = (ErlDrvTermData) &buf[0]; - msg[len++] = (ErlDrvTermData) sizeof(buf); + msg[len++] = (ErlDrvTermData) buf7; + msg[len++] = (ErlDrvTermData) sizeof(buf7); msg[len++] = ERL_DRV_TUPLE; msg[len++] = (ErlDrvTermData) 4; - output_term(msg, len); + msg += len; break; } case 8: msg[0] = ERL_DRV_NIL; - output_term(msg, 1); + msg += 1; break; case 9: msg[0] = ERL_DRV_ATOM; msg[1] = (ErlDrvTermData) driver_mk_atom(""); - output_term(msg, 2); + msg += 2; break; case 10: msg[0] = ERL_DRV_ATOM; msg[1] = (ErlDrvTermData) driver_mk_atom("an_atom"); - output_term(msg, 2); + msg += 2; break; case 11: msg[0] = ERL_DRV_INT; msg[1] = (ErlDrvTermData) -4711; - output_term(msg, 2); + msg += 2; break; case 12: msg[0] = ERL_DRV_UINT; msg[1] = (ErlDrvTermData) 4711; - output_term(msg, 2); + msg += 2; break; case 13: msg[0] = ERL_DRV_PORT; msg[1] = driver_mk_port(erlang_port); - output_term(msg, 2); + msg += 2; break; case 14: { - ErlDrvBinary *dbin = driver_alloc_binary(0); + ErlDrvBinary *dbin = bins[bin_ix++] = driver_alloc_binary(0); msg[0] = ERL_DRV_BINARY; msg[1] = (ErlDrvTermData) dbin; msg[2] = (ErlDrvTermData) 0; msg[3] = (ErlDrvTermData) 0; - output_term(msg, 4); - driver_free_binary(dbin); + msg += 4; break; } case 15: { - char buf[] = "hejsan"; - ErlDrvBinary *dbin = driver_alloc_binary(sizeof(buf)-1); + static const char buf[] = "hejsan"; + ErlDrvBinary *dbin = bins[bin_ix++] = driver_alloc_binary(sizeof(buf)-1); if (dbin) memcpy((void *) dbin->orig_bytes, (void *) buf, sizeof(buf)-1); msg[0] = ERL_DRV_BINARY; msg[1] = (ErlDrvTermData) dbin; msg[2] = (ErlDrvTermData) (dbin ? sizeof(buf)-1 : 0); msg[3] = (ErlDrvTermData) 0; - output_term(msg, 4); - driver_free_binary(dbin); + msg += 4; break; } @@ -261,24 +267,24 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[0] = ERL_DRV_BUF2BINARY; msg[1] = (ErlDrvTermData) NULL; msg[2] = (ErlDrvTermData) 0; - output_term(msg, 3); + msg += 3; break; case 17: { - char buf[] = ""; + static const char buf[] = ""; msg[0] = ERL_DRV_BUF2BINARY; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf)-1; - output_term(msg, 3); + msg += 3; break; } case 18: { - char buf[] = "hoppsan"; + static const char buf[] = "hoppsan"; msg[0] = ERL_DRV_BUF2BINARY; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf)-1; - output_term(msg, 3); + msg += 3; break; } @@ -286,44 +292,44 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[0] = ERL_DRV_STRING; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) 0; - output_term(msg, 3); + msg += 3; break; case 20: { - char buf[] = ""; + static const char buf[] = ""; msg[0] = ERL_DRV_STRING; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf)-1; - output_term(msg, 3); + msg += 3; break; } case 21: { - char buf[] = "hippsan"; + static const char buf[] = "hippsan"; msg[0] = ERL_DRV_STRING; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf)-1; - output_term(msg, 3); + msg += 3; break; } case 22: msg[0] = ERL_DRV_TUPLE; msg[1] = (ErlDrvTermData) 0; - output_term(msg, 2); + msg += 2; break; case 23: msg[0] = ERL_DRV_NIL; msg[1] = ERL_DRV_LIST; msg[2] = (ErlDrvTermData) 1; - output_term(msg, 3); + msg += 3; break; case 24: msg[0] = ERL_DRV_PID; msg[1] = driver_connected(erlang_port); - output_term(msg, 2); + msg += 2; break; case 25: @@ -331,132 +337,131 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) msg[1] = ERL_DRV_STRING_CONS; msg[2] = (ErlDrvTermData) ""; msg[3] = (ErlDrvTermData) 0; - output_term(msg, 4); + msg += 4; break; case 26: { - double my_float = 0.0; + static double my_float = 0.0; msg[0] = ERL_DRV_FLOAT; msg[1] = (ErlDrvTermData) &my_float; - output_term(msg, 2); + msg += 2; break; } case 27: { - char buf[] = {131, 106}; /* [] */ + static char buf[] = {131, 106}; /* [] */ msg[0] = ERL_DRV_EXT2TERM; msg[1] = (ErlDrvTermData) buf; msg[2] = (ErlDrvTermData) sizeof(buf); - output_term(msg, 3); + msg += 3; break; } case 28: { - ErlDrvUInt64 x = ~((ErlDrvUInt64) 0); + ErlDrvUInt64* x = &u64[u64_ix++]; + *x = ~((ErlDrvUInt64) 0); msg[0] = ERL_DRV_UINT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 29: { - ErlDrvUInt64 x = ((ErlDrvUInt64) 4711) << 32; + ErlDrvUInt64* x = &u64[u64_ix++]; + *x = ((ErlDrvUInt64) 4711) << 32; msg[0] = ERL_DRV_UINT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 30: { - ErlDrvUInt64 x = 4711; + ErlDrvUInt64* x = &u64[u64_ix++]; + *x = 4711; msg[0] = ERL_DRV_UINT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 31: { - ErlDrvUInt64 x = 0; + ErlDrvUInt64* x = &u64[u64_ix++]; + *x = 0; msg[0] = ERL_DRV_UINT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 32: { - ErlDrvSInt64 x = ((((ErlDrvUInt64) 0x7fffffff) << 32) - | ((ErlDrvUInt64) 0xffffffff)); + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = ((((ErlDrvUInt64) 0x7fffffff) << 32) | ((ErlDrvUInt64) 0xffffffff)); msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 33: { - ErlDrvSInt64 x = (ErlDrvSInt64) (((ErlDrvUInt64) 4711) << 32); + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = (ErlDrvSInt64) (((ErlDrvUInt64) 4711) << 32); msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 34: { - ErlDrvSInt64 x = 4711; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = 4711; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 35: { - ErlDrvSInt64 x = 0; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = 0; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 36: { - ErlDrvSInt64 x = -1; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = -1; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 37: { - ErlDrvSInt64 x = -4711; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = -4711; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 38: { - ErlDrvSInt64 x = ((ErlDrvSInt64) ((ErlDrvUInt64) 4711) << 32)*-1; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = ((ErlDrvSInt64) ((ErlDrvUInt64) 4711) << 32)*-1; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } case 39: { - ErlDrvSInt64 x = ((ErlDrvSInt64) 1) << 63; + ErlDrvSInt64* x = &s64[s64_ix++]; + *x = ((ErlDrvSInt64) 1) << 63; msg[0] = ERL_DRV_INT64; - msg[1] = (ErlDrvTermData) &x; - output_term(msg, 2); - + msg[1] = (ErlDrvTermData) x; + msg += 2; break; } @@ -464,7 +469,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) case 127: /* Error cases */ { long refc; - ErlDrvBinary* bin = driver_alloc_binary(256); + ErlDrvBinary* bin = bins[bin_ix++] = driver_alloc_binary(256); FAIL_TERM(msg, 0); @@ -537,7 +542,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) refc = driver_binary_get_refc(bin); if (refc > 3) { char sbuf[128]; - sprintf(sbuf, "bad_refc:%d", refc); + sprintf(sbuf, "bad_refc:%ld", refc); driver_failure_atom(erlang_port, sbuf); } driver_free_binary(bin); @@ -644,6 +649,7 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) /* Signal end of test case */ msg[0] = ERL_DRV_NIL; driver_output_term(erlang_port, msg, 1); + return; } break; @@ -651,6 +657,16 @@ static void send_term_drv_run(ErlDrvData port, char *buf, int count) driver_failure_atom(erlang_port, "bad_request"); break; } + if (count > 1) { + *msg++ = ERL_DRV_NIL; + *msg++ = ERL_DRV_LIST; + *msg++ = count + 1; + } + output_term(spec, msg-spec); + if ((bin_ix|s64_ix|u64_ix) > 15) abort(); + while (bin_ix) { + driver_free_binary(bins[--bin_ix]); + } } static void output_term(ErlDrvTermData* msg, int len) diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index ba433d4e11..cd940f3ddf 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -132,6 +132,7 @@ misc_smoke_tests(Config) when is_list(Config) -> ?line true = is_binary(erlang:system_info(procs)), ?line true = is_binary(erlang:system_info(loaded)), ?line true = is_binary(erlang:system_info(dist)), + ?line ok = try erlang:system_info({cpu_topology,erts_get_cpu_topology_error_case}), fail catch error:badarg -> ok end, ?line ok. diff --git a/erts/emulator/test/time_SUITE.erl b/erts/emulator/test/time_SUITE.erl index 2ad1f0d201..095e9dd1af 100644 --- a/erts/emulator/test/time_SUITE.erl +++ b/erts/emulator/test/time_SUITE.erl @@ -34,6 +34,8 @@ consistency/1, now/1, now_unique/1, now_update/1, timestamp/1]). +-export([local_to_univ_utc/1]). + -include("test_server.hrl"). -export([linear_time/1]). @@ -53,7 +55,40 @@ -define(dst_timezone, 2). all(suite) -> [univ_to_local, local_to_univ, - bad_univ_to_local, bad_local_to_univ, consistency, now, timestamp]. + local_to_univ_utc, + bad_univ_to_local, bad_local_to_univ, + consistency, now, timestamp]. + +local_to_univ_utc(suite) -> + []; +local_to_univ_utc(doc) -> + ["Test that DST = true on timezones without DST is ignored"]; +local_to_univ_utc(Config) when is_list(Config) -> + case os:type() of + {unix,_} -> + %% TZ variable has a meaning + ?line {ok, Node} = + test_server:start_node(local_univ_utc,peer, + [{args, "-env TZ UTC"}]), + ?line {{2008,8,1},{0,0,0}} = + rpc:call(Node, + erlang,localtime_to_universaltime, + [{{2008, 8, 1}, {0, 0, 0}}, + false]), + ?line {{2008,8,1},{0,0,0}} = + rpc:call(Node, + erlang,localtime_to_universaltime, + [{{2008, 8, 1}, {0, 0, 0}}, + true]), + ?line [{{2008,8,1},{0,0,0}}] = + rpc:call(Node, + calendar,local_time_to_universal_time_dst, + [{{2008, 8, 1}, {0, 0, 0}}]), + ?line test_server:stop_node(Node), + ok; + _ -> + {skip,"Only valid on Unix"} + end. %% Tests conversion from univeral to local time. diff --git a/erts/emulator/test/trace_call_time_SUITE.erl b/erts/emulator/test/trace_call_time_SUITE.erl new file mode 100644 index 0000000000..7bc91addde --- /dev/null +++ b/erts/emulator/test/trace_call_time_SUITE.erl @@ -0,0 +1,614 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% +%%% Define to run outside of test server +%%% +%%% -define(STANDALONE,1). +%%% +%%% +%%% Define for debug output +%%% +%%% -define(debug,1). + +-module(trace_call_time_SUITE). + +%% Exported end user tests + +-export([seq/3, seq_r/3]). +-export([loaded/1, a_function/1, a_called_function/1, dec/1, nif_dec/1]). + +-define(US_ERROR, 10000). +-define(R_ERROR, 0.8). +-define(SINGLE_CALL_US_TIME, 10). + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Result examination macros + +-define(CT(P,MFA),{trace,P,call,MFA}). +-define(CTT(P, MFA),{trace_ts,P,call,MFA,{_,_,_}}). +-define(RF(P,MFA,V),{trace,P,return_from,MFA,V}). +-define(RFT(P,MFA,V),{trace_ts,P,return_from,MFA,V,{_,_,_}}). +-define(RT(P,MFA),{trace,P,return_to,MFA}). +-define(RTT(P,MFA),{trace_ts,P,return_to,MFA,{_,_,_}}). + +-ifdef(debug). +-define(dbgformat(A,B),io:format(A,B)). +-else. +-define(dbgformat(A,B),noop). +-endif. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +-include("test_server.hrl"). + +%% When run in test server. +-export([all/1, init_per_testcase/2, fin_per_testcase/2, not_run/1]). +-export([basic/1, on_and_off/1, info/1, + pause_and_restart/1, scheduling/1, called_function/1, combo/1, bif/1, nif/1]). + +init_per_testcase(_Case, Config) -> + ?line Dog=test_server:timetrap(test_server:seconds(400)), + erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time,call_count]), + erlang:trace_pattern(on_load, false, [local,meta,call_time,call_count]), + timer:now_diff(now(),now()), + [{watchdog, Dog}|Config]. + +fin_per_testcase(_Case, Config) -> + erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time,call_count]), + erlang:trace_pattern(on_load, false, [local,meta,call_time,call_count]), + erlang:trace(all, false, [all]), + Dog=?config(watchdog, Config), + test_server:timetrap_cancel(Dog), + ok. + +all(doc) -> + ["Test call count tracing of local function calls."]; +all(suite) -> + case test_server:is_native(?MODULE) of + true -> [not_run]; + false -> [basic, on_and_off, info, + pause_and_restart, scheduling, combo, bif, nif, called_function] + end. + +not_run(Config) when is_list(Config) -> + {skipped,"Native code"}. + +basic(suite) -> + []; +basic(doc) -> + ["Tests basic call count trace"]; +basic(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 1000, + %% + ?line 1 = erlang:trace_pattern({?MODULE,seq, '_'}, true, [call_time]), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_time]), + ?line Pid = setup(), + ?line {L, T1} = execute(Pid, fun() -> seq(1, M, fun(X) -> (X+1) end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T1), + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [], none), + + ?line {Lr, T2} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> (X+1) end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T1), + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Pid, 1, 0, 0}], T2/M), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Pid, M, 0, 0}], T2), + ?line L = lists:reverse(Lr), + + %% + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line Pid ! quit, + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +on_and_off(suite) -> + []; +on_and_off(doc) -> + ["Tests turning trace parameters on and off"]; +on_and_off(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 100, + %% + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, true, [call_time]), + ?line Pid = setup(), + ?line {L, T1} = execute(Pid, {?MODULE, seq, [1, M, fun(X) -> X+1 end]}), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T1), + + ?line N = erlang:trace_pattern({?MODULE,'_','_'}, true, [call_time]), + ?line {L, T2} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T2), + + ?line P = erlang:trace_pattern({'_','_','_'}, true, [call_time]), + ?line {L, T3} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid, M, 0, 0}], T3), + + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, false, [call_time]), + ?line ok = check_trace_info({?MODULE, seq, 3}, false, none), + ?line {L, _T4} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, false, none), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [], none), + ?line {Lr, T5} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Pid,M,0,0}], T5), + + ?line N = erlang:trace_pattern({?MODULE,'_','_'}, false, [call_time]), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, false, none), + ?line {Lr, _T6} = execute(Pid, fun() -> seq_r(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, false, none), + ?line L = lists:reverse(Lr), + %% + ?line Pid ! quit, + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +info(suite) -> + []; +info(doc) -> + ["Tests the trace_info BIF"]; +info(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + %% + ?line 1 = erlang:trace_pattern({?MODULE,seq,3}, true, [call_time]), + ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, pause, [call_time]), + ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time), + ?line {all,[_|_]=L} = erlang:trace_info({?MODULE,seq,3}, all), + ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, L), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, restart, [call_time]), + ?line {call_time,[]} = erlang:trace_info({?MODULE,seq,3}, call_time), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, false, [call_time]), + ?line {call_time,false} = erlang:trace_info({?MODULE,seq,3}, call_time), + ?line {all,false} = erlang:trace_info({?MODULE,seq,3}, all), + %% + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +pause_and_restart(suite) -> + []; +pause_and_restart(doc) -> + ["Tests pausing and restarting call time counters"]; +pause_and_restart(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 100, + ?line Pid = setup(), + %% + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, true, [call_time]), + ?line ok = check_trace_info({?MODULE, seq, 3}, [], none), + ?line {L, T1} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T1), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, pause, [call_time]), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T1), + ?line {L, T2} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T2), + ?line 1 = erlang:trace_pattern({?MODULE,seq,'_'}, restart, [call_time]), + ?line ok = check_trace_info({?MODULE, seq, 3}, [], none), + ?line {L, T3} = execute(Pid, fun() -> seq(1, M, fun(X) -> X+1 end) end), + ?line ok = check_trace_info({?MODULE, seq, 3}, [{Pid,M,0,0}], T3), + %% + ?line Pid ! quit, + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +scheduling(suite) -> + []; +scheduling(doc) -> + ["Tests in/out scheduling of call time counters"]; +scheduling(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 1000000, + ?line Np = erlang:system_info(schedulers_online), + ?line F = 12, + + %% setup load processes + %% (single, no internal calls) + + ?line erlang:trace_pattern({?MODULE,loaded,1}, true, [call_time]), + + ?line Pids = [setup() || _ <- lists:seq(1, F*Np)], + ?line {_Ls,T1} = execute(Pids, {?MODULE,loaded,[M]}), + ?line [Pid ! quit || Pid <- Pids], + + %% logic dictates that each process will get ~ 1/F of the schedulers time + + ?line {call_time, CT} = erlang:trace_info({?MODULE,loaded,1}, call_time), + + ?line lists:foreach(fun (Pid) -> + ?line ok = case check_process_time(lists:keysearch(Pid, 1, CT), M, F, T1) of + schedule_time_error -> + test_server:comment("Warning: Failed time ratio"), + ok; + Other -> Other + end + end, Pids), + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +combo(suite) -> + []; +combo(doc) -> + ["Tests combining local call trace and meta trace with call time trace"]; +combo(Config) when is_list(Config) -> + ?line Self = self(), + ?line Nbc = 3, + ?line MetaMs = [{'_',[],[{return_trace}]}], + ?line Flags = lists:sort([call, return_to]), + ?line LocalTracer = spawn_link(fun () -> relay_n(5 + Nbc + 3, Self) end), + ?line MetaTracer = spawn_link(fun () -> relay_n(9 + Nbc + 3, Self) end), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, [], [local]), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_time]), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, MetaMs, [{meta,MetaTracer}]), + ?line 2 = erlang:trace_pattern({?MODULE,seq_r,'_'}, true, [call_count]), + + % bifs + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, [], [local]), + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_time]), + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, MetaMs, [{meta,MetaTracer}]), + %% not implemented + %?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_count]), + + ?line 1 = erlang:trace(Self, true, [{tracer,LocalTracer} | Flags]), + %% + ?line {traced,local} = + erlang:trace_info({?MODULE,seq_r,3}, traced), + ?line {match_spec,[]} = + erlang:trace_info({?MODULE,seq_r,3}, match_spec), + ?line {meta,MetaTracer} = + erlang:trace_info({?MODULE,seq_r,3}, meta), + ?line {meta_match_spec,MetaMs} = + erlang:trace_info({?MODULE,seq_r,3}, meta_match_spec), + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [], none), + + %% check empty trace_info for ?MODULE:seq_r/3 + ?line {all,[_|_]=TraceInfo} = erlang:trace_info({?MODULE,seq_r,3}, all), + ?line {value,{traced,local}} = lists:keysearch(traced, 1, TraceInfo), + ?line {value,{match_spec,[]}} = lists:keysearch(match_spec, 1, TraceInfo), + ?line {value,{meta,MetaTracer}} = lists:keysearch(meta, 1, TraceInfo), + ?line {value,{meta_match_spec,MetaMs}} = lists:keysearch(meta_match_spec, 1, TraceInfo), + ?line {value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfo), + ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, TraceInfo), + + %% check empty trace_info for erlang:term_to_binary/1 + ?line {all, [_|_] = TraceInfoBif} = erlang:trace_info({erlang, term_to_binary, 1}, all), + ?line {value,{traced,local}} = lists:keysearch(traced, 1, TraceInfoBif), + ?line {value,{match_spec,[]}} = lists:keysearch(match_spec, 1, TraceInfoBif), + ?line {value,{meta, MetaTracer}} = lists:keysearch(meta, 1, TraceInfoBif), + ?line {value,{meta_match_spec,MetaMs}} = lists:keysearch(meta_match_spec, 1, TraceInfoBif), + %% not implemented + ?line {value,{call_count,false}} = lists:keysearch(call_count, 1, TraceInfoBif), + %?line {value,{call_count,0}} = lists:keysearch(call_count, 1, TraceInfoBif), + ?line {value,{call_time,[]}} = lists:keysearch(call_time, 1, TraceInfoBif), + + %% + ?line [3,2,1] = seq_r(1, 3, fun(X) -> X+1 end), + ?line T0 = now(), + ?line with_bif(Nbc), + ?line T1 = now(), + ?line TimeB = timer:now_diff(T1,T0), + %% + + ?line List = collect(100), + ?line {MetaR, LocalR} = + lists:foldl( + fun ({P,X}, {M,L}) when P == MetaTracer -> + {[X|M],L}; + ({P,X}, {M,L}) when P == LocalTracer -> + {M,[X|L]} + end, + {[],[]}, + List), + ?line Meta = lists:reverse(MetaR), + ?line Local = lists:reverse(LocalR), + + ?line [?CTT(Self,{?MODULE,seq_r,[1,3,_]}), + ?CTT(Self,{?MODULE,seq_r,[1,3,_,[]]}), + ?CTT(Self,{?MODULE,seq_r,[2,3,_,[1]]}), + ?CTT(Self,{?MODULE,seq_r,[3,3,_,[2,1]]}), + ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]), + ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]), + ?RFT(Self,{?MODULE,seq_r,4},[3,2,1]), + ?RFT(Self,{?MODULE,seq_r,3},[3,2,1]), + ?CTT(Self,{erlang,term_to_binary,[3]}), % bif + ?RFT(Self,{erlang,term_to_binary,1},<<131,97,3>>), + ?CTT(Self,{erlang,term_to_binary,[2]}), + ?RFT(Self,{erlang,term_to_binary,1},<<131,97,2>>) + ] = Meta, + + ?line [?CT(Self,{?MODULE,seq_r,[1,3,_]}), + ?CT(Self,{?MODULE,seq_r,[1,3,_,[]]}), + ?CT(Self,{?MODULE,seq_r,[2,3,_,[1]]}), + ?CT(Self,{?MODULE,seq_r,[3,3,_,[2,1]]}), + ?RT(Self,{?MODULE,combo,1}), + ?CT(Self,{erlang,term_to_binary,[3]}), % bif + ?RT(Self,{?MODULE,with_bif,1}), + ?CT(Self,{erlang,term_to_binary,[2]}), + ?RT(Self,{?MODULE,with_bif,1}) + ] = Local, + + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Self,1,0,0}], 1), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Self,3,0,0}], 1), + ?line ok = check_trace_info({?MODULE, seq_r, 3}, [{Self,1,0,0}], 1), + ?line ok = check_trace_info({?MODULE, seq_r, 4}, [{Self,3,0,0}], 1), + ?line ok = check_trace_info({erlang, term_to_binary, 1}, [{self(), Nbc - 1, 0, 0}], TimeB), + %% + ?line erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time]), + ?line erlang:trace_pattern(on_load, false, [local,meta,call_time]), + ?line erlang:trace(all, false, [all]), + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +bif(suite) -> + []; +bif(doc) -> + ["Tests tracing of bifs"]; +bif(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 1000000, + %% + ?line 2 = erlang:trace_pattern({erlang, binary_to_term, '_'}, true, [call_time]), + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, true, [call_time]), + ?line Pid = setup(), + ?line {L, T1} = execute(Pid, fun() -> with_bif(M) end), + + ?line ok = check_trace_info({erlang, binary_to_term, 1}, [{Pid, M - 1, 0, 0}], T1/2), + ?line ok = check_trace_info({erlang, term_to_binary, 1}, [{Pid, M - 1, 0, 0}], T1/2), + + % disable term2binary + + ?line 2 = erlang:trace_pattern({erlang, term_to_binary, '_'}, false, [call_time]), + + ?line {L, T2} = execute(Pid, fun() -> with_bif(M) end), + + ?line ok = check_trace_info({erlang, binary_to_term, 1}, [{Pid, M*2 - 2, 0, 0}], T1/2 + T2), + ?line ok = check_trace_info({erlang, term_to_binary, 1}, false, none), + + %% + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line Pid ! quit, + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +nif(suite) -> + []; +nif(doc) -> + ["Tests tracing of nifs"]; +nif(Config) when is_list(Config) -> + load_nif(Config), + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 1000000, + %% + ?line 1 = erlang:trace_pattern({?MODULE, nif_dec, '_'}, true, [call_time]), + ?line 1 = erlang:trace_pattern({?MODULE, with_nif, '_'}, true, [call_time]), + ?line Pid = setup(), + ?line {L, T1} = execute(Pid, fun() -> with_nif(M) end), + + % the nif is called M - 1 times, the last time the function with 'with_nif' + % returns ok and does not call the nif. + ?line ok = check_trace_info({?MODULE, nif_dec, 1}, [{Pid, M-1, 0, 0}], T1/5*4), + ?line ok = check_trace_info({?MODULE, with_nif, 1}, [{Pid, M, 0, 0}], T1/5), + + %% + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line Pid ! quit, + ok. + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +called_function(suite) -> + []; +called_function(doc) -> + ["Tests combining nested function calls and that the time accumulates to the right function"]; +called_function(Config) when is_list(Config) -> + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ?line M = 2100, + ?line Pid = setup(), + %% + ?line 1 = erlang:trace_pattern({?MODULE,a_function,'_'}, true, [call_time]), + ?line {L, T1} = execute(Pid, {?MODULE, a_function, [M]}), + ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M, 0, 0}], T1), + + ?line 1 = erlang:trace_pattern({?MODULE,a_called_function,'_'}, true, [call_time]), + ?line {L, T2} = execute(Pid, {?MODULE, a_function, [M]}), + ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M+M, 0, 0}], T1 + M*?SINGLE_CALL_US_TIME), + ?line ok = check_trace_info({?MODULE, a_called_function, 1}, [{Pid, M, 0, 0}], T2), + + + ?line 1 = erlang:trace_pattern({?MODULE,dec,'_'}, true, [call_time]), + ?line {L, T3} = execute(Pid, {?MODULE, a_function, [M]}), + ?line ok = check_trace_info({?MODULE, a_function, 1}, [{Pid, M+M+M, 0, 0}], T1 + (M+M)*?SINGLE_CALL_US_TIME), + ?line ok = check_trace_info({?MODULE, a_called_function, 1}, [{Pid, M+M, 0, 0}], T2 + M*?SINGLE_CALL_US_TIME ), + ?line ok = check_trace_info({?MODULE, dec, 1}, [{Pid, M, 0, 0}], T3), + + ?line Pid ! quit, + ?line P = erlang:trace_pattern({'_','_','_'}, false, [call_time]), + ok. + +%%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% The Tests +%%% + +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Local helpers + + +load_nif(Config) -> + ?line Path = ?config(data_dir, Config), + ?line ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0). + + +%% Stack recursive seq +seq(Stop, Stop, Succ) when is_function(Succ) -> + [Stop]; +seq(Start, Stop, Succ) when is_function(Succ) -> + [Start | seq(Succ(Start), Stop, Succ)]. + + +a_function(1) -> a_called_function(1); +a_function(N) when N > 1 -> a_function(a_called_function(N)). + +a_called_function(N) -> dec(N). + +with_bif(1) -> ok; +with_bif(N) -> + with_bif(erlang:binary_to_term(erlang:term_to_binary(N)) - 1). + +with_nif(0) -> error; +with_nif(1) -> ok; +with_nif(N) -> + with_nif(?MODULE:nif_dec(N)). + + +nif_dec(N) -> 0. + +dec(N) -> + loaded(10000), + N - 1. + +loaded(N) when N > 1 -> loaded(N - 1); +loaded(_) -> 5. + + +%% Tail recursive seq, result list is reversed +seq_r(Start, Stop, Succ) when is_function(Succ) -> + seq_r(Start, Stop, Succ, []). + +seq_r(Stop, Stop, _, R) -> + [Stop | R]; +seq_r(Start, Stop, Succ, R) -> + seq_r(Succ(Start), Stop, Succ, [Start | R]). + +% Check call time tracing data and print mismatches +check_trace_info(Mfa, [{Pid, C,_,_}] = Expect, Time) -> + case erlang:trace_info(Mfa, call_time) of + % Time tests are somewhat problematic. We want to know if Time (EXPECTED_TIME) and S*1000000 + Us (ACTUAL_TIME) + % is the same. + % If the ratio EXPECTED_TIME/ACTUAL_TIME is ~ 1 or if EXPECTED_TIME - ACTUAL_TIME is near zero, the test is ok. + {call_time,[{Pid,C,S,Us}]} when S >= 0, Us >= 0, abs(1 - Time/(S*1000000 + Us)) < ?R_ERROR; abs(Time - S*1000000 - Us) < ?US_ERROR -> + ok; + {call_time,[{Pid,C,S,Us}]} -> + Sum = S*1000000 + Us, + io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~w s. ~w us. = ~w us. - ~w -> delta ~w (ratio ~.2f, should be 1.0)~n", + [Mfa, Expect, Time, S, Us, Sum, Time, Sum - Time, Time/Sum]), + time_error; + Other -> + io:format("Expected ~p -> {call_time, ~p (Time ~p us)}~n - got ~p~n", [ Mfa, Expect, Time, Other]), + time_count_error + end; +check_trace_info(Mfa, Expect, _) -> + case erlang:trace_info(Mfa, call_time) of + {call_time, Expect} -> + ok; + Other -> + io:format("Expected ~p -> {call_time, ~p}~n - got ~p~n", [Mfa, Expect, Other]), + result_not_expected_error + end. + + +%check process time +check_process_time({value,{Pid, M, S, Us}}, M, F, Time) -> + ?line Sum = S*1000000 + Us, + if + abs(1 - (F/(Time/Sum))) < ?R_ERROR -> + ok; + true -> + io:format("- Pid ~p, Got ratio ~.2f, expected ratio ~w~n", [Pid, Time/Sum,F]), + schedule_time_error + end; +check_process_time(Other, M, _, _) -> + io:format(" - Got ~p, expected count ~w~n", [Other, M]), + error. + + + +%% Message relay process +relay_n(0, _) -> + ok; +relay_n(N, Dest) -> + receive Msg -> + Dest ! {self(), Msg}, + relay_n(N-1, Dest) + end. + + + +%% Collect received messages +collect(Time) -> + Ref = erlang:start_timer(Time, self(), done), + L = lists:reverse(collect([], Ref)), + ?dbgformat("Got: ~p~n",[L]), + L. + +collect(A, 0) -> + receive + Mess -> + collect([Mess | A], 0) + after 0 -> + A + end; +collect(A, Ref) -> + receive + {timeout, Ref, done} -> + collect(A, 0); + Mess -> + collect([Mess | A], Ref) + end. + +setup() -> + Pid = spawn_link(fun() -> loop() end), + ?line 1 = erlang:trace(Pid, true, [call]), + Pid. + +execute(Pids, Mfa) when is_list(Pids) -> + T0 = now(), + [P ! {self(), execute, Mfa} || P <- Pids], + As = [receive {P, answer, Answer} -> Answer end || P <- Pids], + T1 = now(), + {As, timer:now_diff(T1,T0)}; +execute(P, Mfa) -> + T0 = now(), + P ! {self(), execute, Mfa}, + A = receive {P, answer, Answer} -> Answer end, + T1 = now(), + {A, timer:now_diff(T1,T0)}. + + + +loop() -> + receive + quit -> + ok; + {Pid, execute, Fun } when is_function(Fun) -> + Pid ! {self(), answer, erlang:apply(Fun, [])}, + loop(); + {Pid, execute, {M, F, A}} -> + Pid ! {self(), answer, erlang:apply(M, F, A)}, + loop() + end. diff --git a/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src b/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src new file mode 100644 index 0000000000..2b2a35bd2c --- /dev/null +++ b/erts/emulator/test/trace_call_time_SUITE_data/Makefile.src @@ -0,0 +1,6 @@ + +NIF_LIBS = trace_nif@dll@ + +all: $(NIF_LIBS) + +@SHLIB_RULES@ diff --git a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c new file mode 100644 index 0000000000..33b346aab7 --- /dev/null +++ b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c @@ -0,0 +1,37 @@ +#include "erl_nif.h" + + +static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + return 0; +} + +static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + return 0; +} + +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +{ + return 0; +} + +static void unload(ErlNifEnv* env, void* priv_data) +{ +} + +static ERL_NIF_TERM nif_dec_1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + int x = 0; + enif_get_uint(env, argv[0], &x); + return enif_make_int(env, x - 1); +} + + + +static ErlNifFunc nif_funcs[] = +{ + {"nif_dec", 1, nif_dec_1} +}; + +ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,reload,upgrade,unload) diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index de19a2e35b..e7c57142c0 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -27,6 +27,7 @@ my $outdir = "."; # Directory for output files. my $verbose = 0; my $hot = 1; my $num_file_opcodes = 0; +my $wordsize = 32; # This is shift counts and mask for the packer. my $WHOLE_WORD = ''; @@ -36,12 +37,20 @@ my @pack_mask; $pack_instr[2] = ['6', 'i']; $pack_instr[3] = ['0', '0', 'i']; +$pack_instr[4] = ['6', '6', '6', 'i']; # Only for 64 bit wordsize $pack_shift[2] = ['0', 'BEAM_LOOSE_SHIFT']; $pack_shift[3] = ['0', 'BEAM_TIGHT_SHIFT', '(2*BEAM_TIGHT_SHIFT)']; +$pack_shift[4] = ['0', 'BEAM_LOOSE_SHIFT', # Only for 64 bit wordsize + '(2*BEAM_LOOSE_SHIFT)', + '(3*BEAM_LOOSE_SHIFT)']; $pack_mask[2] = ['BEAM_LOOSE_MASK', $WHOLE_WORD]; $pack_mask[3] = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK']; +$pack_mask[4] = ['BEAM_LOOSE_MASK', # Only for 64 bit wordsize + 'BEAM_LOOSE_MASK', + 'BEAM_LOOSE_MASK', + $WHOLE_WORD]; # There are two types of instructions: generic and specific. # The generic instructions are those generated by the Beam compiler. @@ -80,6 +89,8 @@ my %cold_code; my @unnumbered_generic; my %unnumbered; +my %is_transformed; + # # Code transformations. # @@ -118,7 +129,8 @@ my %arg_size = ('r' => 0, # x(0) - x register zero 't' => 1, # untagged integer -- can be packed 'b' => 1, # pointer to bif 'A' => 1, # arity value - 'P' => 1, # byte offset into tuple + 'P' => 1, # byte offset into tuple or stack + 'Q' => 1, # like 'P', but packable 'h' => 1, # character 'l' => 1, # float reg 'q' => 1, # literal term @@ -157,6 +169,7 @@ my @tag_type; $type_bit{'U'} = $type_bit{'u'}; $type_bit{'e'} = $type_bit{'u'}; $type_bit{'P'} = $type_bit{'u'}; + $type_bit{'Q'} = $type_bit{'u'}; } # @@ -169,6 +182,7 @@ while (@ARGV && $ARGV[0] =~ /^-(.*)/) { ($target = \&emulator_output), next if /^emulator/; ($target = \&compiler_output), next if /^compiler/; ($outdir = shift), next if /^outdir/; + ($wordsize = shift), next if /^wordsize/; ($verbose = 1), next if /^v/; die "$0: Bad option: -$_\n"; } @@ -474,8 +488,9 @@ sub emulator_output { $gen_transform_offset{$key} : -1; my($spec_op) = $gen_to_spec{$key}; my($num_specific) = $num_specific{$key}; - defined $spec_op or $tr != -1 or + defined $spec_op or $obsolete[$gen_opnum{$name,$arity}] or + $is_transformed{$name,$arity} or error("instruction $key has no specific instruction"); $spec_op = -1 unless defined $spec_op; &init_item($name, $arity, $spec_op, $num_specific, $tr, $min_window{$key}); @@ -498,12 +513,14 @@ sub emulator_output { print "#define NUM_SPECIFIC_OPS ", scalar(@op_to_name), "\n"; print "\n"; print "#ifdef ARCH_64\n"; + print "# define BEAM_WIDE_MASK 0xFFFFUL\n"; print "# define BEAM_LOOSE_MASK 0x1FFFUL\n"; print "#if HALFWORD_HEAP\n"; print "# define BEAM_TIGHT_MASK 0x1FFCUL\n"; print "#else\n"; print "# define BEAM_TIGHT_MASK 0x1FF8UL\n"; print "#endif\n"; + print "# define BEAM_WIDE_SHIFT 32\n"; print "# define BEAM_LOOSE_SHIFT 16\n"; print "# define BEAM_TIGHT_SHIFT 16\n"; print "#else\n"; @@ -796,6 +813,7 @@ sub basic_generator { 'I' => 1, 't' => 1, 'P' => 1, + 'Q' => 1, ); # Pick up the macro to use and its flags (if any). @@ -916,7 +934,18 @@ sub basic_generator { $var_decls .= "BeamInstr tmp_packed2;" if $macro_code =~ /tmp_packed2/; if ($flags =~ /-nonext/) { - $code = "$macro_code\n"; + $code = join("\n", + "{ $var_decls", + $macro_code, + "}"); + } elsif ($flags =~ /-goto:(\S*)/) { + my $goto = $1; + $code = join("\n", + "{ $var_decls", + $macro_code, + "I += $size + 1;", + "goto $goto;", + "}"); } else { $code = join("\n", "{ $var_decls", @@ -935,18 +964,31 @@ sub basic_generator { sub do_pack { my(@args) = @_; - my($i); my($packable_args) = 0; + my @is_packable; # Packability (boolean) for each argument. + my $wide_packing = 0; # # Count the number of packable arguments. If we encounter any 's' or 'd' # arguments, packing is not possible. # - for ($i = 0; $i < @args; $i++) { - if ($args[$i] =~ /[xyt]/) { + my $packable_types = "xytQ"; + foreach my $arg (@args) { + if ($arg =~ /^[$packable_types]/) { $packable_args++; - } elsif ($args[$i] =~ /[sd]/) { + push @is_packable, 1; + } elsif ($arg =~ /^I/ and $wordsize == 64 and $packable_args < 2) { + $wide_packing = 1; + push @is_packable, 1; + if (++$packable_args == 2) { + # We can only pack two arguments. Turn off packing + # for the rest of the arguments. + $packable_types = "\xFF"; + } + } elsif ($arg =~ /^[sd]/) { return ('', '', @args); + } else { + push @is_packable, 0; } } @@ -962,10 +1004,27 @@ sub do_pack { # beginning). my($up) = ''; # Pack commands (storing back while # moving forward). - my($args_per_word) = $packable_args < 4 ? $packable_args : 2; - my(@shift) = @{$pack_shift[$args_per_word]}; - my(@mask) = @{$pack_mask[$args_per_word]}; - my(@pack_instr) = @{$pack_instr[$args_per_word]}; + my $args_per_word; + if ($packable_args < 4 or $wordsize == 64) { + $args_per_word = $packable_args; + } else { + # 4 packable argument, 32 bit wordsize. Need 2 words. + $args_per_word = 2; + } + + my @shift; + my @mask; + my @instr; + + if ($wide_packing) { + @shift = ('0', 'BEAM_WIDE_SHIFT'); + @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD); + @instr = ('w', 'i'); + } else { + @shift = @{$pack_shift[$args_per_word]}; + @mask = @{$pack_mask[$args_per_word]}; + @instr = @{$pack_instr[$args_per_word]}; + } # # Now generate the packing instructions. One complication is that @@ -979,10 +1038,10 @@ sub do_pack { my($ap) = 0; # Argument number within word. my($tmpnum) = 1; # Number of temporary variable. my($expr) = ''; - for ($i = 0; $i < @args; $i++) { + for (my $i = 0; $i < @args; $i++) { my($reg) = $args[$i]; my($this_size) = $arg_size{$reg}; - if ($reg =~ /[xyt]/) { + if ($is_packable[$i]) { $this_size = 0; $did_some_packing = 1; @@ -993,7 +1052,7 @@ sub do_pack { $this_size = 1; } - $down = "$pack_instr[$ap]$down"; + $down = "$instr[$ap]$down"; my($unpack) = &make_unpack($tmpnum, $shift[$ap], $mask[$ap]); $args[$i] = "pack:$this_size:$reg" . "b($unpack)"; @@ -1103,6 +1162,10 @@ sub compile_transform { if ($obsolete[$gen_opnum{$name,$arity}]) { error("obsolete function must not be used in transformations"); } + + if ($src) { + $is_transformed{$name,$arity} = 1; + } [$name,$arity,@ops]; } @@ -1291,13 +1354,28 @@ sub tr_gen_from { my($var, $type, $type_val, $cond, $val) = @$op; if ($type ne '' && $type ne '*') { - my($types) = ''; - my($type_mask) = 0; - foreach (split('', $type)) { - $types .= "$_ "; - $type_mask |= $type_bit{$_}; + # + # The is_bif, is_not_bif, and is_func instructions have + # their own built-in type test and don't need to + # be guarded with a type test instruction. + # + unless ($cond eq 'is_bif' or + $cond eq 'is_not_bif' or + $cond eq 'is_func') { + my($types) = ''; + my($type_mask) = 0; + foreach (split('', $type)) { + $types .= "$_ "; + $type_mask |= $type_bit{$_}; + } + if ($cond ne 'is_eq') { + push(@code, &make_op($types, 'is_type', $type_mask)); + } else { + $cond = ''; + push(@code, &make_op($types, 'is_type_eq', + $type_mask, $val)); + } } - push(@code, &make_op($types, 'is_type', $type_mask)); } if ($cond eq 'is_func') { diff --git a/erts/emulator/utils/count b/erts/emulator/utils/count new file mode 100755 index 0000000000..617f5c25e8 --- /dev/null +++ b/erts/emulator/utils/count @@ -0,0 +1,127 @@ +%% -*- erlang -*- +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-mode(compile). + +main(_) -> + DisDir = "./dis", + ok = filelib:ensure_dir(filename:join(DisDir, "dummy")), + io:format("Dissambling to ~s\n", [DisDir]), + ok = file:set_cwd(DisDir), + Path = code:get_path() -- ["."], + Beams0 = [filelib:wildcard(filename:join(Dir, "*.beam")) || + Dir <- Path], + Beams = lists:append(Beams0), + Mods0 = [list_to_atom(filename:rootname(filename:basename(F))) || + F <- Beams], + Mods = lists:usort(Mods0), + start_sem(), + Ps = [begin + {_,Ref} = spawn_monitor(fun() -> count(M) end), + Ref + end || M <- Mods], + [put(list_to_atom(I), 0) || I <- erts_debug:instructions()], + Res = wait_for_all(Ps, 1), + OutFile = "count", + {ok,Out} = file:open(OutFile, [write]), + [io:format(Out, "~s ~p\n", [I,C]) || {I,C} <- Res], + ok = file:close(Out), + io:format("\nResult written to ~s\n", + [filename:join(DisDir, OutFile)]), + ok. + +wait_for_all([], _) -> + lists:reverse(lists:keysort(2, get())); +wait_for_all([_|_]=Ps, I) -> + receive + {'DOWN',Ref,process,_,Result} -> + io:format("\r~p", [I]), + [increment(Key, Count) || {Key,Count} <- Result], + wait_for_all(Ps -- [Ref], I+1) + end. + +count(M) -> + down(), + erts_debug:df(M), + {ok,Fd} = file:open(atom_to_list(M) ++ ".dis", [read,raw]), + count_is(Fd), + ok = file:close(Fd), + exit(get()). + +count_is(Fd) -> + case file:read_line(Fd) of + {ok,Line} -> + count_instr(Line), + count_is(Fd); + eof -> + ok + end. + +count_instr([$\s|T]) -> + count_instr_1(T, []); +count_instr([_|T]) -> + count_instr(T); +count_instr([]) -> + %% Empty line. + ok. + +count_instr_1([$\s|_], Acc) -> + Instr = list_to_atom(lists:reverse(Acc)), + increment(Instr, 1); +count_instr_1([H|T], Acc) -> + count_instr_1(T, [H|Acc]). + +increment(Key, Inc) -> + case get(Key) of + undefined -> + put(Key, Inc); + Count -> + put(Key, Count+Inc) + end. + +%%% +%%% Counting sempahore to limit the number of processes that +%%% can run concurrently. +%%% + +down() -> + sem ! {down,self()}, + receive + sem_taken -> ok + end. + +start_sem() -> + spawn(fun() -> + register(sem, self()), + process_flag(trap_exit, true), + do_sem(erlang:system_info(schedulers)+1) end). + +do_sem(0) -> + receive + {'EXIT',_,_} -> + do_sem(1) + end; +do_sem(C) -> + receive + {down,Pid} -> + link(Pid), + Pid ! sem_taken, + do_sem(C-1) + end. diff --git a/erts/emulator/zlib/zutil.h b/erts/emulator/zlib/zutil.h index d560382691..a8872e1c88 100644 --- a/erts/emulator/zlib/zutil.h +++ b/erts/emulator/zlib/zutil.h @@ -142,6 +142,7 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #ifdef WIN32 # ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ # define OS_CODE 0x0b +# define F_OPEN(name, mode) _wfopen((WCHAR *)(name), (WCHAR *)(mode)) /* Unicode */ # endif #endif diff --git a/erts/epmd/src/Makefile.in b/erts/epmd/src/Makefile.in index 498756b468..2f5296a5e8 100644 --- a/erts/epmd/src/Makefile.in +++ b/erts/epmd/src/Makefile.in @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1998-2009. All Rights Reserved. +# Copyright Ericsson AB 1998-2010. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -49,15 +49,31 @@ include ../epmd.mk BINDIR = $(ERL_TOP)/bin/$(TARGET) OBJDIR = $(ERL_TOP)/erts/obj$(TYPEMARKER)/$(TARGET) +ERTS_INCL = -I$(ERL_TOP)/erts/include \ + -I$(ERL_TOP)/erts/include/$(TARGET) \ + -I$(ERL_TOP)/erts/include/internal \ + -I$(ERL_TOP)/erts/include/internal/$(TARGET) + +# On windows we always need reentrant libraries. +ifeq ($(TARGET),win32) +ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal_r$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ +else +ifeq ($(findstring vxworks,$(TARGET)),vxworks) +ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ +else +ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ -lm +endif +endif CC = @CC@ WFLAGS = @WFLAGS@ -CFLAGS = @CFLAGS@ @DEFS@ $(TYPE_FLAGS) $(WFLAGS) +CFLAGS = @CFLAGS@ @DEFS@ $(TYPE_FLAGS) $(WFLAGS) $(ERTS_INCL) LD = @LD@ -LIBS = @LIBS@ +LIBS = @LIBS@ $(ERTS_INTERNAL_LIBS) LDFLAGS = @LDFLAGS@ + # ---------------------------------------------------- # Release directory specification # ---------------------------------------------------- @@ -90,7 +106,7 @@ EPMD_OBJS = $(OBJDIR)/epmd.o \ #--------------------------------- -all: $(BINDIR)/$(EPMD) +all: erts_lib $(BINDIR)/$(EPMD) docs: @@ -109,9 +125,12 @@ clean: $(BINDIR)/$(EPMD): $(EPMD_OBJS) $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(EPMD_OBJS) $(LIBS) -$(OBJDIR)/%.o: %.c +$(OBJDIR)/%.o: %.c epmd.h epmd_int.h $(CC) $(CFLAGS) $(EPMD_FLAGS) -o $@ -c $< +erts_lib: + cd $(ERL_TOP)/erts/lib_src && $(MAKE) $(TYPE) + include $(ERL_TOP)/make/otp_release_targets.mk release_spec: all diff --git a/erts/epmd/src/epmd.c b/erts/epmd/src/epmd.c index c509c49b39..65ff0cd6b2 100644 --- a/erts/epmd/src/epmd.c +++ b/erts/epmd/src/epmd.c @@ -23,6 +23,7 @@ #endif #include "epmd.h" /* Renamed from 'epmd_r4.h' */ #include "epmd_int.h" +#include "erl_printf.h" #ifdef HAVE_STDLIB_H # include <stdlib.h> @@ -33,6 +34,7 @@ static void usage(EpmdVars *); static void run_daemon(EpmdVars*); static int get_port_no(void); +static int check_relaxed(void); #ifdef __WIN32__ static int has_console(void); #endif @@ -161,6 +163,7 @@ int main(int argc, char** argv) g->silent = 0; g->is_daemon = 0; + g->brutal_kill = check_relaxed(); g->packet_timeout = CLOSE_TIMEOUT; /* Default timeout */ g->delay_accept = 0; g->delay_write = 0; @@ -196,6 +199,9 @@ int main(int argc, char** argv) } else if (strcmp(argv[0], "-daemon") == 0) { g->is_daemon = 1; argv++; argc--; + } else if (strcmp(argv[0], "-relaxed_command_check") == 0) { + g->brutal_kill = 1; + argv++; argc--; } else if (strcmp(argv[0], "-kill") == 0) { if (argc == 1) kill_epmd(g); @@ -226,11 +232,17 @@ int main(int argc, char** argv) else usage(g); epmd_cleanup_exit(g,0); + } else if (strcmp(argv[0], "-stop") == 0) { + if (argc == 2) + stop_cli(g, argv[1]); + else + usage(g); + epmd_cleanup_exit(g,0); } else usage(g); } - dbg_printf(g,0,"epmd running - daemon = %d",g->is_daemon); + dbg_printf(g,1,"epmd running - daemon = %d",g->is_daemon); #ifndef NO_SYSCONF if ((g->max_conn = sysconf(_SC_OPEN_MAX)) <= 0) @@ -382,9 +394,10 @@ static void run_daemon(EpmdVars *g) static void usage(EpmdVars *g) { fprintf(stderr, "usage: epmd [-d|-debug] [DbgExtra...] [-port No] [-daemon]\n"); - fprintf(stderr, " [-d|-debug] [-port No] [-names|-kill]\n\n"); - fprintf(stderr, "See the Erlang epmd manual page for info about the usage.\n"); - fprintf(stderr, "The -port and DbgExtra options are\n\n"); + fprintf(stderr, " [-relaxed_command_check]\n"); + fprintf(stderr, " epmd [-d|-debug] [-port No] [-names|-kill|-stop name]\n\n"); + fprintf(stderr, "See the Erlang epmd manual page for info about the usage.\n\n"); + fprintf(stderr, "Regular options\n"); fprintf(stderr, " -port No\n"); fprintf(stderr, " Let epmd listen to another port than default %d\n", EPMD_PORT_NO); @@ -394,8 +407,16 @@ static void usage(EpmdVars *g) fprintf(stderr, " the standard error stream. It will shorten\n"); fprintf(stderr, " the number of saved used node names to 5.\n\n"); fprintf(stderr, " If you give more than one debug flag you may\n"); - fprintf(stderr, " get more debugging information.\n\n"); - fprintf(stderr, " -packet_timout Seconds\n"); + fprintf(stderr, " get more debugging information.\n"); + fprintf(stderr, " -daemon\n"); + fprintf(stderr, " Start epmd detached (as a daemon)\n"); + fprintf(stderr, " -relaxed_command_check\n"); + fprintf(stderr, " Allow this instance of epmd to be killed with\n"); + fprintf(stderr, " epmd -kill even if there " + "are registered nodes.\n"); + fprintf(stderr, " Also allows forced unregister (epmd -stop).\n"); + fprintf(stderr, "\nDbgExtra options\n"); + fprintf(stderr, " -packet_timeout Seconds\n"); fprintf(stderr, " Set the number of seconds a connection can be\n"); fprintf(stderr, " inactive before epmd times out and closes the\n"); fprintf(stderr, " connection (default 60).\n\n"); @@ -407,6 +428,18 @@ static void usage(EpmdVars *g) fprintf(stderr, " -delay_write Seconds\n"); fprintf(stderr, " Also a simulation of a busy server. Inserts\n"); fprintf(stderr, " a delay before a reply is sent.\n"); + fprintf(stderr, "\nInteractive options\n"); + fprintf(stderr, " -names\n"); + fprintf(stderr, " List names registered with the currently " + "running epmd\n"); + fprintf(stderr, " -kill\n"); + fprintf(stderr, " Kill the currently runniing epmd\n"); + fprintf(stderr, " (only allowed if -names show empty database or\n"); + fprintf(stderr, " -relaxed_command_check was given when epmd was started).\n"); + fprintf(stderr, " -stop Name\n"); + fprintf(stderr, " Forcibly unregisters a name with epmd\n"); + fprintf(stderr, " (only allowed if -relaxed_command_check was given when \n"); + fprintf(stderr, " epmd was started).\n"); epmd_cleanup_exit(g,1); } @@ -426,20 +459,20 @@ static void usage(EpmdVars *g) * args... Arguments to print out according to the format * */ - +#define DEBUG_BUFFER_SIZE 2048 static void dbg_gen_printf(int onsyslog,int perr,int from_level, EpmdVars *g,const char *format, va_list args) { time_t now; char *timestr; - char buf[2048]; + char buf[DEBUG_BUFFER_SIZE]; if (g->is_daemon) { #ifndef NO_SYSLOG if (onsyslog) { - vsprintf(buf, format, args); + erts_vsnprintf(buf, DEBUG_BUFFER_SIZE, format, args); syslog(LOG_ERR,"epmd: %s",buf); } #endif @@ -450,9 +483,10 @@ static void dbg_gen_printf(int onsyslog,int perr,int from_level, time(&now); timestr = (char *)ctime(&now); - sprintf(buf, "epmd: %.*s: ", (int) strlen(timestr)-1, timestr); + erts_snprintf(buf, DEBUG_BUFFER_SIZE, "epmd: %.*s: ", + (int) strlen(timestr)-1, timestr); len = strlen(buf); - vsprintf(buf + len, format, args); + erts_vsnprintf(buf + len, DEBUG_BUFFER_SIZE - len, format, args); if (perr == 1) perror(buf); else @@ -539,4 +573,9 @@ static int get_port_no(void) char* port_str = getenv("ERL_EPMD_PORT"); return (port_str != NULL) ? atoi(port_str) : EPMD_PORT_NO; } +static int check_relaxed(void) +{ + char* port_str = getenv("ERL_EPMD_RELAXED_COMMAND_CHECK"); + return (port_str != NULL) ? 1 : 0; +} diff --git a/erts/epmd/src/epmd.h b/erts/epmd/src/epmd.h index 9e939ee38e..5d6e9ac165 100644 --- a/erts/epmd/src/epmd.h +++ b/erts/epmd/src/epmd.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. + * Copyright Ericsson AB 1998-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -17,21 +17,18 @@ * %CopyrightEnd% */ -/* The port number is now defined in a makefile */ +/* The port number is defined in a makefile */ /* Definitions of message codes */ -#define EPMD_ALIVE_REQ 'a' -#define EPMD_ALIVE_OK_RESP 'Y' -#define EPMD_PORT_REQ 'p' +/* Registration and queries */ +#define EPMD_ALIVE2_REQ 'x' +#define EPMD_PORT2_REQ 'z' +#define EPMD_ALIVE2_RESP 'y' +#define EPMD_PORT2_RESP 'w' #define EPMD_NAMES_REQ 'n' + +/* Interactive client command codes */ #define EPMD_DUMP_REQ 'd' #define EPMD_KILL_REQ 'k' #define EPMD_STOP_REQ 's' - -/* New epmd messages */ - -#define EPMD_ALIVE2_REQ 'x' /* 120 */ -#define EPMD_PORT2_REQ 'z' /* 122 */ -#define EPMD_ALIVE2_RESP 'y' /* 121 */ -#define EPMD_PORT2_RESP 'w' /* 119 */ diff --git a/erts/epmd/src/epmd_cli.c b/erts/epmd/src/epmd_cli.c index c12f711bc5..7c60ba0420 100644 --- a/erts/epmd/src/epmd_cli.c +++ b/erts/epmd/src/epmd_cli.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2009. All Rights Reserved. + * Copyright Ericsson AB 1998-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -42,7 +42,47 @@ void kill_epmd(EpmdVars *g) epmd_cleanup_exit(g,1); } if ((rval = read_fill(fd,buf,2)) == 2) { - printf("Killed\n"); + if (buf[0] == 'O' && buf[1] == 'K') { + printf("Killed\n"); + } else { + printf("Killing not allowed - living nodes in database.\n"); + } + epmd_cleanup_exit(g,0); + } else if (rval < 0) { + printf("epmd: failed to read answer from local epmd\n"); + epmd_cleanup_exit(g,1); + } else { /* rval is now 0 or 1 */ + buf[rval] = '\0'; + printf("epmd: local epmd responded with <%s>\n", buf); + epmd_cleanup_exit(g,1); + } +} + +void stop_cli(EpmdVars *g, char *name) +{ + char buf[1024]; + int fd, rval, bsize; + + bsize = strlen(name); + if (bsize > 1000) { + printf("epmd: Name too long!"); + epmd_cleanup_exit(g, 1); + } + + fd = conn_to_epmd(g); + bsize++; + put_int16(bsize, buf); + buf[2] = EPMD_STOP_REQ; + bsize += 2; + strcpy(buf+3, name); + + if (write(fd, buf, bsize) != bsize) { + printf("epmd: Can't write to epmd\n"); + epmd_cleanup_exit(g,1); + } + if ((rval = read_fill(fd,buf,7)) == 7) { + buf[7] = '\000'; + printf("%s\n", buf); epmd_cleanup_exit(g,0); } else if (rval < 0) { printf("epmd: failed to read answer from local epmd\n"); diff --git a/erts/epmd/src/epmd_int.h b/erts/epmd/src/epmd_int.h index 84578c72f8..c2558d52a1 100644 --- a/erts/epmd/src/epmd_int.h +++ b/erts/epmd/src/epmd_int.h @@ -1,3 +1,4 @@ +/* -*- c-indent-level: 2; c-continued-statement-offset: 2 -*- */ /* * %CopyrightBegin% * @@ -141,6 +142,10 @@ # define EADDRINUSE WSAEADDRINUSE #endif +#if defined(__WIN32__) && !defined(ECONNABORTED) +# define ECONNABORTED WSAECONNABORTED +#endif + #ifndef SOMAXCONN # define SOMAXCONN 128 #endif @@ -241,8 +246,10 @@ typedef struct { int fd; /* File descriptor */ - unsigned open:1; /* TRUE if open */ - unsigned keep:1; /* Don't close when sent reply */ + unsigned char open; /* TRUE if open */ + unsigned char keep; /* Don't close when sent reply */ + unsigned char local_peer; /* The peer of this connection is via + loopback interface */ unsigned got; /* # of bytes we have got */ unsigned want; /* Number of bytes we want */ char *buf; /* The remaining buffer */ @@ -282,6 +289,7 @@ typedef struct { int debug; int silent; int is_daemon; + int brutal_kill; unsigned packet_timeout; unsigned delay_accept; unsigned delay_write; @@ -303,6 +311,7 @@ void epmd_call(EpmdVars*,int); void run(EpmdVars*); void epmd_cleanup_exit(EpmdVars*, int); int epmd_conn_close(EpmdVars*,Connection*); +void stop_cli(EpmdVars *g, char *name); #ifdef DONT_USE_MAIN int start_epmd(char *,char *,char *,char *,char *,char *,char *,char *,char *,char *); diff --git a/erts/epmd/src/epmd_srv.c b/erts/epmd/src/epmd_srv.c index 021f8207eb..3499ab2934 100644 --- a/erts/epmd/src/epmd_srv.c +++ b/erts/epmd/src/epmd_srv.c @@ -2,7 +2,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -39,46 +39,16 @@ * server keeps the socket open where the request for registration was * made. * - * The protocol is briefly documented in "erl_ext_dist.txt". All requests - * to this server are done with a packet + * The protocol is briefly documented in the ERTS User's Guide, see + * http://www.erlang.org/doc/apps/erts/erl_dist_protocol.html + * + * All requests to this server are done with a packet * * 2 n * +--------+---------+ * | Length | Request | * +--------+---------+ * - * In all but one case there is only one request for each connection made - * to this server so we can safely close the socket after sending the - * reply. The exception is ALIVE_REQ where we keep the connection - * open without sending any data. When we receive a "close" this is - * an indication that the Erlang node was terminated. The termination - * may have been "normal" or caused by a crash. The operating system - * ensure that the connection is closed either way. - * - * Reading is done non-blocking, i.e. we call a "read" only if we are - * told by the "select" function that there are data to read. - * - * Two databases are used: One node database where the registered names - * of the nodes are stored, and one connection database where the state - * of sockets and the data buffers is stored. - * - * Incomplete packets are thrown away after a timout. The Erlang node - * doing the request is responsible for completing in it in a reasonable time. - * - * Note that if the server gets busy it may not have time to - * process all requests for connection. The "accept()" function - * will on most operating systems silently refuse to accept more - * than 5 outstanding requests. It is the client's responsibility - * to retry the request a number of times with random time interval. - * The "-debug" flag will insert a delay so you can test this - * behaviour. - * - * FIXME: In this code we assume that the packets we send on each - * socket is so small that a "write()" never block - * - * FIXME: We never restarts a read or write that was terminated - * by an interrupt. Do we need to? - * */ /* We use separate data structures for node names and connections @@ -98,7 +68,6 @@ static int conn_open(EpmdVars*,int); static int conn_close_fd(EpmdVars*,int); static void node_init(EpmdVars*); -static Node *node_reg(EpmdVars*,char*,int,int); static Node *node_reg2(EpmdVars*,char*, int, int, unsigned char, unsigned char, int, int, int, char*); static int node_unreg(EpmdVars*,char*); static int node_unreg_sock(EpmdVars*,int); @@ -188,8 +157,10 @@ void run(EpmdVars *g) dbg_printf(g,2,"starting"); - listen(listensock, SOMAXCONN); - + if(listen(listensock, SOMAXCONN) < 0) { + dbg_perror(g,"failed to listen on socket"); + epmd_cleanup_exit(g,1); + } FD_ZERO(&g->orig_read_mask); FD_SET(listensock,&g->orig_read_mask); @@ -210,8 +181,16 @@ void run(EpmdVars *g) timeout.tv_sec = (g->packet_timeout < IDLE_TIMEOUT) ? 1 : IDLE_TIMEOUT; timeout.tv_usec = 0; - if ((ret = select(g->max_conn,&read_mask,(fd_set *)0,(fd_set *)0,&timeout)) < 0) + if ((ret = select(g->max_conn,&read_mask,(fd_set *)0,(fd_set *)0,&timeout)) < 0) { dbg_perror(g,"error in select "); + switch (errno) { + case EAGAIN: + case EINTR: + break; + default: + epmd_cleanup_exit(g,1); + } + } else { time_t now; if (ret == 0) { @@ -295,11 +274,9 @@ static void do_read(EpmdVars *g,Connection *s) s->fd,val); dbg_print_buf(g,s->buf,val); - /* FIXME: Shouldn't be needed to close down.... */ node_unreg_sock(g,s->fd); epmd_conn_close(g,s); } - /* FIXME: We always close, probably the right thing to do */ return; } @@ -384,12 +361,21 @@ static int do_accept(EpmdVars *g,int listensock) if (msgsock < 0) { dbg_perror(g,"error in accept"); - return EPMD_FALSE; + switch (errno) { + case EAGAIN: + case ECONNABORTED: + case EINTR: + return EPMD_FALSE; + default: + epmd_cleanup_exit(g,1); + } } return conn_open(g,msgsock); } +/* buf is actually one byte larger than bsize, + giving place for null termination */ static void do_request(g, fd, s, buf, bsize) EpmdVars *g; int fd; @@ -400,117 +386,23 @@ static void do_request(g, fd, s, buf, bsize) char wbuf[OUTBUF_SIZE]; /* Buffer for writing */ int i; - /* - * Terminate packet as a C string. Needed for requests received from Erlang - * nodes with lower version than R3A. - */ - - buf[bsize] = '\0'; + buf[bsize] = '\0'; /* Needed for strcmp in PORT2 and STOP requests + buf is always large enough */ switch (*buf) { - case EPMD_ALIVE_REQ: - dbg_printf(g,1,"** got ALIVE_REQ"); - - /* The packet has the format "axxyyyyyy" where xx is port, given - in network byte order, and yyyyyy is symname, possibly null - terminated. */ - - if (buf[bsize - 1] == '\000') /* Skip null termination */ - bsize--; - - if (bsize <= 3) - { - dbg_printf(g,0,"packet to small for request ALIVE_REQ (%d)", bsize); - return; - } - - for (i = 3; i < bsize; i++) - if (buf[i] == '\000') - { - dbg_printf(g,0,"node name contains ascii 0 in ALIVE_REQ"); - return; - } - - { - Node *node; - int eport; - char *name = &buf[3]; /* points to node name */ - - eport = get_int16(&buf[1]); - - if ((node = node_reg(g, name, fd, eport)) == NULL) - return; - - wbuf[0] = EPMD_ALIVE_OK_RESP; - put_int16(node->creation, wbuf+1); - - if (g->delay_write) /* Test of busy server */ - sleep(g->delay_write); - - if (reply(g, fd, wbuf, 3) != 3) - { - dbg_tty_printf(g,1,"failed to send ALIVE_OK_RESP for \"%s\"",name); - return; - } - - dbg_tty_printf(g,1,"** sent ALIVE_OK_RESP for \"%s\"",name); - s->keep = EPMD_TRUE; /* Don't close on inactivity */ - } - break; - - case EPMD_PORT_REQ: - dbg_printf(g,1,"** got PORT_REQ"); - - if (buf[bsize - 1] == '\000') /* Skip null termination */ - bsize--; - - if (bsize <= 1) - { - dbg_printf(g,0,"packet to small for request PORT_REQ (%d)", bsize); - return; - } - - for (i = 1; i < bsize; i++) - if (buf[i] == '\000') - { - dbg_printf(g,0,"node name contains ascii 0 in PORT_REQ"); - return; - } - - { - char *name = &buf[1]; /* Points to node name */ - Node *node; - - for (node = g->nodes.reg; node; node = node->next) - { - if (strcmp(node->symname, name) == 0) - { - put_int16(node->port,wbuf); - if (reply(g, fd, wbuf, 2) != 2) - { - dbg_tty_printf(g,1,"failed to send PORT_RESP for %s: %d", - name,node->port); - return; - } - dbg_tty_printf(g,1,"** sent PORT_RESP for %s: %d", - name,node->port); - return; - } - } - dbg_tty_printf(g,1,"Closed on PORT_REQ for %s",name); - } - /* FIXME: How about an answer if no port? Is a close enough? */ - break; - case EPMD_ALIVE2_REQ: dbg_printf(g,1,"** got ALIVE2_REQ"); + if (!s->local_peer) { + dbg_printf(g,0,"ALIVE2_REQ from non local address"); + return; + } /* The packet has the format "axxyyyyyy" where xx is port, given in network byte order, and yyyyyy is symname, possibly null terminated. */ - if (bsize <= 13) + if (bsize <= 13) /* at least one character for the node name */ { dbg_printf(g,0,"packet to small for request ALIVE2_REQ (%d)",bsize); return; @@ -533,7 +425,17 @@ static void do_request(g, fd, s, buf, bsize) highvsn = get_int16(&buf[5]); lowvsn = get_int16(&buf[7]); namelen = get_int16(&buf[9]); + if (namelen + 13 > bsize) { + dbg_printf(g,0,"Node name size error in ALIVE2_REQ"); + return; + } extralen = get_int16(&buf[11+namelen]); + + if (extralen + namelen + 13 > bsize) { + dbg_printf(g,0,"Extra info size error in ALIVE2_REQ"); + return; + } + for (i = 11 ; i < 11 + namelen; i ++) if (buf[i] == '\000') { dbg_printf(g,0,"node name contains ascii 0 in ALIVE2_REQ"); @@ -576,7 +478,7 @@ static void do_request(g, fd, s, buf, bsize) if (bsize <= 1) { - dbg_printf(g,0,"packet to small for request PORT2_REQ (%d)", bsize); + dbg_printf(g,0,"packet too small for request PORT2_REQ (%d)", bsize); return; } @@ -664,6 +566,10 @@ static void do_request(g, fd, s, buf, bsize) case EPMD_DUMP_REQ: dbg_printf(g,1,"** got DUMP_REQ"); + if (!s->local_peer) { + dbg_printf(g,0,"DUMP_REQ from non local address"); + return; + } { Node *node; @@ -713,7 +619,19 @@ static void do_request(g, fd, s, buf, bsize) break; case EPMD_KILL_REQ: + if (!s->local_peer) { + dbg_printf(g,0,"KILL_REQ from non local address"); + return; + } dbg_printf(g,1,"** got KILL_REQ"); + + if (!g->brutal_kill && (g->nodes.reg != NULL)) { + dbg_printf(g,0,"Disallowed KILL_REQ, live nodes"); + if (reply(g, fd,"NO",2) != 2) + dbg_printf(g,0,"failed to send reply to KILL_REQ"); + return; + } + if (reply(g, fd,"OK",2) != 2) dbg_printf(g,0,"failed to send reply to KILL_REQ"); dbg_tty_printf(g,1,"epmd killed"); @@ -723,9 +641,18 @@ static void do_request(g, fd, s, buf, bsize) case EPMD_STOP_REQ: dbg_printf(g,1,"** got STOP_REQ"); + if (!s->local_peer) { + dbg_printf(g,0,"STOP_REQ from non local address"); + return; + } + if (!g->brutal_kill) { + dbg_printf(g,0,"Disallowed STOP_REQ, no relaxed_command_check"); + return; + } + if (bsize <= 1 ) { - dbg_printf(g,0,"packet to small for request STOP_REQ (%d)",bsize); + dbg_printf(g,0,"packet too small for request STOP_REQ (%d)",bsize); return; } @@ -810,6 +737,14 @@ static int conn_open(EpmdVars *g,int fd) for (i = 0; i < g->max_conn; i++) { if (g->conn[i].open == EPMD_FALSE) { + struct sockaddr_in si; +#ifdef HAVE_SOCKLEN_T + socklen_t st; +#else + int st; +#endif + st = sizeof(si); + g->active_conn++; s = &g->conn[i]; @@ -819,6 +754,20 @@ static int conn_open(EpmdVars *g,int fd) s->fd = fd; s->open = EPMD_TRUE; s->keep = EPMD_FALSE; + + /* Determine if connection is from localhost */ + if (getpeername(s->fd,(struct sockaddr*) &si,&st) || + st < sizeof(si)) { + /* Failure to get peername is regarder as non local host */ + s->local_peer = EPMD_FALSE; + } else { + s->local_peer = + ((((unsigned) ntohl(si.sin_addr.s_addr)) & 0xFF000000U) == + 0x7F000000U); /* Only 127.x.x.x allowed, no false positives */ + } + dbg_tty_printf(g,2,(s->local_peer) ? "Local peer connected" : + "Non-local peer connected"); + s->want = 0; /* Currently unknown */ s->got = 0; s->mod_time = current_time(g); /* Note activity */ @@ -887,7 +836,7 @@ static void node_init(EpmdVars *g) /* We have got a close on a connection and it may be a - EPMD_ALIVE_CLOSE_REQ. Note that this call shouild be called + EPMD_ALIVE_CLOSE_REQ. Note that this call should be called *before* calling conn_close() */ static int node_unreg(EpmdVars *g,char *name) @@ -975,11 +924,6 @@ static int node_unreg_sock(EpmdVars *g,int fd) * Perhaps use the oldest or something. */ -static Node *node_reg(EpmdVars *g,char *name,int fd, int port) -{ - return node_reg2(g, name, fd, port, 0, 0, 0, 0, 0, NULL); -} - static Node *node_reg2(EpmdVars *g, char* name, int fd, @@ -1005,6 +949,11 @@ static Node *node_reg2(EpmdVars *g, dbg_printf(g,0,"node name is too long (%d) %s", strlen(name), name); return NULL; } + if (extralen > MAXSYMLEN) + { + dbg_printf(g,0,"extra data is too long (%d) %s", strlen(name), name); + return NULL; + } /* Fail if it is already registered */ diff --git a/erts/epmd/test/epmd_SUITE.erl b/erts/epmd/test/epmd_SUITE.erl index 91e09faf75..da69412e12 100644 --- a/erts/epmd/test/epmd_SUITE.erl +++ b/erts/epmd/test/epmd_SUITE.erl @@ -63,7 +63,13 @@ alive_req_too_large/1, returns_valid_empty_extra/1, - returns_valid_populated_extra_with_nulls/1 + returns_valid_populated_extra_with_nulls/1, + buffer_overrun/1, + buffer_overrun_1/1, + buffer_overrun_2/1, + no_nonlocal_register/1, + no_nonlocal_kill/1, + no_live_killing/1 ]). @@ -82,11 +88,8 @@ -define(REG_REPEAT_LIM,1000). % Message codes in epmd protocol --define(EPMD_ALIVE_REQ, $a). -define(EPMD_ALIVE2_REQ, $x). --define(EPMD_ALIVE_OK_RESP, $Y). -define(EPMD_ALIVE2_RESP, $y). --define(EPMD_PORT_REQ, $p). -define(EPMD_PORT_PLEASE2_REQ, $z). -define(EPMD_PORT2_RESP, $w). -define(EPMD_NAMES_REQ, $n). @@ -124,7 +127,15 @@ all(suite) -> alive_req_too_large, returns_valid_empty_extra, - returns_valid_populated_extra_with_nulls + returns_valid_populated_extra_with_nulls, + + buffer_overrun, + %buffer_overrun_1, + %buffer_overrun_2, + + no_nonlocal_register, + no_nonlocal_kill, + no_live_killing ]. %% @@ -132,7 +143,7 @@ all(suite) -> %% init_per_testcase(_Func, Config) -> - Dog = test_server:timetrap(?SHORT_TEST_TIMEOUT), + Dog = test_server:timetrap(?MEDIUM_TEST_TIMEOUT), cleanup(), [{watchdog, Dog} | Config]. @@ -148,7 +159,7 @@ register_name(doc) -> ["Register a name"]; register_name(suite) -> []; -register_name(Config) when list(Config) -> +register_name(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = register_node("foobar"), ?line ok = close(Sock), % Unregister @@ -158,7 +169,7 @@ register_names_1(doc) -> ["Register and unregister two nodes"]; register_names_1(suite) -> []; -register_names_1(Config) when list(Config) -> +register_names_1(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock1} = register_node("foobar"), ?line {ok,Sock2} = register_node("foozap"), @@ -170,7 +181,7 @@ register_names_2(doc) -> ["Register and unregister two nodes"]; register_names_2(suite) -> []; -register_names_2(Config) when list(Config) -> +register_names_2(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock1} = register_node("foobar"), ?line {ok,Sock2} = register_node("foozap"), @@ -182,7 +193,7 @@ register_duplicate_name(doc) -> ["Two nodes with the same name"]; register_duplicate_name(suite) -> []; -register_duplicate_name(Config) when list(Config) -> +register_duplicate_name(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = register_node("foobar"), ?line error = register_node("foobar"), @@ -192,22 +203,9 @@ register_duplicate_name(Config) when list(Config) -> % Internal function to register a node name, no close, i.e. unregister register_node(Name) -> - register_node(Name,?DUMMY_PORT). - -register_node(Name, Port) -> - case send_req([?EPMD_ALIVE_REQ, put16(Port), Name]) of - {ok,Sock} -> - case recv(Sock,3) of - {ok, [?EPMD_ALIVE_OK_RESP,_D1,_D0]} -> - {ok,Sock}; - Other -> - test_server:format("recv on sock ~w: ~p~n", - [Sock,Other]), - error - end; - error -> - error - end. + register_node_v2(?DUMMY_PORT,$M,0,5,5,Name,""). +register_node(Name,Port) -> + register_node_v2(Port,$M,0,5,5,Name,""). register_node_v2(Port, NodeType, Prot, HVsn, LVsn, Name, Extra) -> Req = [?EPMD_ALIVE2_REQ, put16(Port), NodeType, Prot, @@ -254,7 +252,7 @@ parse_port2_resp(Resp) -> hvsn=HVsn,lvsn=LVsn, node_name=binary_to_list(NodeName), extra=binary_to_list(Extra)}}; - Other -> + _Other -> test_server:format("invalid port2 resp: ~p~n", [Resp]), error @@ -266,7 +264,7 @@ name_with_null_inside(doc) -> ["Register a name with a null char in it"]; name_with_null_inside(suite) -> []; -name_with_null_inside(Config) when list(Config) -> +name_with_null_inside(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line error = register_node("foo\000bar"), ok. @@ -277,11 +275,9 @@ name_null_terminated(doc) -> ["Register a name with terminating null byte"]; name_null_terminated(suite) -> []; -name_null_terminated(Config) when list(Config) -> +name_null_terminated(Config) when is_list(Config) -> ?line ok = epmdrun(), - ?line {ok,Sock} = register_node("foobar\000"), - ?line error = register_node("foobar"), - ?line ok = close(Sock), % Unregister + ?line error = register_node("foobar\000"), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -290,7 +286,7 @@ stupid_names_req(doc) -> ["Read names from epmd in a stupid way"]; stupid_names_req(suite) -> []; -stupid_names_req(Config) when list(Config) -> +stupid_names_req(Config) when is_list(Config) -> Dog = ?config(watchdog, Config), test_server:timetrap_cancel(Dog), LongDog = test_server:timetrap(?MEDIUM_TEST_TIMEOUT), @@ -394,15 +390,15 @@ get_port_nr(doc) -> ["Register a name on a port and ask about port nr"]; get_port_nr(suite) -> []; -get_port_nr(Config) when list(Config) -> - port_request([?EPMD_PORT_REQ,"foo"]). +get_port_nr(Config) when is_list(Config) -> + port_request([?EPMD_PORT_PLEASE2_REQ,"foo"]). slow_get_port_nr(doc) -> ["Register with slow write and ask about port nr"]; slow_get_port_nr(suite) -> []; -slow_get_port_nr(Config) when list(Config) -> - port_request([?EPMD_PORT_REQ,d,$f,d,$o,d,$o]). +slow_get_port_nr(Config) when is_list(Config) -> + port_request([?EPMD_PORT_PLEASE2_REQ,d,$f,d,$o,d,$o]). % Internal function used above @@ -413,9 +409,18 @@ port_request(M) -> ?line {ok,RSock} = register_node("foo", Port), ?line {ok,Sock} = connect(), ?line ok = send(Sock,[size16(M),M]), - R = put16(Port), - ?line {ok,R} = recv(Sock, length(R)), - ?line ok = close(RSock), + ?line case recv_until_sock_closes(Sock) of + {ok, Resp} -> + ?line close(RSock), + ?line {ok,Rec} = parse_port2_resp(Resp), + ?line Port = Rec#node_info.port, + ok; + Other -> + ?line close(RSock), + ?line test_server:format("recv on sock ~w: ~p~n", + [Sock,Other]), + ?line throw({error,Other}) + end, ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -424,8 +429,8 @@ unregister_others_name_1(doc) -> ["Unregister name of other node"]; unregister_others_name_1(suite) -> []; -unregister_others_name_1(Config) when list(Config) -> - ?line ok = epmdrun(), +unregister_others_name_1(Config) when is_list(Config) -> + ?line ok = epmdrun("-relaxed_command_check"), ?line {ok,RSock} = register_node("foo"), ?line {ok,Sock} = connect(), M = [?EPMD_STOP_REQ,"foo"], @@ -441,8 +446,8 @@ unregister_others_name_2(doc) -> ["Unregister name of other node"]; unregister_others_name_2(suite) -> []; -unregister_others_name_2(Config) when list(Config) -> - ?line ok = epmdrun(), +unregister_others_name_2(Config) when is_list(Config) -> + ?line ok = epmdrun("-relaxed_command_check"), ?line {ok,Sock} = connect(), M = [?EPMD_STOP_REQ,"xxx42"], ?line ok = send(Sock,[size16(M),M]), @@ -456,7 +461,7 @@ register_overflow(doc) -> ["Register too many, clean and redo 10 times"]; register_overflow(suite) -> []; -register_overflow(Config) when list(Config) -> +register_overflow(Config) when is_list(Config) -> Dog = ?config(watchdog, Config), test_server:timetrap_cancel(Dog), LongDog = test_server:timetrap(?LONG_TEST_TIMEOUT), @@ -546,7 +551,7 @@ no_data(doc) -> ["Open but send no data"]; no_data(suite) -> []; -no_data(Config) when list(Config) -> +no_data(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = connect(), sleep(?LONG_PAUSE), @@ -559,7 +564,7 @@ one_byte(doc) -> ["Send one byte only"]; one_byte(suite) -> []; -one_byte(Config) when list(Config) -> +one_byte(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = connect(), ?line ok = send(Sock,[0]), @@ -573,7 +578,7 @@ two_bytes(doc) -> ["Send packet size only"]; two_bytes(suite) -> []; -two_bytes(Config) when list(Config) -> +two_bytes(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = connect(), ?line ok = send(Sock,[put16(3)]), @@ -587,7 +592,7 @@ partial_packet(doc) -> ["Got only part of a packet"]; partial_packet(suite) -> []; -partial_packet(Config) when list(Config) -> +partial_packet(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = connect(), ?line ok = send(Sock,[put16(100),"only a few bytes"]), @@ -601,7 +606,7 @@ zero_length(doc) -> ["Invalid zero packet size"]; zero_length(suite) -> []; -zero_length(Config) when list(Config) -> +zero_length(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = connect(), ?line ok = send(Sock,[0,0,0,0,0,0,0,0,0,0]), @@ -615,15 +620,20 @@ too_large(doc) -> ["Invalid large packet"]; too_large(suite) -> []; -too_large(Config) when list(Config) -> +too_large(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = connect(), Size = 63000, M = lists:duplicate(Size, $z), ?line ok = send(Sock,[put16(Size),M]), sleep(?MEDIUM_PAUSE), - ?line closed = recv(Sock,1), - ok. + % With such a large packet, even the writes can fail as the + % daemon closes before everything is delivered -> econnaborted + case recv(Sock,1) of + closed -> ok; + {error,econnaborted} -> ok; + Other -> exit({unexpected,Other}) + end. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -631,10 +641,11 @@ alive_req_too_small_1(doc) -> ["Try to register but not enough data"]; alive_req_too_small_1(suite) -> []; -alive_req_too_small_1(Config) when list(Config) -> +alive_req_too_small_1(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = connect(), - M = [?EPMD_ALIVE_REQ, 42], + M = [?EPMD_ALIVE2_REQ, put16(?DUMMY_PORT),$M,0, put16(5), + put16(5),put16(0)], ?line ok = send(Sock, [size16(M), M]), sleep(?MEDIUM_PAUSE), ?line closed = recv(Sock,1), @@ -646,10 +657,11 @@ alive_req_too_small_2(doc) -> ["Try to register but not enough data"]; alive_req_too_small_2(suite) -> []; -alive_req_too_small_2(Config) when list(Config) -> +alive_req_too_small_2(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = connect(), - M = [?EPMD_ALIVE_REQ, put16(?DUMMY_PORT)], + M = [?EPMD_ALIVE2_REQ, put16(?DUMMY_PORT),$M,0, put16(5), + put16(5)], ?line ok = send(Sock, [size16(M), M]), sleep(?MEDIUM_PAUSE), ?line closed = recv(Sock,1), @@ -661,7 +673,7 @@ alive_req_too_large(doc) -> ["Try to register but node name too large"]; alive_req_too_large(suite) -> []; -alive_req_too_large(Config) when list(Config) -> +alive_req_too_large(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = connect(), L = [ @@ -678,10 +690,12 @@ alive_req_too_large(Config) when list(Config) -> "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" ], - M = [?EPMD_ALIVE_REQ, put16(?DUMMY_PORT), L], + S = length(lists:flatten(L)), + M = [?EPMD_ALIVE2_REQ, put16(?DUMMY_PORT),$M,0, put16(5), + put16(5), put16(S),L,put16(0)], ?line ok = send(Sock, [size16(M), M]), sleep(?MEDIUM_PAUSE), - ?line closed = recv(Sock,1), + ?line {ok,[?EPMD_ALIVE2_RESP,1]} = recv(Sock,2), ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -690,7 +704,7 @@ returns_valid_empty_extra(doc) -> ["Check that an empty extra is prefixed by a two byte length"]; returns_valid_empty_extra(suite) -> []; -returns_valid_empty_extra(Config) when list(Config) -> +returns_valid_empty_extra(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = register_node_v2(4711, 72, 0, 5, 5, "foo", []), ?line {ok,#node_info{extra=[]}} = port_please_v2("foo"), @@ -703,7 +717,7 @@ returns_valid_populated_extra_with_nulls(doc) -> ["Check a populated extra with embedded null characters"]; returns_valid_populated_extra_with_nulls(suite) -> []; -returns_valid_populated_extra_with_nulls(Config) when list(Config) -> +returns_valid_populated_extra_with_nulls(Config) when is_list(Config) -> ?line ok = epmdrun(), ?line {ok,Sock} = register_node_v2(4711, 72, 0, 5, 5, "foo", "ABC\000\000"), ?line {ok,#node_info{extra="ABC\000\000"}} = port_please_v2("foo"), @@ -711,6 +725,167 @@ returns_valid_populated_extra_with_nulls(Config) when list(Config) -> ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +buffer_overrun(suite) -> + [buffer_overrun_1,buffer_overrun_2]. + +buffer_overrun_1(suite) -> + []; +buffer_overrun_1(doc) -> + ["Test security vulnerability in fake extra lengths in alive2_req"]; +buffer_overrun_1(Config) when is_list(Config) -> + ?line ok = epmdrun(), + ?line true = alltrue([hostile(N) || N <- lists:seq(1,10000)]), + ok. +buffer_overrun_2(suite) -> + []; +buffer_overrun_2(doc) -> + ["Test security vulnerability in fake extra lengths in alive2_req"]; +buffer_overrun_2(Config) when is_list(Config) -> + ?line ok = epmdrun(), + ?line [false | Rest] = [hostile2(N) || N <- lists:seq(255,10000)], + ?line true = alltrue(Rest), + ok. +hostile(N) -> + try + Bin= <<$x:8,4747:16,$M:8,0:8,5:16,5:16,5:16,"gurka",N:16>>, + S = size(Bin), + {ok,E}=connect_sturdy(), + gen_tcp:send(E,[<<S:16>>,Bin]), + closed = recv(E,1), + gen_tcp:close(E), + true + catch + _:_ -> + false + end. +hostile2(N) -> + try + B2 = list_to_binary(lists:duplicate(N,255)), + Bin= <<$x:8,4747:16,$M:8,0:8,5:16,5:16,5:16,"gurka",N:16,B2/binary>>, + S = size(Bin), + {ok,E}=connect_sturdy(), + gen_tcp:send(E,[<<S:16>>,Bin]), + Z = recv(E,2), + gen_tcp:close(E), + (Z =:= closed) or (Z =:= {ok, [$y,1]}) + catch + _A:_B -> + false + end. + +alltrue([]) -> + true; +alltrue([true|T]) -> + alltrue(T); +alltrue([_|_]) -> + false. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +no_nonlocal_register(suite) -> + []; +no_nonlocal_register(doc) -> + ["Ensure that we cannot register throug a nonlocal connection"]; +no_nonlocal_register(Config) when is_list(Config) -> + ?line ok = epmdrun(), + ?line {ok,Ifs} = inet:getiflist(), + ?line Addr0 = [ inet:ifget(I, [addr]) || I <- Ifs ], + ?line Addr1 = [ A || {ok,[{addr,A}]} <- Addr0], + ?line Addr = lists:filter(fun({127,_,_,_}) -> + false; + (_) -> + true + end,Addr1), + %% Now we should have all non loopback interface addresses, + %% none should accept a alive2 registration. + ?line Res = lists:map(fun(Ad={A1,A2,A3,A4}) -> + try + Name = "gurka_"++ + integer_to_list(A1)++"_"++ + integer_to_list(A2)++"_"++ + integer_to_list(A3)++"_"++ + integer_to_list(A4), + Bname = list_to_binary(Name), + NameS = byte_size(Bname), + ?line Bin= <<$x:8,4747:16,$M:8,0:8,5:16, + 5:16,NameS:16,Bname/binary, + 0:16>>, + ?line S = size(Bin), + {ok, E} = connect(Ad), + gen_tcp:send(E,[<<S:16>>,Bin]), + closed = recv(E,1), + gen_tcp:close(E), + true + catch + _:_ -> + false + end + end, Addr), + erlang:display(Res), + ?line true = alltrue(Res), + ok. + +no_nonlocal_kill(suite) -> + []; +no_nonlocal_kill(doc) -> + ["Ensure that we cannot kill through nonlocal connection"]; +no_nonlocal_kill(Config) when is_list(Config) -> + ?line ok = epmdrun(), + ?line {ok,Ifs} = inet:getiflist(), + ?line Addr0 = [ inet:ifget(I, [addr]) || I <- Ifs ], + ?line Addr1 = [ A || {ok,[{addr,A}]} <- Addr0], + ?line Addr = lists:filter(fun({127,_,_,_}) -> + false; + (_) -> + true + end,Addr1), + %% Now we should have all non loopback interface addresses, + %% none should accept a alive2 registration. + ?line Res = lists:map(fun(Ad) -> + try + {ok, E} = connect(Ad), + M = [?EPMD_KILL_REQ], + send(E, [size16(M), M]), + closed = recv(E,2), + gen_tcp:close(E), + sleep(?MEDIUM_PAUSE), + {ok, E2} = connect(Ad), + gen_tcp:close(E2), + true + catch + _:_ -> + false + end + end, Addr), + erlang:display(Res), + ?line true = alltrue(Res), + ok. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +no_live_killing(doc) -> + ["Dont allow killing with live nodes or any unregistering w/o -relaxed_command_check"]; +no_live_killing(suite) -> + []; +no_live_killing(Config) when is_list(Config) -> + ?line ok = epmdrun(), + ?line {ok,RSock} = register_node("foo"), + ?line {ok,Sock} = connect(), + ?line M = [?EPMD_KILL_REQ], + ?line ok = send(Sock,[size16(M),M]), + ?line {ok,"NO"} = recv(Sock,2), + ?line close(Sock), + ?line {ok,Sock2} = connect(), + ?line M2 = [?EPMD_STOP_REQ,"foo"], + ?line ok = send(Sock2,[size16(M2),M2]), + ?line closed = recv(Sock2,1), + ?line close(Sock2), + ?line close(RSock), + ?line sleep(?MEDIUM_PAUSE), + ?line {ok,Sock3} = connect(), + ?line M3 = [?EPMD_KILL_REQ], + ?line ok = send(Sock3,[size16(M3),M3]), + ?line {ok,"OK"} = recv(Sock3,2), + ?line close(Sock3), + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % Terminate all tests with killing epmd. cleanup() -> @@ -731,16 +906,24 @@ cleanup() -> % Normal debug start of epmd epmdrun() -> + epmdrun([]). +epmdrun(Args) -> case os:find_executable(epmd) of false -> {error, {could_not_find_epmd_in_path}}; Path -> - epmdrun(Path) + epmdrun(Path,Args) end. -epmdrun(Epmd) -> +epmdrun(Epmd,Args0) -> %% test_server:format("epmdrun() => Epmd = ~p",[Epmd]), - osrun(Epmd ++ " " ?EPMDARGS " -port " ++ integer_to_list(?PORT)). + Args = case Args0 of + [] -> + []; + O -> + " "++O + end, + osrun(Epmd ++ Args ++ " " ?EPMDARGS " -port " ++ integer_to_list(?PORT)). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -753,20 +936,27 @@ osrun(Cmd) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % Wrappers of TCP functions -% These two functions is the interface for connect. +% These functions is the interface for connect. % Passive mode is the default connect() -> - connect(?PORT, passive). + connect("localhost",?PORT, passive). + +connect(Addr) -> + connect(Addr,?PORT, passive). connect_active() -> - connect(?PORT, active). + connect("localhost",?PORT, active). +%% Retry after 15 seconds, to avoid TIME_WAIT socket exhaust. +connect_sturdy() -> + connect("localhost",?PORT, passive, 15000, 3). % Try a few times before giving up - -connect(Port, Mode) -> - case connect_repeat(?CONN_RETRY, Port, Mode) of +connect(Addr, Port, Mode) -> + connect(Addr, Port, Mode, ?CONN_SLEEP, ?CONN_RETRY). +connect(Addr, Port, Mode, Sleep, Retry) -> + case connect_repeat(Addr, Retry, Port, Mode, Sleep) of {ok,Sock} -> {ok,Sock}; {error,timeout} -> @@ -783,25 +973,25 @@ connect(Port, Mode) -> % Try a few times before giving up. Pause a small time between % each try. -connect_repeat(1, Port, Mode) -> - connect_mode(Port, Mode); -connect_repeat(Retry, Port, Mode) -> - case connect_mode(Port, Mode) of +connect_repeat(Addr, 1, Port, Mode, _Sleep) -> + connect_mode(Addr,Port, Mode); +connect_repeat(Addr,Retry, Port, Mode, Sleep) -> + case connect_mode(Addr,Port, Mode) of {ok,Sock} -> {ok,Sock}; {error,Reason} -> test_server:format("connect: error: ~w~n",[Reason]), - timer:sleep(?CONN_SLEEP), - connect_repeat(Retry - 1, Port, Mode); + timer:sleep(Sleep), + connect_repeat(Addr, Retry - 1, Port, Mode, Sleep); Any -> test_server:format("connect: unknown message: ~w~n",[Any]), exit(1) end. -connect_mode(Port, active) -> - gen_tcp:connect("localhost", Port, [{packet, 0}], ?CONN_TIMEOUT); -connect_mode(Port, passive) -> - gen_tcp:connect("localhost", Port, [{packet, 0}, {active, false}], +connect_mode(Addr,Port, active) -> + gen_tcp:connect(Addr, Port, [{packet, 0}], ?CONN_TIMEOUT); +connect_mode(Addr, Port, passive) -> + gen_tcp:connect(Addr, Port, [{packet, 0}, {active, false}], ?CONN_TIMEOUT). @@ -858,9 +1048,9 @@ send(Sock, SendSpec) -> send([], RevBytes, _Sock) -> {ok,RevBytes}; -send([Byte | Spec], RevBytes, Sock) when integer(Byte) -> +send([Byte | Spec], RevBytes, Sock) when is_integer(Byte) -> send(Spec, [Byte | RevBytes], Sock); -send([List | Spec], RevBytes, Sock) when list(List) -> +send([List | Spec], RevBytes, Sock) when is_list(List) -> case send(List, RevBytes, Sock) of {ok,Left} -> send(Spec, Left, Sock); diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in index d2a5080c68..4754328c0b 100644 --- a/erts/etc/common/Makefile.in +++ b/erts/etc/common/Makefile.in @@ -98,7 +98,7 @@ endif ifeq ($(TARGET),win32) ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal_r$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ else -ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ +ERTS_INTERNAL_LIBS=-L../../lib/internal/$(TARGET) -lerts_internal$(ERTS_LIB_TYPEMARKER) @ERTS_INTERNAL_X_LIBS@ -lm endif # ---------------------------------------------------- @@ -178,7 +178,7 @@ MC_OUTPUTS= \ MT_FLAG="-MD" endif INET_GETHOST = $(BINDIR)/inet_gethost.exe -INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe +INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe $(BINDIR)/ct_run.exe INSTALL_SRC = $(WINETC)/start_erl.c $(WINETC)/Nmakefile.start_erl ERLEXECDIR=. INSTALL_LIBS = @@ -211,7 +211,7 @@ ERLSRV_OBJECTS= MC_OUTPUTS= INET_GETHOST = $(BINDIR)/inet_gethost@EXEEXT@ INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer@EXEEXT@ $(BINDIR)/dialyzer@EXEEXT@ \ - $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ \ + $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ $(BINDIR)/ct_run@EXEEXT@ \ $(BINDIR)/run_erl $(BINDIR)/to_erl $(BINDIR)/dyn_erl INSTALL_EMBEDDED_DATA = ../unix/start.src ../unix/start_erl.src INSTALL_TOP = Install @@ -274,6 +274,7 @@ endif rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/dyn_erl.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/safe_string.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/typer.o + rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/ct_run.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/vxcall.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/erl.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/werl.o @@ -326,29 +327,36 @@ $(OBJDIR)/$(ERLEXEC).o: $(ERLEXECDIR)/$(ERLEXEC).c $(CC) -I$(EMUDIR) $(CFLAGS) -o $@ -c $(ERLEXECDIR)/$(ERLEXEC).c endif $(BINDIR)/erlc@EXEEXT@: $(OBJDIR)/erlc.o - $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/erlc.o -L$(OBJDIR) $(LIBS) + $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/erlc.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) $(OBJDIR)/erlc.o: erlc.c $(CC) $(CFLAGS) -o $@ -c erlc.c $(BINDIR)/dialyzer@EXEEXT@: $(OBJDIR)/dialyzer.o - $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/dialyzer.o -L$(OBJDIR) $(LIBS) + $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/dialyzer.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) $(OBJDIR)/dialyzer.o: dialyzer.c $(CC) $(CFLAGS) -o $@ -c dialyzer.c $(BINDIR)/typer@EXEEXT@: $(OBJDIR)/typer.o - $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/typer.o -L$(OBJDIR) $(LIBS) + $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/typer.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) $(OBJDIR)/typer.o: typer.c $(CC) $(CFLAGS) -o $@ -c typer.c $(BINDIR)/escript@EXEEXT@: $(OBJDIR)/escript.o - $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/escript.o -L$(OBJDIR) $(LIBS) + $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/escript.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) $(OBJDIR)/escript.o: escript.c $(CC) $(CFLAGS) -o $@ -c escript.c +$(BINDIR)/ct_run@EXEEXT@: $(OBJDIR)/ct_run.o + $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/ct_run.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) + +$(OBJDIR)/ct_run.o: ct_run.c + $(CC) $(CFLAGS) -o $@ -c ct_run.c + + #------------------------------------------------------------------------ # Windows specific targets # The windows platform is quite different from the others. erl/werl are small C programs diff --git a/erts/etc/common/ct_run.c b/erts/etc/common/ct_run.c new file mode 100644 index 0000000000..7aaab716f7 --- /dev/null +++ b/erts/etc/common/ct_run.c @@ -0,0 +1,545 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ +/* + * Purpose: Common Test front-end. + */ +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#ifdef __WIN32__ +#include <winbase.h> +#endif + +#include <ctype.h> + +#define NO 0 +#define YES 1 + +#define ASIZE(a) (sizeof(a)/sizeof(a[0])) + +static int debug = 0; /* Bit flags for debug printouts. */ + +static char** eargv_base; /* Base of vector. */ +static char** eargv; /* First argument for erl. */ + +static int eargc; /* Number of arguments in eargv. */ + +#ifdef __WIN32__ +# define QUOTE(s) possibly_quote(s) +# define IS_DIRSEP(c) ((c) == '/' || (c) == '\\') +# define ERL_NAME "erl.exe" +#else +# define QUOTE(s) s +# define IS_DIRSEP(c) ((c) == '/') +# define ERL_NAME "erl" +#endif + +#define UNSHIFT(s) eargc++, eargv--; eargv[0] = QUOTE(s) +#define PUSH(s) eargv[eargc++] = QUOTE(s) +#define PUSH2(s, t) PUSH(s); PUSH(t) +#define PUSH3(s, t, u) PUSH2(s, t); PUSH(u) +#define PUSH4(s, t, u, v) PUSH2(s, t); PUSH2(u, v) + +/* + * The possible modes to start Common Test + */ + +#define NORMAL_MODE 0 +#define VTS_MODE 1 +#define CT_SHELL_MODE 2 +#define MASTER_MODE 3 +#define ERL_SHELL_MODE 4 + +/* + * Distribution + */ + +#define SHORT_NAME 0 +#define FULL_NAME 1 + +/* + * Local functions. + */ + +static void error(char* format, ...); +static char* emalloc(size_t size); +static char* strsave(char* string); +static void push_words(char* src); +static int run_erlang(char* name, char** argv); +static char* get_default_emulator(char* progname); +static void print_deprecation_warning(char *progname); +#ifdef __WIN32__ +static char* possibly_quote(char* arg); +#endif + +/* + * Supply a strerror() function if libc doesn't. + */ +#ifndef HAVE_STRERROR + +extern int sys_nerr; + +#ifndef SYS_ERRLIST_DECLARED +extern const char * const sys_errlist[]; +#endif /* !SYS_ERRLIST_DECLARED */ + +char *strerror(int errnum) +{ + static char *emsg[1024]; + + if (errnum != 0) { + if (errnum > 0 && errnum < sys_nerr) + sprintf((char *) &emsg[0], "(%s)", sys_errlist[errnum]); + else + sprintf((char *) &emsg[0], "errnum = %d ", errnum); + } + else { + emsg[0] = '\0'; + } + return (char *) &emsg[0]; +} +#endif /* !HAVE_STRERROR */ + +int +main(int argc, char** argv) +{ + int eargv_size; + int eargc_base; /* How many arguments in the base of eargv. */ + char* emulator; + char nodename[100]; + char browser[100]; + int ct_mode; + int dist_mode; + int cnt; + int erl_args; + char** argv0 = argv; + + print_deprecation_warning(argv[0]); + + emulator = get_default_emulator(argv[0]); + + /* + * Allocate the argv vector to be used for arguments to Erlang. + * Arrange for starting to pushing information in the middle of + * the array, to allow easy addition of commands in the beginning. + */ + + eargv_size = argc*4+100; + eargv_base = (char **) emalloc(eargv_size*sizeof(char*)); + eargv = eargv_base; + eargc = 0; + push_words(emulator); + eargc_base = eargc; + eargv = eargv + eargv_size/2; + eargc = 0; + + strcpy(nodename, "ct"); + dist_mode = SHORT_NAME; + browser[0] = '\0'; + ct_mode = NORMAL_MODE; + erl_args = argc; + cnt = 1; + + /* + * Check various flags before building command line + */ + + while (cnt < argc) { + if (strcmp(argv[1], "-erl_args") == 0) { + erl_args = cnt; + } + else if (strcmp(argv[1], "-sname") == 0) { + strncpy(nodename, argv[2], sizeof(nodename)); + nodename[sizeof(nodename)-1] = '\0'; + cnt++, argv++; + } + else if (strcmp(argv[1], "-name") == 0) { + strncpy(nodename, argv[2], sizeof(nodename)); + nodename[sizeof(nodename)-1] = '\0'; + dist_mode = FULL_NAME; + cnt++, argv++; + } + else { + if (cnt < erl_args) { + if (strcmp(argv[1], "-vts") == 0) { + ct_mode = VTS_MODE; + } + else if (strcmp(argv[1], "-browser") == 0) { + strncpy(browser, argv[2], sizeof(browser)); + browser[sizeof(browser)-1] = '\0'; + cnt++, argv++; + } + else if (strcmp(argv[1], "-shell") == 0) { + ct_mode = CT_SHELL_MODE; + } + else if (strcmp(argv[1], "-ctmaster") == 0) { + strcpy(nodename, "ct_master"); + ct_mode = MASTER_MODE; + } + else if (strcmp(argv[1], "-ctname") == 0) { + strncpy(nodename, argv[2], sizeof(nodename)); + nodename[sizeof(nodename)-1] = '\0'; + ct_mode = ERL_SHELL_MODE; + cnt++, argv++; + } + } + } + cnt++, argv++; + } + + argv = argv0; + + /* + * Push initial arguments. + */ + + if (dist_mode == FULL_NAME) { + PUSH2("-name", nodename); + } + else { + PUSH2("-sname", nodename); + } + + /* + * Push everything else + */ + + if (ct_mode == VTS_MODE) { + PUSH4("-s", "webtool", "script_start", "vts"); + if (browser[0] != '\0') PUSH(browser); + PUSH3("-s", "ct_run", "script_start"); + } + else if (ct_mode == CT_SHELL_MODE) { + PUSH3("-s", "ct_run", "script_start"); + } + else if (ct_mode == NORMAL_MODE) { + PUSH3("-s", "ct_run", "script_start"); + PUSH3("-s", "erlang", "halt"); + } + + cnt = 1; + while (cnt < argc) { + if (strcmp(argv[1], "-erl_args") == 0) { + PUSH("-ct_erl_args"); + } + else if ((strcmp(argv[1], "-sname") == 0) || (strcmp(argv[1], "-name") == 0)) { + cnt++, argv++; + } + else if (cnt < erl_args) { + if (strcmp(argv[1], "-config") == 0) + PUSH("-ct_config"); + else if (strcmp(argv[1], "-decrypt_key") == 0) + PUSH("-ct_decrypt_key"); + else if (strcmp(argv[1], "-decrypt_file") == 0) + PUSH("-ct_decrypt_file"); + else + PUSH(argv[1]); + } + else { + PUSH(argv[1]); + } + cnt++, argv++; + } + + /* + * Move up the commands for invoking the emulator and adjust eargv + * accordingly. + */ + + while (--eargc_base >= 0) { + UNSHIFT(eargv_base[eargc_base]); + } + + /* + * Invoke Erlang with the collected options. + */ + + PUSH(NULL); + + return run_erlang(eargv[0], eargv); +} + +static void +push_words(char* src) +{ + char sbuf[MAXPATHLEN]; + char* dst; + + dst = sbuf; + while ((*dst++ = *src++) != '\0') { + if (isspace((int)*src)) { + *dst = '\0'; + PUSH(strsave(sbuf)); + dst = sbuf; + do { + src++; + } while (isspace((int)*src)); + } + } + if (sbuf[0]) + PUSH(strsave(sbuf)); +} +#ifdef __WIN32__ +char *make_commandline(char **argv) +{ + static char *buff = NULL; + static int siz = 0; + int num = 0; + char **arg, *p; + + if (*argv == NULL) { + return ""; + } + for (arg = argv; *arg != NULL; ++arg) { + num += strlen(*arg)+1; + } + if (!siz) { + siz = num; + buff = malloc(siz*sizeof(char)); + } else if (siz < num) { + siz = num; + buff = realloc(buff,siz*sizeof(char)); + } + p = buff; + for (arg = argv; *arg != NULL; ++arg) { + strcpy(p,*arg); + p+=strlen(*arg); + *p++=' '; + } + *(--p) = '\0'; + + if (debug) { + printf("Processed commandline:%s\n",buff); + } + return buff; +} + +int my_spawnvp(char **argv) +{ + STARTUPINFO siStartInfo; + PROCESS_INFORMATION piProcInfo; + DWORD ec; + + memset(&siStartInfo,0,sizeof(STARTUPINFO)); + siStartInfo.cb = sizeof(STARTUPINFO); + siStartInfo.dwFlags = STARTF_USESTDHANDLES; + siStartInfo.hStdInput = GetStdHandle(STD_INPUT_HANDLE); + siStartInfo.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE); + siStartInfo.hStdError = GetStdHandle(STD_ERROR_HANDLE); + siStartInfo.wShowWindow = SW_HIDE; + siStartInfo.dwFlags |= STARTF_USESHOWWINDOW; + + + if (!CreateProcess(NULL, + make_commandline(argv), + NULL, + NULL, + TRUE, + 0, + NULL, + NULL, + &siStartInfo, + &piProcInfo)) { + return -1; + } + CloseHandle(piProcInfo.hThread); + + WaitForSingleObject(piProcInfo.hProcess,INFINITE); + if (!GetExitCodeProcess(piProcInfo.hProcess,&ec)) { + return 0; + } + return (int) ec; +} +#endif /* __WIN32__ */ + + +static int +run_erlang(char* progname, char** argv) +{ +#ifdef __WIN32__ + int status; +#endif + + if (debug) { + int i = 0; + while (argv[i] != NULL) + printf(" %s", argv[i++]); + printf("\n"); + } + +#ifdef __WIN32__ + /* + * Alas, we must wait here for the program to finish. + * Otherwise, the shell from which we were executed will think + * we are finished and print a prompt and read keyboard input. + */ + + status = my_spawnvp(argv)/*_spawnvp(_P_WAIT,progname,argv)*/; + if (status == -1) { + fprintf(stderr, "ct_run: Error executing '%s': %d", progname, + GetLastError()); + } + return status; +#else + execvp(progname, argv); + error("Error %d executing \'%s\'.", errno, progname); + return 2; +#endif +} + +static void +error(char* format, ...) +{ + char sbuf[1024]; + va_list ap; + + va_start(ap, format); + erts_vsnprintf(sbuf, sizeof(sbuf), format, ap); + va_end(ap); + fprintf(stderr, "ct_run: %s\n", sbuf); + exit(1); +} + +static char* +emalloc(size_t size) +{ + char *p = malloc(size); + if (p == NULL) + error("Insufficient memory"); + return p; +} + +static char* +strsave(char* string) +{ + char* p = emalloc(strlen(string)+1); + strcpy(p, string); + return p; +} + +/* Instead of making sure basename exists, we do our own */ +static char *simple_basename(char *path) +{ + char *ptr; + for (ptr = path; *ptr != '\0'; ++ptr) { + if (*ptr == '/' || *ptr == '\\') { + path = ptr + 1; + } + } + return path; +} + +static void print_deprecation_warning(char* progpath) +{ + char *basename = simple_basename(progpath); + if(strcmp(basename,"run_test") == 0 || + strcmp(basename, "run_test.exe") == 0) { + printf("---***---\nDeprecated: run_test is deprecated and will be removed in R16B,\n please use ct_run instead\n---***---\n"); + } +} + +static char* +get_default_emulator(char* progname) +{ + char sbuf[MAXPATHLEN]; + char* s; + + if (strlen(progname) >= sizeof(sbuf)) + return ERL_NAME; + + strcpy(sbuf, progname); + for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { + if (IS_DIRSEP(*s)) { + strcpy(s+1, ERL_NAME); +#ifdef __WIN32__ + if (_access(sbuf, 0) != -1) { + return strsave(sbuf); + } +#else + if (access(sbuf, 1) != -1) { + return strsave(sbuf); + } +#endif + break; + } + } + return ERL_NAME; +} + +#ifdef __WIN32__ +static char* +possibly_quote(char* arg) +{ + int mustQuote = NO; + int n = 0; + char* s; + char* narg; + + if (arg == NULL) { + return arg; + } + + /* + * Scan the string to find out if it needs quoting and return + * the original argument if not. + */ + + for (s = arg; *s; s++, n++) { + switch(*s) { + case ' ': + mustQuote = YES; + continue; + case '"': + mustQuote = YES; + n++; + continue; + case '\\': + if(s[1] == '"') + n++; + continue; + default: + continue; + } + } + if (!mustQuote) { + return arg; + } + + /* + * Insert the quotes and put a backslash in front of every quote + * inside the string. + */ + + s = narg = emalloc(n+2+1); + for (*s++ = '"'; *arg; arg++, s++) { + if (*arg == '"' || (*arg == '\\' && arg[1] == '"')) { + *s++ = '\\'; + } + *s = *arg; + } + if (s[-1] == '\\') { + *s++ ='\\'; + } + *s++ = '"'; + *s = '\0'; + return narg; +} +#endif /* __WIN32__ */ diff --git a/erts/etc/common/dialyzer.c b/erts/etc/common/dialyzer.c index 4b4c1124f1..4453e63f1c 100644 --- a/erts/etc/common/dialyzer.c +++ b/erts/etc/common/dialyzer.c @@ -147,6 +147,9 @@ main(int argc, char** argv) env = get_env("DIALYZER_EMULATOR"); emulator = env ? env : get_default_emulator(argv[0]); + if (strlen(emulator) >= MAXPATHLEN) + error("Value of environment variable DIALYZER_EMULATOR is too large"); + /* * Allocate the argv vector to be used for arguments to Erlang. * Arrange for starting to pushing information in the middle of @@ -228,7 +231,7 @@ main(int argc, char** argv) static void push_words(char* src) { - char sbuf[1024]; + char sbuf[MAXPATHLEN]; char* dst; dst = sbuf; @@ -360,7 +363,7 @@ error(char* format, ...) va_list ap; va_start(ap, format); - vsprintf(sbuf, format, ap); + erts_vsnprintf(sbuf, sizeof(sbuf), format, ap); va_end(ap); fprintf(stderr, "dialyzer: %s\n", sbuf); exit(1); @@ -389,6 +392,9 @@ get_default_emulator(char* progname) char sbuf[MAXPATHLEN]; char* s; + if (strlen(progname) >= sizeof(sbuf)) + return ERL_NAME; + strcpy(sbuf, progname); for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { if (IS_DIRSEP(*s)) { diff --git a/erts/etc/common/erlc.c b/erts/etc/common/erlc.c index 09aca19e6c..cd137435d1 100644 --- a/erts/etc/common/erlc.c +++ b/erts/etc/common/erlc.c @@ -148,10 +148,6 @@ int main(int argc, char** argv) { char cwd[MAXPATHLEN]; /* Current working directory. */ - char** rpc_eargv; /* Pointer to the beginning of arguments - * if calling a running Erlang system - * via erl_rpc(). - */ int eargv_size; int eargc_base; /* How many arguments in the base of eargv. */ char* emulator; @@ -160,6 +156,9 @@ main(int argc, char** argv) env = get_env("ERLC_EMULATOR"); emulator = env ? env : get_default_emulator(argv[0]); + if (strlen(emulator) >= MAXPATHLEN) + error("Value of environment variable ERLC_EMULATOR is too large"); + /* * Allocate the argv vector to be used for arguments to Erlang. * Arrange for starting to pushing information in the middle of @@ -170,7 +169,7 @@ main(int argc, char** argv) * base of the eargv vector, and move it up later. */ - eargv_size = argc*4+100; + eargv_size = argc*6+100; eargv_base = (char **) emalloc(eargv_size*sizeof(char*)); eargv = eargv_base; eargc = 0; @@ -189,7 +188,6 @@ main(int argc, char** argv) PUSH2("-mode", "minimal"); PUSH2("-boot", "start_clean"); PUSH3("-s", "erl_compile", "compile_cmdline"); - rpc_eargv = eargv+eargc; /* * Push standard arguments to Erlang. @@ -419,7 +417,7 @@ process_opt(int* pArgc, char*** pArgv, int offset) static void push_words(char* src) { - char sbuf[1024]; + char sbuf[MAXPATHLEN]; char* dst; dst = sbuf; @@ -595,7 +593,7 @@ error(char* format, ...) va_list ap; va_start(ap, format); - vsprintf(sbuf, format, ap); + erts_vsnprintf(sbuf, sizeof(sbuf), format, ap); va_end(ap); fprintf(stderr, "erlc: %s\n", sbuf); exit(1); @@ -624,6 +622,9 @@ get_default_emulator(char* progname) char sbuf[MAXPATHLEN]; char* s; + if (strlen(progname) >= sizeof(sbuf)) + return ERL_NAME; + strcpy(sbuf, progname); for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { if (IS_DIRSEP(*s)) { diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index f79f5cc978..60b3af7db7 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -120,6 +120,7 @@ static char *plusM_other_switches[] = { static char *pluss_val_switches[] = { "bt", "ct", + "wt", "ss", NULL }; @@ -131,6 +132,18 @@ static char *plush_val_switches[] = { NULL }; +/* +r arguments with values */ +static char *plusr_val_switches[] = { + "g", + NULL +}; + +/* +z arguments with values */ +static char *plusz_val_switches[] = { + "dbbl", + NULL +}; + /* * Define sleep(seconds) in terms of Sleep() on Windows. @@ -302,7 +315,7 @@ free_env_val(char *value) } /* - * Add the arcitecture suffix to the program name if needed, + * Add the architecture suffix to the program name if needed, * except on Windows, where we insert it just before ".DLL". */ static char* @@ -553,7 +566,7 @@ int main(int argc, char **argv) usage("+MYm"); } emu = add_extra_suffixes(emu, emu_type); - sprintf(tmpStr, "%s" DIRSEP "%s" BINARY_EXT, bindir, emu); + erts_snprintf(tmpStr, sizeof(tmpStr), "%s" DIRSEP "%s" BINARY_EXT, bindir, emu); emu = strsave(tmpStr); add_Eargs(emu); /* Will be argv[0] -- necessary! */ @@ -564,12 +577,12 @@ int main(int argc, char **argv) s = get_env("PATH"); if (!s) { - sprintf(tmpStr, "%s" PATHSEP "%s" DIRSEP "bin", bindir, rootdir); + erts_snprintf(tmpStr, sizeof(tmpStr), "%s" PATHSEP "%s" DIRSEP "bin", bindir, rootdir); } else if (strstr(s, bindir) == NULL) { - sprintf(tmpStr, "%s" PATHSEP "%s" DIRSEP "bin" PATHSEP "%s", bindir, + erts_snprintf(tmpStr, sizeof(tmpStr), "%s" PATHSEP "%s" DIRSEP "bin" PATHSEP "%s", bindir, rootdir, s); } else { - sprintf(tmpStr, "%s", s); + erts_snprintf(tmpStr, sizeof(tmpStr), "%s", s); } free_env_val(s); set_env("PATH", tmpStr); @@ -707,7 +720,7 @@ int main(int argc, char **argv) error("-man not supported on Windows"); #else argv[i] = "man"; - sprintf(tmpStr, "%s/man", rootdir); + erts_snprintf(tmpStr, sizeof(tmpStr), "%s/man", rootdir); set_env("MANPATH", tmpStr); execvp("man", argv+i); error("Could not execute the 'man' command."); @@ -872,6 +885,21 @@ int main(int argc, char **argv) i++; } break; + case 'r': + if (!is_one_of_strings(&argv[i][2], + plusr_val_switches)) + goto the_default; + else { + if (i+1 >= argc + || argv[i+1][0] == '-' + || argv[i+1][0] == '+') + usage(argv[i]); + argv[i][0] = '-'; + add_Eargs(argv[i]); + add_Eargs(argv[i+1]); + i++; + } + break; case 's': if (!is_one_of_strings(&argv[i][2], pluss_val_switches)) @@ -887,6 +915,20 @@ int main(int argc, char **argv) i++; } break; + case 'z': + if (!is_one_of_strings(&argv[i][2], plusz_val_switches)) { + goto the_default; + } else { + if (i+1 >= argc + || argv[i+1][0] == '-' + || argv[i+1][0] == '+') + usage(argv[i]); + argv[i][0] = '-'; + add_Eargs(argv[i]); + add_Eargs(argv[i+1]); + i++; + } + break; default: the_default: argv[i][0] = '-'; /* Change +option to -option. */ @@ -1069,11 +1111,12 @@ usage_aux(void) "[-hybrid] " #endif "[-make] [-man [manopts] MANPAGE] [-x] [-emu_args] " - "[-args_file FILENAME] " - "[+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] [+h HEAP_SIZE_OPTION] [+K BOOLEAN] " + "[-args_file FILENAME] [+A THREADS] [+a SIZE] [+B[c|d|i]] [+c] " + "[+h HEAP_SIZE_OPTION] [+K BOOLEAN] " "[+l] [+M<SUBSWITCH> <ARGUMENT>] [+P MAX_PROCS] [+R COMPAT_REL] " - "[+r] [+s SCHEDULER_OPTION] [+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] [+W<i|w>] " - "[args ...]\n"); + "[+r] [+rg READER_GROUPS_LIMIT] [+s SCHEDULER_OPTION] " + "[+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] " + "[+W<i|w>] [+z MISC_OPTION] [args ...]\n"); exit(1); } @@ -1122,10 +1165,10 @@ start_epmd(char *epmd) if (!epmd) { epmd = epmd_cmd; #ifdef __WIN32__ - sprintf(epmd_cmd, "%s" DIRSEP "epmd", bindir); + erts_snprintf(epmd_cmd, sizeof(epmd_cmd), "%s" DIRSEP "epmd", bindir); arg1 = "-daemon"; #else - sprintf(epmd_cmd, "%s" DIRSEP "epmd -daemon", bindir); + erts_snprintf(epmd_cmd, sizeof(epmd_cmd), "%s" DIRSEP "epmd -daemon", bindir); #endif } #ifdef __WIN32__ @@ -1201,7 +1244,7 @@ void error(char* format, ...) va_list ap; va_start(ap, format); - vsprintf(sbuf, format, ap); + erts_vsnprintf(sbuf, sizeof(sbuf), format, ap); va_end(ap); fprintf(stderr, "erlexec: %s\n", sbuf); exit(1); @@ -1281,14 +1324,14 @@ static void get_start_erl_data(char *file) if (env) reldir = strsave(env); else { - sprintf(tmpbuffer, "%s/releases", rootdir); + erts_snprintf(tmpbuffer, sizeof(tmpbuffer), "%s/releases", rootdir); reldir = strsave(tmpbuffer); } free_env_val(env); if (file == NULL) - sprintf(start_erl_data, "%s/start_erl.data", reldir); + erts_snprintf(start_erl_data, sizeof(start_erl_data), "%s/start_erl.data", reldir); else - sprintf(start_erl_data, "%s", file); + erts_snprintf(start_erl_data, sizeof(start_erl_data), "%s", file); fp = _open(start_erl_data, _O_RDONLY ); if( fp == -1 ) error( "open failed on %s",start_erl_data ); @@ -1318,16 +1361,16 @@ static void get_start_erl_data(char *file) } bindir = emalloc(512); - sprintf(bindir,"%s/erts-%s/bin",rootdir,tmpbuffer); + erts_snprintf(bindir,512,"%s/erts-%s/bin",rootdir,tmpbuffer); /* BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin */ tprogname = progname; progname = emalloc(strlen(tprogname) + 20); - sprintf(progname,"%s -start_erl",tprogname); + erts_snprintf(progname,strlen(tprogname) + 20,"%s -start_erl",tprogname); boot_script = emalloc(512); config_script = emalloc(512); - sprintf(boot_script, "%s/%s/start", reldir, otpstring); - sprintf(config_script, "%s/%s/sys", reldir, otpstring); + erts_snprintf(boot_script, 512, "%s/%s/start", reldir, otpstring); + erts_snprintf(config_script, 512, "%s/%s/sys", reldir, otpstring); } @@ -1335,7 +1378,7 @@ static void get_start_erl_data(char *file) static char *replace_filename(char *path, char *new_base) { int plen = strlen(path); - char *res = malloc((plen+strlen(new_base)+1)*sizeof(char)); + char *res = emalloc((plen+strlen(new_base)+1)*sizeof(char)); char *p; strcpy(res,path); @@ -1350,7 +1393,7 @@ static char *path_massage(char *long_path) { char *p; - p = malloc(MAX_PATH+1); + p = emalloc(MAX_PATH+1); strcpy(p, long_path); GetShortPathName(p, p, MAX_PATH); return p; @@ -1486,7 +1529,8 @@ get_parameters(int argc, char** argv) /* Determine bindir from absolute path to executable */ char *p; char buffer[PATH_MAX]; - strcpy(buffer, argv[0]); + strncpy(buffer, argv[0], sizeof(buffer)); + buffer[sizeof(buffer)-1] = '\0'; for (p = buffer+strlen(buffer)-1 ; p >= buffer && *p != '/'; --p) ; @@ -1499,7 +1543,8 @@ get_parameters(int argc, char** argv) /* Determine rootdir from absolute path to bindir */ char *p; char buffer[PATH_MAX]; - strcpy(buffer, bindir); + strncpy(buffer, bindir, sizeof(buffer)); + buffer[sizeof(buffer)-1] = '\0'; for (p = buffer+strlen(buffer)-1; p >= buffer && *p != '/'; --p) ; diff --git a/erts/etc/common/escript.c b/erts/etc/common/escript.c index 1bc5eb7651..6ed79c91e3 100644 --- a/erts/etc/common/escript.c +++ b/erts/etc/common/escript.c @@ -151,6 +151,9 @@ find_prog(char *origpath) char relpath[PMAX]; char abspath[PMAX]; + if (strlen(origpath) >= sizeof(relpath)) + error("Path too long"); + strcpy(relpath, origpath); if (strstr(relpath, DIRSEPSTR) == NULL) { @@ -180,19 +183,21 @@ find_prog(char *origpath) end = strstr(beg, PATHSEPSTR); if (end != NULL) { sz = end - beg; - strncpy(dir, beg, sz); - dir[sz] = '\0'; } else { sz = strlen(beg); - strcpy(dir, beg); look_for_sep = FALSE; } + if (sz >= sizeof(dir)) { + beg = end + 1; + continue; + } + strncpy(dir, beg, sz); + dir[sz] = '\0'; beg = end + 1; #ifdef __WIN32__ - strcpy(wildcard, dir); - strcat(wildcard, DIRSEPSTR); - strcat(wildcard, relpath); /* basename */ + erts_snprintf(wildcard, sizeof(wildcard), "%s" DIRSEPSTR "%s", + dir, relpath /* basename */); dir_handle = FindFirstFile(wildcard, &find_data); if (dir_handle == INVALID_HANDLE_VALUE) { /* Try next directory in path */ @@ -217,9 +222,8 @@ find_prog(char *origpath) if (strcmp(origpath, dirp->d_name) == 0) { /* Wow we found the executable. */ - strcpy(relpath, dir); - strcat(relpath, DIRSEPSTR); - strcat(relpath, dirp->d_name); + erts_snprintf(relpath, sizeof(relpath), "%s" DIRSEPSTR "%s", + dir, dirp->d_name); closedir(dp); look_for_sep = FALSE; break; @@ -291,7 +295,7 @@ append_shebang_args(char* scriptname) /* Find end of arg */ end = beg; - while (end && end[0] != ' ') { + while (end && end < (linebuf+LINEBUFSZ-1) && end[0] != ' ') { if (end[0] == '\n') { newline = TRUE; end[0]= '\0'; @@ -335,13 +339,16 @@ main(int argc, char** argv) emulator = get_default_emulator(argv[0]); } + if (strlen(emulator) >= PMAX) + error("Value of environment variable ESCRIPT_EMULATOR is too large"); + /* * Allocate the argv vector to be used for arguments to Erlang. * Arrange for starting to pushing information in the middle of * the array, to allow easy addition of commands in the beginning. */ - eargv_size = argc*4+1000; + eargv_size = argc*4+1000+LINEBUFSZ/2; eargv_base = (char **) emalloc(eargv_size*sizeof(char*)); eargv = eargv_base; eargc = 0; @@ -387,7 +394,8 @@ main(int argc, char** argv) if (argc <= 1) { error("Missing filename\n"); } - strcpy(scriptname, argv[1]); + strncpy(scriptname, argv[1], sizeof(scriptname)); + scriptname[sizeof(scriptname)-1] = '\0'; argc--; argv++; } else { @@ -395,16 +403,17 @@ main(int argc, char** argv) int len; #endif absname = find_prog(argv[0]); - strcpy(scriptname, absname); - efree(absname); #ifdef __WIN32__ - len = strlen(scriptname); - if (len >= 4 && _stricmp(scriptname+len-4, ".exe") == 0) { - scriptname[len-4] = '\0'; + len = strlen(absname); + if (len >= 4 && _stricmp(absname+len-4, ".exe") == 0) { + absname[len-4] = '\0'; } #endif - strcat(scriptname, ".escript"); + erts_snprintf(scriptname, sizeof(scriptname), "%s.escript", + absname); + efree(absname); + } /* @@ -455,7 +464,7 @@ main(int argc, char** argv) static void push_words(char* src) { - char sbuf[1024]; + char sbuf[PMAX]; char* dst; dst = sbuf; @@ -584,7 +593,7 @@ error(char* format, ...) va_list ap; va_start(ap, format); - vsprintf(sbuf, format, ap); + erts_vsnprintf(sbuf, sizeof(sbuf), format, ap); va_end(ap); fprintf(stderr, "escript: %s\n", sbuf); exit(1); @@ -619,6 +628,9 @@ get_default_emulator(char* progname) char sbuf[MAXPATHLEN]; char* s; + if (strlen(progname) >= sizeof(sbuf)) + return ERL_NAME; + strcpy(sbuf, progname); for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { if (IS_DIRSEP(*s)) { diff --git a/erts/etc/common/heart.c b/erts/etc/common/heart.c index 4f738947b7..3e19e5f386 100644 --- a/erts/etc/common/heart.c +++ b/erts/etc/common/heart.c @@ -375,7 +375,8 @@ main(int argc, char **argv) _setmode(erlin_fd,_O_BINARY); _setmode(erlout_fd,_O_BINARY); #endif - strcpy(program_name, argv[0]); + strncpy(program_name, argv[0], sizeof(program_name)); + program_name[sizeof(program_name)-1] = '\0'; notify_ack(erlout_fd); cmd[0] = '\0'; do_terminate(message_loop(erlin_fd,erlout_fd)); @@ -728,7 +729,11 @@ heart_cmd_reply(int fd, char *s) struct msg m; int len = strlen(s) + 1; /* Include \0 */ - /* FIXME if s >= MSG_BODY_SIZE error */ + /* if s >= MSG_BODY_SIZE, return a write + * failure immediately. + */ + if (len > sizeof(m.fill)) + return -1; m.op = HEART_CMD; m.len = htons(len + 2); /* Include Op */ diff --git a/erts/etc/common/inet_gethost.c b/erts/etc/common/inet_gethost.c index d3ff4874ac..8bd9368aa1 100644 --- a/erts/etc/common/inet_gethost.c +++ b/erts/etc/common/inet_gethost.c @@ -59,15 +59,14 @@ #define WIN32_LEAN_AND_MEAN #include <winsock2.h> #include <windows.h> +#include <ws2tcpip.h> #include <process.h> #include <stdio.h> #include <stdlib.h> /* These are not used even if they would exist which they should not */ -#undef HAVE_GETADDRINFO #undef HAVE_GETIPNODEBYNAME #undef HAVE_GETHOSTBYNAME2 -#undef HAVE_GETNAMEINFO #undef HAVE_GETIPNODEBYADDR #else /* Unix */ @@ -1761,7 +1760,7 @@ static int worker_loop(void) struct addrinfo hints; memset(&hints, 0, sizeof(hints)); - hints.ai_flags = (AI_CANONNAME|AI_V4MAPPED|AI_ADDRCONFIG); + hints.ai_flags = AI_CANONNAME; hints.ai_socktype = SOCK_STREAM; hints.ai_family = AF_INET6; DEBUGF(5, ("Starting getaddrinfo(%s, ...)", data)); diff --git a/erts/etc/common/typer.c b/erts/etc/common/typer.c index c2567cb8b4..de48daf002 100644 --- a/erts/etc/common/typer.c +++ b/erts/etc/common/typer.c @@ -175,7 +175,7 @@ main(int argc, char** argv) static void push_words(char* src) { - char sbuf[1024]; + char sbuf[MAXPATHLEN]; char* dst; dst = sbuf; @@ -307,7 +307,7 @@ error(char* format, ...) va_list ap; va_start(ap, format); - vsprintf(sbuf, format, ap); + erts_vsnprintf(sbuf, sizeof(sbuf), format, ap); va_end(ap); fprintf(stderr, "typer: %s\n", sbuf); exit(1); @@ -336,6 +336,9 @@ get_default_emulator(char* progname) char sbuf[MAXPATHLEN]; char* s; + if (strlen(progname) >= sizeof(sbuf)) + return ERL_NAME; + strcpy(sbuf, progname); for (s = sbuf+strlen(sbuf); s >= sbuf; s--) { if (IS_DIRSEP(*s)) { diff --git a/erts/etc/unix/Install.src b/erts/etc/unix/Install.src index 83f9690782..8f40c43874 100644 --- a/erts/etc/unix/Install.src +++ b/erts/etc/unix/Install.src @@ -89,8 +89,12 @@ cp -p $ERL_ROOT/erts-%I_VSN%/bin/erl . cp -p $ERL_ROOT/erts-%I_VSN%/bin/erlc . cp -p $ERL_ROOT/erts-%I_VSN%/bin/dialyzer . cp -p $ERL_ROOT/erts-%I_VSN%/bin/typer . +cp -p $ERL_ROOT/erts-%I_VSN%/bin/ct_run . cp -p $ERL_ROOT/erts-%I_VSN%/bin/escript . +# Remove in R16B +ln -s ct_run run_test + # # Set a soft link to epmd # This should not be done for an embedded system! diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 9dab9fcfcc..73b1bafbe0 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -66,6 +66,7 @@ core= GDB= GDBBP= +GDBARGS= TYPE= EMU_TYPE= debug= @@ -280,16 +281,11 @@ else # Set annotation level for gdb in emacs 22 and higher. emacs_major=`$EMACS --version | head -1 | sed 's,^[^0-9]*\([0-9]*\).*,\1,g'` if [ '!' -z "$emacs_major" -a $emacs_major -gt 21 ]; then - # Hack - wait for etp-commands to be loaded and then set - # annotation level, could be done more beautifully than with sit-for... - gdbcmd="$gdbcmd \ - (sit-for 1) \ - (insert-string \"set annotate 3\") \ - (comint-send-input)" + GDBARGS="--annotate=3 " fi gdbcmd="$gdbcmd $GDBBP \ (insert-string \"source $ROOTDIR/erts/etc/unix/etp-commands\") \ (comint-send-input)" # Fire up gdb in emacs... - exec $EMACS --eval "(progn (gdb \"gdb $EMU\") $gdbcmd)" + exec $EMACS --eval "(progn (gdb \"gdb $GDBARGS$EMU\") $gdbcmd)" fi diff --git a/erts/etc/unix/format_man_pages b/erts/etc/unix/format_man_pages index 2c4f6eee4f..93dcdcd8fa 100644 --- a/erts/etc/unix/format_man_pages +++ b/erts/etc/unix/format_man_pages @@ -3,7 +3,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2009. All Rights Reserved. +# Copyright Ericsson AB 1996-2010. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -59,34 +59,21 @@ esac # Create the 'cat' directories (probably not needed) # -cd $ERL_ROOT +cd $ERL_ROOT/man -if [ ! -d man/cat1 ] -then - mkdir man/cat1 -fi +for d in 0 1 2 3 4 5 6 7 8 9 +do + if [ ! -d cat$d ] + then + mkdir cat$d + fi -if [ ! -d man/cat3 ] -then - mkdir man/cat3 -fi - -if [ ! -d man/cat4 ] -then - mkdir man/cat4 -fi - -if [ ! -d man/cat6 ] -then - mkdir man/cat6 -fi +done # # Cleanup old formatting # -cd $ERL_ROOT/man - rm -f whatis windex # Remove old cat files diff --git a/erts/etc/unix/run_erl.c b/erts/etc/unix/run_erl.c index 4bb148df98..cadff12c6f 100644 --- a/erts/etc/unix/run_erl.c +++ b/erts/etc/unix/run_erl.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -75,6 +75,9 @@ #ifdef HAVE_SYS_IOCTL_H # include <sys/ioctl.h> #endif +#if defined(__sun) && defined(__SVR4) +# include <stropts.h> +#endif #include "run_erl.h" #include "safe_string.h" /* sn_printf, strn_cpy, strn_cat, etc */ @@ -864,8 +867,12 @@ static int open_pty_master(char **ptyslave) /* Use the posix_openpt if working, as this guarantees creation of the slave device properly. */ -#ifdef HAVE_WORKING_POSIX_OPENPT +#if defined(HAVE_WORKING_POSIX_OPENPT) || (defined(__sun) && defined(__SVR4)) +# ifdef HAVE_WORKING_POSIX_OPENPT if ((mfd = posix_openpt(O_RDWR)) >= 0) { +# elif defined(__sun) && defined(__SVR4) + if ((mfd = open("/dev/ptmx", O_RDWR)) >= 0) { +# endif if ((*ptyslave = ptsname(mfd)) != NULL && grantpt(mfd) == 0 && unlockpt(mfd) == 0) { @@ -981,8 +988,28 @@ static int open_pty_slave(char *name) return -1; } +#if defined(__sun) && defined(__SVR4) + /* Load the necessary STREAMS modules for Solaris */ + if ((ioctl(sfd, I_FIND, "ldterm")) < 0) { + ERROR0(LOG_ERR, "Failed to find ldterm STREAMS module"); + return -1; + } + if (ioctl(sfd, I_PUSH, "ptem") < 0) { + ERROR0(LOG_ERR, "Failed to push ptem STREAMS module"); + return -1; + } + if (ioctl(sfd, I_PUSH, "ldterm") < 0) { + ERROR0(LOG_ERR, "Failed to push ldterm STREAMS module"); + return -1; + } + if (ioctl(sfd, I_PUSH, "ttcompat") < 0) { + ERROR0(LOG_ERR, "Failed to push ttcompat STREAMS module"); + return -1; + } +#endif + #ifdef DEBUG - if (tcgetattr(sfd, &tty_rmode) , 0) { + if (tcgetattr(sfd, &tty_rmode) < 0) { fprintf(stderr, "Cannot get terminals current mode\n"); exit(-1); } diff --git a/erts/etc/win32/Install.c b/erts/etc/win32/Install.c index 4a559cd8a2..6e60512f6d 100644 --- a/erts/etc/win32/Install.c +++ b/erts/etc/win32/Install.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* @@ -45,8 +45,8 @@ int main(int argc, char **argv) InitSection *ini_section; HANDLE module = GetModuleHandle(NULL); char *binaries[] = { "erl.exe", "werl.exe", "erlc.exe", - "dialyzer.exe", "typer.exe", - "escript.exe", NULL }; + "dialyzer.exe", "typer.exe", + "escript.exe", "ct_run.exe", NULL }; char *scripts[] = { "start_clean.boot", "start_sasl.boot", NULL }; char fromname[MAX_PATH]; char toname[MAX_PATH]; @@ -172,6 +172,20 @@ int main(int argc, char **argv) } } + // Remove in R16B + sprintf(fromname,"%s\\%s",bin_dir,"ct_run.exe"); + sprintf(toname,"%s\\%s",bin_dir,"run_test.exe"); + if (GetFileAttributes(fromname) == 0xFFFFFFFF) { + fprintf(stderr,"Could not find file %s\n", + fromname); + exit(1); + } + if (!CopyFile(fromname,toname,FALSE)) { + fprintf(stderr,"Could not copy file %s to %s\n", + fromname,toname); + fprintf(stderr,"Continuing installation anyway...\n"); + } + for (i = 0; scripts[i] != NULL; ++i) { sprintf(fromname,"%s\\%s",release_dir,scripts[i]); sprintf(toname,"%s\\%s",bin_dir,scripts[i]); diff --git a/erts/etc/win32/cygwin_tools/vc/ld.sh b/erts/etc/win32/cygwin_tools/vc/ld.sh index ac39bf871c..406c63ffee 100755 --- a/erts/etc/win32/cygwin_tools/vc/ld.sh +++ b/erts/etc/win32/cygwin_tools/vc/ld.sh @@ -3,7 +3,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2002-2009. All Rights Reserved. +# Copyright Ericsson AB 2002-2010. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -53,7 +53,7 @@ while test -n "$1" ; do STDLIB_FORCED=true; STDLIB=LIBCMTD.LIB;; -lsocket) - DEFAULT_LIBRARIES="$DEFAULT_LIBRARIES WS2_32.LIB";; + DEFAULT_LIBRARIES="$DEFAULT_LIBRARIES WS2_32.LIB IPHLPAPI.LIB";; -l*) y=`echo $x | sed 's,^-l\(.*\),\1,g'`; MPATH=`cygpath -m $y`; @@ -158,7 +158,7 @@ else fi p=$$ -CMD="$linktype -nologo -incremental:no $CMD $STDLIB $DEFAULT_LIBRARIES" +CMD="$linktype -nologo -incremental:no -largeaddressaware $CMD $STDLIB $DEFAULT_LIBRARIES" if [ "X$LD_SH_DEBUG_LOG" != "X" ]; then echo ld.sh "$SAVE" >>$LD_SH_DEBUG_LOG echo link.exe $CMD >>$LD_SH_DEBUG_LOG @@ -167,6 +167,10 @@ eval link.exe "$CMD" >/tmp/link.exe.${p}.1 2>/tmp/link.exe.${p}.2 RES=$? CMANIFEST=`cygpath $MANIFEST` if [ "$RES" = "0" -a -f "$CMANIFEST" ]; then + # Add stuff to manifest to turn off "virtualization" + sed -n -i '1h;1!H;${;g;s,<trustInfo.*</trustInfo>.,,g;p;}' $CMANIFEST + sed -i "s/<\/assembly>/ <ms_asmv2:trustInfo xmlns:ms_asmv2=\"urn:schemas-microsoft-com:asm.v2\">\n <ms_asmv2:security>\n <ms_asmv2:requestedPrivileges>\n <ms_asmv2:requestedExecutionLevel level=\"AsInvoker\" uiAccess=\"false\"\/>\n <\/ms_asmv2:requestedPrivileges>\n <\/ms_asmv2:security>\n <\/ms_asmv2:trustInfo>\n<\/assembly>/" $CMANIFEST + eval mt.exe -nologo -manifest "$MANIFEST" -outputresource:"$OUTPUTRES" >>/tmp/link.exe.${p}.1 2>>/tmp/link.exe.${p}.2 RES=$? if [ "$RES" != "0" ]; then diff --git a/erts/etc/win32/cygwin_tools/vc/rc.sh b/erts/etc/win32/cygwin_tools/vc/rc.sh index 6a6921c49e..054c672e64 100755 --- a/erts/etc/win32/cygwin_tools/vc/rc.sh +++ b/erts/etc/win32/cygwin_tools/vc/rc.sh @@ -2,20 +2,20 @@ # set -x # # %CopyrightBegin% -# -# Copyright Ericsson AB 2002-2009. All Rights Reserved. -# +# +# Copyright Ericsson AB 2002-2010. All Rights Reserved. +# # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in # compliance with the License. You should have received a copy of the # Erlang Public License along with this software. If not, it can be # retrieved online at http://www.erlang.org/. -# +# # Software distributed under the License is distributed on an "AS IS" # basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See # the License for the specific language governing rights and limitations # under the License. -# +# # %CopyrightEnd% # # Save the command line for debug outputs @@ -80,7 +80,7 @@ if [ "X$RC_SH_DEBUG_LOG" != "X" ]; then fi eval $RCC "$CMD" >/tmp/rc.exe.${p}.1 2>/tmp/rc.exe.${p}.2 RES=$? -tail +2 /tmp/rc.exe.${p}.2 >&2 +tail -n +2 /tmp/rc.exe.${p}.2 >&2 cat /tmp/rc.exe.${p}.1 rm -f /tmp/rc.exe.${p}.2 /tmp/rc.exe.${p}.1 exit $RES diff --git a/erts/etc/win32/nsis/Makefile b/erts/etc/win32/nsis/Makefile index ebb3ad9a96..981a232c69 100644 --- a/erts/etc/win32/nsis/Makefile +++ b/erts/etc/win32/nsis/Makefile @@ -45,6 +45,7 @@ WTARGET_DIR=$(shell (cygpath -d $(TARGET_DIR) 2>/dev/null || cygpath -d $(TARGET REDIST_FILE=$(shell (sh ./find_redist.sh || echo "")) REDIST_DLL_VERSION=$(shell (sh ./dll_version_helper.sh || echo "")) +REDIST_DLL_NAME=$(shell (sh ./dll_version_helper.sh -n || echo "")) release_spec: @NSIS_VER=`makensis /hdrinfo | head -1 | awk '{print $$2}'`; \ @@ -73,6 +74,7 @@ release_spec: cp $(REDIST_FILE) $(RELEASE_PATH)/vcredist_x86.exe;\ echo '!define HAVE_REDIST_FILE 1' >> $(VERSION_HEADER); \ echo '!define REDIST_DLL_VERSION "$(REDIST_DLL_VERSION)"' >> $(VERSION_HEADER);\ + echo '!define REDIST_DLL_NAME "$(REDIST_DLL_NAME)"' >> $(VERSION_HEADER);\ fi;\ if [ -f $(RELEASE_PATH)/docs/doc/index.html ];\ then \ diff --git a/erts/etc/win32/nsis/dll_version_helper.sh b/erts/etc/win32/nsis/dll_version_helper.sh index e0047dea8b..571ee3e39e 100755 --- a/erts/etc/win32/nsis/dll_version_helper.sh +++ b/erts/etc/win32/nsis/dll_version_helper.sh @@ -2,7 +2,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2007-2009. All Rights Reserved. +# Copyright Ericsson AB 2007-2010. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -41,9 +41,15 @@ if [ '!' -f hello.exe.manifest ]; then exit 0 fi VERSION=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*version=.\([0-9\.]*\).*,\1,g' | grep -v '<'` +NAME=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*name=.[A-Za-z\.]*\([0-9]*\).*,msvcr\1.dll,g' | grep -v '<'` rm -f hello.c hello.obj hello.exe hello.exe.manifest -if [ -z "$VERSION" ]; then +if [ "$1" = "-n" ]; then + ASKEDFOR=$NAME +else + ASKEDFOR=$VERSION +fi +if [ -z "$ASKEDFOR" ]; then exit 1 fi -echo $VERSION +echo $ASKEDFOR exit 0 diff --git a/erts/etc/win32/nsis/erlang20.nsi b/erts/etc/win32/nsis/erlang20.nsi index 43e5d91604..941e8e6f5d 100644 --- a/erts/etc/win32/nsis/erlang20.nsi +++ b/erts/etc/win32/nsis/erlang20.nsi @@ -311,23 +311,23 @@ FunctionEnd Function .onInit
SectionGetFlags 0 $MYTEMP
-; MessageBox MB_YESNO "Found $SYSDIR\msvcr80.dll" IDYES FoundLbl
- IfFileExists $SYSDIR\msvcr80.dll MaybeFoundInSystemLbl
+ ;MessageBox MB_YESNO "Found $SYSDIR\${REDIST_DLL_NAME}" IDYES FoundLbl
+ IfFileExists $SYSDIR\${REDIST_DLL_NAME} MaybeFoundInSystemLbl
SearchSxsLbl:
FindFirst $0 $1 $WINDIR\WinSxS\x86*
LoopLbl:
StrCmp $1 "" NotFoundLbl
- IfFileExists $WINDIR\WinSxS\$1\msvcr80.dll MaybeFoundInSxsLbl
+ IfFileExists $WINDIR\WinSxS\$1\${REDIST_DLL_NAME} MaybeFoundInSxsLbl
FindNext $0 $1
Goto LoopLbl
MaybeFoundInSxsLbl:
- GetDllVersion $WINDIR\WinSxS\$1\msvcr80.dll $R0 $R1
+ GetDllVersion $WINDIR\WinSxS\$1\${REDIST_DLL_NAME} $R0 $R1
Call DllVersionGoodEnough
FindNext $0 $1
IntCmp 2 $R0 LoopLbl
Goto FoundLbl
MaybeFoundInSystemLbl:
- GetDllVersion $SYSDIR\msvcr80.dll $R0 $R1
+ GetDllVersion $SYSDIR\${REDIST_DLL_NAME} $R0 $R1
Call DllVersionGoodEnough
IntCmp 2 $R0 SearchSxSLbl
FoundLbl:
diff --git a/erts/etc/win32/nsis/find_redist.sh b/erts/etc/win32/nsis/find_redist.sh index c5572839c5..153977ded5 100755 --- a/erts/etc/win32/nsis/find_redist.sh +++ b/erts/etc/win32/nsis/find_redist.sh @@ -2,7 +2,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2007-2009. All Rights Reserved. +# Copyright Ericsson AB 2007-2010. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -107,16 +107,56 @@ for x in cl bin vc; do fi BPATH="$NBPATH" done +BPATH_LIST=$BPATH + +# rc.exe is in the Microsoft SDK directory of VS2008 +RCPATH=`lookup_prog_in_path rc` +fail=false +if [ '!' -z "$RCPATH" ]; then + BPATH=$RCPATH + for x in rc bin v6.0A ; do + NBPATH=`remove_path_element $x "$BPATH"` + if [ "$NBPATH" = "$BPATH" ]; then + fail=true + break; + fi + BPATH="$NBPATH" + done + if [ $fail = false ]; then + BPATH_LIST="$BPATH_LIST $BPATH" + fi +fi + +# Frantic search through two roots with different +# version directories. We want to be very specific about the +# directory structures as we woildnt want to find the wrong +# redistributables... + #echo $BPATH -for x in sdk v2.0 bootstrapper packages vcredist_x86 vcredist_x86.exe; do - #echo "x=$x" - #echo "BPATH=$BPATH" - NBPATH=`add_path_element $x "$BPATH"` - if [ "$NBPATH" = "$BPATH" ]; then - echo "Failed to locate vcredist_x86.exe because directory structure was unexpected" >&2 - exit 3 +for BP in $BPATH_LIST; do + for verdir in "sdk v2.0" "sdk v3.5" "v6.0A"; do + BPATH=$BP + fail=false + for x in $verdir bootstrapper packages vcredist_x86 vcredist_x86.exe; do + #echo "x=$x" + #echo "BPATH=$BPATH" + NBPATH=`add_path_element $x "$BPATH"` + if [ "$NBPATH" = "$BPATH" ]; then + fail=true + break; + fi + BPATH="$NBPATH" + done + if [ $fail = false ]; then + break; + fi + done + if [ $fail = false ]; then + echo $BPATH + exit 0 fi - BPATH="$NBPATH" done -echo $BPATH -exit 0
\ No newline at end of file + +echo "Failed to locate vcredist_x86.exe because directory structure was unexpected" >&2 +exit 3 + diff --git a/erts/include/internal/erl_misc_utils.h b/erts/include/internal/erl_misc_utils.h index 82e9ba3798..507e1726f4 100644 --- a/erts/include/internal/erl_misc_utils.h +++ b/erts/include/internal/erl_misc_utils.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -34,7 +34,7 @@ typedef struct { erts_cpu_info_t *erts_cpu_info_create(void); void erts_cpu_info_destroy(erts_cpu_info_t *cpuinfo); -void erts_cpu_info_update(erts_cpu_info_t *cpuinfo); +int erts_cpu_info_update(erts_cpu_info_t *cpuinfo); int erts_get_cpu_configured(erts_cpu_info_t *cpuinfo); int erts_get_cpu_online(erts_cpu_info_t *cpuinfo); int erts_get_cpu_available(erts_cpu_info_t *cpuinfo); @@ -50,4 +50,9 @@ int erts_unbind_from_cpu_str(char *str); int erts_milli_sleep(long); +#ifdef __WIN32__ +int erts_map_win_error_to_errno(DWORD win_error); +int erts_get_last_win_errno(void); +#endif + #endif /* #ifndef ERL_MISC_UTILS_H_ */ diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h new file mode 100644 index 0000000000..1caf4d0567 --- /dev/null +++ b/erts/include/internal/ethr_atomics.h @@ -0,0 +1,726 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: The ethread atomic API + * Author: Rickard Green + */ + +#ifndef ETHR_ATOMIC_H__ +#define ETHR_ATOMIC_H__ + +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATOMIC_PROTOTYPES__ +#endif + +#ifndef ETHR_HAVE_NATIVE_ATOMICS +/* + * No native atomic implementation available. :( + * Use fallback... + */ +typedef ethr_sint32_t ethr_atomic32_t; +typedef ethr_sint_t ethr_atomic_t; +#else +/* + * Map ethread native atomics to ethread API atomics. + * + * We do at least have a native atomic implementation that + * can handle integers of a size larger than or equal to + * the size of pointers. + */ + +/* -- Pointer size atomics -- */ + +#undef ETHR_NAINT_T__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_NATMC_ADDR_FUNC__ +#if ETHR_SIZEOF_PTR == 8 +# if defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr +typedef ethr_native_atomic64_t ethr_atomic_t; +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# else +# error "Missing native atomic implementation" +# endif +#elif ETHR_SIZEOF_PTR == 4 +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr +# ifdef ETHR_HAVE_NATIVE_ATOMIC32 +typedef ethr_native_atomic32_t ethr_atomic_t; +# define ETHR_NAINT_T__ ethr_sint32_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +typedef ethr_native_atomic64_t ethr_atomic_t; +# define ETHR_NATMC_T__ ethr_native_atomic64_t +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# else +# error "Missing native atomic implementation" +# endif +#endif + +/* -- 32-bit atomics -- */ + +#undef ETHR_NAINT32_T__ +#undef ETHR_NATMC32_FUNC__ +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +typedef ethr_native_atomic32_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint32_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +typedef ethr_native_atomic64_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint64_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X +#else +# error "Missing native atomic implementation" +#endif + +#endif + +#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ +ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *); +void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read(ethr_atomic_t *); +ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *); +ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *); +void ethr_atomic_inc(ethr_atomic_t *); +void ethr_atomic_dec(ethr_atomic_t *); +ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *); +void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_dec_relb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); + +ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *); +void ethr_atomic32_init(ethr_atomic32_t *, ethr_sint32_t); +void ethr_atomic32_set(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *); +void ethr_atomic32_inc(ethr_atomic32_t *); +void ethr_atomic32_dec(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *, ethr_sint32_t); +void ethr_atomic32_add(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *, + ethr_sint32_t, + ethr_sint32_t); +ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *); +void ethr_atomic32_set_relb(ethr_atomic32_t *, ethr_sint32_t); +void ethr_atomic32_dec_relb(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *, + ethr_sint32_t, + ethr_sint32_t); +ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *, + ethr_sint32_t, + ethr_sint32_t); +#endif + +int ethr_init_atomics(void); + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#ifndef ETHR_HAVE_NATIVE_ATOMICS +/* + * Fallbacks for atomics used in absence of a native implementation. + */ + +#define ETHR_ATOMIC_ADDR_BITS 10 +#define ETHR_ATOMIC_ADDR_SHIFT 6 + +typedef struct { + union { + ethr_spinlock_t lck; + char buf[ETHR_CACHE_LINE_SIZE]; + } u; +} ethr_atomic_protection_t; + +extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; + +#define ETHR_ATOMIC_PTR2LCK__(PTR) \ +(ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ + & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) + + +#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ +do { \ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ + ethr_spin_lock(slp__); \ + { EXPS; } \ + ethr_spin_unlock(slp__); \ +} while (0) + +#endif + +/* + * --- Pointer size atomics --------------------------------------------------- + */ + +static ETHR_INLINE ethr_sint_t * +ETHR_INLINE_FUNC_NAME_(ethr_atomic_addr)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t *) ETHR_NATMC_ADDR_FUNC__(var); +#else + return (ethr_sint_t *) var; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) incr); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) i); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(inc)(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(dec)(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, + ethr_sint_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, + (ETHR_NAINT_T__) mask); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, + ethr_sint_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, + (ETHR_NAINT_T__) mask); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, + (ETHR_NAINT_T__) new); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, + (ETHR_NAINT_T__) new, + (ETHR_NAINT_T__) exp); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, + { + res = *var; + if (__builtin_expect(res == exp, 1)) + *var = new; + }); + return res; +#endif +} + +/* + * Important memory barrier requirements. + * + * The following atomic operations *must* supply a memory barrier of + * at least the type specified by its suffix: + * _acqb = acquire barrier + * _relb = release barrier + */ + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, + ethr_sint_t val) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(dec_relb)(var); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, + (ETHR_NAINT_T__) new, + (ETHR_NAINT_T__) exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, + (ETHR_NAINT_T__) new, + (ETHR_NAINT_T__) exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#endif +} + +/* + * --- 32-bit atomics --------------------------------------------------------- + */ + +static ETHR_INLINE ethr_sint32_t * +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_addr)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic32_addr(var); +#else + return (ethr_sint32_t *) var; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_init)(ethr_atomic32_t *var, + ethr_sint32_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(ethr_atomic32_t *var, ethr_sint32_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add)(ethr_atomic32_t *var, + ethr_sint32_t incr) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) incr); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add_read)(ethr_atomic32_t *var, + ethr_sint32_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) + ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) i); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(inc)(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(dec)(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_band)(ethr_atomic32_t *var, + ethr_sint32_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) + ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) mask); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_bor)(ethr_atomic32_t *var, + ethr_sint32_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return + (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, + (ETHR_NAINT32_T__) mask); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_xchg)(ethr_atomic32_t *var, + ethr_sint32_t new) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, + (ETHR_NAINT32_T__) new); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, + (ETHR_NAINT32_T__) new, + (ETHR_NAINT32_T__) exp); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, + { + res = *var; + if (__builtin_expect(res == exp, 1)) + *var = new; + }); + return res; +#endif +} + +/* + * Important memory barrier requirements. + * + * The following atomic operations *must* supply a memory barrier of + * at least the type specified by its suffix: + * _acqb = acquire barrier + * _relb = release barrier + */ + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_acqb)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read_acqb)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(var); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set_relb)(ethr_atomic32_t *var, + ethr_sint32_t val) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(var, val); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_relb)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(dec_relb)(var); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(var); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read_relb)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_acqb)(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) + ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, + (ETHR_NAINT32_T__) new, + (ETHR_NAINT32_T__) exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_relb)(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) + ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, + (ETHR_NAINT32_T__) new, + (ETHR_NAINT32_T__) exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp); +#endif +} + + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#undef ETHR_NAINT_T__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_NATMC_ADDR_FUNC__ + +#undef ETHR_NAINT32_T__ +#undef ETHR_NATMC32_FUNC__ + +#endif diff --git a/erts/include/internal/ethr_internal.h b/erts/include/internal/ethr_internal.h new file mode 100644 index 0000000000..e9c3daf783 --- /dev/null +++ b/erts/include/internal/ethr_internal.h @@ -0,0 +1,67 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Internal ethread exports + * Author: Rickard Green + */ + +#ifndef ETHR_INTERNAL_H__ +#define ETHR_INTERNAL_H__ + +#include "erl_misc_utils.h" + +extern ethr_memory_allocators ethr_mem__; +extern erts_cpu_info_t *ethr_cpu_info__; +extern size_t ethr_pagesize__; +extern size_t ethr_min_stack_size__; /* kilo words */ +extern size_t ethr_max_stack_size__; /* kilo words */ +extern int ethr_not_completely_inited__; +extern int ethr_not_inited__; + +extern void *(*ethr_thr_prepare_func__)(void); +extern void (*ethr_thr_parent_func__)(void *); +extern void (*ethr_thr_child_func__)(void *); + +#define ETHR_PAGE_ALIGN(SZ) \ + (((((size_t) (SZ)) - 1)/ethr_pagesize__ + 1)*ethr_pagesize__) +#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1) +#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024) + +#undef ETHR_STACK_GUARD_SIZE +#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE +# define ETHR_STACK_GUARD_SIZE (ethr_pagesize__) +#endif + +/* implemented in lib_src/<thr-lib>/ethread.c */ +int ethr_set_tse__(ethr_ts_event *tsep); +ethr_ts_event *ethr_get_tse__(void); +ETHR_PROTO_NORETURN__ ethr_abort__(void); +#ifdef ETHR_WIN32_THREADS +int ethr_win_get_errno__(void); +#endif + +/* implemented in lib_src/common/ethread_aux.c */ +int ethr_init_common__(ethr_init_data *id); +int ethr_late_init_common__(ethr_late_init_data *lid); +void ethr_run_exit_handlers__(void); +void ethr_ts_event_destructor__(void *vtsep); + + +#endif diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h new file mode 100644 index 0000000000..fadaf1e2a4 --- /dev/null +++ b/erts/include/internal/ethr_mutex.h @@ -0,0 +1,674 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Mutex, rwmutex and condition variable implementation + * Author: Rickard Green + */ + +#ifndef ETHR_MUTEX_H__ +#define ETHR_MUTEX_H__ + +#define ETHR_RWMUTEX_INITIALIZED 0x99999999 +#define ETHR_MUTEX_INITIALIZED 0x77777777 +#define ETHR_COND_INITIALIZED 0x55555555 + +#if 0 +# define ETHR_MTX_HARD_DEBUG +#endif + +#if 0 +# define ETHR_MTX_CHK_EXCL +#if 1 +# define ETHR_MTX_CHK_NON_EXCL +#endif +#endif + +#ifdef ETHR_MTX_HARD_DEBUG +# ifdef __GNUC__ +# warning ETHR_MTX_HARD_DEBUG +# endif +/*# define ETHR_MTX_HARD_DEBUG_LFS*/ +/*# define ETHR_MTX_HARD_DEBUG_FENCE*/ +/*# define ETHR_MTX_HARD_DEBUG_Q*/ +# define ETHR_MTX_HARD_DEBUG_WSQ + +# if !defined(ETHR_MTX_HARD_DEBUG_WSQ) && defined(ETHR_MTX_HARD_DEBUG_Q) +# define ETHR_MTX_HARD_DEBUG_WSQ +# endif +#endif + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) + +#ifdef ETHR_DEBUG +# ifndef ETHR_MTX_CHK_EXCL +# define ETHR_MTX_CHK_EXCL +# endif +# ifndef ETHR_MTX_CHK_NON_EXCL +# define ETHR_MTX_CHK_NON_EXCL +# endif +#endif + +#if 0 +# define ETHR_MTX_Q_LOCK_SPINLOCK__ +# define ETHR_MTX_QLOCK_TYPE__ ethr_spinlock_t +#elif defined(ETHR_PTHREADS) +# define ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__ +# define ETHR_MTX_QLOCK_TYPE__ pthread_mutex_t +#elif defined(ETHR_WIN32_THREADS) +# define ETHR_MTX_Q_LOCK_CRITICAL_SECTION__ +# define ETHR_MTX_QLOCK_TYPE__ CRITICAL_SECTION +#else +# error Need a qlock implementation +#endif + +#define ETHR_RWMTX_W_FLG__ (((ethr_sint32_t) 1) << 31) +#define ETHR_RWMTX_W_WAIT_FLG__ (((ethr_sint32_t) 1) << 30) +#define ETHR_RWMTX_R_WAIT_FLG__ (((ethr_sint32_t) 1) << 29) + +/* frequent read kind */ +#define ETHR_RWMTX_R_FLG__ (((ethr_sint32_t) 1) << 28) +#define ETHR_RWMTX_R_ABRT_UNLCK_FLG__ (((ethr_sint32_t) 1) << 27) +#define ETHR_RWMTX_R_PEND_UNLCK_MASK__ (ETHR_RWMTX_R_ABRT_UNLCK_FLG__ - 1) + +/* normal kind */ +#define ETHR_RWMTX_RS_MASK__ (ETHR_RWMTX_R_WAIT_FLG__ - 1) + +#define ETHR_RWMTX_WAIT_FLGS__ \ + (ETHR_RWMTX_W_WAIT_FLG__|ETHR_RWMTX_R_WAIT_FLG__) + +#define ETHR_CND_WAIT_FLG__ ETHR_RWMTX_R_WAIT_FLG__ + +#ifdef ETHR_DEBUG +#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V) \ + ETHR_ASSERT(!((V) & ~(ETHR_RWMTX_W_FLG__ \ + | ETHR_RWMTX_W_WAIT_FLG__ \ + | ETHR_RWMTX_R_WAIT_FLG__ \ + | ETHR_RWMTX_RS_MASK__))) +#else +#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V) +#endif + +#define ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(MTX) \ + ETHR_DBG_CHK_UNUSED_FLG_BITS(ethr_atomic32_read(&(MTX)->mtxb.flgs)) + +struct ethr_mutex_base_ { +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long pre_fence; +#endif + ethr_atomic32_t flgs; + short aux_scnt; + short main_scnt; + ETHR_MTX_QLOCK_TYPE__ qlck; + ethr_ts_event *q; +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + int ws; +#endif +#ifdef ETHR_MTX_CHK_EXCL + ethr_atomic32_t exclusive; +#endif +#ifdef ETHR_MTX_CHK_NON_EXCL + ethr_atomic32_t non_exclusive; +#endif +#ifdef ETHR_MTX_HARD_DEBUG_LFS + ethr_atomic32_t hdbg_lfs; +#endif +}; + +#endif + +typedef struct { + int main_spincount; + int aux_spincount; +} ethr_mutex_opt; + +typedef struct { + int main_spincount; + int aux_spincount; +} ethr_cond_opt; + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + +typedef struct ethr_mutex_ ethr_mutex; +struct ethr_mutex_ { + struct ethr_mutex_base_ mtxb; +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long post_fence; +#endif +#if ETHR_XCHK + int initialized; +#endif +}; + +typedef struct ethr_cond_ ethr_cond; +struct ethr_cond_ { +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + struct { + long pre_fence; + } mtxb; /* mtxb allows us to use same macro as for mutex and rwmutex... */ +#endif + ETHR_MTX_QLOCK_TYPE__ qlck; + ethr_ts_event *q; + short aux_scnt; + short main_scnt; +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long post_fence; +#endif +#if ETHR_XCHK + int initialized; +#endif +}; + +#else /* pthread */ + +typedef struct ethr_mutex_ ethr_mutex; +struct ethr_mutex_ { + pthread_mutex_t pt_mtx; +#if ETHR_XCHK + int initialized; +#endif +}; + +typedef struct ethr_cond_ ethr_cond; +struct ethr_cond_ { + pthread_cond_t pt_cnd; +#if ETHR_XCHK + int initialized; +#endif +}; + +#endif /* pthread */ + +int ethr_mutex_init_opt(ethr_mutex *, ethr_mutex_opt *); +int ethr_mutex_init(ethr_mutex *); +int ethr_mutex_destroy(ethr_mutex *); +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) +int ethr_mutex_trylock(ethr_mutex *); +void ethr_mutex_lock(ethr_mutex *); +void ethr_mutex_unlock(ethr_mutex *); +#endif +int ethr_cond_init_opt(ethr_cond *, ethr_cond_opt *); +int ethr_cond_init(ethr_cond *); +int ethr_cond_destroy(ethr_cond *); +void ethr_cond_signal(ethr_cond *); +void ethr_cond_broadcast(ethr_cond *); +int ethr_cond_wait(ethr_cond *, ethr_mutex *); + +typedef enum { + ETHR_RWMUTEX_TYPE_NORMAL, + ETHR_RWMUTEX_TYPE_FREQUENT_READ, + ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ +} ethr_rwmutex_type; + +typedef enum { + ETHR_RWMUTEX_LONG_LIVED, + ETHR_RWMUTEX_SHORT_LIVED, + ETHR_RWMUTEX_UNKNOWN_LIVED +} ethr_rwmutex_lived; + +typedef struct { + ethr_rwmutex_type type; + ethr_rwmutex_lived lived; + int main_spincount; + int aux_spincount; +} ethr_rwmutex_opt; + +#define ETHR_RWMUTEX_OPT_DEFAULT_INITER \ + {ETHR_RWMUTEX_TYPE_NORMAL, ETHR_RWMUTEX_UNKNOWN_LIVED, -1, -1} + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + +typedef union { + struct { + ethr_atomic32_t readers; + int waiting_readers; + int byte_offset; + ethr_rwmutex_lived lived; + } data; + char align__[ETHR_CACHE_LINE_SIZE]; +} ethr_rwmtx_readers_array__; + +typedef struct ethr_rwmutex_ ethr_rwmutex; +struct ethr_rwmutex_ { + struct ethr_mutex_base_ mtxb; + ethr_rwmutex_type type; + ethr_ts_event *rq_end; + union { + ethr_rwmtx_readers_array__ *ra; + int rs; + } tdata; +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + long post_fence; +#endif +#if ETHR_XCHK + int initialized; +#endif +}; + +#else /* pthread_rwlock */ + +typedef struct ethr_rwmutex_ ethr_rwmutex; +struct ethr_rwmutex_ { + pthread_rwlock_t pt_rwlock; +#if ETHR_XCHK + int initialized; +#endif +}; + +#endif /* pthread_rwlock */ + +int ethr_rwmutex_set_reader_group(int); +int ethr_rwmutex_init_opt(ethr_rwmutex *, ethr_rwmutex_opt *); +int ethr_rwmutex_init(ethr_rwmutex *); +int ethr_rwmutex_destroy(ethr_rwmutex *); +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) \ + || !defined(ETHR_TRY_INLINE_FUNCS) \ + || defined(ETHR_MUTEX_IMPL__) +int ethr_rwmutex_tryrlock(ethr_rwmutex *); +void ethr_rwmutex_rlock(ethr_rwmutex *); +void ethr_rwmutex_runlock(ethr_rwmutex *); +int ethr_rwmutex_tryrwlock(ethr_rwmutex *); +void ethr_rwmutex_rwlock(ethr_rwmutex *); +void ethr_rwmutex_rwunlock(ethr_rwmutex *); +#endif + +#ifdef ETHR_MTX_HARD_DEBUG +#define ETHR_MTX_HARD_ASSERT(A) \ + ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A))) +#else +#define ETHR_MTX_HARD_ASSERT(A) ((void) 1) +#endif + +#ifdef ETHR_MTX_HARD_DEBUG_LFS +# define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB) \ +do { \ + ethr_atomic32_init(&(MTXB)->hdbg_lfs, 0); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB) \ +do { \ + ethr_sint32_t val__; \ + ETHR_COMPILER_BARRIER; \ + val__ = ethr_atomic32_inc_read(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ > 0); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if ((RES) == 0) \ + ETHR_MTX_HARD_DEBUG_LFS_RLOCK((MTXB)); \ + else \ + ETHR_MTX_HARD_ASSERT((RES) == EBUSY); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB) \ +do { \ + ethr_sint32_t val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ >= 0); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB) \ +do { \ + ethr_sint32_t val__; \ + ETHR_COMPILER_BARRIER; \ + val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ == -1); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if ((RES) == 0) \ + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK((MTXB)); \ + else \ + ETHR_MTX_HARD_ASSERT((RES) == EBUSY); \ +} while (0) +# define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB) \ +do { \ + ethr_sint32_t val__ = ethr_atomic32_inctest(&(MTXB)->hdbg_lfs); \ + ETHR_MTX_HARD_ASSERT(val__ == 0); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +#else +# define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES) +# define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB) +# define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES) +# define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB) +#endif + +#ifdef ETHR_MTX_HARD_DEBUG_FENCE + +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_MTX_HARD_DEBUG_PRE_FENCE 0xdeadbeefdeadbeefL +# define ETHR_MTX_HARD_DEBUG_POST_FENCE 0xdeaddeaddeaddeadL +#else +# define ETHR_MTX_HARD_DEBUG_PRE_FENCE 0xdeaddeadL +# define ETHR_MTX_HARD_DEBUG_POST_FENCE 0xdeaddeadL +#endif + +#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X) \ +do { \ + ETHR_COMPILER_BARRIER; \ + ETHR_MTX_HARD_ASSERT((X)->mtxb.pre_fence == ETHR_MTX_HARD_DEBUG_PRE_FENCE);\ + ETHR_MTX_HARD_ASSERT((X)->post_fence == ETHR_MTX_HARD_DEBUG_POST_FENCE); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) \ +do { \ + (X)->mtxb.pre_fence = ETHR_MTX_HARD_DEBUG_PRE_FENCE; \ + (X)->post_fence = ETHR_MTX_HARD_DEBUG_POST_FENCE; \ +} while (0) +#else +#define ETHR_MTX_HARD_DEBUG_FENCE_CHK(X) +#define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) +#endif + +#ifdef ETHR_MTX_CHK_EXCL + +#if !defined(ETHR_DEBUG) && defined(__GNUC__) +#warning "check exclusive is enabled" +#endif + +# define ETHR_MTX_CHK_EXCL_INIT__(MTXB) \ + ethr_atomic32_init(&(MTXB)->exclusive, 0) + +# define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if (!ethr_atomic32_read(&(MTXB)->exclusive)) \ + ethr_assert_failed(__FILE__, __LINE__, __func__,\ + "is exclusive"); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if (ethr_atomic32_read(&(MTXB)->exclusive)) \ + ethr_assert_failed(__FILE__, __LINE__, __func__,\ + "is not exclusive"); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB) \ +do { \ + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL((MTXB)); \ + ethr_atomic32_set(&(MTXB)->exclusive, 1); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB) \ +do { \ + ETHR_MTX_CHK_EXCL_IS_EXCL((MTXB)); \ + ethr_atomic32_set(&(MTXB)->exclusive, 0); \ + ETHR_COMPILER_BARRIER; \ +} while (0) + +#ifdef ETHR_MTX_CHK_NON_EXCL + +#if !defined(ETHR_DEBUG) && defined(__GNUC__) +#warning "check non-exclusive is enabled" +#endif + +# define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB) \ + ethr_atomic32_init(&(MTXB)->non_exclusive, 0) +# define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if (!ethr_atomic32_read(&(MTXB)->non_exclusive)) \ + ethr_assert_failed(__FILE__, __LINE__, __func__,\ + "is non-exclusive"); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if (ethr_atomic32_read(&(MTXB)->non_exclusive)) \ + ethr_assert_failed(__FILE__, __LINE__, __func__,\ + "is not non-exclusive"); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + ethr_atomic32_inc(&(MTXB)->non_exclusive); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO) \ +do { \ + ETHR_COMPILER_BARRIER; \ + ethr_atomic32_add(&(MTXB)->non_exclusive, (NO)); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + ethr_atomic32_dec(&(MTXB)->non_exclusive); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +#else +# define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB) +#endif + +#else +# define ETHR_MTX_CHK_EXCL_INIT__(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB) +# define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB) +#endif + +# define ETHR_MTX_CHK_EXCL_INIT(MTXB) \ +do { \ + ETHR_MTX_CHK_EXCL_INIT__((MTXB)); \ + ETHR_MTX_CHK_NON_EXCL_INIT__((MTXB)); \ +} while (0) + + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + +#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000 +#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE 800 +#define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC 50 +#define ETHR_MTX_DEFAULT_AUX_SPINCOUNT 50 + +#define ETHR_CND_DEFAULT_MAIN_SPINCOUNT 0 +#define ETHR_CND_DEFAULT_AUX_SPINCOUNT 0 + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +void ethr_mutex_lock_wait__(ethr_mutex *, ethr_sint32_t); +void ethr_mutex_unlock_wake__(ethr_mutex *, ethr_sint32_t); + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +{ + ethr_sint32_t act; + int res; + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); + + act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); + res = (act == 0) ? 0 : EBUSY; + +#ifdef ETHR_MTX_CHK_EXCL + if (res == 0) + ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb); +#endif + + ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&mtx->mtxb, res); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); + + ETHR_COMPILER_BARRIER; + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +{ + ethr_sint32_t act; + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); + + act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); + if (act != 0) + ethr_mutex_lock_wait__(mtx, act); + + ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb); + + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); + + ETHR_COMPILER_BARRIER; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +{ + ethr_sint32_t act; + ETHR_COMPILER_BARRIER; + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&mtx->mtxb); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); + + ETHR_MTX_CHK_EXCL_UNSET_EXCL(&mtx->mtxb); + + act = ethr_atomic32_cmpxchg_relb(&mtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) + ethr_mutex_unlock_wake__(mtx, act); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#else /* pthread_mutex */ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) +{ + int res; + res = pthread_mutex_trylock(&mtx->pt_mtx); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) +{ + int res = pthread_mutex_lock(&mtx->pt_mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) +{ + int res = pthread_mutex_unlock(&mtx->pt_mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* pthread_mutex */ + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + +#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000 +#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE 800 +#define ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC 50 +#define ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT 50 + +#else /* pthread_rwlock */ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_rdlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE int +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_trywrlock(&rwmtx->pt_rwlock); + if (res != 0 && res != EBUSY) + ETHR_FATAL_ERROR__(res); + return res; +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_wrlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) +{ + int res = pthread_rwlock_unlock(&rwmtx->pt_rwlock); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#endif /* pthread_rwlock */ + +int ethr_mutex_lib_init(int); +int ethr_mutex_lib_late_init(int, int); + +#endif /* #ifndef ETHR_MUTEX_H__ */ diff --git a/erts/include/internal/ethr_optimized_fallbacks.h b/erts/include/internal/ethr_optimized_fallbacks.h new file mode 100644 index 0000000000..8e04692856 --- /dev/null +++ b/erts/include/internal/ethr_optimized_fallbacks.h @@ -0,0 +1,209 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: "Optimized" fallbacks used when native ops are missing + * Author: Rickard Green + */ + +#ifndef ETHR_OPTIMIZED_FALLBACKS_H__ +#define ETHR_OPTIMIZED_FALLBACKS_H__ + +#ifdef ETHR_HAVE_NATIVE_ATOMICS +#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 +#endif + +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS +#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 +#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) +/* --- Optimized spinlocks using pthread spinlocks -------------------------- */ +#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 + +typedef pthread_spinlock_t ethr_opt_spinlock_t; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE int +ethr_opt_spinlock_init(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_init((pthread_spinlock_t *) lock, 0); +} + +static ETHR_INLINE int +ethr_opt_spinlock_destroy(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_destroy((pthread_spinlock_t *) lock); +} + + +static ETHR_INLINE int +ethr_opt_spin_unlock(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_unlock((pthread_spinlock_t *) lock); +} + +static ETHR_INLINE int +ethr_opt_spin_lock(ethr_opt_spinlock_t *lock) +{ + return pthread_spin_lock((pthread_spinlock_t *) lock); +} + +#endif + +#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +/* --- Native spinlocks using native atomics -------------------------------- */ +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 + +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +typedef ethr_native_atomic32_t ethr_native_spinlock_t; +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +typedef ethr_native_atomic64_t ethr_native_spinlock_t; +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#else +# error "Missing native atomic implementation" +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE void +ethr_native_spinlock_init(ethr_native_spinlock_t *lock) +{ + ETHR_NATMC_FUNC__(init)(lock, 0); +} + +static ETHR_INLINE void +ethr_native_spin_unlock(ethr_native_spinlock_t *lock) +{ + ETHR_COMPILER_BARRIER; + ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == 1); + ETHR_NATMC_FUNC__(set_relb)(lock, 0); +} + +static ETHR_INLINE void +ethr_native_spin_lock(ethr_native_spinlock_t *lock) +{ + while (ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, 1, 0) != 0) { + while (ETHR_NATMC_FUNC__(read)(lock) != 0) + ETHR_SPIN_BODY; + } + ETHR_COMPILER_BARRIER; +} + +#endif + +#undef ETHR_NATMC_FUNC__ + +#endif + + +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS +#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 +#elif defined(ETHR_HAVE_NATIVE_ATOMICS) +/* --- Native rwspinlocks using native atomics ------------------------------ */ +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 +#define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 + +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +typedef ethr_native_atomic32_t ethr_native_rwlock_t; +# define ETHR_NAINT_T__ ethr_sint32_t +# define ETHR_WLOCK_FLAG__ (((ethr_sint32_t) 1) << 30) +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +typedef ethr_native_atomic64_t ethr_native_rwlock_t; +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_WLOCK_FLAG__ (((ethr_sint64_t) 1) << 62) +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#else +# error "Missing native atomic implementation" +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE void +ethr_native_rwlock_init(ethr_native_rwlock_t *lock) +{ + ETHR_NATMC_FUNC__(init)(lock, 0); +} + +static ETHR_INLINE void +ethr_native_read_unlock(ethr_native_rwlock_t *lock) +{ + ETHR_COMPILER_BARRIER; +#ifdef DEBUG + ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) >= 0); +#endif + ETHR_NATMC_FUNC__(dec_relb)(lock); +} + +static ETHR_INLINE void +ethr_native_read_lock(ethr_native_rwlock_t *lock) +{ + ETHR_NAINT_T__ act, exp = 0; + while (1) { + act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp+1, exp); + if (act == exp) + break; + while (act & ETHR_WLOCK_FLAG__) { + ETHR_SPIN_BODY; + act = ETHR_NATMC_FUNC__(read)(lock); + } + exp = act; + } + ETHR_COMPILER_BARRIER; +} + +static ETHR_INLINE void +ethr_native_write_unlock(ethr_native_rwlock_t *lock) +{ + ETHR_COMPILER_BARRIER; + ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == ETHR_WLOCK_FLAG__); + ETHR_NATMC_FUNC__(set_relb)(lock, 0); +} + +static ETHR_INLINE void +ethr_native_write_lock(ethr_native_rwlock_t *lock) +{ + ETHR_NAINT_T__ act, exp = 0; + while (1) { + act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp); + if (act == exp) + break; + ETHR_SPIN_BODY; + exp = act & ~ETHR_WLOCK_FLAG__; + } + act |= ETHR_WLOCK_FLAG__; + /* Wait for readers to leave */ + while (act != ETHR_WLOCK_FLAG__) { + ETHR_SPIN_BODY; + act = ETHR_NATMC_FUNC__(read_acqb)(lock); + } + ETHR_COMPILER_BARRIER; +} + +#endif + +#undef ETHR_NAINT_T__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_WLOCK_FLAG__ + +#endif + +#endif diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 934a79c6f9..4cd95faf6a 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -33,27 +33,15 @@ #include <stdlib.h> #include "erl_errno.h" -/* - * Extra memory barrier requirements: - * - ethr_atomic_or_old() needs to enforce a memory barrier sufficient - * for a lock operation. - * - ethr_atomic_and_old() needs to enforce a memory barrier sufficient - * for an unlock operation. - * - ethr_atomic_cmpxchg() needs to enforce a memory barrier sufficient - * for a lock and unlock operation. - */ - - -#undef ETHR_USE_RWMTX_FALLBACK #undef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -#undef ETHR_HAVE_OPTIMIZED_LOCKS - -typedef struct { - long tv_sec; - long tv_nsec; -} ethr_timeval; +#undef ETHR_HAVE_OPTIMIZED_SPINLOCK +#undef ETHR_HAVE_OPTIMIZED_RWSPINLOCK #if defined(DEBUG) +# define ETHR_DEBUG +#endif + +#if defined(ETHR_DEBUG) # undef ETHR_XCHK # define ETHR_XCHK 1 #else @@ -68,47 +56,57 @@ typedef struct { #elif defined(__WIN32__) # define ETHR_INLINE __forceinline #endif -#if defined(DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \ +#if defined(ETHR_DEBUG) || !defined(ETHR_INLINE) || ETHR_XCHK \ || (defined(__GNUC__) && defined(ERTS_MIXED_CYGWIN_VC)) # undef ETHR_INLINE # define ETHR_INLINE # undef ETHR_TRY_INLINE_FUNCS #endif -#ifdef ETHR_FORCE_INLINE_FUNCS -# define ETHR_TRY_INLINE_FUNCS -#endif -#if !defined(ETHR_DISABLE_NATIVE_IMPLS) \ - && (defined(PURIFY) || defined(VALGRIND) || defined(ERTS_MIXED_CYGWIN_VC)) +#if !defined(ETHR_DISABLE_NATIVE_IMPLS) && (defined(PURIFY)||defined(VALGRIND)) # define ETHR_DISABLE_NATIVE_IMPLS #endif -#define ETHR_RWMUTEX_INITIALIZED 0x99999999 -#define ETHR_MUTEX_INITIALIZED 0x77777777 -#define ETHR_COND_INITIALIZED 0x55555555 +/* Assume 64-byte cache line size */ +#define ETHR_CACHE_LINE_SIZE ((ethr_uint_t) 64) +#define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1) -#define ETHR_CACHE_LINE_SIZE 64 +#define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \ + (((((SZ) - 1) / ETHR_CACHE_LINE_SIZE) + 1) * ETHR_CACHE_LINE_SIZE) -#ifdef ETHR_INLINE_FUNC_NAME_ -# define ETHR_CUSTOM_INLINE_FUNC_NAME_ -#else +#ifndef ETHR_INLINE_FUNC_NAME_ # define ETHR_INLINE_FUNC_NAME_(X) X #endif -#define ETHR_COMPILER_BARRIER ethr_compiler_barrier() -#ifdef __GNUC__ -# undef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER __asm__ __volatile__("":::"memory") +#if !defined(__func__) +# if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# if !defined(__GNUC__) || __GNUC__ < 2 +# define __func__ "[unknown_function]" +# else +# define __func__ __FUNCTION__ +# endif +# endif #endif -#ifdef DEBUG +int ethr_assert_failed(const char *file, int line, const char *func, char *a); +#ifdef ETHR_DEBUG #define ETHR_ASSERT(A) \ - ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, #A))) -int ethr_assert_failed(char *f, int l, char *a); + ((void) ((A) ? 1 : ethr_assert_failed(__FILE__, __LINE__, __func__, #A))) #else #define ETHR_ASSERT(A) ((void) 1) #endif +#if defined(__GNUC__) +# define ETHR_PROTO_NORETURN__ void __attribute__((noreturn)) +# define ETHR_IMPL_NORETURN__ void +#elif defined(__WIN32__) && defined(_MSC_VER) +# define ETHR_PROTO_NORETURN__ __declspec(noreturn) void +# define ETHR_IMPL_NORETURN__ __declspec(noreturn) void +#else +# define ETHR_PROTO_NORETURN__ void +# define ETHR_IMPL_NORETURN__ void +#endif + #if defined(ETHR_PTHREADS) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * The pthread implementation * @@ -118,7 +116,9 @@ int ethr_assert_failed(char *f, int l, char *a); #error "_GNU_SOURCE not defined. Please, compile all files with -D_GNU_SOURCE." #endif -#if defined(ETHR_HAVE_MIT_PTHREAD_H) +#if defined(ETHR_NEED_NPTL_PTHREAD_H) +#include <nptl/pthread.h> +#elif defined(ETHR_HAVE_MIT_PTHREAD_H) #include <pthread/mit/pthread.h> #elif defined(ETHR_HAVE_PTHREAD_H) #include <pthread.h> @@ -128,130 +128,23 @@ int ethr_assert_failed(char *f, int l, char *a); typedef pthread_t ethr_tid; -typedef struct ethr_mutex_ ethr_mutex; -struct ethr_mutex_ { - pthread_mutex_t pt_mtx; - int is_rec_mtx; - ethr_mutex *prev; - ethr_mutex *next; -#if ETHR_XCHK - int initialized; -#endif -}; - -typedef struct ethr_cond_ ethr_cond; -struct ethr_cond_ { - pthread_cond_t pt_cnd; -#if ETHR_XCHK - int initialized; -#endif -}; +typedef pthread_key_t ethr_tsd_key; -#ifndef ETHR_HAVE_PTHREAD_RWLOCK_INIT -#define ETHR_USE_RWMTX_FALLBACK -#else -typedef struct ethr_rwmutex_ ethr_rwmutex; -struct ethr_rwmutex_ { - pthread_rwlock_t pt_rwlock; -#if ETHR_XCHK - int initialized; -#endif -}; -#endif +#define ETHR_HAVE_ETHR_SIG_FUNCS 1 -/* Static initializers */ -#if ETHR_XCHK -#define ETHR_MUTEX_XCHK_INITER , ETHR_MUTEX_INITIALIZED -#define ETHR_COND_XCHK_INITER , ETHR_COND_INITIALIZED -#else -#define ETHR_MUTEX_XCHK_INITER -#define ETHR_COND_XCHK_INITER +#if defined(PURIFY) || defined(VALGRIND) +# define ETHR_FORCE_PTHREAD_RWLOCK +# define ETHR_FORCE_PTHREAD_MUTEX #endif -#define ETHR_MUTEX_INITER {PTHREAD_MUTEX_INITIALIZER, 0, NULL, NULL ETHR_MUTEX_XCHK_INITER} -#define ETHR_COND_INITER {PTHREAD_COND_INITIALIZER ETHR_COND_XCHK_INITER} - -#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE) \ - || defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP) -# define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1 -# ifdef PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP -# define ETHR_REC_MUTEX_INITER \ - {PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP, 1, NULL, NULL ETHR_MUTEX_XCHK_INITER} -# endif -#else -# undef ETHR_HAVE_ETHR_REC_MUTEX_INIT +#if !defined(ETHR_FORCE_PTHREAD_RWLOCK) +# define ETHR_USE_OWN_RWMTX_IMPL__ #endif -#ifndef ETHR_HAVE_PTHREAD_ATFORK -# define ETHR_NO_FORKSAFETY 1 +#if !defined(ETHR_FORCE_PTHREAD_MUTEX) && 0 +# define ETHR_USE_OWN_MTX_IMPL__ #endif -typedef pthread_key_t ethr_tsd_key; - -#define ETHR_HAVE_ETHR_SIG_FUNCS 1 - -#ifdef ETHR_TRY_INLINE_FUNCS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) -{ - return pthread_mutex_trylock(&mtx->pt_mtx); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) -{ - return pthread_mutex_lock(&mtx->pt_mtx); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) -{ - return pthread_mutex_unlock(&mtx->pt_mtx); -} - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_tryrdlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_rdlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_runlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_unlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_tryrwlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_trywrlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_wrlock(&rwmtx->pt_rwlock); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) -{ - return pthread_rwlock_unlock(&rwmtx->pt_rwlock); -} - -#endif /* ETHR_HAVE_PTHREAD_RWLOCK_INIT */ - -#endif /* ETHR_TRY_INLINE_FUNCS */ - #elif defined(ETHR_WIN32_THREADS) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * The native win32 threads implementation * @@ -273,412 +166,89 @@ ETHR_INLINE_FUNC_NAME_(ethr_rwmutex_rwunlock)(ethr_rwmutex *rwmtx) # undef WIN32_LEAN_AND_MEAN #endif -/* Types */ -typedef long ethr_tid; /* thread id type */ -typedef struct { - volatile int initialized; - CRITICAL_SECTION cs; -#if ETHR_XCHK - int is_rec_mtx; -#endif -} ethr_mutex; - -typedef struct cnd_wait_event__ cnd_wait_event_; - -typedef struct { - volatile int initialized; - CRITICAL_SECTION cs; - cnd_wait_event_ *queue; - cnd_wait_event_ *queue_end; -} ethr_cond; - -#define ETHR_USE_RWMTX_FALLBACK - -/* Static initializers */ - -#define ETHR_MUTEX_INITER {0} -#define ETHR_COND_INITER {0} - -#define ETHR_REC_MUTEX_INITER ETHR_MUTEX_INITER - -#define ETHR_HAVE_ETHR_REC_MUTEX_INIT 1 - -typedef DWORD ethr_tsd_key; - -#undef ETHR_HAVE_ETHR_SIG_FUNCS - -#ifdef ETHR_TRY_INLINE_FUNCS -int ethr_fake_static_mutex_init(ethr_mutex *mtx); - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) -{ - if (!mtx->initialized) { - int res = ethr_fake_static_mutex_init(mtx); - if (res != 0) - return res; - } - return TryEnterCriticalSection(&mtx->cs) ? 0 : EBUSY; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) -{ - if (!mtx->initialized) { - int res = ethr_fake_static_mutex_init(mtx); - if (res != 0) - return res; - } - EnterCriticalSection(&mtx->cs); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) -{ - LeaveCriticalSection(&mtx->cs); - return 0; -} - -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ +#if defined(_MSC_VER) -#ifdef ERTS_MIXED_CYGWIN_VC - -/* atomics */ - -#ifdef _MSC_VER -# if _MSC_VER < 1300 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 /* Dont trust really old compilers */ -# else -# if defined(_M_IX86) -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else /* I.e. IA64 */ -# if _MSC_VER >= 1400 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 -# endif -# endif -# endif -# if _MSC_VER >= 1400 -# include <intrin.h> -# undef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -# endif -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_InterlockedAnd) -#pragma intrinsic(_InterlockedOr) -#else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 +#if ETHR_SIZEOF_LONG == 4 +#define ETHR_HAVE_INT32_T 1 +typedef long ethr_sint32_t; +typedef unsigned long ethr_uint32_t; #endif -#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 -#define ETHR_HAVE_OPTIMIZED_LOCKS 1 - -typedef struct { - volatile LONG value; -} ethr_atomic_t; - -typedef struct { - volatile LONG locked; -} ethr_spinlock_t; - -typedef struct { - volatile LONG counter; -} ethr_rwlock_t; -#define ETHR_WLOCK_FLAG__ (((LONG) 1) << 30) - -#ifdef ETHR_TRY_INLINE_FUNCS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - var->value = (LONG) i; -#else - (void) InterlockedExchange(&var->value, (LONG) i); +#if ETHR_SIZEOF___INT64 == 8 +#define ETHR_HAVE_INT64_T 1 +typedef __int64 ethr_sint64_t; +typedef unsigned __int64 ethr_uint64_t; #endif - return 0; -} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - var->value = (LONG) i; -#else - (void) InterlockedExchange(&var->value, (LONG) i); #endif - return 0; -} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - *i = var->value; -#else - *i = InterlockedExchangeAdd(&var->value, (LONG) 0); -#endif - return 0; -} +struct ethr_join_data_; -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ - (void) InterlockedExchangeAdd(&var->value, (LONG) incr); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var, - long i, - long *testp) -{ - *testp = InterlockedExchangeAdd(&var->value, (LONG) i); - *testp += i; - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) -{ - (void) InterlockedIncrement(&var->value); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) -{ - (void) InterlockedDecrement(&var->value); - return 0; -} +/* Types */ +typedef struct { + long id; + struct ethr_join_data_ *jdata; +} ethr_tid; /* thread id type */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp) -{ - *testp = (long) InterlockedIncrement(&var->value); - return 0; -} +typedef DWORD ethr_tsd_key; -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp) -{ - *testp = (long) InterlockedDecrement(&var->value); - return 0; -} +#undef ETHR_HAVE_ETHR_SIG_FUNCS -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - * - * According to msdn _InterlockedAnd() provides a full - * memory barrier. - */ - *old = (long) _InterlockedAnd(&var->value, mask); - return 0; -} +#define ETHR_USE_OWN_RWMTX_IMPL__ +#define ETHR_USE_OWN_MTX_IMPL__ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - * - * According to msdn _InterlockedOr() provides a full - * memory barrier. - */ - *old = (long) _InterlockedOr(&var->value, mask); - return 0; -} +#define ETHR_YIELD() (Sleep(0), 0) -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - long new, - long expected, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - * - * According to msdn _InterlockedCompareExchange() provides a full - * memory barrier. - */ - *old = _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) expected); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new, - long *old) -{ - *old = (long) InterlockedExchange(&var->value, (LONG) new); - return 0; -} - -/* - * According to msdn InterlockedExchange() provides a full - * memory barrier. - */ +#else /* No supported thread lib found */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - lock->locked = (LONG) 0; +#ifdef ETHR_NO_SUPP_THR_LIB_NOT_FATAL +#define ETHR_NO_THREAD_LIB #else - (void) InterlockedExchange(&lock->locked, (LONG) 0); +#error "No supported thread lib found" #endif - return 0; -} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) -{ - return 0; -} - - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) -{ - ETHR_COMPILER_BARRIER; - { -#ifdef DEBUG - LONG old = -#endif - InterlockedExchange(&lock->locked, (LONG) 0); -#ifdef DEBUG - ETHR_ASSERT(old == 1); #endif - } - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) -{ - LONG old; - do { - old = InterlockedExchange(&lock->locked, (LONG) 1); - } while (old != (LONG) 0); - ETHR_COMPILER_BARRIER; - return 0; -} -/* - * According to msdn InterlockedIncrement, InterlockedDecrement, - * and InterlockedExchangeAdd(), _InterlockedAnd, and _InterlockedOr - * provides full memory barriers. - */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock) -{ -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - lock->counter = (LONG) 0; -#else - (void) InterlockedExchange(&lock->counter, (LONG) 0); +#ifndef ETHR_HAVE_INT32_T +#if ETHR_SIZEOF_INT == 4 +#define ETHR_HAVE_INT32_T 1 +typedef int ethr_sint32_t; +typedef unsigned int ethr_uint32_t; +#elif ETHR_SIZEOF_LONG == 4 +#define ETHR_HAVE_INT32_T 1 +typedef long ethr_sint32_t; +typedef unsigned long ethr_uint32_t; #endif - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) -{ - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock) -{ - ETHR_COMPILER_BARRIER; - { -#ifdef DEBUG - LONG old = #endif - InterlockedDecrement(&lock->counter); - ETHR_ASSERT(old != 0); - } - return 0; -} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock) -{ - while (1) { - LONG old = InterlockedIncrement(&lock->counter); - if ((old & ETHR_WLOCK_FLAG__) == 0) - break; /* Got read lock */ - /* Restore and wait for writers to unlock */ - old = InterlockedDecrement(&lock->counter); - while (old & ETHR_WLOCK_FLAG__) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - old = lock->counter; -#else - old = InterlockedExchangeAdd(&lock->counter, (LONG) 0); +#ifndef ETHR_HAVE_INT64_T +#if ETHR_SIZEOF_INT == 8 +#define ETHR_HAVE_INT64_T 1 +typedef int ethr_sint64_t; +typedef unsigned int ethr_uint64_t; +#elif ETHR_SIZEOF_LONG == 8 +#define ETHR_HAVE_INT64_T 1 +typedef long ethr_sint64_t; +typedef unsigned long ethr_uint64_t; +#elif ETHR_SIZEOF_LONG_LONG == 8 +#define ETHR_HAVE_INT64_T 1 +typedef long long ethr_sint64_t; +typedef unsigned long long ethr_uint64_t; #endif - } - } - ETHR_COMPILER_BARRIER; - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock) -{ - ETHR_COMPILER_BARRIER; - { -#ifdef DEBUG - LONG old = #endif - _InterlockedAnd(&lock->counter, ~ETHR_WLOCK_FLAG__); - ETHR_ASSERT(old & ETHR_WLOCK_FLAG__); - } - return 0; -} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) -{ - LONG old; - do { - old = _InterlockedOr(&lock->counter, ETHR_WLOCK_FLAG__); - } while (old & ETHR_WLOCK_FLAG__); - /* We got the write part of the lock; wait for readers to unlock */ - while ((old & ~ETHR_WLOCK_FLAG__) != 0) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - old = lock->counter; -#else - old = InterlockedExchangeAdd(&lock->counter, (LONG) 0); +#if ETHR_SIZEOF_PTR == 4 +#ifndef ETHR_HAVE_INT32_T +#error "No 32-bit integer type found" #endif - ETHR_ASSERT(old & ETHR_WLOCK_FLAG__); - } - ETHR_COMPILER_BARRIER; - return 0; -} - -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ - -#endif /* #ifdef ERTS_MIXED_CYGWIN_VC */ - -#else /* No supported thread lib found */ - -#ifdef ETHR_NO_SUPP_THR_LIB_NOT_FATAL -#define ETHR_NO_THREAD_LIB -#else -#error "No supported thread lib found" +typedef ethr_sint32_t ethr_sint_t; +typedef ethr_uint32_t ethr_uint_t; +#elif ETHR_SIZEOF_PTR == 8 +#ifndef ETHR_HAVE_INT64_T +#error "No 64-bit integer type found" #endif - +typedef ethr_sint64_t ethr_sint_t; +typedef ethr_uint64_t ethr_uint_t; #endif /* __builtin_expect() is needed by both native atomics code @@ -688,131 +258,169 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) #endif /* For CPU-optimised atomics, spinlocks, and rwlocks. */ -#if !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(__GNUC__) -# if ETHR_SIZEOF_PTR == 4 -# if defined(__i386__) -# include "i386/ethread.h" -# elif (defined(__powerpc__) || defined(__ppc__)) && !defined(__powerpc64__) -# include "ppc32/ethread.h" -# elif defined(__sparc__) -# include "sparc32/ethread.h" -# elif defined(__tile__) -# include "tile/ethread.h" +#if !defined(ETHR_DISABLE_NATIVE_IMPLS) +# if defined(__GNUC__) +# if defined(ETHR_PREFER_GCC_NATIVE_IMPLS) +# include "gcc/ethread.h" +# elif defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS) +# include "libatomic_ops/ethread.h" # endif -# elif ETHR_SIZEOF_PTR == 8 -# if defined(__x86_64__) -# include "x86_64/ethread.h" -# elif defined(__sparc__) && defined(__arch64__) -# include "sparc64/ethread.h" +# ifndef ETHR_HAVE_NATIVE_ATOMICS +# if ETHR_SIZEOF_PTR == 4 +# if defined(__i386__) +# include "i386/ethread.h" +# elif (defined(__powerpc__)||defined(__ppc__))&&!defined(__powerpc64__) +# include "ppc32/ethread.h" +# elif defined(__sparc__) +# include "sparc32/ethread.h" +# elif defined(__tile__) +# include "tile/ethread.h" +# endif +# elif ETHR_SIZEOF_PTR == 8 +# if defined(__x86_64__) +# include "x86_64/ethread.h" +# elif defined(__sparc__) && defined(__arch64__) +# include "sparc64/ethread.h" +# endif +# endif +# include "gcc/ethread.h" +# include "libatomic_ops/ethread.h" # endif +# elif defined(ETHR_HAVE_LIBATOMIC_OPS) +# include "libatomic_ops/ethread.h" +# elif defined(ETHR_WIN32_THREADS) +# include "win/ethread.h" # endif -#endif /* !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(__GNUC__) */ - -#ifdef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -# undef ETHR_HAVE_NATIVE_ATOMICS -#endif -#ifdef ETHR_HAVE_OPTIMIZED_LOCKS -# undef ETHR_HAVE_NATIVE_LOCKS -#endif +#endif /* !ETHR_DISABLE_NATIVE_IMPLS */ -#ifdef ETHR_HAVE_NATIVE_ATOMICS -#define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 -#endif -#ifdef ETHR_HAVE_NATIVE_LOCKS -#define ETHR_HAVE_OPTIMIZED_LOCKS 1 +#if defined(__GNUC__) +# ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") +# endif +# ifndef ETHR_SPIN_BODY +# if defined(__i386__) || defined(__x86_64__) +# define ETHR_SPIN_BODY __asm__ __volatile__("rep;nop" : : : "memory") +# elif defined(__ia64__) +# define ETHR_SPIN_BODY __asm__ __volatile__("hint @pause" : : : "memory") +# elif defined(__sparc__) +# define ETHR_SPIN_BODY __asm__ __volatile__("membar #LoadLoad") +# else +# define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER +# endif +# endif +#elif defined(ETHR_WIN32_THREADS) +# ifndef ETHR_COMPILER_BARRIER +# include <intrin.h> +# pragma intrinsic(_ReadWriteBarrier) +# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +# endif +# ifndef ETHR_SPIN_BODY +# define ETHR_SPIN_BODY do {YieldProcessor();ETHR_COMPILER_BARRIER;} while(0) +# endif #endif -typedef struct { - unsigned open; - ethr_mutex mtx; - ethr_cond cnd; -} ethr_gate; +#define ETHR_YIELD_AFTER_BUSY_LOOPS 50 -#ifdef ETHR_HAVE_NATIVE_ATOMICS +#ifndef ETHR_HAVE_NATIVE_ATOMICS /* - * Map ethread native atomics to ethread API atomics. + * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, + * i.e. when our lock based atomic fallback is used, a noop is sufficient. */ -typedef ethr_native_atomic_t ethr_atomic_t; +#define ETHR_MEMORY_BARRIER do { } while (0) +#define ETHR_WRITE_MEMORY_BARRIER do { } while (0) +#define ETHR_READ_MEMORY_BARRIER do { } while (0) +#define ETHR_READ_DEPEND_MEMORY_BARRIER do { } while (0) #endif -#ifdef ETHR_HAVE_NATIVE_LOCKS -/* - * Map ethread native spinlocks to ethread API spinlocks. - */ -typedef ethr_native_spinlock_t ethr_spinlock_t; -/* - * Map ethread native rwlocks to ethread API rwlocks. - */ -typedef ethr_native_rwlock_t ethr_rwlock_t; +#ifndef ETHR_WRITE_MEMORY_BARRIER +# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER +# define ETHR_WRITE_MEMORY_BARRIER_IS_FULL #endif - -#ifdef ETHR_USE_RWMTX_FALLBACK -typedef struct { - ethr_mutex mtx; - ethr_cond rcnd; - ethr_cond wcnd; - unsigned readers; - unsigned waiting_readers; - unsigned waiting_writers; -#if ETHR_XCHK - int initialized; +#ifndef ETHR_READ_MEMORY_BARRIER +# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER +# define ETHR_READ_MEMORY_BARRIER_IS_FULL #endif -} ethr_rwmutex; +#ifndef ETHR_READ_DEPEND_MEMORY_BARRIER +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER +# define ETHR_READ_DEPEND_MEMORY_BARRIER_IS_COMPILER_BARRIER #endif -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -typedef long ethr_atomic_t; -#endif +#define ETHR_FATAL_ERROR__(ERR) \ + ethr_fatal_error__(__FILE__, __LINE__, __func__, (ERR)) -#ifndef ETHR_HAVE_OPTIMIZED_LOCKS +ETHR_PROTO_NORETURN__ ethr_fatal_error__(const char *file, + int line, + const char *func, + int err); -#if defined(ETHR_WIN32_THREADS) -typedef struct { - CRITICAL_SECTION cs; -} ethr_spinlock_t; -typedef struct { - CRITICAL_SECTION cs; - unsigned counter; -} ethr_rwlock_t; +void ethr_compiler_barrier_fallback(void); +#ifndef ETHR_COMPILER_BARRIER +# define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() +#endif -int ethr_do_spinlock_init(ethr_spinlock_t *lock); -int ethr_do_rwlock_init(ethr_rwlock_t *lock); +#ifndef ETHR_SPIN_BODY +# define ETHR_SPIN_BODY ETHR_COMPILER_BARRIER +#endif -#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31) +#ifndef ETHR_YIELD +# if defined(ETHR_HAVE_SCHED_YIELD) +# ifdef ETHR_HAVE_SCHED_H +# include <sched.h> +# endif +# include <errno.h> +# if defined(ETHR_SCHED_YIELD_RET_INT) +# define ETHR_YIELD() (sched_yield() < 0 ? errno : 0) +# else +# define ETHR_YIELD() (sched_yield(), 0) +# endif +# elif defined(ETHR_HAVE_PTHREAD_YIELD) +# if defined(ETHR_PTHREAD_YIELD_RET_INT) +# define ETHR_YIELD() pthread_yield() +# else +# define ETHR_YIELD() (pthread_yield(), 0) +# endif +# else +# define ETHR_YIELD() (ethr_compiler_barrier(), 0) +# endif +#endif + +#include "ethr_optimized_fallbacks.h" -#elif defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) -typedef struct { - pthread_spinlock_t spnlck; -} ethr_spinlock_t; typedef struct { - pthread_spinlock_t spnlck; - unsigned counter; -} ethr_rwlock_t; -#define ETHR_RWLOCK_WRITERS (((unsigned) 1) << 31) + void *(*thread_create_prepare_func)(void); + void (*thread_create_parent_func)(void *); + void (*thread_create_child_func)(void *); +} ethr_init_data; -#else /* ethr mutex/rwmutex */ +#define ETHR_INIT_DATA_DEFAULT_INITER {NULL, NULL, NULL} typedef struct { - ethr_mutex mtx; -} ethr_spinlock_t; + void *(*alloc)(size_t); + void *(*realloc)(void *, size_t); + void (*free)(void *); +} ethr_memory_allocator; + +#define ETHR_MEM_ALLOC_DEF_INITER__ {NULL, NULL, NULL} typedef struct { - ethr_rwmutex rwmtx; -} ethr_rwlock_t; + ethr_memory_allocator std; + ethr_memory_allocator sl; + ethr_memory_allocator ll; +} ethr_memory_allocators; -#endif /* end mutex/rwmutex */ -#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */ +#define ETHR_MEM_ALLOCS_DEF_INITER__ \ + {ETHR_MEM_ALLOC_DEF_INITER__, \ + ETHR_MEM_ALLOC_DEF_INITER__, \ + ETHR_MEM_ALLOC_DEF_INITER__} typedef struct { - void *(*alloc)(size_t); - void *(*realloc)(void *, size_t); - void (*free)(void *); - void *(*thread_create_prepare_func)(void); - void (*thread_create_parent_func)(void *); - void (*thread_create_child_func)(void *); -} ethr_init_data; + ethr_memory_allocators mem; + int reader_groups; + int main_threads; +} ethr_late_init_data; -#define ETHR_INIT_DATA_DEFAULT_INITER {malloc, realloc, free, NULL, NULL, NULL} +#define ETHR_LATE_INIT_DATA_DEFAULT_INITER \ + {ETHR_MEM_ALLOCS_DEF_INITER__, 0, 0} typedef struct { int detached; /* boolean (default false) */ @@ -821,18 +429,14 @@ typedef struct { #define ETHR_THR_OPTS_DEFAULT_INITER {0, -1} -#if defined(ETHR_CUSTOM_INLINE_FUNC_NAME_) || !defined(ETHR_TRY_INLINE_FUNCS) -# define ETHR_NEED_MTX_PROTOTYPES__ -# define ETHR_NEED_RWMTX_PROTOTYPES__ -# define ETHR_NEED_SPINLOCK_PROTOTYPES__ -# define ETHR_NEED_ATOMIC_PROTOTYPES__ -#endif -#if !defined(ETHR_NEED_RWMTX_PROTOTYPES__) && defined(ETHR_USE_RWMTX_FALLBACK) -# define ETHR_NEED_RWMTX_PROTOTYPES__ +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +# define ETHR_NEED_SPINLOCK_PROTOTYPES__ +# define ETHR_NEED_RWSPINLOCK_PROTOTYPES__ #endif int ethr_init(ethr_init_data *); +int ethr_late_init(ethr_late_init_data *); int ethr_install_exit_handler(void (*funcp)(void)); int ethr_thr_create(ethr_tid *, void * (*)(void *), void *, ethr_thr_opts *); int ethr_thr_join(ethr_tid, void **); @@ -840,79 +444,12 @@ int ethr_thr_detach(ethr_tid); void ethr_thr_exit(void *); ethr_tid ethr_self(void); int ethr_equal_tids(ethr_tid, ethr_tid); -int ethr_mutex_init(ethr_mutex *); -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT -int ethr_rec_mutex_init(ethr_mutex *); -#endif -int ethr_mutex_destroy(ethr_mutex *); -int ethr_mutex_set_forksafe(ethr_mutex *); -int ethr_mutex_unset_forksafe(ethr_mutex *); -#ifdef ETHR_NEED_MTX_PROTOTYPES__ -int ethr_mutex_trylock(ethr_mutex *); -int ethr_mutex_lock(ethr_mutex *); -int ethr_mutex_unlock(ethr_mutex *); -#endif -int ethr_cond_init(ethr_cond *); -int ethr_cond_destroy(ethr_cond *); -int ethr_cond_signal(ethr_cond *); -int ethr_cond_broadcast(ethr_cond *); -int ethr_cond_wait(ethr_cond *, ethr_mutex *); -int ethr_cond_timedwait(ethr_cond *, ethr_mutex *, ethr_timeval *); - -int ethr_rwmutex_init(ethr_rwmutex *); -int ethr_rwmutex_destroy(ethr_rwmutex *); -#ifdef ETHR_NEED_RWMTX_PROTOTYPES__ -int ethr_rwmutex_tryrlock(ethr_rwmutex *); -int ethr_rwmutex_rlock(ethr_rwmutex *); -int ethr_rwmutex_runlock(ethr_rwmutex *); -int ethr_rwmutex_tryrwlock(ethr_rwmutex *); -int ethr_rwmutex_rwlock(ethr_rwmutex *); -int ethr_rwmutex_rwunlock(ethr_rwmutex *); -#endif - -#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ -int ethr_atomic_init(ethr_atomic_t *, long); -int ethr_atomic_set(ethr_atomic_t *, long); -int ethr_atomic_read(ethr_atomic_t *, long *); -int ethr_atomic_inctest(ethr_atomic_t *, long *); -int ethr_atomic_dectest(ethr_atomic_t *, long *); -int ethr_atomic_inc(ethr_atomic_t *); -int ethr_atomic_dec(ethr_atomic_t *); -int ethr_atomic_addtest(ethr_atomic_t *, long, long *); -int ethr_atomic_add(ethr_atomic_t *, long); -int ethr_atomic_and_old(ethr_atomic_t *, long, long *); -int ethr_atomic_or_old(ethr_atomic_t *, long, long *); -int ethr_atomic_xchg(ethr_atomic_t *, long, long *); -int ethr_atomic_cmpxchg(ethr_atomic_t *, long, long, long *); -#endif - -#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__ -int ethr_spinlock_init(ethr_spinlock_t *); -int ethr_spinlock_destroy(ethr_spinlock_t *); -int ethr_spin_unlock(ethr_spinlock_t *); -int ethr_spin_lock(ethr_spinlock_t *); -int ethr_rwlock_init(ethr_rwlock_t *); -int ethr_rwlock_destroy(ethr_rwlock_t *); -int ethr_read_unlock(ethr_rwlock_t *); -int ethr_read_lock(ethr_rwlock_t *); -int ethr_write_unlock(ethr_rwlock_t *); -int ethr_write_lock(ethr_rwlock_t *); -#endif - -int ethr_time_now(ethr_timeval *); int ethr_tsd_key_create(ethr_tsd_key *); int ethr_tsd_key_delete(ethr_tsd_key); int ethr_tsd_set(ethr_tsd_key, void *); void *ethr_tsd_get(ethr_tsd_key); -int ethr_gate_init(ethr_gate *); -int ethr_gate_destroy(ethr_gate *); -int ethr_gate_close(ethr_gate *); -int ethr_gate_let_through(ethr_gate *, unsigned); -int ethr_gate_wait(ethr_gate *); -int ethr_gate_swait(ethr_gate *, int); - #ifdef ETHR_HAVE_ETHR_SIG_FUNCS #include <signal.h> int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset); @@ -921,528 +458,274 @@ int ethr_sigwait(const sigset_t *set, int *sig); void ethr_compiler_barrier(void); -#ifdef ETHR_TRY_INLINE_FUNCS - -#ifdef ETHR_HAVE_NATIVE_ATOMICS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) -{ - ethr_native_atomic_init(var, i); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) -{ - ethr_native_atomic_set(var, i); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i) -{ - *i = ethr_native_atomic_read(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ - ethr_native_atomic_add(var, incr); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *var, - long i, - long *testp) -{ - *testp = ethr_native_atomic_add_return(var, i); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) -{ - ethr_native_atomic_inc(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) -{ - ethr_native_atomic_dec(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *var, long *testp) -{ - *testp = ethr_native_atomic_inc_return(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *var, long *testp) -{ - *testp = ethr_native_atomic_dec_return(var); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - *old = ethr_native_atomic_and_retold(var, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - *old = ethr_native_atomic_or_retold(var, mask); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new, - long *old) -{ - *old = ethr_native_atomic_xchg(var, new); - return 0; -} - -/* - * If *var == *old, replace *old with new, else do nothing. - * In any case return the original value of *var in *old. - */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - long new, - long expected, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - *old = ethr_native_atomic_cmpxchg(var, new, expected); - return 0; -} +#if defined(ETHR_HAVE_NATIVE_SPINLOCKS) +typedef ethr_native_spinlock_t ethr_spinlock_t; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) +typedef ethr_opt_spinlock_t ethr_spinlock_t; +#elif defined(__WIN32__) +typedef CRITICAL_SECTION ethr_spinlock_t; +#else +typedef pthread_mutex_t ethr_spinlock_t; +#endif -#endif /* ETHR_HAVE_NATIVE_ATOMICS */ +#ifdef ETHR_NEED_SPINLOCK_PROTOTYPES__ +int ethr_spinlock_init(ethr_spinlock_t *); +int ethr_spinlock_destroy(ethr_spinlock_t *); +void ethr_spin_unlock(ethr_spinlock_t *); +void ethr_spin_lock(ethr_spinlock_t *); +#endif -#ifdef ETHR_HAVE_NATIVE_LOCKS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spinlock_init(lock); return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + return ethr_opt_spinlock_init((ethr_opt_spinlock_t *) lock); +#elif defined(__WIN32__) + if (!InitializeCriticalSectionAndSpinCount((CRITICAL_SECTION *) lock, INT_MAX)) + return ethr_win_get_errno__(); + return 0; +#else + return pthread_mutex_init((pthread_mutex_t *) lock, NULL); +#endif } static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + return ethr_opt_spinlock_destroy((ethr_opt_spinlock_t *) lock); +#elif defined(__WIN32__) + DeleteCriticalSection((CRITICAL_SECTION *) lock); + return 0; +#else + return pthread_mutex_destroy((pthread_mutex_t *) lock); +#endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_unlock(lock); - return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + int err = ethr_opt_spin_unlock((ethr_opt_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#elif defined(__WIN32__) + LeaveCriticalSection((CRITICAL_SECTION *) lock); +#else + int err = pthread_mutex_unlock((pthread_mutex_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) { +#ifdef ETHR_HAVE_NATIVE_SPINLOCKS ethr_native_spin_lock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock) -{ - ethr_native_rwlock_init(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) -{ - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock) -{ - ethr_native_read_unlock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock) -{ - ethr_native_read_lock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock) -{ - ethr_native_write_unlock(lock); - return 0; -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) -{ - ethr_native_write_lock(lock); - return 0; +#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCKS) + int err = ethr_opt_spin_lock((ethr_opt_spinlock_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#elif defined(__WIN32__) + EnterCriticalSection((CRITICAL_SECTION *) lock); +#else + int err = pthread_mutex_lock((pthread_mutex_t *) lock); + if (err) + ETHR_FATAL_ERROR__(err); +#endif } -#endif /* ETHR_HAVE_NATIVE_LOCKS */ - #endif /* ETHR_TRY_INLINE_FUNCS */ -/* - * Fallbacks for atomics used in absence of optimized implementation. - */ -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS +#include "ethr_atomics.h" -#define ETHR_ATOMIC_ADDR_BITS 4 -#define ETHR_ATOMIC_ADDR_SHIFT 3 +typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */ -typedef struct { - union { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - pthread_spinlock_t spnlck; +#if defined(ETHR_WIN32_THREADS) +# include "win/ethr_event.h" #else - ethr_mutex mtx; -#endif - char buf[ETHR_CACHE_LINE_SIZE]; - } u; -} ethr_atomic_protection_t; - -extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; - - -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - -#define ETHR_ATOMIC_PTR2LCK__(PTR) \ -(ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.spnlck) - - -#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ -do { \ - pthread_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - int res__ = pthread_spin_lock(slp__); \ - if (res__ != 0) \ - return res__; \ - { EXPS; } \ - return pthread_spin_unlock(slp__); \ -} while (0) - -#else /* ethread mutex */ +# include "pthread/ethr_event.h" +#endif + +int ethr_set_main_thr_status(int, int); +int ethr_get_main_thr_status(int *); + +struct ethr_ts_event_ { + ethr_ts_event *next; + ethr_ts_event *prev; + ethr_event event; + void *udata; + ethr_atomic32_t uaflgs; + unsigned uflgs; + unsigned iflgs; /* for ethr lib only */ + short rgix; /* for ethr lib only */ + short mtix; /* for ethr lib only */ +}; -#define ETHR_ATOMIC_PTR2LCK__(PTR) \ -(ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.mtx) +#define ETHR_TS_EV_ETHREAD (((unsigned) 1) << 0) +#define ETHR_TS_EV_INITED (((unsigned) 1) << 1) +#define ETHR_TS_EV_TMP (((unsigned) 1) << 2) +#define ETHR_TS_EV_MAIN_THR (((unsigned) 1) << 3) -#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ -do { \ - ethr_mutex *mtxp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - int res__ = ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(mtxp__); \ - if (res__ != 0) \ - return res__; \ - { EXPS; } \ - return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(mtxp__); \ -} while (0) +int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp); +int ethr_free_ts_event__(ethr_ts_event *tsep); +int ethr_make_ts_event__(ethr_ts_event **tsepp); -#endif /* end ethread mutex */ +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__) +ethr_ts_event *ethr_get_ts_event(void); +void ethr_leave_ts_event(ethr_ts_event *); +#endif -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_PTHREADS) -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) -{ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i); -} +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__) -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) -{ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = (ethr_atomic_t) i); -} +extern pthread_key_t ethr_ts_event_key__; -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var, long *i) +static ETHR_INLINE ethr_ts_event * +ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *i = (long) *var); + ethr_ts_event *tsep = pthread_getspecific(ethr_ts_event_key__); + if (!tsep) { + int res = ethr_make_ts_event__(&tsep); + if (res != 0) + ETHR_FATAL_ERROR__(res); + ETHR_ASSERT(tsep); + } + return tsep; } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inctest)(ethr_atomic_t *incp, long *testp) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *testp = (long) ++(*incp)); -} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dectest)(ethr_atomic_t *decp, long *testp) -{ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, *testp = (long) --(*decp)); } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); -} - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_addtest)(ethr_atomic_t *incp, - long i, - long *testp) -{ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, *incp += i; *testp = *incp); -} +#endif -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *incp) -{ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(incp, ++(*incp)); -} +#elif defined(ETHR_WIN32_THREADS) -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *decp) -{ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(decp, --(*decp)); -} +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHREAD_IMPL__) -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_and_old)(ethr_atomic_t *var, - long mask, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var &= mask); -} +extern DWORD ethr_ts_event_key__; -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_or_old)(ethr_atomic_t *var, - long mask, - long *old) +static ETHR_INLINE ethr_ts_event * +ETHR_INLINE_FUNC_NAME_(ethr_get_ts_event)(void) { - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var |= mask); + ethr_ts_event *tsep = TlsGetValue(ethr_ts_event_key__); + if (!tsep) { + int res = ethr_get_tmp_ts_event__(&tsep); + if (res != 0) + ETHR_FATAL_ERROR__(res); + ETHR_ASSERT(tsep); + } + return tsep; } -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new, - long *old) +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_leave_ts_event)(ethr_ts_event *tsep) { - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *old = *var; *var = new); -} - -/* - * If *var == *old, replace *old with new, else do nothing. - * In any case return the original value of *var in *old. - */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - long new, - long expected, - long *old) -{ - /* - * See "Extra memory barrier requirements" note at the top - * of the file. - */ - ETHR_ATOMIC_OP_FALLBACK_IMPL__( - var, - long old_val = *var; - *old = old_val; - if (__builtin_expect(old_val == expected, 1)) - *var = new; - ); - return 0; + if (tsep->iflgs & ETHR_TS_EV_TMP) { + int res = ethr_free_ts_event__(tsep); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } } -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ -#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */ - -/* - * Fallbacks for spin locks, and rw spin locks used in absence of - * optimized implementation. - */ -#ifndef ETHR_HAVE_OPTIMIZED_LOCKS - -#ifdef ETHR_TRY_INLINE_FUNCS - -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_init)(ethr_spinlock_t *lock) -{ -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_init(&lock->spnlck, 0); -#else - return ethr_mutex_init(&lock->mtx); #endif -} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spinlock_destroy)(ethr_spinlock_t *lock) -{ -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_destroy(&lock->spnlck); -#else - return ethr_mutex_destroy(&lock->mtx); #endif -} +#include "ethr_mutex.h" /* Need atomic declarations and tse */ -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_unlock)(ethr_spinlock_t *lock) -{ -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_unlock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS +typedef ethr_native_rwlock_t ethr_rwlock_t; #else - return ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(&lock->mtx); +typedef ethr_rwmutex ethr_rwlock_t; #endif -} -static ETHR_INLINE int -ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) -{ -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_lock(&lock->spnlck); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(&lock->mtx); +#ifdef ETHR_NEED_RWSPINLOCK_PROTOTYPES__ +int ethr_rwlock_init(ethr_rwlock_t *); +int ethr_rwlock_destroy(ethr_rwlock_t *); +void ethr_read_unlock(ethr_rwlock_t *); +void ethr_read_lock(ethr_rwlock_t *); +void ethr_write_unlock(ethr_rwlock_t *); +void ethr_write_lock(ethr_rwlock_t *); #endif -} -#ifdef ETHR_USE_RWMTX_FALLBACK -#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) X -#else -#define ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(X) ETHR_INLINE_FUNC_NAME_(X) -#endif +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_rwlock_init)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - lock->counter = 0; - return pthread_spin_init(&lock->spnlck, 0); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_rwlock_init(lock); + return 0; #else - return ethr_rwmutex_init(&lock->rwmtx); + return ethr_rwmutex_init_opt((ethr_rwmutex *) lock, NULL); #endif } static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - return pthread_spin_destroy(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + return 0; #else - return ethr_rwmutex_destroy(&lock->rwmtx); + return ethr_rwmutex_destroy((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_read_unlock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - int res = pthread_spin_lock(&lock->spnlck); - if (res != 0) - return res; - lock->counter--; - return pthread_spin_unlock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_read_unlock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_runlock)(&lock->rwmtx); + ethr_rwmutex_runlock((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_read_lock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - int locked = 0; - do { - int res = pthread_spin_lock(&lock->spnlck); - if (res != 0) - return res; - if ((lock->counter & ETHR_RWLOCK_WRITERS) == 0) { - lock->counter++; - locked = 1; - } - res = pthread_spin_unlock(&lock->spnlck); - if (res != 0) - return res; - } while (!locked); - return 0; +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_read_lock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rlock)(&lock->rwmtx); + ethr_rwmutex_rlock((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_write_unlock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - lock->counter = 0; - return pthread_spin_unlock(&lock->spnlck); +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_write_unlock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwunlock)(&lock->rwmtx); + ethr_rwmutex_rwunlock((ethr_rwmutex *) lock); #endif } -static ETHR_INLINE int +static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) { -#if defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) - while (1) { - int res = pthread_spin_lock(&lock->spnlck); - if (res != 0) - return res; - lock->counter |= ETHR_RWLOCK_WRITERS; - if (lock->counter == ETHR_RWLOCK_WRITERS) - return 0; - res = pthread_spin_unlock(&lock->spnlck); - if (res != 0) - return res; - } +#ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS + ethr_native_write_lock(lock); #else - return ETHR_RWLOCK_RWMTX_FALLBACK_NAME_(ethr_rwmutex_rwlock)(&lock->rwmtx); + ethr_rwmutex_rwlock((ethr_rwmutex *) lock); #endif } -#endif /* #ifdef ETHR_TRY_INLINE_FUNCS */ - -#endif /* ETHR_HAVE_OPTIMIZED_LOCKS */ - -#if defined(ETHR_HAVE_OPTIMIZED_LOCKS) || defined(ETHR_HAVE_PTHREAD_SPIN_LOCK) -# define ETHR_HAVE_OPTIMIZED_SPINLOCK -#endif +#endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* #ifndef ETHREAD_H__ */ diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index e5b4946a53..f394d790d2 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -1,25 +1,40 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ /* Define to the size of pointers */ #undef ETHR_SIZEOF_PTR +/* Define to the size of int */ +#undef ETHR_SIZEOF_INT + +/* Define to the size of long */ +#undef ETHR_SIZEOF_LONG + +/* Define to the size of long long */ +#undef ETHR_SIZEOF_LONG_LONG + +/* Define to the size of __int64 */ +#undef ETHR_SIZEOF___INT64 + +/* Define if bigendian */ +#undef ETHR_BIGENDIAN + /* Define if you want to disable native ethread implementations */ #undef ETHR_DISABLE_NATIVE_IMPLS @@ -29,26 +44,97 @@ /* Define if you have pthreads */ #undef ETHR_PTHREADS +/* Define if you need the <nptl/pthread.h> header file. */ +#undef ETHR_NEED_NPTL_PTHREAD_H + /* Define if you have the <pthread.h> header file. */ #undef ETHR_HAVE_PTHREAD_H /* Define if the pthread.h header file is in pthread/mit directory. */ #undef ETHR_HAVE_MIT_PTHREAD_H -/* Define if you have the pthread_mutexattr_settype function. */ -#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE +/* Define if you have the pthread_spin_lock function. */ +#undef ETHR_HAVE_PTHREAD_SPIN_LOCK + +/* Define if you want to force usage of pthread rwlocks */ +#undef ETHR_FORCE_PTHREAD_RWLOCK -/* Define if you have the pthread_mutexattr_setkind_np function. */ -#undef ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP +/* Define if you have the pthread_rwlockattr_setkind_np() function. */ +#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP -/* Define if you have the pthread_atfork function. */ -#undef ETHR_HAVE_PTHREAD_ATFORK +/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock + attribute. */ +#undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP -/* Define if you have the pthread_spin_lock function. */ -#undef ETHR_HAVE_PTHREAD_SPIN_LOCK +/* Define if you have a linux futex implementation. */ +#undef ETHR_HAVE_LINUX_FUTEX + +/* Define if you have gcc atomic operations */ +#undef ETHR_HAVE_GCC_ATOMIC_OPS + +/* Define if you prefer gcc native ethread implementations */ +#undef ETHR_PREFER_GCC_NATIVE_IMPLS + +/* Define if you have the <sched.h> header file. */ +#undef ETHR_HAVE_SCHED_H + +/* Define if you have the sched_yield() function. */ +#undef ETHR_HAVE_SCHED_YIELD + +/* Define if you have the pthread_yield() function. */ +#undef ETHR_HAVE_PTHREAD_YIELD + +/* Define if pthread_yield() returns an int. */ +#undef ETHR_PTHREAD_YIELD_RET_INT + +/* Define if sched_yield() returns an int. */ +#undef ETHR_SCHED_YIELD_RET_INT + +/* Define if you want compatibilty with x86 processors before pentium4. */ +#undef ETHR_PRE_PENTIUM4_COMPAT + +/* Define if you have the pthread_rwlockattr_setkind_np() function. */ +#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP + +/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock + attribute. */ +#undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP + +/* Define if you have gcc atomic operations */ +#undef ETHR_HAVE_GCC_ATOMIC_OPS + +/* Define if you prefer gcc native ethread implementations */ +#undef ETHR_PREFER_GCC_NATIVE_IMPLS + +/* Define if you have libatomic_ops atomic operations */ +#undef ETHR_HAVE_LIBATOMIC_OPS + +/* Define if you prefer libatomic_ops native ethread implementations */ +#undef ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS + +/* Define to the size of AO_t if libatomic_ops is used */ +#undef ETHR_SIZEOF_AO_T + +/* Define if you have _InterlockedCompareExchange64() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 + +/* Define if you have _InterlockedDecrement64() */ +#undef ETHR_HAVE__INTERLOCKEDDECREMENT64 + +/* Define if you have _InterlockedIncrement64() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT64 + +/* Define if you have _InterlockedExchangeAdd64() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 + +/* Define if you have _InterlockedExchange64() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGE64 + +/* Define if you have _InterlockedAnd64() */ +#undef ETHR_HAVE__INTERLOCKEDAND64 -/* Define if you have a pthread_rwlock implementation that can be used */ -#undef ETHR_HAVE_PTHREAD_RWLOCK_INIT +/* Define if you have _InterlockedOr64() */ +#undef ETHR_HAVE__INTERLOCKEDOR64 /* Define if you want to turn on extra sanity checking in the ethread library */ #undef ETHR_XCHK diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h new file mode 100644 index 0000000000..16935084b1 --- /dev/null +++ b/erts/include/internal/gcc/ethr_atomic.h @@ -0,0 +1,290 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support using gcc's builtins + * Author: Rickard Green + */ + +#undef ETHR_INCLUDE_ATOMIC_IMPL__ +#if !defined(ETHR_GCC_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +#define ETHR_GCC_ATOMIC32_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_GCC_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +#define ETHR_GCC_ATOMIC64_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#endif + +#ifdef ETHR_INCLUDE_ATOMIC_IMPL__ + +#ifndef ETHR_GCC_ATOMIC_COMMON__ +#define ETHR_GCC_ATOMIC_COMMON__ + +#define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 0 +#if defined(__i386__) || defined(__x86_64__) || defined(__sparc__) \ + || defined(__powerpc__) || defined(__ppc__) || defined(__mips__) +# undef ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ +# define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 1 +#endif + +#if defined(__x86_64__) || (defined(__i386__) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 +#else +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 +#endif + +/* + * According to the documentation this is what we want: + * #define ETHR_MEMORY_BARRIER __sync_synchronize() + * However, __sync_synchronize() is known to erroneously be + * a noop on at least some platforms with some gcc versions. + * This has suposedly been fixed in some gcc version, but we + * don't know from which version. Therefore, we only use + * it when it has been verified to work. Otherwise + * we use a workaround. + */ +#if defined(__mips__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2)) +/* __sync_synchronize() has been verified to work here */ +#define ETHR_MEMORY_BARRIER __sync_synchronize() +#define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize() +#elif defined(__x86_64__) || (defined(__i386__) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +/* Use fence instructions directly instead of workaround */ +#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") +#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory") +#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory") +#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory") +#else +/* Workaround */ +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile ethr_sint32_t x___ = 0; \ + (void) __sync_val_compare_and_swap(&x___, (ethr_sint32_t) 0, (ethr_sint32_t) 1); \ +} while (0) +#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#endif + +#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") + +#endif /* ETHR_GCC_ATOMIC_COMMON__ */ + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +#define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t +#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t +#else +#error "Unsupported integer size" +#endif + +typedef struct { + volatile ETHR_AINT_T__ counter; +} ETHR_ATMC_T__; + + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->counter; +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +{ +#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ + var->counter = value; +#else + /* + * Unfortunately no __sync_store() or similar exist in the gcc atomic + * op interface. We therefore have to simulate it this way... + */ + ETHR_AINT_T__ act = 0, exp; + do { + exp = act; + act = __sync_val_compare_and_swap(&var->counter, exp, value); + } while (act != exp); +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +{ + ETHR_NATMC_FUNC__(set)(var, value); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) +{ +#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ + return var->counter; +#else + /* + * Unfortunately no __sync_fetch() or similar exist in the gcc atomic + * op interface. We therefore have to simulate it this way... + */ + return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0); +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +{ + (void) __sync_add_and_fetch(&var->counter, incr); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +{ + return __sync_add_and_fetch(&var->counter, incr); +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) +{ + (void) __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1); +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) +{ + (void) __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) +{ + return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +{ + return __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +{ + return __sync_fetch_and_and(&var->counter, mask); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +{ + return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return __sync_val_compare_and_swap(&var->counter, old, new); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +{ + ETHR_AINT_T__ exp, act = 0; + do { + exp = act; + act = __sync_val_compare_and_swap(&var->counter, exp, new); + } while (act != exp); + return act; +} + +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) +{ +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_AINT_T__ val = var->counter; + ETHR_COMPILER_BARRIER; + return val; +#else + return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0); +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_COMPILER_BARRIER; + var->counter = i; +#else + (void) ETHR_NATMC_FUNC__(xchg)(var, i); +#endif +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +{ + return ETHR_NATMC_FUNC__(inc_return)(var); +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +{ + ETHR_NATMC_FUNC__(dec)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) +{ + return ETHR_NATMC_FUNC__(dec_return)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} + +#endif + +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ +#undef ETHR_AINT_SUFFIX__ + +#endif diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h new file mode 100644 index 0000000000..392a1aa2b2 --- /dev/null +++ b/erts/include/internal/gcc/ethread.h @@ -0,0 +1,40 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomic ethread support when using gcc + * Author: Rickard Green + */ + +#ifndef ETHREAD_GCC_H__ +#define ETHREAD_GCC_H__ + +#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_GCC_ATOMIC_OPS) +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#define ETHR_ATOMIC_WANT_32BIT_IMPL__ +#include "ethr_atomic.h" +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_ATOMIC_WANT_64BIT_IMPL__ +# include "ethr_atomic.h" +#endif + +#endif + +#endif diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h index 3291ad38e5..4e402f261a 100644 --- a/erts/include/internal/i386/atomic.h +++ b/erts/include/internal/i386/atomic.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -23,133 +23,265 @@ * * This code requires a 486 or newer processor. */ -#ifndef ETHREAD_I386_ATOMIC_H -#define ETHREAD_I386_ATOMIC_H -/* An atomic is an aligned long accessed via locked operations. - */ -typedef struct { - volatile long counter; -} ethr_native_atomic_t; +#undef ETHR_INCLUDE_ATOMIC_IMPL__ +#if !defined(ETHR_X86_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +#define ETHR_X86_ATOMIC32_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_X86_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +#define ETHR_X86_ATOMIC64_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#endif + +#ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#ifdef ETHR_TRY_INLINE_FUNCS +#ifndef ETHR_X86_ATOMIC_COMMON__ +#define ETHR_X86_ATOMIC_COMMON__ -#ifdef __x86_64__ -#define LONG_SUFFIX "q" +#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 + +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) +#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") +#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory") +#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory") +#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory") +#else +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile ethr_sint32_t x___ = 0; \ + __asm__ __volatile__("lock; incl %0" : "=m"(x___) : "m"(x___) : "memory"); \ +} while (0) +#endif + +#endif /* ETHR_X86_ATOMIC_COMMON__ */ + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +#define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t +#define ETHR_AINT_SUFFIX__ "l" +#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t +#define ETHR_AINT_SUFFIX__ "q" #else -#define LONG_SUFFIX "l" +#error "Unsupported integer size" #endif +/* An atomic is an aligned ETHR_AINT_T__ accessed via locked operations. + */ +typedef struct { + volatile ETHR_AINT_T__ counter; +} ETHR_ATMC_T__; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->counter; +} + static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { var->counter = i; } -#define ethr_native_atomic_set(v, i) ethr_native_atomic_init((v), (i)) -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + var->counter = i; +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { return var->counter; } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { __asm__ __volatile__( - "lock; add" LONG_SUFFIX " %1, %0" + "lock; add" ETHR_AINT_SUFFIX__ " %1, %0" : "=m"(var->counter) : "ir"(incr), "m"(var->counter)); } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) { __asm__ __volatile__( - "lock; inc" LONG_SUFFIX " %0" + "lock; inc" ETHR_AINT_SUFFIX__ " %0" : "=m"(var->counter) : "m"(var->counter)); } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) { __asm__ __volatile__( - "lock; dec" LONG_SUFFIX " %0" + "lock; dec" ETHR_AINT_SUFFIX__ " %0" : "=m"(var->counter) : "m"(var->counter)); } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - long tmp; + ETHR_AINT_T__ tmp; tmp = incr; __asm__ __volatile__( - "lock; xadd" LONG_SUFFIX " %0, %1" /* xadd didn't exist prior to the 486 */ + "lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */ : "=r"(tmp) : "m"(var->counter), "0"(tmp)); /* now tmp is the atomic's previous value */ return tmp + incr; } -#define ethr_native_atomic_inc_return(var) ethr_native_atomic_add_return((var), 1) -#define ethr_native_atomic_dec_return(var) ethr_native_atomic_add_return((var), -1) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) +{ + return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) 1); +} -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +{ + return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) -1); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { __asm__ __volatile__( - "lock; cmpxchg" LONG_SUFFIX " %2, %3" + "lock; cmpxchg" ETHR_AINT_SUFFIX__ " %2, %3" : "=a"(old), "=m"(var->counter) : "r"(new), "m"(var->counter), "0"(old) : "cc", "memory"); /* full memory clobber to make this a compiler barrier */ return old; } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - long tmp, old; + ETHR_AINT_T__ tmp, old; tmp = var->counter; do { old = tmp; - tmp = ethr_native_atomic_cmpxchg(var, tmp & mask, tmp); + tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp & mask, tmp); } while (__builtin_expect(tmp != old, 0)); /* now tmp is the atomic's previous value */ return tmp; } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - long tmp, old; + ETHR_AINT_T__ tmp, old; tmp = var->counter; do { old = tmp; - tmp = ethr_native_atomic_cmpxchg(var, tmp | mask, tmp); + tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp | mask, tmp); } while (__builtin_expect(tmp != old, 0)); /* now tmp is the atomic's previous value */ return tmp; } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) { - long tmp = val; + ETHR_AINT_T__ tmp = val; __asm__ __volatile__( - "xchg" LONG_SUFFIX " %0, %1" + "xchg" ETHR_AINT_SUFFIX__ " %0, %1" : "=r"(tmp) : "m"(var->counter), "0"(tmp)); /* now tmp is the atomic's previous value */ return tmp; } -#undef LONG_SUFFIX +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) +{ + ETHR_AINT_T__ val; +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) + val = var->counter; +#else + val = ETHR_NATMC_FUNC__(add_return)(var, 0); +#endif + __asm__ __volatile__("" : : : "memory"); + return val; +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + __asm__ __volatile__("" : : : "memory"); +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) + var->counter = i; +#else + (void) ETHR_NATMC_FUNC__(xchg)(var, i); +#endif +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +{ + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); + __asm__ __volatile__("" : : : "memory"); + return res; +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +{ + __asm__ __volatile__("" : : : "memory"); + ETHR_NATMC_FUNC__(dec)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) +{ + __asm__ __volatile__("" : : : "memory"); + return ETHR_NATMC_FUNC__(dec_return)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} #endif /* ETHR_TRY_INLINE_FUNCS */ -#endif /* ETHREAD_I386_ATOMIC_H */ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ +#undef ETHR_AINT_SUFFIX__ + +#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h index fad8b108fa..b5a17caefb 100644 --- a/erts/include/internal/i386/ethread.h +++ b/erts/include/internal/i386/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,11 +24,17 @@ #ifndef ETHREAD_I386_ETHREAD_H #define ETHREAD_I386_ETHREAD_H +#define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "atomic.h" +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_ATOMIC_WANT_64BIT_IMPL__ +# include "atomic.h" +#endif #include "spinlock.h" #include "rwlock.h" #define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_LOCKS 1 +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #endif /* ETHREAD_I386_ETHREAD_H */ diff --git a/erts/include/internal/i386/rwlock.h b/erts/include/internal/i386/rwlock.h index c009be8ef1..be47f459ce 100644 --- a/erts/include/internal/i386/rwlock.h +++ b/erts/include/internal/i386/rwlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -31,7 +31,7 @@ typedef struct { volatile int lock; } ethr_native_rwlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) #define ETHR_RWLOCK_OFFSET (1<<24) diff --git a/erts/include/internal/i386/spinlock.h b/erts/include/internal/i386/spinlock.h index 2b4832e26a..0325324895 100644 --- a/erts/include/internal/i386/spinlock.h +++ b/erts/include/internal/i386/spinlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -31,7 +31,7 @@ typedef struct { volatile unsigned int lock; } ethr_native_spinlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) @@ -46,7 +46,7 @@ ethr_native_spin_unlock(ethr_native_spinlock_t *lock) * On i386 this needs to be a locked operation * to avoid Pentium Pro errata 66 and 92. */ -#if defined(__x86_64__) +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) __asm__ __volatile__("" : : : "memory"); *(unsigned char*)&lock->lock = 0; #else diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h new file mode 100644 index 0000000000..d56693dbf8 --- /dev/null +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -0,0 +1,350 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support using libatomic_ops + * Author: Rickard Green + */ + +#ifndef ETHR_LIBATOMIC_OPS_ATOMIC_H__ +#define ETHR_LIBATOMIC_OPS_ATOMIC_H__ + +#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#if (defined(__i386__) && !defined(ETHR_PRE_PENTIUM4_COMPAT)) \ + || defined(__x86_64__) +#define AO_USE_PENTIUM4_INSTRS +#endif + +#include "atomic_ops.h" + +/* + * libatomic_ops can be downloaded from: + * http://www.hpl.hp.com/research/linux/atomic_ops/ + * + * These operations need to be defined by libatomic_ops; + * otherwise, we won't compile: + * - AO_nop_full() + * - AO_load() + * - AO_store() + * - AO_compare_and_swap() + * + * The `AO_t' type also have to be at least as large as the `void *' type. + */ + +#if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR +#error The AO_t type is too small +#endif + +#if ETHR_SIZEOF_AO_T == 4 +#define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t +#define ETHR_AINT_SUFFIX__ "l" +#elif ETHR_SIZEOF_AO_T == 8 +#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t +#define ETHR_AINT_SUFFIX__ "q" +#else +#error "Unsupported integer size" +#endif + +#if ETHR_SIZEOF_AO_T == 8 +typedef union { + volatile AO_t counter; + ethr_sint32_t sint32[2]; +} ETHR_ATMC_T__; +#else +typedef struct { + volatile AO_t counter; +} ETHR_ATMC_T__; +#endif + +#define ETHR_MEMORY_BARRIER AO_nop_full() +#ifdef AO_HAVE_nop_write +# define ETHR_WRITE_MEMORY_BARRIER AO_nop_write() +#else +# define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#endif +#ifdef AO_HAVE_nop_read +# define ETHR_READ_MEMORY_BARRIER AO_nop_read() +#else +# define ETHR_READ_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#endif +#ifdef AO_NO_DD_ORDERING +# define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#else +# define ETHR_READ_DEPEND_MEMORY_BARRIER AO_compiler_barrier() +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->counter; +} + +#if ETHR_SIZEOF_AO_T == 8 +/* + * We also need to provide an ethr_native_atomic32_addr(), since + * this 64-bit implementation will be used implementing 32-bit + * native atomics. + */ + +static ETHR_INLINE ethr_sint32_t * +ethr_native_atomic32_addr(ETHR_ATMC_T__ *var) +{ + ETHR_ASSERT(((void *) &var->sint32[0]) == ((void *) &var->counter)); +#ifdef ETHR_BIGENDIAN + return &var->sint32[1]; +#else + return &var->sint32[0]; +#endif +} + +#endif /* ETHR_SIZEOF_AO_T == 8 */ + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +{ + AO_store(&var->counter, (AO_t) value); +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +{ + ETHR_NATMC_FUNC__(set)(var, value); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__) AO_load(&var->counter); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +{ +#ifdef AO_HAVE_fetch_and_add + return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr; +#else + while (1) { + AO_t exp = AO_load(&var->counter); + AO_t new = exp + (AO_t) incr; + if (AO_compare_and_swap(&var->counter, exp, new)) + return (ETHR_AINT_T__) new; + } +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +{ + (void) ETHR_NATMC_FUNC__(add_return)(var, incr); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) +{ +#ifdef AO_HAVE_fetch_and_add1 + return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1; +#else + return ETHR_NATMC_FUNC__(add_return)(var, 1); +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) +{ + (void) ETHR_NATMC_FUNC__(inc_return)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +{ +#ifdef AO_HAVE_fetch_and_sub1 + return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1; +#else + return ETHR_NATMC_FUNC__(add_return)(var, -1); +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) +{ + (void) ETHR_NATMC_FUNC__(dec_return)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +{ + while (1) { + AO_t exp = AO_load(&var->counter); + AO_t new = exp & ((AO_t) mask); + if (AO_compare_and_swap(&var->counter, exp, new)) + return (ETHR_AINT_T__) exp; + } +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +{ + while (1) { + AO_t exp = AO_load(&var->counter); + AO_t new = exp | ((AO_t) mask); + if (AO_compare_and_swap(&var->counter, exp, new)) + return (ETHR_AINT_T__) exp; + } +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) +{ + ETHR_AINT_T__ act; + do { + if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (ETHR_AINT_T__) AO_load(&var->counter); + } while (act == exp); + return act; +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +{ + while (1) { + AO_t exp = AO_load(&var->counter); + if (AO_compare_and_swap(&var->counter, exp, (AO_t) new)) + return (ETHR_AINT_T__) exp; + } +} + +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) +{ +#ifdef AO_HAVE_load_acquire + return (ETHR_AINT_T__) AO_load_acquire(&var->counter); +#else + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); + ETHR_MEMORY_BARRIER; + return res; +#endif +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +{ +#ifdef AO_HAVE_fetch_and_add1_acquire + return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1; +#else + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1); + ETHR_MEMORY_BARRIER; + return res; +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +{ +#ifdef AO_HAVE_store_release + AO_store_release(&var->counter, (AO_t) value); +#else + ETHR_MEMORY_BARRIER; + ETHR_NATMC_FUNC__(set)(var, value); +#endif +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) +{ +#ifdef AO_HAVE_fetch_and_sub1_release + return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1; +#else + ETHR_MEMORY_BARRIER; + return ETHR_NATMC_FUNC__(dec_return)(var); +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +{ + (void) ETHR_NATMC_FUNC__(dec_return_relb)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) +{ +#ifdef AO_HAVE_compare_and_swap_acquire + ETHR_AINT_T__ act; + do { + if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (ETHR_AINT_T__) AO_load(&var->counter); + } while (act == exp); + AO_nop_full(); + return act; +#else + ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); + ETHR_MEMORY_BARRIER; + return act; +#endif +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) +{ +#ifdef AO_HAVE_compare_and_swap_release + ETHR_AINT_T__ act; + do { + if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; + act = (ETHR_AINT_T__) AO_load(&var->counter); + } while (act == exp); + return act; +#else + ETHR_MEMORY_BARRIER; + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); +#endif +} + + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ + +#endif /* !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) */ + +#endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */ diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h new file mode 100644 index 0000000000..ee73ba73bc --- /dev/null +++ b/erts/include/internal/libatomic_ops/ethread.h @@ -0,0 +1,30 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support using libatomic_ops + * Author: Rickard Green + */ + +#ifndef ETHREAD_LIBATOMIC_OPS_H__ +#define ETHREAD_LIBATOMIC_OPS_H__ + +#include "ethr_atomic.h" + +#endif diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index fa701c6a92..522f433649 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -28,30 +28,39 @@ #ifndef ETHREAD_PPC_ATOMIC_H #define ETHREAD_PPC_ATOMIC_H +#define ETHR_HAVE_NATIVE_ATOMIC32 1 + typedef struct { - volatile int counter; -} ethr_native_atomic_t; + volatile ethr_sint32_t counter; +} ethr_native_atomic32_t; + +#define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory") +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) -#ifdef ETHR_TRY_INLINE_FUNCS +static ETHR_INLINE ethr_sint32_t * +ethr_native_atomic32_addr(ethr_native_atomic32_t *var) +{ + return (ethr_sint32_t *) &var->counter; +} static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, int i) +ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i) { var->counter = i; } -#define ethr_native_atomic_set(v, i) ethr_native_atomic_init((v), (i)) +#define ethr_native_atomic32_set(v, i) ethr_native_atomic32_init((v), (i)) -static ETHR_INLINE int -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_read(ethr_native_atomic32_t *var) { return var->counter; } -static ETHR_INLINE int -ethr_native_atomic_add_return(ethr_native_atomic_t *var, int incr) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - int tmp; + ethr_sint32_t tmp; __asm__ __volatile__( "eieio\n\t" @@ -68,16 +77,16 @@ ethr_native_atomic_add_return(ethr_native_atomic_t *var, int incr) } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, int incr) +ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) { /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic_add_return(var, incr); + (void)ethr_native_atomic32_add_return(var, incr); } -static ETHR_INLINE int -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var) { - int tmp; + ethr_sint32_t tmp; __asm__ __volatile__( "eieio\n\t" @@ -94,16 +103,16 @@ ethr_native_atomic_inc_return(ethr_native_atomic_t *var) } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ethr_native_atomic32_inc(ethr_native_atomic32_t *var) { /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic_inc_return(var); + (void)ethr_native_atomic32_inc_return(var); } -static ETHR_INLINE int -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) { - int tmp; + ethr_sint32_t tmp; __asm__ __volatile__( "eieio\n\t" @@ -120,16 +129,16 @@ ethr_native_atomic_dec_return(ethr_native_atomic_t *var) } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ethr_native_atomic32_dec(ethr_native_atomic32_t *var) { /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic_dec_return(var); + (void)ethr_native_atomic32_dec_return(var); } -static ETHR_INLINE int -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, int mask) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - int old, new; + ethr_sint32_t old, new; __asm__ __volatile__( "eieio\n\t" @@ -145,10 +154,10 @@ ethr_native_atomic_and_retold(ethr_native_atomic_t *var, int mask) return old; } -static ETHR_INLINE int -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, int mask) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - int old, new; + ethr_sint32_t old, new; __asm__ __volatile__( "eieio\n\t" @@ -164,10 +173,10 @@ ethr_native_atomic_or_retold(ethr_native_atomic_t *var, int mask) return old; } -static ETHR_INLINE int -ethr_native_atomic_xchg(ethr_native_atomic_t *var, int val) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) { - int tmp; + ethr_sint32_t tmp; __asm__ __volatile__( "eieio\n\t" @@ -182,10 +191,12 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, int val) return tmp; } -static ETHR_INLINE int -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, int new, int expected) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t expected) { - int old; + ethr_sint32_t old; __asm__ __volatile__( "eieio\n\t" @@ -204,6 +215,26 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, int new, int expected) return old; } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE long +ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) +{ + long res = ethr_native_atomic32_read(var); + ETHR_MEMORY_BARRIER; + return res; +} + +#define ethr_native_atomic32_set_relb ethr_native_atomic32_xchg +#define ethr_native_atomic32_inc_return_acqb ethr_native_atomic32_inc_return +#define ethr_native_atomic32_dec_relb ethr_native_atomic32_dec_return +#define ethr_native_atomic32_dec_return_relb ethr_native_atomic32_dec_return + +#define ethr_native_atomic32_cmpxchg_acqb ethr_native_atomic32_cmpxchg +#define ethr_native_atomic32_cmpxchg_relb ethr_native_atomic32_cmpxchg + #endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* ETHREAD_PPC_ATOMIC_H */ diff --git a/erts/include/internal/ppc32/ethread.h b/erts/include/internal/ppc32/ethread.h index d2a72c3dc1..12efc1b653 100644 --- a/erts/include/internal/ppc32/ethread.h +++ b/erts/include/internal/ppc32/ethread.h @@ -29,6 +29,7 @@ #include "rwlock.h" #define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_LOCKS 1 +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #endif /* ETHREAD_PPC32_ETHREAD_H */ diff --git a/erts/include/internal/ppc32/rwlock.h b/erts/include/internal/ppc32/rwlock.h index 9bdab12826..19ec26ab68 100644 --- a/erts/include/internal/ppc32/rwlock.h +++ b/erts/include/internal/ppc32/rwlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -34,7 +34,7 @@ typedef struct { volatile int lock; } ethr_native_rwlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_rwlock_init(ethr_native_rwlock_t *lock) diff --git a/erts/include/internal/ppc32/spinlock.h b/erts/include/internal/ppc32/spinlock.h index 034c20c143..c8460a3e8a 100644 --- a/erts/include/internal/ppc32/spinlock.h +++ b/erts/include/internal/ppc32/spinlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -34,7 +34,7 @@ typedef struct { volatile unsigned int lock; } ethr_native_spinlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) diff --git a/erts/include/internal/pthread/ethr_event.h b/erts/include/internal/pthread/ethr_event.h new file mode 100644 index 0000000000..93da8a0429 --- /dev/null +++ b/erts/include/internal/pthread/ethr_event.h @@ -0,0 +1,135 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#if defined(ETHR_HAVE_LINUX_FUTEX) && defined(ETHR_HAVE_NATIVE_ATOMICS) +/* --- Linux futex implementation of ethread events ------------------------- */ +#define ETHR_LINUX_FUTEX_IMPL__ + +#include <sys/syscall.h> +#include <unistd.h> +#include <linux/futex.h> +#include <sys/time.h> + +#define ETHR_EVENT_OFF_WAITER__ ((ethr_sint32_t) -1) +#define ETHR_EVENT_OFF__ ((ethr_sint32_t) 1) +#define ETHR_EVENT_ON__ ((ethr_sint32_t) 0) + +#if defined(FUTEX_WAIT_PRIVATE) && defined(FUTEX_WAKE_PRIVATE) +# define ETHR_FUTEX_WAIT__ FUTEX_WAIT_PRIVATE +# define ETHR_FUTEX_WAKE__ FUTEX_WAKE_PRIVATE +#else +# define ETHR_FUTEX_WAIT__ FUTEX_WAIT +# define ETHR_FUTEX_WAKE__ FUTEX_WAKE +#endif + +typedef struct { + ethr_atomic32_t futex; +} ethr_event; + +#define ETHR_FUTEX__(FTX, OP, VAL) \ + (-1 == syscall(__NR_futex, \ + (void *) ethr_atomic32_addr((FTX)), \ + (OP), \ + (int) (VAL), \ + NULL, \ + NULL, \ + 0) \ + ? errno : 0) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) +{ + ethr_sint32_t val; + ETHR_WRITE_MEMORY_BARRIER; + val = ethr_atomic32_xchg(&e->futex, ETHR_EVENT_ON__); + if (val == ETHR_EVENT_OFF_WAITER__) { + int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } +} + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) +{ + ethr_atomic32_set(&e->futex, ETHR_EVENT_OFF__); + ETHR_MEMORY_BARRIER; +} + +#endif + +#elif defined(ETHR_PTHREADS) +/* --- Posix mutex/cond implementation of events ---------------------------- */ + +typedef struct { + ethr_atomic32_t state; + pthread_mutex_t mtx; + pthread_cond_t cnd; +} ethr_event; + +#define ETHR_EVENT_OFF_WAITER__ -1L +#define ETHR_EVENT_OFF__ 1L +#define ETHR_EVENT_ON__ 0L + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) +{ + ethr_sint32_t val; + ETHR_WRITE_MEMORY_BARRIER; + val = ethr_atomic32_xchg(&e->state, ETHR_EVENT_ON__); + if (val == ETHR_EVENT_OFF_WAITER__) { + int res = pthread_mutex_lock(&e->mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); + res = pthread_cond_signal(&e->cnd); + if (res != 0) + ETHR_FATAL_ERROR__(res); + res = pthread_mutex_unlock(&e->mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } +} + +static void ETHR_INLINE +ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) +{ + ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__); + ETHR_MEMORY_BARRIER; +} + +#endif + +#endif + +int ethr_event_init(ethr_event *e); +int ethr_event_destroy(ethr_event *e); +int ethr_event_wait(ethr_event *e); +int ethr_event_swait(ethr_event *e, int spincount); +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) +void ethr_event_set(ethr_event *e); +void ethr_event_reset(ethr_event *e); +#endif diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index d6fdc6b2a4..00380dbf07 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -21,45 +21,86 @@ * Native ethread atomics on SPARC V9. * Author: Mikael Pettersson. */ -#ifndef ETHR_SPARC32_ATOMIC_H -#define ETHR_SPARC32_ATOMIC_H -typedef struct { - volatile long counter; -} ethr_native_atomic_t; +#undef ETHR_INCLUDE_ATOMIC_IMPL__ +#if !defined(ETHR_SPARC_V9_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +#define ETHR_SPARC_V9_ATOMIC32_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_SPARC_V9_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +#define ETHR_SPARC_V9_ATOMIC64_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#endif + +#ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#ifdef ETHR_TRY_INLINE_FUNCS +#ifndef ETHR_SPARC_V9_ATOMIC_COMMON__ +#define ETHR_SPARC_V9_ATOMIC_COMMON__ -#if defined(__arch64__) -#define CASX "casx" +#define ETHR_MEMORY_BARRIER \ + __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \ + : : : "memory") + +#endif /* ETHR_SPARC_V9_ATOMIC_COMMON__ */ + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +#define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t +#define ETHR_CAS__ "cas" +#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t +#define ETHR_CAS__ "casx" #else -#define CASX "cas" +#error "Unsupported integer size" #endif +typedef struct { + volatile ETHR_AINT_T__ counter; +} ETHR_ATMC_T__; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->counter; +} + static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { var->counter = i; } -#define ethr_native_atomic_set(v, i) ethr_native_atomic_init((v), (i)) -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + var->counter = i; +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { return var->counter; } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - long old, tmp; + ETHR_AINT_T__ old, tmp; __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); do { old = var->counter; tmp = old+incr; __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(tmp) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); @@ -69,46 +110,46 @@ ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - (void)ethr_native_atomic_add_return(var, incr); + (void)ETHR_NATMC_FUNC__(add_return)(var, incr); } -static ETHR_INLINE long -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { - return ethr_native_atomic_add_return(var, 1); + return ETHR_NATMC_FUNC__(add_return)(var, 1); } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) { - (void)ethr_native_atomic_add_return(var, 1); + (void)ETHR_NATMC_FUNC__(add_return)(var, 1); } -static ETHR_INLINE long -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { - return ethr_native_atomic_add_return(var, -1); + return ETHR_NATMC_FUNC__(add_return)(var, -1); } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) { - (void)ethr_native_atomic_add_return(var, -1); + (void)ETHR_NATMC_FUNC__(add_return)(var, -1); } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - long old, tmp; + ETHR_AINT_T__ old, tmp; __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); do { old = var->counter; tmp = old & mask; __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(tmp) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); @@ -117,17 +158,17 @@ ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) return old; } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - long old, tmp; + ETHR_AINT_T__ old, tmp; __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); do { old = var->counter; tmp = old | mask; __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(tmp) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); @@ -136,17 +177,17 @@ ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) return old; } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) { - long old, new; + ETHR_AINT_T__ old, new; __asm__ __volatile__("membar #LoadLoad|#StoreLoad"); do { old = var->counter; new = val; __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(new) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); @@ -155,12 +196,12 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) return old; } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(new) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); @@ -168,6 +209,69 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) return new; } +/* + * Atomic ops with at least specified barriers. + */ + +/* TODO: relax acquire barriers */ + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) +{ + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); + __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore" : : : "memory"); + return res; +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); + ETHR_NATMC_FUNC__(set)(var, i); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +{ + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); + __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); + return res; +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); + ETHR_NATMC_FUNC__(dec)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); + return ETHR_NATMC_FUNC__(dec_return)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) +{ + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); + return res; +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} + #endif /* ETHR_TRY_INLINE_FUNCS */ -#endif /* ETHR_SPARC32_ATOMIC_H */ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ +#undef ETHR_CAS__ + +#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/sparc32/ethread.h b/erts/include/internal/sparc32/ethread.h index 1d55399640..aea9794390 100644 --- a/erts/include/internal/sparc32/ethread.h +++ b/erts/include/internal/sparc32/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,11 +24,17 @@ #ifndef ETHREAD_SPARC32_ETHREAD_H #define ETHREAD_SPARC32_ETHREAD_H +#define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "atomic.h" +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_ATOMIC_WANT_64BIT_IMPL__ +# include "atomic.h" +#endif #include "spinlock.h" #include "rwlock.h" #define ETHR_HAVE_NATIVE_ATOMICS 1 -#define ETHR_HAVE_NATIVE_LOCKS 1 +#define ETHR_HAVE_NATIVE_SPINLOCKS 1 +#define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #endif /* ETHREAD_SPARC32_ETHREAD_H */ diff --git a/erts/include/internal/sparc32/rwlock.h b/erts/include/internal/sparc32/rwlock.h index 12448e0b06..465ec96866 100644 --- a/erts/include/internal/sparc32/rwlock.h +++ b/erts/include/internal/sparc32/rwlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -29,7 +29,7 @@ typedef struct { volatile int lock; } ethr_native_rwlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_rwlock_init(ethr_native_rwlock_t *lock) diff --git a/erts/include/internal/sparc32/spinlock.h b/erts/include/internal/sparc32/spinlock.h index b4fe48b714..493d514210 100644 --- a/erts/include/internal/sparc32/spinlock.h +++ b/erts/include/internal/sparc32/spinlock.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -29,7 +29,7 @@ typedef struct { volatile unsigned char lock; } ethr_native_spinlock_t; -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 59a9250e7c..48e4c0c6c8 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -24,105 +24,164 @@ #ifndef ETHREAD_TILE_ATOMIC_H #define ETHREAD_TILE_ATOMIC_H +#define ETHR_HAVE_NATIVE_ATOMIC32 1 + #include <atomic.h> /* An atomic is an aligned int accessed via locked operations. */ typedef struct { - volatile long counter; -} ethr_native_atomic_t; + volatile ethr_sint32_t counter; +} ethr_native_atomic32_t; + +#define ETHR_MEMORY_BARRIER __insn_mf() -#ifdef ETHR_TRY_INLINE_FUNCS +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ethr_sint32_t * +ethr_native_atomic32_addr(ethr_native_atomic32_t *var) +{ + return (ethr_sint32_t *) &var->counter; +} static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i) { var->counter = i; } static ETHR_INLINE void -ethr_native_atomic_set(ethr_native_atomic_t *var, long i) +ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i) { - __insn_mf(); atomic_exchange_acq(&var->counter, i); } -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_read(ethr_native_atomic32_t *var) { return var->counter; } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - __insn_mf(); atomic_add(&var->counter, incr); } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ethr_native_atomic32_inc(ethr_native_atomic32_t *var) { - __insn_mf(); atomic_increment(&var->counter); } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ethr_native_atomic32_dec(ethr_native_atomic32_t *var) { - __insn_mf(); atomic_decrement(&var->counter); } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - __insn_mf(); return atomic_exchange_and_add(&var->counter, incr) + incr; } -static ETHR_INLINE long -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var) { - return ethr_native_atomic_add_return(var, 1); + return ethr_native_atomic32_add_return(var, 1); } -static ETHR_INLINE long -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) { - return ethr_native_atomic_add_return(var, -1); + return ethr_native_atomic32_add_return(var, -1); } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - /* Implement a barrier suitable for a mutex unlock. */ - __insn_mf(); return atomic_and_val(&var->counter, mask); } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - __insn_mf(); return atomic_or_val(&var->counter, mask); } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) { - __insn_mf(); return atomic_exchange_acq(&var->counter, val); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long expected) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t expected) { - /* Implement a barrier suitable for a mutex unlock. */ - __insn_mf(); return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); } +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) +{ + ethr_sint32_t res = ethr_native_atomic32_read(var); + ETHR_MEMORY_BARRIER; + return res; +} + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var) +{ + ethr_sint32_t res = ethr_native_atomic32_inc_return(var); + ETHR_MEMORY_BARRIER; + return res; +} + +static ETHR_INLINE void +ethr_native_atomic32_set_relb(ethr_native_atomic32_t *var, ethr_sint32_t val) +{ + ETHR_MEMORY_BARRIER; + ethr_native_atomic32_set(var, val); +} + +static ETHR_INLINE void +ethr_native_atomic32_dec_relb(ethr_native_atomic32_t *var) +{ + ETHR_MEMORY_BARRIER; + ethr_native_atomic32_dec(var); +} + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var) +{ + ETHR_MEMORY_BARRIER; + return ethr_native_atomic32_dec_return(var); +} + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ + return ethr_native_atomic32_cmpxchg(var, new, exp); +} + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ + ETHR_MEMORY_BARRIER; + return ethr_native_atomic32_cmpxchg(var, new, exp); +} + #endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* ETHREAD_TILE_ATOMIC_H */ diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h new file mode 100644 index 0000000000..60def01a7e --- /dev/null +++ b/erts/include/internal/win/ethr_atomic.h @@ -0,0 +1,415 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support when using VC++ + * Author: Rickard Green + */ + +#undef ETHR_INCLUDE_ATOMIC_IMPL__ +#if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +#define ETHR_WIN_ATOMIC32_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +#define ETHR_WIN_ATOMIC64_H__ +#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 +/* _InterlockedCompareExchange64() required... */ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#endif +#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#endif + +#ifdef ETHR_INCLUDE_ATOMIC_IMPL__ + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + +#ifndef ETHR_WIN_ATOMIC_COMMON__ +#define ETHR_WIN_ATOMIC_COMMON__ + +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1 +#else +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0 +#endif + +#if defined(_M_AMD64) || (defined(_M_IX86) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 +#else +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 +#endif +/* + * No configure test checking for interlocked acquire/release + * versions have been written, yet. It should define + * ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS if, and + * only if, all used interlocked operations with barriers + * exists. + * + * Note, that these are pure optimizations for the itanium + * processor. + */ + +#include <intrin.h> +#undef ETHR_COMPILER_BARRIER +#define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +#pragma intrinsic(_ReadWriteBarrier) +#pragma intrinsic(_InterlockedCompareExchange) + +#if defined(_M_AMD64) || (defined(_M_IX86) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +#include <emmintrin.h> +#include <mmintrin.h> +#pragma intrinsic(_mm_mfence) +#define ETHR_MEMORY_BARRIER _mm_mfence() +#pragma intrinsic(_mm_sfence) +#define ETHR_WRITE_MEMORY_BARRIER _mm_sfence() +#pragma intrinsic(_mm_lfence) +#define ETHR_READ_MEMORY_BARRIER _mm_lfence() +#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER + +#else + +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile long x___ = 0; \ + _InterlockedCompareExchange(&x___, (long) 1, (long) 0); \ +} while (0) + +#endif + +#endif /* ETHR_WIN_ATOMIC_COMMON__ */ + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 + +#define ETHR_HAVE_NATIVE_ATOMIC32 1 + +/* + * All used operations available as 32-bit intrinsics + */ + +#pragma intrinsic(_InterlockedDecrement) +#pragma intrinsic(_InterlockedIncrement) +#pragma intrinsic(_InterlockedExchangeAdd) +#pragma intrinsic(_InterlockedExchange) +#pragma intrinsic(_InterlockedAnd) +#pragma intrinsic(_InterlockedOr) +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#pragma intrinsic(_InterlockedExchangeAdd_acq) +#pragma intrinsic(_InterlockedIncrement_acq) +#pragma intrinsic(_InterlockedDecrement_rel) +#pragma intrinsic(_InterlockedCompareExchange_acq) +#pragma intrinsic(_InterlockedCompareExchange_rel) +#endif + +#define ETHR_ILCKD__(X) _Interlocked ## X +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq +#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel +#else +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X +#define ETHR_ILCKD_REL__(X) _Interlocked ## X +#endif + +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t + +#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 + +#define ETHR_HAVE_NATIVE_ATOMIC64 1 + +/* + * _InterlockedCompareExchange64() is required. The other may not + * be available, but if so, we can generate them. + */ +#pragma intrinsic(_InterlockedCompareExchange64) + +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ +#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR) +#else +#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0 +#endif + +#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET) \ +{ \ + __int64 NEW, ACT, EXP; \ + ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR); \ + do { \ + EXP = ACT; \ + { OPS; } \ + ACT = _InterlockedCompareExchange64(PTR, NEW, EXP); \ + } while (ACT != EXP); \ + return RET; \ +} + +#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET) \ +static __forceinline __int64 \ +FUNC(__int64 volatile *ptr) \ +ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) + +#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET) \ +static __forceinline __int64 \ +FUNC(__int64 volatile *ptr, __int64 ARG) \ +ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) + + +#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64 +#pragma intrinsic(_InterlockedDecrement64) +#else +ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp, + new = act - 1, new) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64 +#pragma intrinsic(_InterlockedIncrement64) +#else +ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp, + new = act + 1, new) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 +#pragma intrinsic(_InterlockedExchangeAdd64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp, + new = act + arg, arg, act) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64 +#pragma intrinsic(_InterlockedExchange64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp, + new = arg, arg, act) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDAND64 +#pragma intrinsic(_InterlockedAnd64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp, + new = act & arg, arg, act) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDOR64 +#pragma intrinsic(_InterlockedOr64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp, + new = act | arg, arg, act) +#endif +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#pragma intrinsic(_InterlockedExchangeAdd64_acq) +#pragma intrinsic(_InterlockedIncrement64_acq) +#pragma intrinsic(_InterlockedDecrement64_rel) +#pragma intrinsic(_InterlockedCompareExchange64_acq) +#pragma intrinsic(_InterlockedCompareExchange64_rel) +#endif + +#define ETHR_ILCKD__(X) _Interlocked ## X ## 64 +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq +#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel +#else +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64 +#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64 +#endif + +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t + +#else +#error "Unsupported integer size" +#endif + +typedef struct { + volatile ETHR_AINT_T__ value; +} ETHR_ATMC_T__; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->value; +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ + var->value = i; +#else + (void) ETHR_ILCKD__(Exchange)(&var->value, i); +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ + var->value = i; +#else + (void) ETHR_ILCKD__(Exchange)(&var->value, i); +#endif +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) +{ +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ + return var->value; +#else + return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +{ + (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i; +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) +{ + (void) ETHR_ILCKD__(Increment)(&var->value); +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) +{ + (void) ETHR_ILCKD__(Decrement)(&var->value); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) +{ + return ETHR_ILCKD__(Increment)(&var->value); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +{ + return ETHR_ILCKD__(Decrement)(&var->value); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +{ + return ETHR_ILCKD__(And)(&var->value, mask); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) +{ + return ETHR_ILCKD__(Or)(&var->value, mask); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_ILCKD__(CompareExchange)(&var->value, new, old); +} + + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) +{ + return ETHR_ILCKD__(Exchange)(&var->value, new); +} + +/* + * Atomic ops with at least specified barriers. + */ + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) +{ +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_AINT_T__ val = var->value; + ETHR_COMPILER_BARRIER; + return val; +#else + return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); +#endif +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +{ + return ETHR_ILCKD_ACQ__(Increment)(&var->value); +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_COMPILER_BARRIER; + var->value = i; +#else + (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i); +#endif +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +{ + (void) ETHR_ILCKD_REL__(Decrement)(&var->value); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) +{ + return ETHR_ILCKD_REL__(Decrement)(&var->value); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old); +} + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#undef ETHR_ILCKD__ +#undef ETHR_ILCKD_ACQ__ +#undef ETHR_ILCKD_REL__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ +#undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ +#undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + +#endif /* _MSC_VER */ + +#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h new file mode 100644 index 0000000000..598816b2c6 --- /dev/null +++ b/erts/include/internal/win/ethr_event.h @@ -0,0 +1,64 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#define ETHR_EVENT_OFF_WAITER__ ((long) -1) +#define ETHR_EVENT_OFF__ ((long) 1) +#define ETHR_EVENT_ON__ ((long) 0) + +typedef struct { + volatile long state; + HANDLE handle; +} ethr_event; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) + +#pragma intrinsic(_InterlockedExchange) + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) +{ + /* _InterlockedExchange() imply a full memory barrier which is important */ + long state = _InterlockedExchange(&e->state, ETHR_EVENT_ON__); + if (state == ETHR_EVENT_OFF_WAITER__) { + if (!SetEvent(e->handle)) + ETHR_FATAL_ERROR__(ethr_win_get_errno__()); + } +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) +{ + /* _InterlockedExchange() imply a full memory barrier which is important */ + InterlockedExchange(&e->state, ETHR_EVENT_OFF__); +} + +#endif + +int ethr_event_init(ethr_event *e); +int ethr_event_destroy(ethr_event *e); +int ethr_event_wait(ethr_event *e); +int ethr_event_swait(ethr_event *e, int spincount); +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) +void ethr_event_set(ethr_event *e); +void ethr_event_reset(ethr_event *e); +#endif diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h new file mode 100644 index 0000000000..c01b17cf14 --- /dev/null +++ b/erts/include/internal/win/ethread.h @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomic and spinlock ethread support when using VC++ + * Author: Rickard Green + */ + +#ifndef ETHREAD_WIN_H__ +#define ETHREAD_WIN_H__ + +#define ETHR_ATOMIC_WANT_32BIT_IMPL__ +#include "ethr_atomic.h" +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_ATOMIC_WANT_64BIT_IMPL__ +# include "ethr_atomic.h" +#endif + +#endif diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index 49f5b1f048..757b3b24e2 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -280,8 +280,14 @@ endif # # ethread library # +ETHR_THR_LIB_BASE_DIR=@ETHR_THR_LIB_BASE_DIR@ ifneq ($(strip $(ETHR_LIB_NAME)),) -ETHREAD_LIB_SRC=common/ethread.c +ETHREAD_LIB_SRC=common/ethr_aux.c \ + common/ethr_atomics.c \ + common/ethr_mutex.c \ + common/ethr_cbf.c \ + $(ETHR_THR_LIB_BASE_DIR)/ethread.c \ + $(ETHR_THR_LIB_BASE_DIR)/ethr_event.c ETHREAD_LIB_NAME=ethread$(TYPE_SUFFIX) ifeq ($(USING_VC),yes) @@ -376,16 +382,21 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS) # Object files # +ifeq ($(TYPE)-@GCC@,debug-yes) +$(r_OBJ_DIR)/ethr_aux.o: common/ethr_aux.c + $(CC) $(THR_DEFS) $(CFLAGS) -Wno-unused-function $(INCLUDES) -c $< -o $@ +endif + $(r_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ -$(r_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(r_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ $(OBJ_DIR)/%.o: common/%.c $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ -$(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(CFLAGS) $(INCLUDES) -c $< -o $@ # Win32 specific @@ -393,25 +404,25 @@ $(OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c $(MD_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@ -$(MD_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MD_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MD $(INCLUDES) -c $< -o $@ $(MDd_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@ -$(MDd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MDd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MDd $(INCLUDES) -c $< -o $@ $(MT_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@ -$(MT_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MT_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MT $(INCLUDES) -c $< -o $@ $(MTd_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@ -$(MTd_OBJ_DIR)/%.o: $(ERLANG_OSTYPE)/%.c +$(MTd_OBJ_DIR)/%.o: $(ETHR_THR_LIB_BASE_DIR)/%.c $(CC) $(THR_DEFS) $(CFLAGS) -MTd $(INCLUDES) -c $< -o $@ # @@ -438,6 +449,9 @@ RELEASE_LIBS=$(ERTS_LIBS) INTERNAL_RELEASE_INCLUDES= \ $(ERTS_INCL_INT)/README \ $(ERTS_INCL_INT)/ethread.h \ + $(ERTS_INCL_INT)/ethr_mutex.h \ + $(ERTS_INCL_INT)/ethr_optimized_fallbacks.h \ + $(ERTS_INCL_INT)/ethr_atomics.h \ $(ERTS_INCL_INT)/$(TARGET)/ethread.mk \ $(ERTS_INCL_INT)/$(TARGET)/erts_internal.mk \ $(ERTS_INCL_INT)/$(TARGET)/ethread_header_config.h \ @@ -447,7 +461,8 @@ INTERNAL_RELEASE_INCLUDES= \ $(ERTS_INCL_INT)/erl_misc_utils.h \ $(ERTS_INCL_INT)/erl_errno.h -INTERNAL_X_RELEASE_INCLUDE_DIRS= i386 x86_64 ppc32 sparc32 sparc64 tile +INTERNAL_X_RELEASE_INCLUDE_DIRS= \ + i386 x86_64 ppc32 sparc32 sparc64 tile gcc pthread win libatomic_ops INTERNAL_RELEASE_LIBS= \ ../lib/internal/README \ diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c index f70db86960..4c881993a5 100644 --- a/erts/lib_src/common/erl_misc_utils.c +++ b/erts/lib_src/common/erl_misc_utils.c @@ -21,10 +21,13 @@ #include "config.h" #endif +#if defined(__WIN32__) +# include <windows.h> +#endif + #include "erl_misc_utils.h" #if defined(__WIN32__) -# include <windows.h> #elif defined(VXWORKS) # include <selectLib.h> #else /* UNIX */ @@ -59,8 +62,38 @@ # endif #endif -#ifdef HAVE_SCHED_xETAFFINITY +#if defined(HAVE_SCHED_xETAFFINITY) # include <sched.h> +# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ +#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP, CPUSET) \ + (sched_getaffinity((CPUINFOP)->pid, \ + sizeof(cpu_set_t), \ + (CPUSET)) != 0 ? -errno : 0) +#define ERTS_MU_SET_THR_AFFINITY__(SETP) \ + (sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0) +#elif defined(HAVE_CPUSET_xETAFFINITY) +# include <sys/param.h> +# include <sys/cpuset.h> +# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ +#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP, CPUSET) \ + (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, \ + sizeof(cpuset_t), \ + (CPUSET)) != 0 ? -errno : 0) +#define ERTS_MU_SET_THR_AFFINITY__(CPUSETP) \ + (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, \ + sizeof(cpuset_t), \ + (CPUSETP)) != 0 ? -errno : 0) +# define cpu_set_t cpuset_t +#elif defined(__WIN32__) +# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ +# define cpu_set_t DWORD +# define CPU_SETSIZE (sizeof(DWORD)*8) +# define CPU_ZERO(SETP) (*(SETP) = (DWORD) 0) +# define CPU_SET(CPU, SETP) (*(SETP) |= (((DWORD) 1) << (CPU))) +# define CPU_CLR(CPU, SETP) (*(SETP) &= ~(((DWORD) 1) << (CPU))) +# define CPU_ISSET(CPU, SETP) ((*(SETP) & (((DWORD) 1) << (CPU))) != (DWORD) 0) +#define ERTS_MU_GET_PROC_AFFINITY__ get_proc_affinity +#define ERTS_MU_SET_THR_AFFINITY__ set_thr_affinity #endif #ifdef HAVE_PSET_INFO # include <sys/pset.h> @@ -80,8 +113,33 @@ # define ERTS_SYS_CPU_PATH "/sys/devices/system/cpu" #endif +#ifdef __FreeBSD__ +#include <sys/types.h> +#include <sys/sysctl.h> +#endif + static int read_topology(erts_cpu_info_t *cpuinfo); +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) +static int +cpu_sets_are_eq(cpu_set_t *x, cpu_set_t *y) +{ + int i; + for (i = 0; i < CPU_SETSIZE; i++) { + if (CPU_ISSET(i, x)) { + if (!CPU_ISSET(i, y)) + return 0; + } + else { + if (CPU_ISSET(i, y)) + return 0; + } + } + return 1; +} + +#endif + int erts_milli_sleep(long ms) { @@ -105,30 +163,66 @@ struct erts_cpu_info_t_ { int available; int topology_size; erts_cpu_topology_t *topology; -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) char *affinity_str; char affinity_str_buf[CPU_SETSIZE/4+2]; cpu_set_t cpuset; +#if defined(HAVE_SCHED_xETAFFINITY) pid_t pid; +#endif #elif defined(HAVE_PSET_INFO) processorid_t *cpuids; #endif }; +#if defined(__WIN32__) + +static __forceinline int +get_proc_affinity(erts_cpu_info_t *cpuinfo, cpu_set_t *cpuset) +{ + DWORD pamask, samask; + if (GetProcessAffinityMask(GetCurrentProcess(), &pamask, &samask)) { + *cpuset = (cpu_set_t) pamask; + return 0; + } + else { + *cpuset = (cpu_set_t) 0; + return -erts_get_last_win_errno(); + } +} + +static __forceinline int +set_thr_affinity(cpu_set_t *set) +{ + if (*set == (cpu_set_t) 0) + return -ENOTSUP; + if (SetThreadAffinityMask(GetCurrentThread(), *set) == 0) + return -erts_get_last_win_errno(); + else + return 0; +} + +#endif + erts_cpu_info_t * erts_cpu_info_create(void) { erts_cpu_info_t *cpuinfo = malloc(sizeof(erts_cpu_info_t)); if (!cpuinfo) return NULL; -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) cpuinfo->affinity_str = NULL; +#if defined(HAVE_SCHED_xETAFFINITY) cpuinfo->pid = getpid(); +#endif #elif defined(HAVE_PSET_INFO) cpuinfo->cpuids = NULL; #endif cpuinfo->topology_size = 0; cpuinfo->topology = NULL; + cpuinfo->configured = -1; + cpuinfo->online = -1; + cpuinfo->available = -1; erts_cpu_info_update(cpuinfo); return cpuinfo; } @@ -153,31 +247,40 @@ erts_cpu_info_destroy(erts_cpu_info_t *cpuinfo) } } -void +int erts_cpu_info_update(erts_cpu_info_t *cpuinfo) { - cpuinfo->configured = 0; - cpuinfo->online = 0; - cpuinfo->available = 0; + int changed = 0; + int configured = 0; + int online = 0; + int available = 0; + erts_cpu_topology_t *old_topology; + int old_topology_size; +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + cpu_set_t cpuset; +#endif #ifdef __WIN32__ { + int i; SYSTEM_INFO sys_info; GetSystemInfo(&sys_info); - cpuinfo->configured = (int) sys_info.dwNumberOfProcessors; - + configured = (int) sys_info.dwNumberOfProcessors; + for (i = 0; i < sizeof(DWORD)*8; i++) + if (sys_info.dwActiveProcessorMask & (((DWORD) 1) << i)) + online++; } #elif !defined(NO_SYSCONF) && (defined(_SC_NPROCESSORS_CONF) \ || defined(_SC_NPROCESSORS_ONLN)) #ifdef _SC_NPROCESSORS_CONF - cpuinfo->configured = (int) sysconf(_SC_NPROCESSORS_CONF); - if (cpuinfo->configured < 0) - cpuinfo->configured = 0; + configured = (int) sysconf(_SC_NPROCESSORS_CONF); + if (configured < 0) + configured = 0; #endif #ifdef _SC_NPROCESSORS_ONLN - cpuinfo->online = (int) sysconf(_SC_NPROCESSORS_ONLN); - if (cpuinfo->online < 0) - cpuinfo->online = 0; + online = (int) sysconf(_SC_NPROCESSORS_ONLN); + if (online < 0) + online = 0; #endif #elif defined(HAVE_SYS_SYSCTL_H) && defined(CTL_HW) && (defined(HW_NCPU) \ || defined(HW_AVAILCPU)) @@ -189,71 +292,138 @@ erts_cpu_info_update(erts_cpu_info_t *cpuinfo) len = sizeof(int); mib[0] = CTL_HW; mib[1] = HW_NCPU; - if (sysctl(&mib[0], 2, &cpuinfo->configured, &len, NULL, 0) < 0) - cpuinfo->configured = 0; + if (sysctl(&mib[0], 2, &configured, &len, NULL, 0) < 0) + configured = 0; #endif #ifdef HW_AVAILCPU len = sizeof(int); mib[0] = CTL_HW; mib[1] = HW_AVAILCPU; - if (sysctl(&mib[0], 2, &cpuinfo->online, &len, NULL, 0) < 0) - cpuinfo->online = 0; + if (sysctl(&mib[0], 2, &online, &len, NULL, 0) < 0) + online = 0; #endif } #endif - if (cpuinfo->online > cpuinfo->configured) - cpuinfo->online = cpuinfo->configured; + if (online > configured) + online = configured; -#ifdef HAVE_SCHED_xETAFFINITY - if (sched_getaffinity(cpuinfo->pid, sizeof(cpu_set_t), &cpuinfo->cpuset) == 0) { - int i, c, cn, si; - c = cn = 0; - si = sizeof(cpuinfo->affinity_str_buf) - 1; - cpuinfo->affinity_str_buf[si] = '\0'; - for (i = 0; i < CPU_SETSIZE; i++) { - if (CPU_ISSET(i, &cpuinfo->cpuset)) { - c |= 1 << cn; - cpuinfo->available++; + if (cpuinfo->configured != configured) + changed = 1; + if (cpuinfo->online != online) + changed = 1; + +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + if (ERTS_MU_GET_PROC_AFFINITY__(cpuinfo, &cpuset) == 0) { + if (!changed && !cpu_sets_are_eq(&cpuset, &cpuinfo->cpuset)) + changed = 1; + + if (!changed) + available = cpuinfo->available; + else { + int i, c, cn, si; + + memcpy((void *) &cpuinfo->cpuset, + (void *) &cpuset, + sizeof(cpu_set_t)); + + c = cn = 0; + si = sizeof(cpuinfo->affinity_str_buf) - 1; + cpuinfo->affinity_str_buf[si] = '\0'; + for (i = 0; i < CPU_SETSIZE; i++) { + if (CPU_ISSET(i, &cpuinfo->cpuset)) { + c |= 1 << cn; + available++; + } + cn++; + if (cn == 4) { + cpuinfo->affinity_str_buf[--si] = (c < 10 + ? '0' + c + : 'A' + c - 10); + c = cn = 0; + } } - cn++; - if (cn == 4) { + if (c) cpuinfo->affinity_str_buf[--si] = (c < 10 ? '0' + c : 'A' + c - 10); - c = cn = 0; - } + while (cpuinfo->affinity_str_buf[si] == '0') + si++; + cpuinfo->affinity_str = &cpuinfo->affinity_str_buf[si]; } - if (c) - cpuinfo->affinity_str_buf[--si] = (c < 10 - ? '0' + c - : 'A' + c - 10); - while (cpuinfo->affinity_str_buf[si] == '0') - si++; - cpuinfo->affinity_str = &cpuinfo->affinity_str_buf[si]; } #elif defined(HAVE_PSET_INFO) { - uint_t numcpus = cpuinfo->configured; - if (cpuinfo->cpuids) - free(cpuinfo->cpuids); - cpuinfo->cpuids = malloc(sizeof(processorid_t)*numcpus); - if (cpuinfo->cpuids) { - if (pset_info(PS_MYID, NULL, &numcpus, &cpuinfo->cpuids) == 0) - cpuinfo->available = (int) numcpus; - if (cpuinfo->available < 0) { - free(cpuinfo->cpuid); - cpuinfo->available = 0; + processorid_t *cpuids; + uint_t numcpus = configured; + cpuids = malloc(sizeof(processorid_t)*numcpus); + if (cpuids) { + if (pset_info(PS_MYID, NULL, &numcpus, &cpuids) == 0) + available = (int) numcpus; + if (available < 0) { + free(cpuids); + cpuids = NULL; + available = 0; } } + if (!cpuids) { + if (cpuinfo->cpuids) + changed = 1; + } + else { + if (cpuinfo->cpuids) + changed = 1; + if (memcmp((void *) cpuinfo->cpuids, + (void *) cpuids, + sizeof(processorid_t)*numcpus) != 0) + changed = 1; + + } + if (!changed) { + if (cpuids) + free(cpuids); + } + else { + if (cpuinfo->cpuids) + free(cpuinfo->cpuids); + cpuinfo->cpuids = cpuids; + } } #endif - if (cpuinfo->available > cpuinfo->online) - cpuinfo->available = cpuinfo->online; + if (available > online) + available = online; + + if (cpuinfo->available != available) + changed = 1; + + cpuinfo->configured = configured; + cpuinfo->online = online; + cpuinfo->available = available; + + old_topology = cpuinfo->topology; + old_topology_size = cpuinfo->topology_size; + cpuinfo->topology = NULL; read_topology(cpuinfo); + if (cpuinfo->topology_size != old_topology_size + || (old_topology_size != 0 + && memcmp((void *) cpuinfo->topology, + (void *) old_topology, + (sizeof(erts_cpu_topology_t) + * old_topology_size)) != 0)) { + changed = 1; + if (old_topology) + free(old_topology); + } + else { + if (cpuinfo->topology) + free(cpuinfo->topology); + cpuinfo->topology = old_topology; + } + + return changed; } int @@ -289,7 +459,7 @@ erts_get_cpu_available(erts_cpu_info_t *cpuinfo) char * erts_get_unbind_from_cpu_str(erts_cpu_info_t *cpuinfo) { -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) if (!cpuinfo) return "false"; return cpuinfo->affinity_str; @@ -303,7 +473,7 @@ erts_get_available_cpu(erts_cpu_info_t *cpuinfo, int no) { if (!cpuinfo || no < 1 || cpuinfo->available < no) return -EINVAL; -#ifdef HAVE_SCHED_xETAFFINITY +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) { cpu_set_t *allowed = &cpuinfo->cpuset; int ix, n; @@ -335,8 +505,8 @@ int erts_is_cpu_available(erts_cpu_info_t *cpuinfo, int id) { if (cpuinfo && 0 <= id) { -#ifdef HAVE_SCHED_xETAFFINITY - if (id <= CPU_SETSIZE) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + if (id < CPU_SETSIZE) return CPU_ISSET(id, &cpuinfo->cpuset); #elif defined(HAVE_PROCESSOR_BIND) int no; @@ -388,7 +558,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu) */ if (!cpuinfo) return -EINVAL; -#ifdef HAVE_SCHED_xETAFFINITY +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) { cpu_set_t bind_set; if (cpu < 0) @@ -398,9 +568,7 @@ erts_bind_to_cpu(erts_cpu_info_t *cpuinfo, int cpu) CPU_ZERO(&bind_set); CPU_SET(cpu, &bind_set); - if (sched_setaffinity(0, sizeof(cpu_set_t), &bind_set) != 0) - return -errno; - return 0; + return ERTS_MU_SET_THR_AFFINITY__(&bind_set); } #elif defined(HAVE_PROCESSOR_BIND) if (cpu < 0) @@ -418,10 +586,8 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo) { if (!cpuinfo) return -EINVAL; -#if defined(HAVE_SCHED_xETAFFINITY) - if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuinfo->cpuset) != 0) - return -errno; - return 0; +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) + return ERTS_MU_SET_THR_AFFINITY__(&cpuinfo->cpuset); #elif defined(HAVE_PROCESSOR_BIND) if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0) return -errno; @@ -434,7 +600,7 @@ erts_unbind_from_cpu(erts_cpu_info_t *cpuinfo) int erts_unbind_from_cpu_str(char *str) { -#if defined(HAVE_SCHED_xETAFFINITY) +#if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) char *c = str; int cpus = 0; int shft = 0; @@ -486,9 +652,7 @@ erts_unbind_from_cpu_str(char *str) if (!cpus) return -EINVAL; - if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuset) != 0) - return -errno; - return 0; + return ERTS_MU_SET_THR_AFFINITY__(&cpuset); #elif defined(HAVE_PROCESSOR_BIND) if (processor_bind(P_LWPID, P_MYID, PBIND_NONE, NULL) != 0) return -errno; @@ -541,6 +705,56 @@ cpu_cmp(const void *vx, const void *vy) return 0; } +static void +adjust_processor_nodes(erts_cpu_info_t *cpuinfo, int no_nodes) +{ + erts_cpu_topology_t *prev, *this, *last; + if (no_nodes > 1) { + int processor = -1; + int processor_node = 0; + int node = -1; + + qsort(cpuinfo->topology, + cpuinfo->topology_size, + sizeof(erts_cpu_topology_t), + pn_cmp); + + prev = NULL; + this = &cpuinfo->topology[0]; + last = &cpuinfo->topology[cpuinfo->configured-1]; + while (1) { + if (processor == this->processor) { + if (node != this->node) + processor_node = 1; + } + else { + if (processor_node) { + make_processor_node: + while (prev->processor == processor) { + prev->processor_node = prev->node; + prev->node = -1; + if (prev == &cpuinfo->topology[0]) + break; + prev--; + } + processor_node = 0; + } + processor = this->processor; + node = this->node; + } + if (this == last) { + if (processor_node) { + prev = this; + goto make_processor_node; + } + break; + } + prev = this++; + } + } +} + + #ifdef __linux__ static int @@ -594,9 +808,6 @@ read_topology(erts_cpu_info_t *cpuinfo) errno = 0; - if (cpuinfo->topology) - free(cpuinfo->topology); - if (cpuinfo->configured < 1) goto error; @@ -710,49 +921,7 @@ read_topology(erts_cpu_info_t *cpuinfo) cpuinfo->topology = t; } - if (no_nodes > 1) { - int processor = -1; - int processor_node = 0; - int node = -1; - - qsort(cpuinfo->topology, - cpuinfo->topology_size, - sizeof(erts_cpu_topology_t), - pn_cmp); - - prev = NULL; - this = &cpuinfo->topology[0]; - last = &cpuinfo->topology[cpuinfo->configured-1]; - while (1) { - if (processor == this->processor) { - if (node != this->node) - processor_node = 1; - } - else { - if (processor_node) { - make_processor_node: - while (prev->processor == processor) { - prev->processor_node = prev->node; - prev->node = -1; - if (prev == &cpuinfo->topology[0]) - break; - prev--; - } - processor_node = 0; - } - processor = this->processor; - node = this->node; - } - if (this == last) { - if (processor_node) { - prev = this; - goto make_processor_node; - } - break; - } - prev = this++; - } - } + adjust_processor_nodes(cpuinfo, no_nodes); qsort(cpuinfo->topology, cpuinfo->topology_size, @@ -849,9 +1018,6 @@ read_topology(erts_cpu_info_t *cpuinfo) errno = 0; - if (cpuinfo->topology) - free(cpuinfo->topology); - if (cpuinfo->configured < 1) goto error; @@ -938,6 +1104,8 @@ read_topology(erts_cpu_info_t *cpuinfo) } } + adjust_processor_nodes(cpuinfo, 1); + error: if (res == 0) { @@ -956,6 +1124,519 @@ read_topology(erts_cpu_info_t *cpuinfo) } +#elif defined(__WIN32__) + +/* + * We cannot use Relation* out of the box since all of them are not + * always part of the LOGICAL_PROCESSOR_RELATIONSHIP enum. They are + * however documented as follows... + */ +#define ERTS_MU_RELATION_PROCESSOR_CORE 0 /* RelationProcessorCore */ +#define ERTS_MU_RELATION_NUMA_NODE 1 /* RelationNumaNode */ +#define ERTS_MU_RELATION_CACHE 2 /* RelationCache */ +#define ERTS_MU_RELATION_PROCESSOR_PACKAGE 3 /* RelationProcessorPackage */ + +static __forceinline int +rel_cmp_val(int r) +{ + switch (r) { + case ERTS_MU_RELATION_NUMA_NODE: return 0; + case ERTS_MU_RELATION_PROCESSOR_PACKAGE: return 1; + case ERTS_MU_RELATION_PROCESSOR_CORE: return 2; + default: /* currently not used */ return 3; + } +} + +static int +slpi_cmp(const void *vx, const void *vy) +{ + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION x, y; + x = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION) vx; + y = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION) vy; + + if ((int) x->Relationship != (int) y->Relationship) + return (rel_cmp_val((int) x->Relationship) + - rel_cmp_val((int) y->Relationship)); + + switch ((int) x->Relationship) { + case ERTS_MU_RELATION_NUMA_NODE: + if (x->NumaNode.NodeNumber == y->NumaNode.NodeNumber) + break; + return ((int) x->NumaNode.NodeNumber) - ((int) y->NumaNode.NodeNumber); + case ERTS_MU_RELATION_PROCESSOR_CORE: + case ERTS_MU_RELATION_PROCESSOR_PACKAGE: + default: + break; + } + + if (x->ProcessorMask == y->ProcessorMask) + return 0; + return x->ProcessorMask < y->ProcessorMask ? -1 : 1; +} + +typedef BOOL (WINAPI *glpi_t)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); + +static int +read_topology(erts_cpu_info_t *cpuinfo) +{ + int res = 0; + glpi_t glpi; + int *core_id = NULL; + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION slpip = NULL; + int wix, rix, max_l, l, packages, nodes, no_slpi; + DWORD slpi_size = 0; + + + glpi = (glpi_t) GetProcAddress(GetModuleHandle("kernel32"), + "GetLogicalProcessorInformation"); + if (!glpi) + return -ENOTSUP; + + cpuinfo->topology = NULL; + + if (cpuinfo->configured < 1 || sizeof(ULONG_PTR)*8 < cpuinfo->configured) + goto error; + + while (1) { + DWORD werr; + if (TRUE == glpi(slpip, &slpi_size)) + break; + werr = GetLastError(); + if (werr != ERROR_INSUFFICIENT_BUFFER) { + res = -erts_map_win_error_to_errno(werr); + goto error; + } + if (slpip) + free(slpip); + slpip = malloc(slpi_size); + if (!slpip) { + res = -ENOMEM; + goto error; + } + } + + no_slpi = (int) slpi_size/sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); + + qsort(slpip, + no_slpi, + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION), + slpi_cmp); + + /* + * Now numa node relations appear before package relations which + * appear before core relations which appear before relations + * we aren't interested in... + */ + + max_l = 0; + packages = 0; + nodes = 0; + for (rix = 0; rix < no_slpi; rix++) { + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION this = &slpip[rix]; + for (l = sizeof(ULONG_PTR)*8 - 1; l > 0; l--) { + if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) { + if (max_l < l) + max_l = l; + break; + } + } + if ((int) slpip[rix].Relationship == ERTS_MU_RELATION_PROCESSOR_PACKAGE) + packages++; + if ((int) slpip[rix].Relationship == ERTS_MU_RELATION_NUMA_NODE) + nodes++; + } + + if (!packages) { + packages = 1; + } + core_id = malloc(sizeof(int)*packages); + if (!core_id) { + res = -ENOMEM; + goto error; + } + + for (rix = 0; rix < packages; rix++) + core_id[rix] = 0; + + cpuinfo->topology_size = max_l + 1; + cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t) + * cpuinfo->topology_size); + if (!cpuinfo->topology) { + res = -ENOMEM; + goto error; + } + + for (wix = 0; wix < cpuinfo->topology_size; wix++) { + cpuinfo->topology[wix].node = -1; + cpuinfo->topology[wix].processor = -1; + cpuinfo->topology[wix].processor_node = -1; + cpuinfo->topology[wix].core = -1; + cpuinfo->topology[wix].thread = -1; + cpuinfo->topology[wix].logical = -1; + } + + nodes = 0; + packages = 0; + + for (rix = 0; rix < no_slpi; rix++) { + + switch ((int) slpip[rix].Relationship) { + case ERTS_MU_RELATION_NUMA_NODE: + for (l = 0; l < sizeof(ULONG_PTR)*8; l++) { + if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) { + cpuinfo->topology[l].logical = l; + cpuinfo->topology[l].node = slpip[rix].NumaNode.NodeNumber; + } + } + nodes++; + break; + case ERTS_MU_RELATION_PROCESSOR_PACKAGE: + for (l = 0; l < sizeof(ULONG_PTR)*8; l++) { + if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) { + cpuinfo->topology[l].logical = l; + cpuinfo->topology[l].processor = packages; + } + } + packages++; + break; + case ERTS_MU_RELATION_PROCESSOR_CORE: { + int thread = 0; + int processor = -1; + for (l = 0; l < sizeof(ULONG_PTR)*8; l++) { + /* + * Nodes and packages may not be supported; pretend + * that there are one if this is the case... + */ + if (slpip[rix].ProcessorMask & (((ULONG_PTR) 1) << l)) { + if (!nodes) { + cpuinfo->topology[l].node = 0; + } + if (!packages) { + cpuinfo->topology[l].processor = 0; + } + if (processor < 0) { + processor = cpuinfo->topology[l].processor; + if (processor < 0) { + res = -EINVAL; + goto error; + } + } + else if (processor != cpuinfo->topology[l].processor) { + res = -EINVAL; + goto error; + } + cpuinfo->topology[l].logical = l; + cpuinfo->topology[l].thread = thread; + cpuinfo->topology[l].core = core_id[processor]; + thread++; + } + } + core_id[processor]++; + break; + } + default: + /* + * We have reached the end of the relationships + * that we (currently) are interested in... + */ + goto relationships_done; + } + } + + relationships_done: + + /* + * There may be unused entries; remove them... + */ + for (rix = wix = 0; rix < cpuinfo->topology_size; rix++) { + if (cpuinfo->topology[rix].logical >= 0) { + if (wix != rix) + cpuinfo->topology[wix] = cpuinfo->topology[rix]; + wix++; + } + } + + if (cpuinfo->topology_size != wix) { + erts_cpu_topology_t *new = cpuinfo->topology; + new = realloc(cpuinfo->topology, + sizeof(erts_cpu_topology_t)*wix); + if (!new) { + res = -ENOMEM; + goto error; + } + cpuinfo->topology = new; + cpuinfo->topology_size = wix; + } + + res = wix; + + adjust_processor_nodes(cpuinfo, nodes); + + qsort(cpuinfo->topology, + cpuinfo->topology_size, + sizeof(erts_cpu_topology_t), + cpu_cmp); + + if (res < cpuinfo->online) + res = -EINVAL; + + error: + + if (res <= 0) { + cpuinfo->topology_size = 0; + if (cpuinfo->topology) { + free(cpuinfo->topology); + cpuinfo->topology = NULL; + } + } + + if (slpip) + free(slpip); + if (core_id) + free(core_id); + + return res; +} + +#elif defined(__FreeBSD__) + +/** + * FreeBSD topology detection is based on kern.sched.topology_spec XML as + * exposed by the ULE scheduler and described in SMP(4). It is available in + * 8.0 and higher. + * + * Threads are identified in this XML chunk with a THREAD flag. The function + * (simplistically) distinguishes cores and processors by the amount of cache + * they share (0 => processor, otherwise => core). Nodes are not identified + * (ULE doesn't handle NUMA yet, I believe). + */ + +/** + * Recursively parse a topology_spec <group> tag. + */ +static +const char* parse_topology_spec_group(erts_cpu_info_t *cpuinfo, const char* xml, int parentCacheLevel, int* processor_p, int* core_p, int* index_procs_p) { + int error = 0; + int cacheLevel = parentCacheLevel; + const char* next_group_start = strstr(xml + 1, "<group"); + int is_thread_group = 0; + const char* next_cache_level; + const char* next_thread_flag; + const char* next_group_end; + const char* next_children; + const char* next_children_end; + + /* parse the cache level */ + next_cache_level = strstr(xml, "cache-level=\""); + if (next_cache_level && (next_group_start == NULL || next_cache_level < next_group_start)) { + sscanf(next_cache_level, "cache-level=\"%i\"", &cacheLevel); + } + + /* parse the threads flag */ + next_thread_flag = strstr(xml, "THREAD"); + if (next_thread_flag && (next_group_start == NULL || next_thread_flag < next_group_start)) + is_thread_group = 1; + + /* Determine if it's a leaf with the position of the next children tag */ + next_group_end = strstr(xml, "</group>"); + next_children = strstr(xml, "<children>"); + next_children_end = strstr(xml, "</children>"); + if (next_children == NULL || next_group_end < next_children) { + do { + const char* next_cpu_start; + const char* next_cpu_cdata; + const char* next_cpu_end; + int cpu_str_size; + char* cpu_str; + char* cpu_crsr; + char* brkb; + int thread = 0; + int index_procs = *index_procs_p; + + next_cpu_start = strstr(xml, "<cpu"); + if (!next_cpu_start) { + error = 1; + break; + } + next_cpu_cdata = strstr(next_cpu_start, ">") + 1; + if (!next_cpu_cdata) { + error = 1; + break; + } + next_cpu_end = strstr(next_cpu_cdata, "</cpu>"); + if (!next_cpu_end) { + error = 1; + break; + } + cpu_str_size = next_cpu_end - next_cpu_cdata; + cpu_str = (char*) malloc(cpu_str_size + 1); + memcpy(cpu_str, (const char*) next_cpu_cdata, cpu_str_size); + cpu_str[cpu_str_size] = 0; + for (cpu_crsr = strtok_r(cpu_str, " \t,", &brkb); cpu_crsr; cpu_crsr = strtok_r(NULL, " \t,", &brkb)) { + int cpu_id; + if (index_procs >= cpuinfo->configured) { + void* t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t) * (index_procs + 1))); + if (t) { + cpuinfo->topology = t; + } else { + error = 1; + break; + } + } + cpu_id = atoi(cpu_crsr); + cpuinfo->topology[index_procs].node = -1; + cpuinfo->topology[index_procs].processor = *processor_p; + cpuinfo->topology[index_procs].processor_node = -1; + cpuinfo->topology[index_procs].core = *core_p; + cpuinfo->topology[index_procs].thread = thread; + cpuinfo->topology[index_procs].logical = cpu_id; + if (is_thread_group) { + thread++; + } else { + *core_p = (*core_p)++; + } + index_procs++; + } + *index_procs_p = index_procs; + free(cpu_str); + } while (0); + xml = next_group_end; + } else { + while (next_group_start != NULL && next_group_start < next_children_end) { + xml = parse_topology_spec_group(cpuinfo, next_group_start, cacheLevel, processor_p, core_p, index_procs_p); + if (!xml) + break; + next_group_start = strstr(xml, "<group"); + next_children_end = strstr(xml, "</children>"); + } + } + + if (cacheLevel == 0) { + *core_p = 0; + *processor_p = (*processor_p)++; + } else { + *core_p = (*core_p)++; + } + + if (error) + xml = NULL; + + return xml; +} + +/** + * Parse the topology_spec. Return the number of CPUs or 0 if parsing failed. + */ +static +int parse_topology_spec(erts_cpu_info_t *cpuinfo, const char* xml) { + int res = 1; + int index_procs = 0; + int core = 0; + int processor = 0; + xml = strstr(xml, "<groups"); + if (!xml) + return -1; + + xml += 7; + xml = strstr(xml, "<group"); + while (xml) { + xml = parse_topology_spec_group(cpuinfo, xml, 0, &processor, &core, &index_procs); + if (!xml) { + res = 0; + break; + } + xml = strstr(xml, "<group"); + } + + if (res) + res = index_procs; + + return res; +} + +static int +read_topology(erts_cpu_info_t *cpuinfo) +{ + int ix; + int res = 0; + size_t topology_spec_size = 0; + void* topology_spec = NULL; + + errno = 0; + + if (cpuinfo->configured < 1) + goto error; + + cpuinfo->topology_size = cpuinfo->configured; + cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t) + * cpuinfo->configured); + if (!cpuinfo->topology) { + res = -ENOMEM; + goto error; + } + + for (ix = 0; ix < cpuinfo->configured; ix++) { + cpuinfo->topology[ix].node = -1; + cpuinfo->topology[ix].processor = -1; + cpuinfo->topology[ix].processor_node = -1; + cpuinfo->topology[ix].core = -1; + cpuinfo->topology[ix].thread = -1; + cpuinfo->topology[ix].logical = -1; + } + + if (!sysctlbyname("kern.sched.topology_spec", NULL, &topology_spec_size, NULL, 0)) { + topology_spec = malloc(topology_spec_size); + if (!topology_spec) { + res = -ENOMEM; + goto error; + } + + if (sysctlbyname("kern.sched.topology_spec", topology_spec, &topology_spec_size, NULL, 0)) { + goto error; + } + + res = parse_topology_spec(cpuinfo, topology_spec); + if (!res || res < cpuinfo->online) + res = 0; + else { + cpuinfo->topology_size = res; + + if (cpuinfo->topology_size != cpuinfo->configured) { + void *t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t) + * cpuinfo->topology_size)); + if (t) + cpuinfo->topology = t; + } + + adjust_processor_nodes(cpuinfo, 1); + + qsort(cpuinfo->topology, + cpuinfo->topology_size, + sizeof(erts_cpu_topology_t), + cpu_cmp); + } + } + +error: + + if (res == 0) { + cpuinfo->topology_size = 0; + if (cpuinfo->topology) { + free(cpuinfo->topology); + cpuinfo->topology = NULL; + } + if (errno) + res = -errno; + else + res = -EINVAL; + } + + if (topology_spec) + free(topology_spec); + + return res; +} + #else static int @@ -965,3 +1646,98 @@ read_topology(erts_cpu_info_t *cpuinfo) } #endif + +#if defined(__WIN32__) + +int +erts_map_win_error_to_errno(DWORD win_error) +{ + switch (win_error) { + case ERROR_INVALID_FUNCTION: return EINVAL; /* 1 */ + case ERROR_FILE_NOT_FOUND: return ENOENT; /* 2 */ + case ERROR_PATH_NOT_FOUND: return ENOENT; /* 3 */ + case ERROR_TOO_MANY_OPEN_FILES: return EMFILE; /* 4 */ + case ERROR_ACCESS_DENIED: return EACCES; /* 5 */ + case ERROR_INVALID_HANDLE: return EBADF; /* 6 */ + case ERROR_ARENA_TRASHED: return ENOMEM; /* 7 */ + case ERROR_NOT_ENOUGH_MEMORY: return ENOMEM; /* 8 */ + case ERROR_INVALID_BLOCK: return ENOMEM; /* 9 */ + case ERROR_BAD_ENVIRONMENT: return E2BIG; /* 10 */ + case ERROR_BAD_FORMAT: return ENOEXEC; /* 11 */ + case ERROR_INVALID_ACCESS: return EINVAL; /* 12 */ + case ERROR_INVALID_DATA: return EINVAL; /* 13 */ + case ERROR_OUTOFMEMORY: return ENOMEM; /* 14 */ + case ERROR_INVALID_DRIVE: return ENOENT; /* 15 */ + case ERROR_CURRENT_DIRECTORY: return EACCES; /* 16 */ + case ERROR_NOT_SAME_DEVICE: return EXDEV; /* 17 */ + case ERROR_NO_MORE_FILES: return ENOENT; /* 18 */ + case ERROR_WRITE_PROTECT: return EACCES; /* 19 */ + case ERROR_BAD_UNIT: return EACCES; /* 20 */ + case ERROR_NOT_READY: return EACCES; /* 21 */ + case ERROR_BAD_COMMAND: return EACCES; /* 22 */ + case ERROR_CRC: return EACCES; /* 23 */ + case ERROR_BAD_LENGTH: return EACCES; /* 24 */ + case ERROR_SEEK: return EACCES; /* 25 */ + case ERROR_NOT_DOS_DISK: return EACCES; /* 26 */ + case ERROR_SECTOR_NOT_FOUND: return EACCES; /* 27 */ + case ERROR_OUT_OF_PAPER: return EACCES; /* 28 */ + case ERROR_WRITE_FAULT: return EACCES; /* 29 */ + case ERROR_READ_FAULT: return EACCES; /* 30 */ + case ERROR_GEN_FAILURE: return EACCES; /* 31 */ + case ERROR_SHARING_VIOLATION: return EACCES; /* 32 */ + case ERROR_LOCK_VIOLATION: return EACCES; /* 33 */ + case ERROR_WRONG_DISK: return EACCES; /* 34 */ + case ERROR_SHARING_BUFFER_EXCEEDED: return EACCES; /* 36 */ + case ERROR_BAD_NETPATH: return ENOENT; /* 53 */ + case ERROR_NETWORK_ACCESS_DENIED: return EACCES; /* 65 */ + case ERROR_BAD_NET_NAME: return ENOENT; /* 67 */ + case ERROR_FILE_EXISTS: return EEXIST; /* 80 */ + case ERROR_CANNOT_MAKE: return EACCES; /* 82 */ + case ERROR_FAIL_I24: return EACCES; /* 83 */ + case ERROR_INVALID_PARAMETER: return EINVAL; /* 87 */ + case ERROR_NO_PROC_SLOTS: return EAGAIN; /* 89 */ + case ERROR_DRIVE_LOCKED: return EACCES; /* 108 */ + case ERROR_BROKEN_PIPE: return EPIPE; /* 109 */ + case ERROR_DISK_FULL: return ENOSPC; /* 112 */ + case ERROR_INVALID_TARGET_HANDLE: return EBADF; /* 114 */ + case ERROR_WAIT_NO_CHILDREN: return ECHILD; /* 128 */ + case ERROR_CHILD_NOT_COMPLETE: return ECHILD; /* 129 */ + case ERROR_DIRECT_ACCESS_HANDLE: return EBADF; /* 130 */ + case ERROR_NEGATIVE_SEEK: return EINVAL; /* 131 */ + case ERROR_SEEK_ON_DEVICE: return EACCES; /* 132 */ + case ERROR_DIR_NOT_EMPTY: return ENOTEMPTY;/* 145 */ + case ERROR_NOT_LOCKED: return EACCES; /* 158 */ + case ERROR_BAD_PATHNAME: return ENOENT; /* 161 */ + case ERROR_MAX_THRDS_REACHED: return EAGAIN; /* 164 */ + case ERROR_LOCK_FAILED: return EACCES; /* 167 */ + case ERROR_ALREADY_EXISTS: return EEXIST; /* 183 */ + case ERROR_INVALID_STARTING_CODESEG: return ENOEXEC; /* 188 */ + case ERROR_INVALID_STACKSEG: return ENOEXEC; /* 189 */ + case ERROR_INVALID_MODULETYPE: return ENOEXEC; /* 190 */ + case ERROR_INVALID_EXE_SIGNATURE: return ENOEXEC; /* 191 */ + case ERROR_EXE_MARKED_INVALID: return ENOEXEC; /* 192 */ + case ERROR_BAD_EXE_FORMAT: return ENOEXEC; /* 193 */ + case ERROR_ITERATED_DATA_EXCEEDS_64k: return ENOEXEC; /* 194 */ + case ERROR_INVALID_MINALLOCSIZE: return ENOEXEC; /* 195 */ + case ERROR_DYNLINK_FROM_INVALID_RING: return ENOEXEC; /* 196 */ + case ERROR_IOPL_NOT_ENABLED: return ENOEXEC; /* 197 */ + case ERROR_INVALID_SEGDPL: return ENOEXEC; /* 198 */ + case ERROR_AUTODATASEG_EXCEEDS_64k: return ENOEXEC; /* 199 */ + case ERROR_RING2SEG_MUST_BE_MOVABLE: return ENOEXEC; /* 200 */ + case ERROR_RELOC_CHAIN_XEEDS_SEGLIM: return ENOEXEC; /* 201 */ + case ERROR_INFLOOP_IN_RELOC_CHAIN: return ENOEXEC; /* 202 */ + case ERROR_FILENAME_EXCED_RANGE: return ENOENT; /* 206 */ + case ERROR_NESTING_NOT_ALLOWED: return EAGAIN; /* 215 */ + case ERROR_NOT_ENOUGH_QUOTA: return ENOMEM; /* 1816 */ + default: return EINVAL; + } +} + +int +erts_get_last_win_errno(void) +{ + return erts_map_win_error_to_errno(GetLastError()); +} + + +#endif diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c new file mode 100644 index 0000000000..94557d904a --- /dev/null +++ b/erts/lib_src/common/ethr_atomics.c @@ -0,0 +1,402 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: The ethread atomic API + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_ATOMIC_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" + +#ifndef ETHR_HAVE_NATIVE_ATOMICS +ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; +#endif + +int +ethr_init_atomics(void) +{ +#ifndef ETHR_HAVE_NATIVE_ATOMICS + { + int i; + for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { + int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); + if (res != 0) + return res; + } + } +#endif + return 0; +} + +/* + * --- Pointer size atomics --------------------------------------------------- + */ + +ethr_sint_t * +ethr_atomic_addr(ethr_atomic_t *var) +{ + ETHR_ASSERT(var); + return ethr_atomic_addr__(var); +} + +void +ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t i) +{ + ETHR_ASSERT(var); + ethr_atomic_init__(var, i); +} + +void +ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set__(var, i); +} + +ethr_sint_t +ethr_atomic_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read__(var); +} + +ethr_sint_t +ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_add_read__(var, incr); +} + +ethr_sint_t +ethr_atomic_inc_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read__(var); +} + +ethr_sint_t +ethr_atomic_dec_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read__(var); +} + +void +ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_add__(var, incr); +} + +void +ethr_atomic_inc(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_inc__(var); +} + +void +ethr_atomic_dec(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec__(var); +} + +ethr_sint_t +ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_band__(var, mask); +} + +ethr_sint_t +ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_bor__(var, mask); +} + +ethr_sint_t +ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t new) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_xchg__(var, new); +} + +ethr_sint_t +ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t expected) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg__(var, new, expected); +} + +ethr_sint_t +ethr_atomic_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_acqb__(var); +} + +ethr_sint_t +ethr_atomic_inc_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read_acqb__(var); +} + +void +ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set_relb__(var, i); +} + +void +ethr_atomic_dec_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec_relb__(var); +} + +ethr_sint_t +ethr_atomic_dec_read_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read_relb__(var); +} + +ethr_sint_t +ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_acqb__(var, new, exp); +} + +ethr_sint_t +ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_relb__(var, new, exp); +} + + +/* + * --- 32-bit atomics --------------------------------------------------------- + */ + +ethr_sint32_t * +ethr_atomic32_addr(ethr_atomic32_t *var) +{ + ETHR_ASSERT(var); + return ethr_atomic32_addr__(var); +} + +void +ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t i) +{ + ETHR_ASSERT(var); + ethr_atomic32_init__(var, i); +} + +void +ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_set__(var, i); +} + +ethr_sint32_t +ethr_atomic32_read(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_read__(var); +} + + +ethr_sint32_t +ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_add_read__(var, incr); +} + +ethr_sint32_t +ethr_atomic32_inc_read(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_inc_read__(var); +} + +ethr_sint32_t +ethr_atomic32_dec_read(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_dec_read__(var); +} + +void +ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_add__(var, incr); +} + +void +ethr_atomic32_inc(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_inc__(var); +} + +void +ethr_atomic32_dec(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_dec__(var); +} + +ethr_sint32_t +ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_read_band__(var, mask); +} + +ethr_sint32_t +ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_read_bor__(var, mask); +} + +ethr_sint32_t +ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t new) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_xchg__(var, new); +} + +ethr_sint32_t +ethr_atomic32_cmpxchg(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t expected) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_cmpxchg__(var, new, expected); +} + +ethr_sint32_t +ethr_atomic32_read_acqb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_read_acqb__(var); +} + +ethr_sint32_t +ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_inc_read_acqb__(var); +} + +void +ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_set_relb__(var, i); +} + +void +ethr_atomic32_dec_relb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_dec_relb__(var); +} + +ethr_sint32_t +ethr_atomic32_dec_read_relb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_dec_read_relb__(var); +} + +ethr_sint32_t +ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_cmpxchg_acqb__(var, new, exp); +} + +ethr_sint32_t +ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_cmpxchg_relb__(var, new, exp); +} + diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c new file mode 100644 index 0000000000..2c3e25a805 --- /dev/null +++ b/erts/lib_src/common/ethr_aux.c @@ -0,0 +1,582 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: A Thread library for use in the ERTS and other OTP + * applications. + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include <stdio.h> + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_AUX_IMPL__ +#define ETHR_ATOMIC_IMPL__ /* Needed in order to pull in + native atomic implementations + for optimized fallbacks of + spinlocks and rwspinlocks */ +#include "ethread.h" +#include "ethr_internal.h" +#include <string.h> +#include <limits.h> + +#ifndef __WIN32__ +#include <unistd.h> +#endif + +#define ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE 100 +#define ERTS_TS_EV_ALLOC_POOL_SIZE 25 + +erts_cpu_info_t *ethr_cpu_info__; + +int ethr_not_completely_inited__ = 1; +int ethr_not_inited__ = 1; + +ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__; + +void *(*ethr_thr_prepare_func__)(void) = NULL; +void (*ethr_thr_parent_func__)(void *) = NULL; +void (*ethr_thr_child_func__)(void *) = NULL; + +typedef struct ethr_xhndl_list_ ethr_xhndl_list; +struct ethr_xhndl_list_ { + ethr_xhndl_list *next; + void (*funcp)(void); +}; + +size_t ethr_pagesize__; +size_t ethr_min_stack_size__; /* kilo words */ +size_t ethr_max_stack_size__; /* kilo words */ + +ethr_rwmutex xhndl_rwmtx; +ethr_xhndl_list *xhndl_list; + +static int main_threads; + +static int init_ts_event_alloc(void); + +int +ethr_init_common__(ethr_init_data *id) +{ + int res; + if (id) { + ethr_thr_prepare_func__ = id->thread_create_prepare_func; + ethr_thr_parent_func__ = id->thread_create_parent_func; + ethr_thr_child_func__ = id->thread_create_child_func; + } + + ethr_cpu_info__ = erts_cpu_info_create(); + if (!ethr_cpu_info__) + return ENOMEM; + +#ifdef _SC_PAGESIZE + ethr_pagesize__ = (size_t) sysconf(_SC_PAGESIZE); +#elif defined(HAVE_GETPAGESIZE) + ethr_pagesize__ = (size_t) getpagesize(); +#else + ethr_pagesize__ = (size_t) 4*1024; /* Guess 4 KB */ +#endif + + /* User needs at least 4 KB */ + ethr_min_stack_size__ = 4*1024; +#if SIZEOF_VOID_P == 8 + /* Double that on 64-bit archs */ + ethr_min_stack_size__ *= 2; +#endif + /* On some systems as much as about 4 KB is used by the system */ + ethr_min_stack_size__ += 4*1024; + /* There should be room for signal handlers */ +#ifdef SIGSTKSZ + ethr_min_stack_size__ += SIGSTKSZ; +#else + ethr_min_stack_size__ += ethr_pagesize__; +#endif + /* The system may think that we need more stack */ +#if defined(PTHREAD_STACK_MIN) + if (ethr_min_stack_size__ < PTHREAD_STACK_MIN) + ethr_min_stack_size__ = PTHREAD_STACK_MIN; +#elif defined(_SC_THREAD_STACK_MIN) + { + size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN); + if (ethr_min_stack_size__ < thr_min_stk_sz) + ethr_min_stack_size__ = thr_min_stk_sz; + } +#endif + /* The guard is at least on some platforms included in the stack size + passed when creating threads */ +#ifdef ETHR_STACK_GUARD_SIZE + ethr_min_stack_size__ += ETHR_STACK_GUARD_SIZE; +#endif + ethr_min_stack_size__ = ETHR_PAGE_ALIGN(ethr_min_stack_size__); + + ethr_min_stack_size__ = ETHR_B2KW(ethr_min_stack_size__); + + ethr_max_stack_size__ = 32*1024*1024; +#if SIZEOF_VOID_P == 8 + ethr_max_stack_size__ *= 2; +#endif + ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__); + + res = ethr_init_atomics(); + if (res != 0) + return res; + + res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__)); + if (res != 0) + return res; + + xhndl_list = NULL; + + return 0; +} + +int +ethr_late_init_common__(ethr_late_init_data *lid) +{ + ethr_ts_event *tsep = NULL; + int reader_groups; + int res; + int i; + ethr_memory_allocator *m[] = {ðr_mem__.std, + ðr_mem__.sl, + ðr_mem__.ll}; + if (lid) + ethr_mem__ = lid->mem; + if (!ethr_mem__.std.alloc + || !ethr_mem__.std.realloc + || !ethr_mem__.std.free) { + ethr_mem__.std.alloc = malloc; + ethr_mem__.std.realloc = realloc; + ethr_mem__.std.free = free; + } + for (i = 0; i < sizeof(m)/sizeof(m[0]); i++) { + if (!m[i]->alloc || !m[i]->realloc || !m[i]->free) { + m[i]->alloc = ethr_mem__.std.alloc; + m[i]->realloc = ethr_mem__.std.realloc; + m[i]->free = ethr_mem__.std.free; + } + + } + res = init_ts_event_alloc(); + if (res != 0) + return res; + res = ethr_make_ts_event__(&tsep); + if (res == 0) + tsep->iflgs |= ETHR_TS_EV_ETHREAD; + if (!lid) { + main_threads = 0; + reader_groups = 0; + } + else { + if (lid->main_threads < 0 || USHRT_MAX < lid->main_threads) + return res; + main_threads = lid->main_threads; + reader_groups = lid->reader_groups; + } + res = ethr_mutex_lib_late_init(reader_groups, main_threads); + if (res != 0) + return res; + ethr_not_completely_inited__ = 0; /* Need it for + rwmutex_init */ + res = ethr_rwmutex_init(&xhndl_rwmtx); + ethr_not_completely_inited__ = 1; + if (res != 0) + return res; + return 0; +} + +int +ethr_install_exit_handler(void (*funcp)(void)) +{ + ethr_xhndl_list *xhp; + +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + + if (!funcp) + return EINVAL; + + xhp = (ethr_xhndl_list *) ethr_mem__.std.alloc(sizeof(ethr_xhndl_list)); + if (!xhp) + return ENOMEM; + + ethr_rwmutex_rwlock(&xhndl_rwmtx); + + xhp->funcp = funcp; + xhp->next = xhndl_list; + xhndl_list = xhp; + + ethr_rwmutex_rwunlock(&xhndl_rwmtx); + + return 0; +} + +void +ethr_run_exit_handlers__(void) +{ + ethr_xhndl_list *xhp; + + ethr_rwmutex_rlock(&xhndl_rwmtx); + + xhp = xhndl_list; + + ethr_rwmutex_runlock(&xhndl_rwmtx); + + for (; xhp; xhp = xhp->next) + (*xhp->funcp)(); +} + +/* + * Thread specific event alloc, etc. + * + * Note that we don't know when it is safe to destroy an event, but + * we know when it is safe to reuse it. ts_event_free() therefore + * never destroys an event (but makes freed events available for + * reuse). + * + * We could easily keep track of the usage of events, and by this + * make it possible to destroy events. We would however suffer a + * performance penalty for this and save very little memory. + */ + +typedef union { + ethr_ts_event ts_ev; + char align[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_ts_event))]; +} ethr_aligned_ts_event; + +static ethr_spinlock_t ts_ev_alloc_lock; +static ethr_ts_event *free_ts_ev; + +static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp) +{ + int i; + ethr_aligned_ts_event *atsev; + atsev = ethr_mem__.std.alloc(sizeof(ethr_aligned_ts_event) * size + + ETHR_CACHE_LINE_SIZE); + if (!atsev) + return NULL; + if ((((ethr_uint_t) atsev) & ETHR_CACHE_LINE_MASK) == 0) + atsev = ((ethr_aligned_ts_event *) + ((((ethr_uint_t) atsev) & ~ETHR_CACHE_LINE_MASK) + + ETHR_CACHE_LINE_SIZE)); + for (i = 1; i < size; i++) { + atsev[i-1].ts_ev.next = &atsev[i].ts_ev; + ethr_atomic32_init(&atsev[i-1].ts_ev.uaflgs, 0); + atsev[i-1].ts_ev.iflgs = 0; + } + ethr_atomic32_init(&atsev[size-1].ts_ev.uaflgs, 0); + atsev[size-1].ts_ev.iflgs = 0; + atsev[size-1].ts_ev.next = NULL; + if (endpp) + *endpp = &atsev[size-1].ts_ev; + return &atsev[0].ts_ev; +} + +static int init_ts_event_alloc(void) +{ + free_ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_DEFAULT_POOL_SIZE, + NULL); + if (!free_ts_ev) + return ENOMEM; + return ethr_spinlock_init(&ts_ev_alloc_lock); +} + +static ethr_ts_event *ts_event_alloc(void) +{ + ethr_ts_event *ts_ev; + ethr_spin_lock(&ts_ev_alloc_lock); + if (free_ts_ev) { + ts_ev = free_ts_ev; + free_ts_ev = ts_ev->next; + ethr_spin_unlock(&ts_ev_alloc_lock); + } + else { + ethr_ts_event *ts_ev_pool_end; + ethr_spin_unlock(&ts_ev_alloc_lock); + + ts_ev = ts_event_pool(ERTS_TS_EV_ALLOC_POOL_SIZE, &ts_ev_pool_end); + if (!ts_ev) + return NULL; + + ethr_spin_lock(&ts_ev_alloc_lock); + ts_ev_pool_end->next = free_ts_ev; + free_ts_ev = ts_ev->next; + ethr_spin_unlock(&ts_ev_alloc_lock); + } + return ts_ev; +} + +static void ts_event_free(ethr_ts_event *ts_ev) +{ + ETHR_ASSERT(!ts_ev->udata); + ethr_spin_lock(&ts_ev_alloc_lock); + ts_ev->next = free_ts_ev; + free_ts_ev = ts_ev; + ethr_spin_unlock(&ts_ev_alloc_lock); +} + +int ethr_make_ts_event__(ethr_ts_event **tsepp) +{ + int res; + ethr_ts_event *tsep = *tsepp; + + if (!tsep) { + tsep = ts_event_alloc(); + if (!tsep) + return ENOMEM; + } + + if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) { + res = ethr_event_init(&tsep->event); + if (res != 0) { + ts_event_free(tsep); + return res; + } + } + + tsep->iflgs = ETHR_TS_EV_INITED; + tsep->udata = NULL; + tsep->rgix = 0; + tsep->mtix = 0; + + res = ethr_set_tse__(tsep); + if (res != 0 && tsepp && *tsepp) { + ts_event_free(tsep); + return res; + } + + if (tsepp) + *tsepp = tsep; + + return 0; +} + +int ethr_get_tmp_ts_event__(ethr_ts_event **tsepp) +{ + int res; + ethr_ts_event *tsep = *tsepp; + + if (!tsep) { + tsep = ts_event_alloc(); + if (!tsep) + return ENOMEM; + } + + if ((tsep->iflgs & ETHR_TS_EV_INITED) == 0) { + res = ethr_event_init(&tsep->event); + if (res != 0) { + ts_event_free(tsep); + return res; + } + } + + tsep->iflgs = ETHR_TS_EV_INITED|ETHR_TS_EV_TMP; + tsep->udata = NULL; + + if (tsepp) + *tsepp = tsep; + + return 0; +} + +int ethr_free_ts_event__(ethr_ts_event *tsep) +{ + ts_event_free(tsep); + return 0; +} + +void ethr_ts_event_destructor__(void *vtsep) +{ + if (vtsep) { + ethr_ts_event *tsep = (ethr_ts_event *) vtsep; + ts_event_free(tsep); + ethr_set_tse__(NULL); + } +} + +int ethr_set_main_thr_status(int on, int no) +{ + ethr_ts_event *tsep = ethr_get_tse__(); + if (!tsep) + return EINVAL; + if (on) { + if (no < 1 || main_threads < no) + return EINVAL; + tsep->mtix = (unsigned short) no; + tsep->iflgs |= ETHR_TS_EV_MAIN_THR; + } + else { + tsep->iflgs &= ~ETHR_TS_EV_MAIN_THR; + tsep->mtix = (unsigned short) 0; + } + return 0; +} + +int ethr_get_main_thr_status(int *on) +{ + ethr_ts_event *tsep = ethr_get_tse__(); + if (!tsep) + *on = 0; + else { + if (tsep->iflgs & ETHR_TS_EV_MAIN_THR) + *on = 1; + else + *on = 0; + } + return 0; +} + +/* Spinlocks and rwspinlocks */ + +int +ethr_spinlock_init(ethr_spinlock_t *lock) +{ +#if ETHR_XCHK + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_spinlock_init__(lock); +} + +int +ethr_spinlock_destroy(ethr_spinlock_t *lock) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_spinlock_destroy__(lock); +} + +void +ethr_spin_unlock(ethr_spinlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_spin_unlock__(lock); +} + +void +ethr_spin_lock(ethr_spinlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_spin_lock__(lock); +} + +int +ethr_rwlock_init(ethr_rwlock_t *lock) +{ +#if ETHR_XCHK + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_rwlock_init__(lock); +} + +int +ethr_rwlock_destroy(ethr_rwlock_t *lock) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!lock) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return ethr_rwlock_destroy__(lock); +} + +void +ethr_read_unlock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_read_unlock__(lock); +} + +void +ethr_read_lock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_read_lock__(lock); +} + +void +ethr_write_unlock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_write_unlock__(lock); +} + +void +ethr_write_lock(ethr_rwlock_t *lock) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(lock); + ethr_write_lock__(lock); +} + +ETHR_IMPL_NORETURN__ ethr_fatal_error__(const char *file, + int line, + const char *func, + int err) +{ + char *errstr; + if (err == ENOTSUP) + errstr = "Operation not supported"; + else { + errstr = strerror(err); + if (!errstr) + errstr = "Unknown error"; + } + fprintf(stderr, "%s:%d: Fatal error in %s(): %s (%d)\n", + file, line, func, errstr, err); + ethr_abort__(); +} + +int ethr_assert_failed(const char *file, int line, const char *func, char *a) +{ + fprintf(stderr, "%s:%d: %s(): Assertion failed: %s\n", file, line, func, a); + ethr_abort__(); + return 0; +} diff --git a/erts/lib_src/common/ethr_cbf.c b/erts/lib_src/common/ethr_cbf.c new file mode 100644 index 0000000000..04feceec89 --- /dev/null +++ b/erts/lib_src/common/ethr_cbf.c @@ -0,0 +1,36 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + +/* + * We keep this function alone in a separate file so the + * compiler wont optimize it away. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "ethread.h" + +void +ethr_compiler_barrier_fallback(void) +{ + +} diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c new file mode 100644 index 0000000000..2ddef32dfc --- /dev/null +++ b/erts/lib_src/common/ethr_mutex.c @@ -0,0 +1,2912 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Mutex, rwmutex and condition variable implementation + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_MUTEX_IMPL__ + +#include <limits.h> +#include "ethread.h" +#include "ethr_internal.h" + +#define ETHR_SPIN_WITH_WAITERS 1 + +#define ETHR_MTX_MAX_FLGS_SPIN 10 + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +static int default_rwmtx_main_spincount; +static int default_rwmtx_aux_spincount; +#endif +#ifdef ETHR_USE_OWN_MTX_IMPL__ +static int default_mtx_main_spincount; +static int default_mtx_aux_spincount; +static int default_cnd_main_spincount; +static int default_cnd_aux_spincount; +#endif + +static int no_spin; + +#ifndef ETHR_USE_OWN_RWMTX_IMPL__ +static pthread_rwlockattr_t write_pref_attr_data; +static pthread_rwlockattr_t *write_pref_attr; +#endif + +#if defined(ETHR_MTX_Q_LOCK_SPINLOCK__) +# define ETHR_MTX_QLOCK_INIT ethr_spinlock_init +# define ETHR_MTX_QLOCK_DESTROY ethr_spinlock_destroy +# define ETHR_MTX_Q_LOCK ethr_spin_lock +# define ETHR_MTX_Q_UNLOCK ethr_spin_unlock +#elif defined(ETHR_MTX_Q_LOCK_PTHREAD_MUTEX__) +# define ETHR_MTX_QLOCK_INIT(QL) pthread_mutex_init((QL), NULL) +# define ETHR_MTX_QLOCK_DESTROY pthread_mutex_destroy +# define ETHR_MTX_Q_LOCK(L) \ +do { \ + int res__ = pthread_mutex_lock(L); \ + if (res__ != 0) \ + ETHR_FATAL_ERROR__(res__); \ +} while (0) +# define ETHR_MTX_Q_UNLOCK(L) \ +do { \ + int res__ = pthread_mutex_unlock(L); \ + if (res__ != 0) \ + ETHR_FATAL_ERROR__(res__); \ +} while (0) +#elif defined(ETHR_MTX_Q_LOCK_CRITICAL_SECTION__) +# define ETHR_MTX_QLOCK_INIT(QL) (InitializeCriticalSection((QL)), 0) +# define ETHR_MTX_QLOCK_DESTROY(QL) (DeleteCriticalSection((QL)), 0) +# define ETHR_MTX_Q_LOCK(QL) EnterCriticalSection((QL)) +# define ETHR_MTX_Q_UNLOCK(QL) LeaveCriticalSection((QL)) +#endif + +int +ethr_mutex_lib_init(int cpu_conf) +{ + int res = 0; + + no_spin = cpu_conf == 1; + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_BASE; + default_mtx_aux_spincount = ETHR_MTX_DEFAULT_AUX_SPINCOUNT; + default_cnd_main_spincount = ETHR_CND_DEFAULT_MAIN_SPINCOUNT; + default_cnd_aux_spincount = ETHR_CND_DEFAULT_AUX_SPINCOUNT; +#endif + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + + default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_BASE; + default_rwmtx_aux_spincount = ETHR_RWMTX_DEFAULT_AUX_SPINCOUNT; + +#else + +#if defined(ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP) \ + && defined(ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP) + res = pthread_rwlockattr_init(&write_pref_attr_data); + if (res != 0) + return res; + res = pthread_rwlockattr_setkind_np( + &write_pref_attr_data, + PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + write_pref_attr = &write_pref_attr_data; +#else + write_pref_attr = NULL; +#endif + +#endif + + return res; +} + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + +#ifdef ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS +#if 0 /* + * When inc and dec are real atomic instructions as on x86, the + * ETHR_RLOCK_WITH_INC_DEC implementations performs better with + * lots of read locks compared to the cmpxchg based implementation. + * It, however, performs worse with lots of mixed reads and writes. + * It could be used for rwlocks that are known to be read locked + * much, but the readers array based implementation outperforms it + * by far. Therefore, it has been disabled, and will probably be + * removed some time in the future. + */ +# define ETHR_RLOCK_WITH_INC_DEC +#endif +#endif + +static int reader_groups_array_size = 0; +static int main_threads_array_size = 0; + +#endif + +int +ethr_mutex_lib_late_init(int no_reader_groups, int no_main_threads) +{ + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + default_mtx_main_spincount += (no_main_threads + * ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_INC); + if (default_mtx_main_spincount > ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX) + default_mtx_main_spincount = ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX; +#endif + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + + default_rwmtx_main_spincount += (no_main_threads + * ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_INC); + if (default_rwmtx_main_spincount > ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX) + default_rwmtx_main_spincount = ETHR_RWMTX_DEFAULT_MAIN_SPINCOUNT_MAX; + + reader_groups_array_size = (no_reader_groups <= 1 + ? 1 + : no_reader_groups + 1); + main_threads_array_size = (no_main_threads <= 1 + ? 1 + : no_main_threads + 1); +#endif + return 0; +} + +int +ethr_rwmutex_set_reader_group(int ix) +{ +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ + ethr_ts_event *tse; + + if (ix < 0 || reader_groups_array_size <= ix) + return EINVAL; + + tse = ethr_get_ts_event(); + + if ((tse->iflgs & ETHR_TS_EV_ETHREAD) == 0) { + ethr_leave_ts_event(tse); + return EINVAL; + } + + tse->rgix = ix; + + ethr_leave_ts_event(tse); +#endif + return 0; +} + +#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ) +static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int); +#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) hard_debug_chk_q__(&(RWMTX)->mtxb,1) +#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) hard_debug_chk_q__(&(MTX)->mtxb, 0) +#else +#define ETHR_RWMTX_HARD_DEBUG_CHK_Q(RWMTX) +#define ETHR_MTX_HARD_DEBUG_CHK_Q(MTX) +#endif + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +static void +rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx, + ethr_sint32_t initial, + int q_locked); +static void +rwmutex_unlock_wake(ethr_rwmutex *rwmtx, + int have_w, + ethr_sint32_t initial, + int transfer_read_lock); +static int +rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, + ethr_sint32_t initial, + ethr_ts_event *tse, + int start_next_ix, + int check_before_try, + int try_write_lock); +#endif + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) + +/* -- Utilities operating both on ordinary mutexes and read write mutexes -- */ + +static ETHR_INLINE void +rwmutex_freqread_wtng_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix = (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ + ? tse->rgix + : tse->mtix); + rwmtx->tdata.ra[ix].data.waiting_readers++; +} + +static ETHR_INLINE void +rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx, + ethr_rwmutex_type type, + int ix, + int inc) +{ + if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0) + ethr_atomic32_add(&rwmtx->tdata.ra[ix].data.readers, inc); + else { + ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0); + ETHR_ASSERT(inc == 1); + ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1); + } +} + +static ETHR_INLINE void +rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_inc: + ethr_atomic32_inc(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_inc; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0); + ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1); + } +} + +#if 0 /* Not used */ + +static ETHR_INLINE void +rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_dec: + ethr_atomic32_dec(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_dec; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0); + } +} + +#endif + +static ETHR_INLINE ethr_sint32_t +rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_dec_read: + return ethr_atomic32_dec_read(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_dec_read; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0); + return (ethr_sint32_t) 0; + } +} + +static ETHR_INLINE ethr_sint32_t +rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse) +{ + int ix; + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + ix = tse->rgix; + atomic_dec_read: + return ethr_atomic32_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers); + } + else { + ix = tse->mtix; + if (ix == 0) + goto atomic_dec_read; + ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic32_set_relb(&rwmtx->tdata.ra[ix].data.readers, + (ethr_sint32_t) 0); + return (ethr_sint32_t) 0; + } +} + +static ETHR_INLINE ethr_sint32_t +rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix) +{ + ethr_sint32_t res = ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers); +#ifdef ETHR_DEBUG + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + ETHR_ASSERT(res >= 0); + break; + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + ETHR_ASSERT(ix == 0 ? res >= 0 : (res == 0 || res == 1)); + break; + default: + ETHR_ASSERT(0); + break; + } +#endif + return res; +} + + +static ETHR_INLINE void +enqueue(ethr_ts_event **queue, + ethr_ts_event *tse_start, + ethr_ts_event *tse_end) +{ + if (!*queue) { + *queue = tse_start; + tse_start->prev = tse_end; + tse_end->next = tse_start; + } + else { + tse_end->next = *queue; + tse_start->prev = (*queue)->prev; + (*queue)->prev->next = tse_start; + (*queue)->prev = tse_end; + } +} + +static ETHR_INLINE void +insert(ethr_ts_event *tse_pred, ethr_ts_event *tse) +{ + tse->next = tse_pred->next; + tse->prev = tse_pred; + tse_pred->next->prev = tse; + tse_pred->next = tse; +} + +static ETHR_INLINE void +dequeue(ethr_ts_event **queue, + ethr_ts_event *tse_start, + ethr_ts_event *tse_end) +{ + if (tse_start->prev == tse_end) { + ETHR_ASSERT(*queue == tse_start && tse_end->next == tse_start); + *queue = NULL; + } + else { + if (*queue == tse_start) + *queue = tse_end->next; + tse_end->next->prev = tse_start->prev; + tse_start->prev->next = tse_end->next; + } +} + +static void +event_wait(struct ethr_mutex_base_ *mtxb, + ethr_ts_event *tse, + int spincount, + ethr_sint32_t type, + int is_rwmtx, + int is_freq_read) +{ + int locked = 0; + ethr_sint32_t act; + int need_try_complete_runlock = 0; + int transfer_read_lock = 0; + + /* Need to enqueue and wait... */ + + tse->uflgs = type; + ethr_atomic32_set(&tse->uaflgs, type); + + ETHR_MTX_Q_LOCK(&mtxb->qlck); + locked = 1; + +#ifdef ETHR_MTX_HARD_DEBUG_Q + hard_debug_chk_q__(mtxb, is_rwmtx); +#endif + + act = ethr_atomic32_read(&mtxb->flgs); + + if (act & type) { + + /* Wait bit already there; enqueue... */ + + ETHR_ASSERT(mtxb->q); + if (type == ETHR_RWMTX_W_WAIT_FLG__) { + enqueue(&mtxb->q, tse, tse); +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtxb->ws++; +#endif + } + else { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_ASSERT(is_rwmtx); + ETHR_ASSERT(rwmtx->rq_end); + insert(rwmtx->rq_end, tse); + rwmtx->rq_end = tse; + if (is_freq_read) + rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse); + else + rwmtx->tdata.rs++; + } + } + else { + + /* Set wait bit */ + + while (1) { + ethr_sint32_t new, exp = act; + need_try_complete_runlock = 0; + transfer_read_lock = 0; + + if (type == ETHR_RWMTX_W_WAIT_FLG__) { + if (is_freq_read && act == ETHR_RWMTX_R_FLG__) + need_try_complete_runlock = 1; + if (act != 0) + new = act | ETHR_RWMTX_W_WAIT_FLG__; + else + new = ETHR_RWMTX_W_FLG__; /* Try to get it */ + } + else { + ETHR_ASSERT(is_rwmtx); + + if (!is_freq_read) { + if (act & (ETHR_RWMTX_W_FLG__| ETHR_RWMTX_W_WAIT_FLG__)) + new = act | ETHR_RWMTX_R_WAIT_FLG__; + else + new = act + 1; /* Try to get it */ + } + else { + new = act | ETHR_RWMTX_R_WAIT_FLG__; + if ((act & (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_W_WAIT_FLG__)) == 0) { + /* Transfer read lock to this thread. */ + transfer_read_lock = 1; + } + } + } + + act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs, new, exp); + if (exp == act) { + if (new & type) { + act = new; + break; + } + else { + /* Got it */ + goto done; + } + } + } + + /* Enqueue */ + + if (type == ETHR_RWMTX_R_WAIT_FLG__) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_ASSERT(is_rwmtx); + ETHR_ASSERT(!rwmtx->rq_end); + rwmtx->rq_end = tse; + if (is_freq_read) + rwmutex_freqread_wtng_rdrs_inc(rwmtx, tse); + else + rwmtx->tdata.rs++; + } +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + else { + mtxb->ws++; + } +#endif + + enqueue(&mtxb->q, tse, tse); + } + +#ifdef ETHR_MTX_HARD_DEBUG_Q + hard_debug_chk_q__(mtxb, is_rwmtx); +#endif + + /* Wait */ + locked = 0; + + ETHR_ASSERT(!(transfer_read_lock && need_try_complete_runlock)); + + if (transfer_read_lock) { + ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type + != ETHR_RWMUTEX_TYPE_NORMAL); + /* + * We are the only one in the queue and we are not write + * locked; rwmutex_transfer_read_lock() will: + * - transfer a read lock to us (since we're first in q) + * - unlock the Q-lock + */ + rwmutex_transfer_read_lock(((ethr_rwmutex *) mtxb), act, 1); + } + else { + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + + if (need_try_complete_runlock) { + ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type + != ETHR_RWMUTEX_TYPE_NORMAL); + /* + * We were the only one in queue when we enqueued, and it + * was seemingly read locked. We need to try to complete a + * runlock otherwise we might be hanging forever. If the + * runlock could be completed we will be dequeued and + * woken by ourselves. + */ + rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb, + act, tse, 0, 1, 0); + } + } + + while (1) { + ethr_event_reset(&tse->event); + + act = ethr_atomic32_read_acqb(&tse->uaflgs); + if (!act) + goto done; /* Got it */ + + ETHR_ASSERT(act == type); + ethr_event_swait(&tse->event, spincount); + /* swait result: 0 || EINTR */ + + act = ethr_atomic32_read_acqb(&tse->uaflgs); + if (!act) + goto done; /* Got it */ + } + + done: + if (locked) + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); +} + +static void +wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx) +{ + ethr_ts_event *tse; + + tse = mtxb->q; + ETHR_ASSERT(tse); + dequeue(&mtxb->q, tse, tse); + + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__); +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtxb->ws--; +#endif +#if defined(ETHR_MTX_HARD_DEBUG_Q) || defined(ETHR_MTX_HARD_DEBUG_WSQ) + hard_debug_chk_q__(mtxb, is_rwmtx); +#endif + + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + + ethr_atomic32_set(&tse->uaflgs, 0); + ethr_event_set(&tse->event); +} + +static ETHR_INLINE int +initial_spincount(struct ethr_mutex_base_ *mtxb) +{ + return (mtxb->aux_scnt < ETHR_MTX_MAX_FLGS_SPIN + ? mtxb->aux_scnt + : ETHR_MTX_MAX_FLGS_SPIN); +} + +static ETHR_INLINE int +update_spincount(struct ethr_mutex_base_ *mtxb, + ethr_ts_event *tse, + int *scnt_state, + int *scnt) +{ + int state = *scnt_state; + if (state <= 0) { + /* Here state is max spincount to do on event negated */ + *scnt = -state; + } + else { + /* Here state is initial spincount made on flags */ + *scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) + ? mtxb->main_scnt + : mtxb->aux_scnt); + if (*scnt <= state) + *scnt = 0; + else { + if (*scnt <= ETHR_MTX_MAX_FLGS_SPIN) + *scnt_state = 0; /* No spin on event */ + else { + /* Spin on event after... */ + *scnt_state = -1*(*scnt - ETHR_MTX_MAX_FLGS_SPIN); + /* ... we have spun on flags */ + *scnt = ETHR_MTX_MAX_FLGS_SPIN; + } + *scnt -= state; + return 0; + } + } + return 1; +} + +int check_readers_array(ethr_rwmutex *rwmtx, + int start_rix, + int length); + +static ETHR_INLINE void +write_lock_wait(struct ethr_mutex_base_ *mtxb, + ethr_sint32_t initial, + int is_rwmtx, + int is_freq_read) +{ + ethr_sint32_t act = initial; + int scnt, start_scnt; + ethr_ts_event *tse = NULL; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + int res; + + ETHR_ASSERT(!is_freq_read || is_rwmtx); + + start_scnt = scnt = initial_spincount(mtxb); + + /* + * Spin trying to write lock for a while. If unsuccessful, + * wait on event. + */ + + while (1) { + while (act != 0) { + + if (is_freq_read && act == ETHR_RWMTX_R_FLG__) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + scnt--; + if (!tse) + tse = ethr_get_ts_event(); + res = rwmutex_try_complete_runlock(rwmtx, act, + tse, 0, 0, + 1); + if (res != EBUSY) + goto done; /* Got it */ + if (scnt <= 0) + goto chk_spin; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + } + + if (scnt <= 0) { + chk_spin: + scnt = 0; + + if (!tse) + tse = ethr_get_ts_event(); + if (update_spincount(mtxb, tse, &start_scnt, &scnt)) { + event_wait(mtxb, tse, scnt, ETHR_RWMTX_W_WAIT_FLG__, + is_rwmtx, is_freq_read); + goto done; + } + } + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + act = ethr_atomic32_read(&mtxb->flgs); + scnt--; + } + + act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs, + ETHR_RWMTX_W_FLG__, + 0); + if (act == 0) + goto done; /* Got it */ + } + + done: + if (tse) + ethr_leave_ts_event(tse); +} + +static int +mtxb_init(struct ethr_mutex_base_ *mtxb, + int def_main_scnt, + int main_scnt, + int def_aux_scnt, + int aux_scnt) +{ + ETHR_MTX_HARD_DEBUG_LFS_INIT(mtxb); +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtxb->ws = 0; +#endif + ETHR_MTX_CHK_EXCL_INIT(mtxb); + if (no_spin) { + mtxb->main_scnt = 0; + mtxb->aux_scnt = 0; + } + else { + if (main_scnt > SHRT_MAX) + mtxb->main_scnt = SHRT_MAX; + else if (main_scnt < 0) + mtxb->main_scnt = def_main_scnt; + else + mtxb->main_scnt = (short) main_scnt; + if (aux_scnt > SHRT_MAX) + mtxb->aux_scnt = SHRT_MAX; + else if (aux_scnt < 0) + mtxb->aux_scnt = def_aux_scnt; + else + mtxb->aux_scnt = (short) aux_scnt; + if (mtxb->main_scnt < mtxb->aux_scnt) + mtxb->main_scnt = mtxb->aux_scnt; + + } + mtxb->q = NULL; + ethr_atomic32_init(&mtxb->flgs, 0); + return ETHR_MTX_QLOCK_INIT(&mtxb->qlck); +} + +static int +mtxb_destroy(struct ethr_mutex_base_ *mtxb) +{ + ethr_sint32_t act; + ETHR_MTX_Q_LOCK(&mtxb->qlck); + act = ethr_atomic32_read(&mtxb->flgs); + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + if (act != 0) + return EINVAL; + return ETHR_MTX_QLOCK_DESTROY(&mtxb->qlck); +} + + +#endif /* ETHR_USE_OWN_RWMTX_IMPL__ || ETHR_USE_OWN_MTX_IMPL__ */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Mutex and condition variable implementation * +\* */ + +#ifdef ETHR_USE_OWN_MTX_IMPL__ + +/* -- Mutex ---------------------------------------------------------------- */ + +int +ethr_mutex_init_opt(ethr_mutex *mtx, ethr_mutex_opt *opt) +{ + int res; +#if ETHR_XCHK + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = ETHR_MUTEX_INITIALIZED; +#endif + ETHR_MTX_HARD_DEBUG_FENCE_INIT(mtx); + res = mtxb_init(&mtx->mtxb, + default_mtx_main_spincount, + opt ? opt->main_spincount : -1, + default_mtx_aux_spincount, + opt ? opt->aux_spincount : -1); +#if ETHR_XCHK + if (res != 0) + mtx->initialized = 0; +#endif + return res; +} + +int +ethr_mutex_init(ethr_mutex *mtx) +{ + return ethr_mutex_init_opt(mtx, NULL); +} + +int +ethr_mutex_destroy(ethr_mutex *mtx) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = 0; +#endif + return mtxb_destroy(&mtx->mtxb); +} + +void +ethr_mutex_lock_wait__(ethr_mutex *mtx, ethr_sint32_t initial) +{ + write_lock_wait(&mtx->mtxb, initial, 0, 0); +} + +void +ethr_mutex_unlock_wake__(ethr_mutex *mtx, ethr_sint32_t initial) +{ + ethr_ts_event *tse; + + ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck); + tse = mtx->mtxb.q; + + ETHR_ASSERT(tse); + ETHR_ASSERT(ethr_atomic32_read(&mtx->mtxb.flgs) + == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)); + ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__); + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + + /* + * If we have multiple waiters, there is no need to modify + * mtxb->flgs; otherwise, we need to clear the write wait bit... + */ + if (tse->next == mtx->mtxb.q) + ethr_atomic32_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__); + + wake_writer(&mtx->mtxb, 0); +} + +/* -- Condition variables -------------------------------------------------- */ + +static void +enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end) +{ + ethr_sint32_t act; + + /* + * `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx' + * is not currently locked by current thread, we almost certainly have a + * hard to debug race condition. There might however be some (strange) + * use for it. POSIX also allow a call to `pthread_cond_signal' or + * `pthread_cond_broadcast' even though the the associated mutex isn't + * locked by the caller. Therefore, we also allow this kind of strange + * usage, but optimize for the case where the mutex is locked by the + * calling thread. + */ + + ETHR_MTX_Q_LOCK(&mtx->mtxb.qlck); + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + { + int dbg_nws__ = 0; + ethr_ts_event *dbg_tse__; + for (dbg_tse__ = tse_start; + dbg_tse__ != tse_end; + dbg_tse__ = dbg_tse__->next) + dbg_nws__++; + mtx->mtxb.ws += dbg_nws__ + 1; + } +#endif + + act = ethr_atomic32_read(&mtx->mtxb.flgs); + ETHR_ASSERT(act == 0 + || act == ETHR_RWMTX_W_FLG__ + || act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)); + if (act & ETHR_RWMTX_W_FLG__) { + /* The normal sane case */ + if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) { + ETHR_ASSERT(!mtx->mtxb.q); + act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs, + (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_W_WAIT_FLG__), + ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) { + /* + * Sigh... this wasn't so sane after all since, the mutex was + * obviously not locked by the current thread.... + */ + ETHR_ASSERT(act == 0); + goto mtx_unlocked; + } + } + +#ifdef ETHR_DEBUG + if (act & ETHR_RWMTX_W_WAIT_FLG__) + ETHR_ASSERT(mtx->mtxb.q); + else + ETHR_ASSERT(!mtx->mtxb.q); +#endif + + enqueue(&mtx->mtxb.q, tse_start, tse_end); + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); + + } + else { + int multi; + mtx_unlocked: + /* Sigh... mutex isn't locked... */ + + multi = tse_start != tse_end; + + while (1) { + ethr_sint32_t new, exp = act; + + if (multi || (act & ETHR_RWMTX_W_FLG__)) + new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__; + else + new = ETHR_RWMTX_W_FLG__; + + act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs, new, exp); + if (exp == act) { + ETHR_ASSERT(!mtx->mtxb.q); + if (act & ETHR_RWMTX_W_FLG__) { + enqueue(&mtx->mtxb.q, tse_start, tse_end); + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); + + } + else { + ETHR_ASSERT(!mtx->mtxb.q); + /* + * Acquired the mutex on behalf of the + * first thread in the queue; wake + * it and enqueue the rest... + */ +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + mtx->mtxb.ws--; +#endif + if (multi) { + enqueue(&mtx->mtxb.q, tse_start->next, tse_end); + ETHR_ASSERT(mtx->mtxb.q); + } + + ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); + ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); + + ethr_atomic32_set(&tse_start->uaflgs, 0); + ethr_event_set(&tse_start->event); + } + break; + } + } + } +} + +int +ethr_cond_init_opt(ethr_cond *cnd, ethr_cond_opt *opt) +{ +#if ETHR_XCHK + if (!cnd) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = ETHR_COND_INITIALIZED; +#endif + ETHR_MTX_HARD_DEBUG_FENCE_INIT(cnd); + cnd->q = NULL; + if (no_spin) { + cnd->main_scnt = 0; + cnd->aux_scnt = 0; + } + else { + if (!opt || opt->main_spincount < 0) + cnd->main_scnt = default_cnd_main_spincount; + else if (opt->main_spincount > SHRT_MAX) + cnd->main_scnt = SHRT_MAX; + else + cnd->main_scnt = (short) opt->main_spincount; + if (!opt || opt->aux_spincount < 0) + cnd->aux_scnt = default_cnd_aux_spincount; + else if (opt->aux_spincount > SHRT_MAX) + cnd->aux_scnt = SHRT_MAX; + else + cnd->aux_scnt = (short) opt->aux_spincount; + if (cnd->main_scnt < cnd->aux_scnt) + cnd->main_scnt = cnd->aux_scnt; + } + ETHR_MTX_QLOCK_INIT(&cnd->qlck); + return 0; +} + +int +ethr_cond_init(ethr_cond *cnd) +{ + return ethr_cond_init_opt(cnd, NULL); +} + +int +ethr_cond_destroy(ethr_cond *cnd) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = 0; +#endif + return ETHR_MTX_QLOCK_DESTROY(&cnd->qlck); +} + +void +ethr_cond_signal(ethr_cond *cnd) +{ + ethr_ts_event *tse; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + + ETHR_MTX_Q_LOCK(&cnd->qlck); + + tse = cnd->q; + + if (!tse) { + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + } + else { + ethr_mutex *mtx = (ethr_mutex *) tse->udata; + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__); + + ethr_atomic32_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__); + + dequeue(&cnd->q, tse, tse); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + tse->next = tse->prev = NULL; + + enqueue_mtx(mtx, tse, tse); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + } +} + +void +ethr_cond_broadcast(ethr_cond *cnd) +{ + int got_all; + ethr_ts_event *tse; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + do { + got_all = 1; + + ETHR_MTX_Q_LOCK(&cnd->qlck); + + tse = cnd->q; + + if (!tse) { + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + } + else { + ethr_mutex *mtx = (ethr_mutex *) tse->udata; + ethr_ts_event *tse_tmp, *tse_end; + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + tse_end = cnd->q->prev; + + tse_tmp = tse; + + do { + + if (mtx == (ethr_mutex *) tse_tmp->udata) { + /* The normal case */ + + ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic32_read(&tse_tmp->uaflgs) + == ETHR_CND_WAIT_FLG__); + + ethr_atomic32_set(&tse_tmp->uaflgs, + ETHR_RWMTX_W_WAIT_FLG__); + } + else { + /* Should be very unusual */ + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + tse_end = tse_tmp->prev; + got_all = 0; + break; + } + + tse_tmp = tse_tmp->next; + + } while (tse_tmp != cnd->q); + + dequeue(&cnd->q, tse, tse_end); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + enqueue_mtx(mtx, tse, tse_end); + } + + } while (!got_all); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); +} + +int +ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) +{ + int woken; + int scnt; + void *udata = NULL; + ethr_ts_event *tse; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + tse = ethr_get_ts_event(); + + scnt = ((tse->iflgs & ETHR_TS_EV_MAIN_THR) + ? cnd->main_scnt + : cnd->aux_scnt); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + + udata = tse->udata; /* Got to restore udata before returning */ + tse->udata = (void *) mtx; + + tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */ + ethr_atomic32_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__); + + ETHR_MTX_Q_LOCK(&cnd->qlck); + + enqueue(&cnd->q, tse, tse); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + ethr_mutex_unlock(mtx); + + /* Wait */ + woken = 0; + while (1) { + ethr_sint32_t act; + + ethr_event_reset(&tse->event); + + act = ethr_atomic32_read_acqb(&tse->uaflgs); + if (!act) + break; /* Mtx locked */ + + /* First time, got EINTR, or spurious wakeup... */ + + ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__ + || act == ETHR_RWMTX_W_WAIT_FLG__); + + if (woken) { + /* + * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already been enqueued + * on the mutex; continue wait until locked... + */ + if (act == ETHR_CND_WAIT_FLG__) { + ETHR_MTX_Q_LOCK(&cnd->qlck); + act = ethr_atomic32_read(&tse->uaflgs); + ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__ + || act == ETHR_RWMTX_W_WAIT_FLG__); + /* + * If act == ETHR_RWMTX_W_WAIT_FLG__, we have already + * enqueued on the mutex; continue wait until locked... + */ + if (act == ETHR_CND_WAIT_FLG__) + dequeue(&cnd->q, tse, tse); + + ETHR_MTX_Q_UNLOCK(&cnd->qlck); + + if (act == ETHR_CND_WAIT_FLG__) { + tse->udata = udata; + ethr_leave_ts_event(tse); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ethr_mutex_lock(mtx); + return EINTR; + } + } + ETHR_ASSERT(act == ETHR_RWMTX_W_WAIT_FLG__); + } + ethr_event_swait(&tse->event, scnt); + /* swait result: 0 || EINTR */ + woken = 1; + } + + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb); + tse->udata = udata; + ethr_leave_ts_event(tse); + return 0; +} + +#else +/* -- pthread mutex and condition variables -------------------------------- */ + +int +ethr_mutex_init(ethr_mutex *mtx) +{ +#if ETHR_XCHK + if (!mtx) { + ETHR_ASSERT(0); + return EINVAL; + } + mtx->initialized = ETHR_MUTEX_INITIALIZED; +#endif + return pthread_mutex_init(&mtx->pt_mtx, NULL); +} + +int +ethr_mutex_destroy(ethr_mutex *mtx) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif +#if ETHR_XCHK + mtx->initialized = 0; +#endif + return pthread_mutex_destroy(&mtx->pt_mtx); +} + +int +ethr_cond_init(ethr_cond *cnd) +{ +#if ETHR_XCHK + if (!cnd) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = ETHR_COND_INITIALIZED; +#endif + return pthread_cond_init(&cnd->pt_cnd, NULL); +} + +int +ethr_cond_destroy(ethr_cond *cnd) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } + cnd->initialized = 0; +#endif + return pthread_cond_destroy(&cnd->pt_cnd); +} + +void +ethr_cond_signal(ethr_cond *cnd) +{ + int res; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + + res = pthread_cond_signal(&cnd->pt_cnd); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +void +ethr_cond_broadcast(ethr_cond *cnd) +{ + int res; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + + res = pthread_cond_broadcast(&cnd->pt_cnd); + if (res != 0) + ETHR_FATAL_ERROR__(res); +} + +int +ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) +{ + int res; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(cnd); + ETHR_ASSERT(cnd->initialized == ETHR_COND_INITIALIZED); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + res = pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx); + if (res != 0 && res != EINTR) + ETHR_FATAL_ERROR__(res); + return res; +} + +#endif /* pthread_mutex */ + +/* -- Exported symbols of inline functions --------------------------------- */ + +int +ethr_mutex_trylock(ethr_mutex *mtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + return ethr_mutex_trylock__(mtx); +} + +void +ethr_mutex_lock(ethr_mutex *mtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + ethr_mutex_lock__(mtx); +} + +void +ethr_mutex_unlock(ethr_mutex *mtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(mtx); + ETHR_ASSERT(mtx->initialized == ETHR_MUTEX_INITIALIZED); + + ethr_mutex_unlock__(mtx); +} + + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ + * Read/Write Mutex * +\* */ + +static void +wake_readers(ethr_rwmutex *rwmtx, int rs) +{ + ethr_ts_event *tse; +#ifdef ETHR_DEBUG + int drs = 0; +#endif + + tse = rwmtx->mtxb.q; + ETHR_ASSERT(tse); + ETHR_ASSERT(rwmtx->rq_end); + dequeue(&rwmtx->mtxb.q, tse, rwmtx->rq_end); + rwmtx->rq_end->next = NULL; + rwmtx->rq_end = NULL; + + ETHR_ASSERT(!rwmtx->mtxb.q + || (ethr_atomic32_read(&rwmtx->mtxb.q->uaflgs) + == ETHR_RWMTX_W_WAIT_FLG__)); + + ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + + while (tse) { + ethr_ts_event *tse_next; + +#ifdef ETHR_DEBUG + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) + == ETHR_RWMTX_R_WAIT_FLG__); + drs++; +#endif + + tse_next = tse->next; /* we aren't allowed to read tse->next + after we have reset uaflgs */ + + ethr_atomic32_set(&tse->uaflgs, 0); + ethr_event_set(&tse->event); + tse = tse_next; + } + + ETHR_ASSERT(rs == drs); +} + +static ETHR_INLINE int +is_w_waiter(ethr_ts_event *tse) +{ + ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__ + || tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__); + return tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__; +} + +static ETHR_INLINE int +multiple_w_waiters(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(rwmtx->mtxb.q); + ETHR_ASSERT(rwmtx->mtxb.q->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + + if (!rwmtx->rq_end) + return rwmtx->mtxb.q->next != rwmtx->mtxb.q; + else { + ETHR_ASSERT(rwmtx->mtxb.q->next != rwmtx->mtxb.q); + if (rwmtx->mtxb.q->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__) + return 1; + ETHR_ASSERT(rwmtx->rq_end->next == rwmtx->mtxb.q + || rwmtx->rq_end->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + return rwmtx->rq_end->next != rwmtx->mtxb.q; + } +} + +int check_readers_array(ethr_rwmutex *rwmtx, + int start_rix, + int length) +{ + int ix = start_rix; + + ETHR_MEMORY_BARRIER; + + do { + ethr_sint32_t act = rwmutex_freqread_rdrs_read(rwmtx, ix); + if (act != 0) + return EBUSY; + ix++; + if (ix == length) + ix = 0; + } while (ix != start_rix); + + return 0; +} + +static void +rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex *rwmtx, + ethr_ts_event *tse, + ethr_sint32_t initial) +{ + ethr_sint32_t act = initial; + + if ((act & (ETHR_RWMTX_W_FLG__| + ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) == 0) { + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) { + /* + * We *need* to try to complete the runlock. + * A writer that just enqueued (not seen by us + * in flag field) may depend on someone else + * completing the runlock. We just took over + * that responsibilty since we modified reader + * groups. + */ + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + } + } + else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_R_WAIT_FLG__) + rwmutex_transfer_read_lock(rwmtx, act, 0); + else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_W_WAIT_FLG__) + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + else { + /* + * Don't know if we got readers or writers + * first in queue; need to peek + */ + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + if (!rwmtx->mtxb.q) + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + else if (is_w_waiter(rwmtx->mtxb.q)) { + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + if ((act & ETHR_RWMTX_W_FLG__) == 0) + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + } + else { + /* + * rwmutex_transfer_read_lock() will + * unlock Q lock. + */ + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + if (act & ETHR_RWMTX_W_FLG__) + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + else + rwmutex_transfer_read_lock(rwmtx, act, 1); + } + } + } +} + +static void +rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx, + ethr_ts_event *tse) +{ + ethr_sint32_t act; + /* + * Restore failed increment + */ + act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + if (act == 0) { + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act); + } +} + +static int +rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, + ethr_sint32_t initial, + ethr_ts_event *tse, + int start_next_ix, + int check_before_try, + int try_write_lock) +{ + ethr_ts_event *tse_tmp; + ethr_sint32_t act = initial; + int six, res, length; + + ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); + + if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) + return try_write_lock ? EBUSY : 0; + + tse_tmp = tse; + if (!tse_tmp) + tse_tmp = ethr_get_ts_event(); + + if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) + goto check_waiters; + + if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { + length = reader_groups_array_size; + six = tse_tmp->rgix; + } + else { + length = main_threads_array_size; + six = tse_tmp->mtix; + } + if (start_next_ix) { + six++; + if (six >= length) + six = 0; + } + + if (!tse) + ethr_leave_ts_event(tse_tmp); + + if (check_before_try) { + res = check_readers_array(rwmtx, six, length); + + ETHR_MEMORY_BARRIER; + + if (res == EBUSY) + return try_write_lock ? EBUSY : 0; + } + + restart: + + while (1) { + ethr_sint32_t exp = act; + ethr_sint32_t new = act+1; + + ETHR_ASSERT((act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) == 0); + + ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) + < ETHR_RWMTX_R_PEND_UNLCK_MASK__); + + act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (exp == act) { + act = new; + break; + } + + if (!try_write_lock) { + if (act == 0 || (act & (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_R_ABRT_UNLCK_FLG__))) + return 0; + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if ((act & ETHR_RWMTX_R_FLG__) == 0) + return 0; + } + else if ((act & ETHR_RWMTX_R_FLG__) == 0) { + if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) + return 0; + goto check_waiters; + } + } + else { + if (act == 0) + goto tryrwlock; + if (act & (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) + return EBUSY; + } + } + + res = check_readers_array(rwmtx, six, length); + + ETHR_MEMORY_BARRIER; + + ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); + + while (1) { + int finished_abort = 0; + ethr_sint32_t exp = act; + ethr_sint32_t new = act; + + new--; + if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) { + if ((new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) == 0) { + new &= ~ETHR_RWMTX_R_ABRT_UNLCK_FLG__; + finished_abort = 1; + } + ETHR_ASSERT(act & ETHR_RWMTX_R_FLG__); + } + else if ((act & ETHR_RWMTX_R_FLG__) && res != EBUSY) { + new &= ~ETHR_RWMTX_R_FLG__; + } + + ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__); + + act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (exp == act) { + act = new; + if (act & ETHR_RWMTX_W_FLG__) + return try_write_lock ? EBUSY : 0; + if (finished_abort && (act & ETHR_RWMTX_WAIT_FLGS__)) + goto restart; + if (act & (ETHR_RWMTX_R_FLG__ + | ETHR_RWMTX_R_ABRT_UNLCK_FLG__ + | ETHR_RWMTX_R_PEND_UNLCK_MASK__)) + return try_write_lock ? EBUSY : 0; + /* Read unlock completed */ + break; + } + } + + /* + * Read unlock completed, but we have to check if + * threads have to be woken (or if we should try + * to write lock it). + */ + + if (act & ETHR_RWMTX_WAIT_FLGS__) { + check_waiters: + rwmutex_unlock_wake(rwmtx, 0, act, 0); + return try_write_lock ? EBUSY : 0; + } + + if (!try_write_lock) + return 0; + + tryrwlock: + /* Try to write lock it */ + + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, + 0); + return act == 0 ? 0 : EBUSY; +} + +#ifdef ETHR_RLOCK_WITH_INC_DEC + +static ETHR_INLINE void +rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx) +{ + ethr_sint32_t act; + /* + * Restore failed increment + */ + act = ethr_atomic32_dec_read(&rwmtx->mtxb.flgs); + if ((act & ETHR_RWMTX_WAIT_FLGS__) + && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { + rwmutex_unlock_wake(rwmtx, 0, act, 0); + } +} + +#endif + +static void +rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial) +{ + ethr_sint32_t act = initial, exp; + int scnt, start_scnt; + ethr_ts_event *tse = NULL; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + start_scnt = scnt = initial_spincount(&rwmtx->mtxb); + + /* + * Spin trying to read lock for a while. If unsuccessful, + * wait on event. + */ + + while (1) { + +#ifdef ETHR_RLOCK_WITH_INC_DEC + rwmutex_incdec_restore_failed_tryrlock(rwmtx); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); +#endif + + while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + if (scnt <= 0) { + tse = ethr_get_ts_event(); + if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { + event_wait(&rwmtx->mtxb, tse, scnt, + ETHR_RWMTX_R_WAIT_FLG__, 1, 0); + goto done; + } + } + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + scnt--; + } + exp = act; + +#ifdef ETHR_RLOCK_WITH_INC_DEC + act = ethr_atomic32_inc_read(&rwmtx->mtxb.flgs); + if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0) + goto done; /* Got it */ +#else + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) + goto done; /* Got it */ +#endif + } + + done: + if (tse) + ethr_leave_ts_event(tse); +} + +static void +rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, + ethr_ts_event *tse); + +static int +rwmutex_freqread_rlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse, int trylock) +{ + int res = 0; + ethr_sint32_t act; + + rwmutex_freqread_rdrs_inc(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + act = ethr_atomic32_read_acqb(&rwmtx->mtxb.flgs); + + if (act != ETHR_RWMTX_R_FLG__) { + int wake_other_readers; + + while (1) { + ethr_sint32_t exp, new; + + wake_other_readers = 0; + + if (act == 0) + new = act | ETHR_RWMTX_R_FLG__; + else if (act == ETHR_RWMTX_R_FLG__) + break; /* Got it */ + else if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); + if (trylock) + res = EBUSY; + else + rwmutex_freqread_rlock_wait(rwmtx, tse); + break; + } + else if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) { + if ((act & ETHR_RWMTX_R_FLG__) == 0) + ETHR_FATAL_ERROR__(EFAULT); + /* + * An aborted runlock, not write locked, and no write + * waiters, i.e., we got it... + */ + if (act & ETHR_RWMTX_R_WAIT_FLG__) + wake_other_readers = 1; + break; + } + else { + new = act | ETHR_RWMTX_R_FLG__; + if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) { + /* + * Someone is doing tryrwlock (no writer and no + * write waiters); we will try to abort that... + */ + new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__; + } + + if (act & ETHR_RWMTX_R_WAIT_FLG__) + wake_other_readers = 1; + } + + exp = act; + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + } + + if (wake_other_readers) + rwmutex_transfer_read_lock(rwmtx, act, 0); + } + + return res; +} + +static void +rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, + ethr_ts_event *tse) +{ + ethr_sint32_t act; + int scnt, start_scnt; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + start_scnt = scnt = initial_spincount(&rwmtx->mtxb); + + /* + * Spin trying to read lock for a while. If unsuccessful, + * wait on event. + */ + + while (1) { + + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + + while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + if (scnt <= 0) { + if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { + event_wait(&rwmtx->mtxb, tse, scnt, + ETHR_RWMTX_R_WAIT_FLG__, 1, 1); + return; /* Got it */ + } + } + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); + } + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + scnt--; + } + + if (rwmutex_freqread_rlock(rwmtx, tse, 1) != EBUSY) + break; /* Got it */ + } +} + +static void +rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial) +{ + write_lock_wait(&rwmtx->mtxb, initial, 1, 0); +} + +static void +rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial) +{ + write_lock_wait(&rwmtx->mtxb, initial, 1, 1); +} + +static ETHR_INLINE void +rwlock_wake_set_flags(ethr_rwmutex *rwmtx, + ethr_sint32_t new_initial, + ethr_sint32_t act_initial) +{ + ethr_sint32_t act, act_mask; + int chk_abrt_flg; + + ETHR_MEMORY_BARRIER; + + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) { + /* r pend unlock mask may vary and must be retained */ + act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__; + if (new_initial & ETHR_RWMTX_R_FLG__) + chk_abrt_flg = 1; + else + chk_abrt_flg = 0; + } + else { +#ifdef ETHR_RLOCK_WITH_INC_DEC + /* rs mask may vary and must be retained */ + act_mask = ETHR_RWMTX_RS_MASK__; + chk_abrt_flg = 0; +#else + /* rs mask always zero */ + ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0); + ethr_atomic32_set(&rwmtx->mtxb.flgs, new_initial); + return; +#endif + } + + act = act_initial; + while (1) { + ethr_sint32_t exp = act; + ethr_sint32_t new = new_initial + (act & act_mask); + if (chk_abrt_flg && (act & act_mask)) + new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__; + act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + exp = act; + } +} + +#ifdef ETHR_DEBUG + +static void +dbg_unlock_wake(ethr_rwmutex *rwmtx, + int have_w, + ethr_ts_event *tse) +{ + ethr_sint32_t exp, act, imask; + + exp = have_w ? ETHR_RWMTX_W_FLG__ : 0; + + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) + imask = ETHR_RWMTX_R_PEND_UNLCK_MASK__; + else { +#ifdef ETHR_RLOCK_WITH_INC_DEC + imask = ETHR_RWMTX_RS_MASK__; +#else + imask = 0; +#endif + } + + ETHR_ASSERT(tse); + + if (is_w_waiter(tse)) { + + exp |= ETHR_RWMTX_W_WAIT_FLG__; + if (rwmtx->rq_end) { + exp |= ETHR_RWMTX_R_WAIT_FLG__; + } + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((exp & ~imask) == (act & ~imask)); + + ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); + + } + else { + + exp |= ETHR_RWMTX_R_WAIT_FLG__; + if (rwmtx->rq_end->next != rwmtx->mtxb.q) + exp |= ETHR_RWMTX_W_WAIT_FLG__; + else if (exp == ETHR_RWMTX_R_WAIT_FLG__) { + if (!have_w) { + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) + imask |= ETHR_RWMTX_R_FLG__; + else + imask |= ETHR_RWMTX_RS_MASK__; + } + } + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((exp & ~imask) == (act & ~imask)); + + ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); + + } +} + +#endif + +static void +rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx, + ethr_sint32_t initial, + int q_locked) +{ + ethr_sint32_t act = initial; + + if (!q_locked) { + ethr_ts_event *tse; + ETHR_ASSERT(initial & ETHR_RWMTX_R_WAIT_FLG__); + ETHR_ASSERT((initial & ETHR_RWMTX_W_FLG__) == 0); + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + tse = rwmtx->mtxb.q; + if ((act & ETHR_RWMTX_W_FLG__) || !tse || is_w_waiter(tse)) { + /* Someone else woke the readers up... */ + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + return; + } + } + + rwmutex_unlock_wake(rwmtx, 0, initial, 1); +} + +static void +rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, ethr_sint32_t initial, + int transfer_read_lock) +{ + ethr_sint32_t new, act = initial; + ethr_ts_event *tse; + + if (transfer_read_lock) { + /* + * - Q already locked + * - Got R waiters first in Q + * - Not W locked + */ + tse = rwmtx->mtxb.q; + + ETHR_ASSERT(act & ETHR_RWMTX_R_WAIT_FLG__); + ETHR_ASSERT((act & (ETHR_RWMTX_W_FLG__)) == 0); + ETHR_ASSERT(tse && !is_w_waiter(tse)); + } + else { + + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if (!have_w) + return; + else { + while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + ethr_sint32_t exp = act; + new = exp & ~ETHR_RWMTX_W_FLG__; + act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + return; + } + } + } + + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + tse = rwmtx->mtxb.q; + + if (!have_w) { + if (!tse) { +#ifdef ETHR_DEBUG + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); +#endif + goto already_served; + } + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + if (act == (ETHR_RWMTX_R_WAIT_FLG__|ETHR_RWMTX_R_FLG__)) { + ETHR_ASSERT(tse && !is_w_waiter(tse)); + } + else if (act & ~ETHR_RWMTX_WAIT_FLGS__) { + already_served: + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + return; + } + } + } + +#ifdef ETHR_DEBUG + dbg_unlock_wake(rwmtx, have_w, tse); +#endif + + if (is_w_waiter(tse)) { + + if (!have_w) { + act = ethr_atomic32_read_bor(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__); + ETHR_ASSERT((act & ~(ETHR_RWMTX_WAIT_FLGS__ + | (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL + ? 0 + : ETHR_RWMTX_R_PEND_UNLCK_MASK__))) == 0); + ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__); + act |= ETHR_RWMTX_W_FLG__; + } + + /* + * If we have multiple write waiters, there + * is no need to modify mtxb->flgs; otherwise, + * we need to clear the write wait bit... + */ + if (!multiple_w_waiters(rwmtx)) { + new = ETHR_RWMTX_W_FLG__; + if (tse->next != rwmtx->mtxb.q) { + ETHR_ASSERT(tse->next->uflgs == ETHR_RWMTX_R_WAIT_FLG__); + new |= ETHR_RWMTX_R_WAIT_FLG__; + } + + rwlock_wake_set_flags(rwmtx, new, act); + } + + wake_writer(&rwmtx->mtxb, 1); + } + else { + int rs; + + if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) { + rs = rwmtx->tdata.rs; + new = (ethr_sint32_t) rs; + rwmtx->tdata.rs = 0; + } + else { + ethr_rwmutex_type type = rwmtx->type; + int length = (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ + ? reader_groups_array_size + : main_threads_array_size); + int ix; + + rs = 0; + for (ix = 0; ix < length; ix++) { + int wrs = rwmtx->tdata.ra[ix].data.waiting_readers; + rwmtx->tdata.ra[ix].data.waiting_readers = 0; + ETHR_ASSERT(wrs >= 0); + if (wrs) { + rs += wrs; + rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs); + } + } + + new = ETHR_RWMTX_R_FLG__; + } + + if (rwmtx->rq_end->next != rwmtx->mtxb.q) + new |= ETHR_RWMTX_W_WAIT_FLG__; + + rwlock_wake_set_flags(rwmtx, new, act); + + wake_readers(rwmtx, rs); + } +} + +static ethr_rwmtx_readers_array__ * +alloc_readers_array(int length, ethr_rwmutex_lived lived) +{ + ethr_rwmtx_readers_array__ *ra; + size_t sz; + void *mem; + + sz = sizeof(ethr_rwmtx_readers_array__) * (length + 1); + + switch (lived) { + case ETHR_RWMUTEX_LONG_LIVED: + mem = ethr_mem__.ll.alloc(sz); + break; + case ETHR_RWMUTEX_SHORT_LIVED: + mem = ethr_mem__.sl.alloc(sz); + break; + default: + mem = ethr_mem__.std.alloc(sz); + break; + } + if (!mem) + return NULL; + + if ((((ethr_uint_t) mem) & ETHR_CACHE_LINE_MASK) == 0) { + ra = (ethr_rwmtx_readers_array__ *) mem; + ra->data.byte_offset = 0; + } + else { + ra = ((ethr_rwmtx_readers_array__ *) + ((((ethr_uint_t) mem) & ~ETHR_CACHE_LINE_MASK) + + ETHR_CACHE_LINE_SIZE)); + ra->data.byte_offset = (int) ((ethr_uint_t) ra + - (ethr_uint_t) mem); + } + ra->data.lived = lived; + return ra; +} + +static void +free_readers_array(ethr_rwmtx_readers_array__ *ra) +{ + void *ptr = (void *) (((char *) ra) - ra->data.byte_offset); + switch (ra->data.lived) { + case ETHR_RWMUTEX_LONG_LIVED: + ethr_mem__.ll.free(ptr); + break; + case ETHR_RWMUTEX_SHORT_LIVED: + ethr_mem__.sl.free(ptr); + break; + default: + ethr_mem__.std.free(ptr); + break; + } +} + +int +ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt) +{ + int res; + ethr_rwmtx_readers_array__ *ra = NULL; +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!rwmtx) { + ETHR_ASSERT(0); + return EINVAL; + } + rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; +#endif + ETHR_MTX_HARD_DEBUG_FENCE_INIT(rwmtx); + rwmtx->rq_end = NULL; + rwmtx->type = opt ? opt->type : ETHR_RWMUTEX_TYPE_NORMAL; + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + if (main_threads_array_size <= reader_groups_array_size) { + /* No point using reader groups... */ + rwmtx->type = ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ; + } + /* Fall through */ + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + int length; + + length = (rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ + ? main_threads_array_size + : reader_groups_array_size); + + if (length == 1) { + /* No point using a frequent reader type... */ + rwmtx->type = ETHR_RWMUTEX_TYPE_NORMAL; + } + else { + int ix; + ra = alloc_readers_array(length, + (opt + ? opt->lived + : ETHR_RWMUTEX_UNKNOWN_LIVED)); + if (!ra) { + res = ENOMEM; + goto error; + } + + rwmtx->tdata.ra = ra; + + for (ix = 0; ix < length; ix++) { + ethr_atomic32_init(&rwmtx->tdata.ra[ix].data.readers, 0); + rwmtx->tdata.ra[ix].data.waiting_readers = 0; + } + break; + } + } + case ETHR_RWMUTEX_TYPE_NORMAL: + rwmtx->tdata.rs = 0; + break; + default: + res = EINVAL; + goto error; + } + res = mtxb_init(&rwmtx->mtxb, + default_rwmtx_main_spincount, + opt ? opt->main_spincount : -1, + default_rwmtx_aux_spincount, + opt ? opt->aux_spincount : -1); + if (res == 0) + return 0; + + error: + + if (ra) + free_readers_array(ra); + +#if ETHR_XCHK + rwmtx->initialized = 0; +#endif + return res; +} + +int +ethr_rwmutex_init(ethr_rwmutex *rwmtx) +{ + return ethr_rwmutex_init_opt(rwmtx, NULL); +} + +int +ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) +{ + int res; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) { + ethr_sint32_t act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + if (act == ETHR_RWMTX_R_FLG__) + rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0); + } + res = mtxb_destroy(&rwmtx->mtxb); + if (res != 0) + return res; + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) + free_readers_array(rwmtx->tdata.ra); +#if ETHR_XCHK + rwmtx->initialized = 0; +#endif + return 0; +} + +#define ETHR_MAX_TRYRLOCK_TRIES 5 + +int +ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) +{ + int res = 0; + ethr_sint32_t act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: { +#ifdef ETHR_RLOCK_WITH_INC_DEC + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) + res = EBUSY; + else { + act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs); + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + rwmutex_incdec_restore_failed_tryrlock(rwmtx); + res = EBUSY; + } + } +#else + ethr_sint32_t exp = 0; + int tries = 0; + + while (1) { + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) { + res = 0; + break; + } + if (tries > ETHR_MAX_TRYRLOCK_TRIES + || (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__))) { + res = EBUSY; + break; + } + tries++; + exp = act; + } +#endif + break; + } + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + ethr_ts_event *tse = ethr_get_ts_event(); + res = rwmutex_freqread_rlock(rwmtx, tse, 1); + ethr_leave_ts_event(tse); + break; + } + } + +#ifdef ETHR_MTX_CHK_EXCL + if (res == 0) { + ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb); + } +#endif + + ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + return res; +} + +void +ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) +{ + ethr_sint32_t act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: { +#ifdef ETHR_RLOCK_WITH_INC_DEC + act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs); + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) + rwmutex_normal_rlock_wait(rwmtx, act); +#else + ethr_sint32_t exp = 0; + + while (1) { + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) + break; + + if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + rwmutex_normal_rlock_wait(rwmtx, act); + break; + } + exp = act; + } +#endif + break; + } + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + ethr_ts_event *tse = ethr_get_ts_event(); + rwmutex_freqread_rlock(rwmtx, tse, 0); + ethr_leave_ts_event(tse); + break; + } + } + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb); + ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); +} + +void +ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) +{ + ethr_sint32_t act; + + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(&rwmtx->mtxb); + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb); + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic32_dec_read_relb(&rwmtx->mtxb.flgs); + if ((act & ETHR_RWMTX_WAIT_FLGS__) + && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { + ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); + rwmutex_unlock_wake(rwmtx, 0, act, 0); + } + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { + ethr_ts_event *tse = ethr_get_ts_event(); + + act = rwmutex_freqread_rdrs_dec_read_relb(rwmtx, tse); + + ETHR_ASSERT(act >= 0); + + ETHR_MEMORY_BARRIER; + + if (act == 0) { + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + if (act != ETHR_RWMTX_R_FLG__) + rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act); + } + + ethr_leave_ts_event(tse); + break; + } + } + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); +} + +int +ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) +{ + int res = 0; + ethr_sint32_t act; + + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + if (act != 0) + res = EBUSY; + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + + res = 0; + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + + do { + + if (act == 0) + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + else if (act == ETHR_RWMTX_R_FLG__) { + res = rwmutex_try_complete_runlock(rwmtx, act, NULL, + 0, 1, 1); + break; + } + else { + res = EBUSY; + break; + } + + } while (act != 0); + + break; + } + +#ifdef ETHR_MTX_CHK_EXCL + if (res == 0) { + ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb); + } +#endif + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + return res; +} + +void +ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) +{ + ethr_sint32_t act; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + if (act != 0) + rwmutex_normal_rwlock_wait(rwmtx, act); + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + + do { + + if (act != 0) { + rwmutex_freqread_rwlock_wait(rwmtx, act); + break; + } + + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + + } while (act != 0); + + break; + } + + ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb); + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + +} + +void +ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) +{ + ethr_sint32_t act; + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb); + + ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_UNSET_EXCL(&rwmtx->mtxb); + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + + switch (rwmtx->type) { + case ETHR_RWMUTEX_TYPE_NORMAL: + act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs, + 0, ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) + rwmutex_unlock_wake(rwmtx, 1, act, 0); + break; + + case ETHR_RWMUTEX_TYPE_FREQUENT_READ: + case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: + act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs, 0, + ETHR_RWMTX_W_FLG__); + if (act != ETHR_RWMTX_W_FLG__) + rwmutex_unlock_wake(rwmtx, 1, act, 0); + break; + } + + ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); +} + +#else +/* -- pthread read/write mutex --------------------------------------------- */ + +int +ethr_rwmutex_init(ethr_rwmutex *rwmtx) +{ +#if ETHR_XCHK + if (!rwmtx) { + ETHR_ASSERT(0); + return EINVAL; + } + rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; +#endif + return pthread_rwlock_init(&rwmtx->pt_rwlock, write_pref_attr); +} + +int +ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt) +{ + return ethr_rwmutex_init(rwmtx); +} + +int +ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) +{ + int res; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + res = pthread_rwlock_destroy(&rwmtx->pt_rwlock); +#if ETHR_XCHK + rwmtx->initialized = 0; +#endif + return res; +} + +/* -- Exported symbols of inline functions --------------------------------- */ + +int +ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + return ethr_rwmutex_tryrlock__(rwmtx); +} + +void +ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ethr_rwmutex_rlock__(rwmtx); +} + +void +ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ethr_rwmutex_runlock__(rwmtx); +} + +int +ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + return ethr_rwmutex_tryrwlock__(rwmtx); +} + +void +ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + return ethr_rwmutex_rwlock__(rwmtx); +} + +void +ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(rwmtx); + ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); + + ethr_rwmutex_rwunlock__(rwmtx); +} + +#endif /* pthread */ + + +#if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) + +#ifdef ETHR_MTX_HARD_DEBUG_Q +static void +hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx) +{ + int res; + ethr_sint32_t flgs = ethr_atomic32_read(&mtxb->flgs); + + ETHR_MTX_HARD_ASSERT(res == 0); + + ETHR_MTX_HARD_ASSERT(!(flgs & ETHR_RWMTX_R_WAIT_FLG__) || is_rwmtx); + + if (!(flgs & ETHR_RWMTX_WAIT_FLGS__)) { + ETHR_MTX_HARD_ASSERT(!mtxb->q); + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end); + ETHR_MTX_HARD_ASSERT(!rwmtx->rs); + } + } + else { + ethr_ts_event *tse; + int ws = 0, rs = 0, rsf = 0, ref = 0; + + ETHR_MTX_HARD_ASSERT(mtxb->q); + + tse = mtxb->q; + + do { + ethr_sint32_t type; + + ETHR_MTX_HARD_ASSERT(tse->next->prev == tse); + ETHR_MTX_HARD_ASSERT(tse->prev->next == tse); + + type = ethr_atomic32_read(&tse->uaflgs); + ETHR_MTX_HARD_ASSERT(type == tse->uflgs); + switch (type) { + case ETHR_RWMTX_W_WAIT_FLG__: + ws++; + break; + case ETHR_RWMTX_R_WAIT_FLG__: { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(is_rwmtx); + if (!rsf) + rsf = 1; + ETHR_MTX_HARD_ASSERT(!ref); + if (rwmtx->rq_end == tse) { + ETHR_MTX_HARD_ASSERT( + tse->next == rwmtx->mtxb.q + || tse->next->uflgs == ETHR_RWMTX_W_WAIT_FLG__); + ref = 1; + } + rs++; + break; + } + default: + ETHR_MTX_HARD_ASSERT(! "invalid wait type found"); + } + + tse = tse->next; + } while (tse != mtxb->q); + + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(rs == rwmtx->rs); + } + +#ifdef ETHR_MTX_HARD_DEBUG_WSQ + ETHR_MTX_HARD_ASSERT(ws == mtxb->ws); +#endif + + if (flgs & ETHR_RWMTX_W_WAIT_FLG__) + ETHR_MTX_HARD_ASSERT(ws); + else + ETHR_MTX_HARD_ASSERT(!ws); + + if (flgs & ETHR_RWMTX_R_WAIT_FLG__) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(is_rwmtx); + ETHR_MTX_HARD_ASSERT(rwmtx->rq_end); + ETHR_MTX_HARD_ASSERT(rsf); + ETHR_MTX_HARD_ASSERT(ref); + ETHR_MTX_HARD_ASSERT(rs); + } + else { + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(!rwmtx->rq_end); + } + ETHR_MTX_HARD_ASSERT(!rsf); + ETHR_MTX_HARD_ASSERT(!ref); + ETHR_MTX_HARD_ASSERT(!rs); + } + } +} + +#elif defined(ETHR_MTX_HARD_DEBUG_WSQ) + +static void +hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx) +{ + int ws = 0; + int rs = 0; + + if (mtxb->q) { + ethr_ts_event *tse = mtxb->q; + do { + switch (tse->uflgs) { + case ETHR_RWMTX_W_WAIT_FLG__: + ws++; + break; + case ETHR_RWMTX_R_WAIT_FLG__: + rs++; + break; + default: + ETHR_MTX_HARD_ASSERT(0); + break; + } + tse = tse->next; + } while (tse != mtxb->q); + } + + ETHR_MTX_HARD_ASSERT(mtxb->ws == ws); + if (is_rwmtx) { + ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + ETHR_MTX_HARD_ASSERT(rwmtx->rs == rs); + } +} + +#endif + +#endif diff --git a/erts/lib_src/common/ethread.c b/erts/lib_src/common/ethread.c deleted file mode 100644 index eb4d0cad20..0000000000 --- a/erts/lib_src/common/ethread.c +++ /dev/null @@ -1,3346 +0,0 @@ -/* - * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. - * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. - * - * %CopyrightEnd% - */ - -/* - * Description: A Thread library for use in the ERTS and other OTP - * applications. - * Author: Rickard Green - */ - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#undef ETHR_STACK_GUARD_SIZE - -#if defined(ETHR_PTHREADS) - -#ifdef ETHR_TIME_WITH_SYS_TIME -# include <time.h> -# include <sys/time.h> -#else -# ifdef ETHR_HAVE_SYS_TIME_H -# include <sys/time.h> -# else -# include <time.h> -# endif -#endif -#include <sys/types.h> -#include <unistd.h> -#include <signal.h> - -#ifdef ETHR_HAVE_PTHREAD_ATTR_SETGUARDSIZE -# define ETHR_STACK_GUARD_SIZE (pagesize) -#endif - -#elif defined(ETHR_WIN32_THREADS) - -#undef WIN32_LEAN_AND_MEAN -#define WIN32_LEAN_AND_MEAN -#include <windows.h> -#include <process.h> -#include <winerror.h> - -#else -#error "Missing thread implementation" -#endif - -#include <limits.h> - -#define ETHR_FORCE_INLINE_FUNCS -#define ETHR_INLINE_FUNC_NAME_(X) X ## __ -#include "ethread.h" - -#ifndef ETHR_HAVE_ETHREAD_DEFINES -#error Missing configure defines -#endif - -/* - * ---------------------------------------------------------------------------- - * Common stuff - * ---------------------------------------------------------------------------- - */ - -#define ETHR_MAX_THREADS 2048 /* Has to be an even power of 2 */ - -static int ethr_not_inited = 1; - -#define ASSERT(A) ETHR_ASSERT((A)) - -static void *(*allocp)(size_t) = malloc; -static void *(*reallocp)(void *, size_t) = realloc; -static void (*freep)(void *) = free; - -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; -#endif - -void *(*thread_create_prepare_func)(void) = NULL; -void (*thread_create_parent_func)(void *) = NULL; -void (*thread_create_child_func)(void *) = NULL; - -typedef struct ethr_xhndl_list_ ethr_xhndl_list; -struct ethr_xhndl_list_ { - ethr_xhndl_list *next; - void (*funcp)(void); -}; - -static size_t pagesize; -#define ETHR_PAGE_ALIGN(SZ) (((((size_t) (SZ)) - 1)/pagesize + 1)*pagesize) -static size_t min_stack_size; /* kilo words */ -static size_t max_stack_size; /* kilo words */ -#define ETHR_B2KW(B) ((((size_t) (B)) - 1)/(sizeof(void *)*1024) + 1) -#define ETHR_KW2B(KW) (((size_t) (KW))*sizeof(void *)*1024) - -ethr_mutex xhndl_mtx; -ethr_xhndl_list *xhndl_list; - -static int -init_common(ethr_init_data *id) -{ - int res; - if (id) { - allocp = id->alloc; - reallocp = id->realloc; - freep = id->free; - thread_create_prepare_func = id->thread_create_prepare_func; - thread_create_parent_func = id->thread_create_parent_func; - thread_create_child_func = id->thread_create_child_func; - } - if (!allocp || !reallocp || !freep) - return EINVAL; - -#ifdef _SC_PAGESIZE - pagesize = (size_t) sysconf(_SC_PAGESIZE); -#elif defined(HAVE_GETPAGESIZE) - pagesize = (size_t) getpagesize(); -#else - pagesize = (size_t) 4*1024; /* Guess 4 KB */ -#endif - - /* User needs at least 4 KB */ - min_stack_size = 4*1024; -#if SIZEOF_VOID_P == 8 - /* Double that on 64-bit archs */ - min_stack_size *= 2; -#endif - /* On some systems as much as about 4 KB is used by the system */ - min_stack_size += 4*1024; - /* There should be room for signal handlers */ -#ifdef SIGSTKSZ - min_stack_size += SIGSTKSZ; -#else - min_stack_size += pagesize; -#endif - /* The system may think that we need more stack */ -#if defined(PTHREAD_STACK_MIN) - if (min_stack_size < PTHREAD_STACK_MIN) - min_stack_size = PTHREAD_STACK_MIN; -#elif defined(_SC_THREAD_STACK_MIN) - { - size_t thr_min_stk_sz = (size_t) sysconf(_SC_THREAD_STACK_MIN); - if (min_stack_size < thr_min_stk_sz) - min_stack_size = thr_min_stk_sz; - } -#endif - /* The guard is at least on some platforms included in the stack size - passed when creating threads */ -#ifdef ETHR_STACK_GUARD_SIZE - min_stack_size += ETHR_STACK_GUARD_SIZE; -#endif - min_stack_size = ETHR_PAGE_ALIGN(min_stack_size); - - min_stack_size = ETHR_B2KW(min_stack_size); - - max_stack_size = 32*1024*1024; -#if SIZEOF_VOID_P == 8 - max_stack_size *= 2; -#endif - max_stack_size = ETHR_B2KW(max_stack_size); - - xhndl_list = NULL; - - res = ethr_mutex_init(&xhndl_mtx); - if (res != 0) - return res; - - res = ethr_mutex_set_forksafe(&xhndl_mtx); - if (res != 0 && res != ENOTSUP) - return res; - - return 0; -} - -int -ethr_install_exit_handler(void (*funcp)(void)) -{ - ethr_xhndl_list *xhp; - int res; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - - if (!funcp) - return EINVAL; - - xhp = (ethr_xhndl_list *) (*allocp)(sizeof(ethr_xhndl_list)); - if (!xhp) - return ENOMEM; - - res = ethr_mutex_lock__(&xhndl_mtx); - if (res != 0) { - (*freep)((void *) xhp); - return res; - } - - xhp->funcp = funcp; - xhp->next = xhndl_list; - xhndl_list = xhp; - - res = ethr_mutex_unlock__(&xhndl_mtx); - if (res != 0) - abort(); - - return res; -} - -static void -run_exit_handlers(void) -{ - int res; - ethr_xhndl_list *xhp; - - res = ethr_mutex_lock__(&xhndl_mtx); - if (res != 0) - abort(); - - xhp = xhndl_list; - - res = ethr_mutex_unlock__(&xhndl_mtx); - if (res != 0) - abort(); - - for (; xhp; xhp = xhp->next) - (*xhp->funcp)(); -} - -#if defined(ETHR_PTHREADS) -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * pthread implementation * -\* */ - -typedef struct { - pthread_mutex_t mtx; - pthread_cond_t cnd; - int initialized; - void *(*thr_func)(void *); - void *arg; - void *prep_func_res; -} thr_wrap_data_; - -static int no_ethreads; -static ethr_mutex no_ethrs_mtx; - -#ifndef ETHR_HAVE_PTHREAD_ATFORK -#define ETHR_HAVE_PTHREAD_ATFORK 0 -#endif - -#if !ETHR_HAVE_PTHREAD_ATFORK -#warning "Cannot enforce fork-safety" -#endif - -/* - * ---------------------------------------------------------------------------- - * Static functions - * ---------------------------------------------------------------------------- - */ - -/* - * Functions with safe_ prefix aborts on failure. To be used when - * we cannot recover after failure. - */ - -static ETHR_INLINE void -safe_mutex_lock(pthread_mutex_t *mtxp) -{ - int res = pthread_mutex_lock(mtxp); - if (res != 0) - abort(); -} - -static ETHR_INLINE void -safe_mutex_unlock(pthread_mutex_t *mtxp) -{ - int res = pthread_mutex_unlock(mtxp); - if (res != 0) - abort(); -} - -static ETHR_INLINE void -safe_cond_signal(pthread_cond_t *cndp) -{ - int res = pthread_cond_signal(cndp); - if (res != 0) - abort(); -} - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -static volatile int rec_mtx_attr_need_init = 1; -static pthread_mutexattr_t rec_mtx_attr; - -static int init_rec_mtx_attr(void); - -#endif - -#if ETHR_HAVE_PTHREAD_ATFORK - -static ethr_mutex forksafe_mtx = ETHR_MUTEX_INITER; - -static void lock_mutexes(void) -{ - ethr_mutex *m = &forksafe_mtx; - do { - - safe_mutex_lock(&m->pt_mtx); - - m = m->next; - - } while (m != &forksafe_mtx); -} - -static void unlock_mutexes(void) -{ - ethr_mutex *m = forksafe_mtx.prev; - do { - - safe_mutex_unlock(&m->pt_mtx); - - m = m->prev; - - } while (m->next != &forksafe_mtx); -} - -#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK - -static void reinit_mutexes(void) -{ - ethr_mutex *m = forksafe_mtx.prev; - do { - pthread_mutexattr_t *attrp = NULL; - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - if (m->is_rec_mtx) { - if (rec_mtx_attr_need_init) { - int res = init_rec_mtx_attr(); - if (res != 0) - abort(); - } - attrp = &rec_mtx_attr; - } -#endif - if (pthread_mutex_init(&m->pt_mtx, attrp) != 0) - abort(); - - m = m->prev; - - } while (m->next != &forksafe_mtx); -} - -#endif - -static int -init_forksafe(void) -{ - static int init_done = 0; - int res = 0; - - if (init_done) - return res; - - forksafe_mtx.prev = &forksafe_mtx; - forksafe_mtx.next = &forksafe_mtx; - - res = pthread_atfork(lock_mutexes, - unlock_mutexes, -#if ETHR_INIT_MUTEX_IN_CHILD_AT_FORK - reinit_mutexes -#else - unlock_mutexes -#endif - ); - - init_done = 1; - return res; -} - -#endif - - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -#if defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETTYPE) - -#define SET_REC_MUTEX_ATTR(AP) \ - pthread_mutexattr_settype((AP), PTHREAD_MUTEX_RECURSIVE); - -#elif defined(ETHR_HAVE_PTHREAD_MUTEXATTR_SETKIND_NP) - -#define SET_REC_MUTEX_ATTR(AP) \ - pthread_mutexattr_setkind_np((AP), PTHREAD_MUTEX_RECURSIVE_NP); - -#else - -#error "Don't know how to set recursive mutex attributes" - -#endif - -static int -init_rec_mtx_attr(void) -{ - int res, mres; - static pthread_mutex_t attrinit_mtx = PTHREAD_MUTEX_INITIALIZER; - - mres = pthread_mutex_lock(&attrinit_mtx); - if (mres != 0) - return mres; - /* Got here under race conditions; check again ... */ - if (!rec_mtx_attr_need_init) - res = 0; - else { - res = pthread_mutexattr_init(&rec_mtx_attr); - if (res == 0) { - res = SET_REC_MUTEX_ATTR(&rec_mtx_attr); - if (res == 0) - rec_mtx_attr_need_init = 0; - else - (void) pthread_mutexattr_destroy(&rec_mtx_attr); - } - } - - mres = pthread_mutex_unlock(&attrinit_mtx); - if (mres != 0) - return mres; - return res; -} - -#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */ - -static ETHR_INLINE void thr_exit_cleanup(void) -{ - run_exit_handlers(); - safe_mutex_lock(&no_ethrs_mtx.pt_mtx); - ASSERT(no_ethreads > 0); - no_ethreads--; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); -} - -static void *thr_wrapper(void *vtwd) -{ - void *res; - thr_wrap_data_ *twd = (thr_wrap_data_ *) vtwd; - void *(*thr_func)(void *) = twd->thr_func; - void *arg = twd->arg; - - safe_mutex_lock(&twd->mtx); - - if (thread_create_child_func) - (*thread_create_child_func)(twd->prep_func_res); - - twd->initialized = 1; - - safe_cond_signal(&twd->cnd); - safe_mutex_unlock(&twd->mtx); - - res = (*thr_func)(arg); - thr_exit_cleanup(); - return res; -} - - -/* - * ---------------------------------------------------------------------------- - * Exported functions - * ---------------------------------------------------------------------------- - */ - -int -ethr_init(ethr_init_data *id) -{ - int res; - - if (!ethr_not_inited) - return EINVAL; - - ethr_not_inited = 0; - - res = init_common(id); - if (res != 0) - goto error; - -#if ETHR_HAVE_PTHREAD_ATFORK - init_forksafe(); -#endif - - no_ethreads = 1; - res = ethr_mutex_init(&no_ethrs_mtx); - if (res != 0) - goto error; - res = ethr_mutex_set_forksafe(&no_ethrs_mtx); - if (res != 0 && res != ENOTSUP) - goto error; - -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS - { - int i; - for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { -#ifdef ETHR_HAVE_PTHREAD_SPIN_LOCK - res = pthread_spin_init(ðr_atomic_protection__[i].u.spnlck, 0); -#else - res = ethr_mutex_init(ðr_atomic_protection__[i].u.mtx); -#endif - if (res != 0) - goto error; - } - } -#endif - - return 0; - - error: - ethr_not_inited = 1; - return res; - -} - -int -ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, - ethr_thr_opts *opts) -{ - thr_wrap_data_ twd; - pthread_attr_t attr; - int res, dres; - int use_stack_size = (opts && opts->suggested_stack_size >= 0 - ? opts->suggested_stack_size - : -1 /* Use system default */); - -#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE - if (use_stack_size < 0) - use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; -#endif - - twd.initialized = 0; - twd.thr_func = func; - twd.arg = arg; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!tid || !func) { - ASSERT(0); - return EINVAL; - } -#endif - - /* Call prepare func if it exist */ - if (thread_create_prepare_func) - twd.prep_func_res = (*thread_create_prepare_func)(); - else - twd.prep_func_res = NULL; - - /* Set som thread attributes */ - res = pthread_attr_init(&attr); - if (res != 0) - goto cleanup_parent_func; - res = pthread_mutex_init(&twd.mtx, NULL); - if (res != 0) - goto cleanup_attr_destroy; - res = pthread_cond_init(&twd.cnd, NULL); - if (res != 0) - goto cleanup_mutex_destroy; - - /* Schedule child thread in system scope (if possible) ... */ - res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); - if (res != 0 && res != ENOTSUP) - goto cleanup_cond_destroy; - - if (use_stack_size >= 0) { - size_t suggested_stack_size = (size_t) use_stack_size; - size_t stack_size; -#ifdef DEBUG - suggested_stack_size /= 2; /* Make sure we got margin */ -#endif -#ifdef ETHR_STACK_GUARD_SIZE - /* The guard is at least on some platforms included in the stack size - passed when creating threads */ - suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE); -#endif - if (suggested_stack_size < min_stack_size) - stack_size = ETHR_KW2B(min_stack_size); - else if (suggested_stack_size > max_stack_size) - stack_size = ETHR_KW2B(max_stack_size); - else - stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); - (void) pthread_attr_setstacksize(&attr, stack_size); - } - -#ifdef ETHR_STACK_GUARD_SIZE - (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE); -#endif - - /* Detached or joinable... */ - res = pthread_attr_setdetachstate(&attr, - (opts && opts->detached - ? PTHREAD_CREATE_DETACHED - : PTHREAD_CREATE_JOINABLE)); - if (res != 0) - goto cleanup_cond_destroy; - - res = pthread_mutex_lock(&twd.mtx); - - if (res != 0) - goto cleanup_cond_destroy; - - safe_mutex_lock(&no_ethrs_mtx.pt_mtx); - if (no_ethreads < ETHR_MAX_THREADS) { - no_ethreads++; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); - } - else { - res = EAGAIN; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); - goto cleanup_mutex_unlock; - } - - res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void *) &twd); - - if (res != 0) { - safe_mutex_lock(&no_ethrs_mtx.pt_mtx); - ASSERT(no_ethreads > 0); - no_ethreads--; - safe_mutex_unlock(&no_ethrs_mtx.pt_mtx); - } - else { - - /* Wait for child to initialize... */ - while (!twd.initialized) { - res = pthread_cond_wait(&twd.cnd, &twd.mtx); - if (res != 0 && res != EINTR) - break; - } - - } - - /* Cleanup... */ - cleanup_mutex_unlock: - dres = pthread_mutex_unlock(&twd.mtx); - if (res == 0) - res = dres; - cleanup_cond_destroy: - dres = pthread_cond_destroy(&twd.cnd); - if (res == 0) - res = dres; - cleanup_mutex_destroy: - dres = pthread_mutex_destroy(&twd.mtx); - if (res == 0) - res = dres; - cleanup_attr_destroy: - dres = pthread_attr_destroy(&attr); - if (res == 0) - res = dres; - cleanup_parent_func: - if (thread_create_parent_func) - (*thread_create_parent_func)(twd.prep_func_res); - - return res; -} - -int -ethr_thr_join(ethr_tid tid, void **res) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_join((pthread_t) tid, res); -} - -int -ethr_thr_detach(ethr_tid tid) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_detach((pthread_t) tid); -} - -void -ethr_thr_exit(void *res) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return; - } -#endif - thr_exit_cleanup(); - pthread_exit(res); -} - -ethr_tid -ethr_self(void) -{ - return (ethr_tid) pthread_self(); -} - -int -ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) -{ - return pthread_equal((pthread_t) tid1, (pthread_t) tid2); -} - - -/* - * Mutex functions - */ - - -int -ethr_mutex_init(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx) { - ASSERT(0); - return EINVAL; - } - mtx->initialized = ETHR_MUTEX_INITIALIZED; -#endif - mtx->prev = NULL; - mtx->next = NULL; - mtx->is_rec_mtx = 0; - return pthread_mutex_init(&mtx->pt_mtx, NULL); -} - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -int -ethr_rec_mutex_init(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx) { - ASSERT(0); - return EINVAL; - } - mtx->initialized = ETHR_MUTEX_INITIALIZED; -#endif - if (rec_mtx_attr_need_init) - init_rec_mtx_attr(); - - mtx->prev = NULL; - mtx->next = NULL; - mtx->is_rec_mtx = 1; - return pthread_mutex_init(&mtx->pt_mtx, &rec_mtx_attr); -} - -#endif /* #if ETHR_HAVE_ETHR_REC_MUTEX_INIT */ - -int -ethr_mutex_destroy(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - if (mtx->next) { - ASSERT(mtx->prev); - ethr_mutex_unset_forksafe(mtx); - } -#if ETHR_XCHK - mtx->initialized = 0; -#endif - return pthread_mutex_destroy(&mtx->pt_mtx); -} - -int ethr_mutex_set_forksafe(ethr_mutex *mtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif -#if ETHR_HAVE_PTHREAD_ATFORK - res = pthread_mutex_lock(&forksafe_mtx.pt_mtx); - if (res != 0) - return res; - if (!forksafe_mtx.next) { - ASSERT(!forksafe_mtx.prev); - init_forksafe(); - } - if (mtx->next) { - /* forksafe already set for this mutex */ - ASSERT(mtx->prev); - } - else { - mtx->next = forksafe_mtx.next; - mtx->prev = &forksafe_mtx; - forksafe_mtx.next->prev = mtx; - forksafe_mtx.next = mtx; - } - - res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx); - -#else /* #if ETHR_HAVE_PTHREAD_ATFORK */ - res = ENOTSUP; -#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */ - return res; -} - -int ethr_mutex_unset_forksafe(ethr_mutex *mtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif -#if ETHR_HAVE_PTHREAD_ATFORK - res = pthread_mutex_lock(&forksafe_mtx.pt_mtx); - if (res != 0) - return res; - if (!forksafe_mtx.next) { - ASSERT(!forksafe_mtx.prev); - init_forksafe(); - } - if (!mtx->next) { - /* forksafe already unset for this mutex */ - ASSERT(!mtx->prev); - } - else { - mtx->prev->next = mtx->next; - mtx->next->prev = mtx->prev; - mtx->next = NULL; - mtx->prev = NULL; - } - res = pthread_mutex_unlock(&forksafe_mtx.pt_mtx); - -#else /* #if ETHR_HAVE_PTHREAD_ATFORK */ - res = ENOTSUP; -#endif /* #if ETHR_HAVE_PTHREAD_ATFORK */ - return res; -} - -int -ethr_mutex_trylock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_trylock__(mtx); -} - -int -ethr_mutex_lock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_lock__(mtx); -} - -int -ethr_mutex_unlock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_unlock__(mtx); -} - -/* - * Condition variable functions - */ - -int -ethr_cond_init(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd) { - ASSERT(0); - return EINVAL; - } - cnd->initialized = ETHR_COND_INITIALIZED; -#endif - return pthread_cond_init(&cnd->pt_cnd, NULL); -} - -int -ethr_cond_destroy(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { - ASSERT(0); - return EINVAL; - } - cnd->initialized = 0; -#endif - return pthread_cond_destroy(&cnd->pt_cnd); -} - -int -ethr_cond_signal(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_cond_signal(&cnd->pt_cnd); -} - -int -ethr_cond_broadcast(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd || cnd->initialized != ETHR_COND_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_cond_broadcast(&cnd->pt_cnd); -} - -int -ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || cnd->initialized != ETHR_COND_INITIALIZED - || !mtx - || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_cond_wait(&cnd->pt_cnd, &mtx->pt_mtx); -} - -int -ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout) -{ - struct timespec to; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || cnd->initialized != ETHR_COND_INITIALIZED - || !mtx - || mtx->initialized != ETHR_MUTEX_INITIALIZED - || !timeout) { - ASSERT(0); - return EINVAL; - } -#endif - - to.tv_sec = timeout->tv_sec; - to.tv_nsec = timeout->tv_nsec; - - return pthread_cond_timedwait(&cnd->pt_cnd, &mtx->pt_mtx, &to); -} - - -#ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT - -int -ethr_rwmutex_init(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx) { - ASSERT(0); - return EINVAL; - } - rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; -#endif - return pthread_rwlock_init(&rwmtx->pt_rwlock, NULL); -} - -int -ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = pthread_rwlock_destroy(&rwmtx->pt_rwlock); -#if ETHR_XCHK - rwmtx->initialized = 0; -#endif - return res; -} - -int -ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_tryrlock__(rwmtx); -} - -int -ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_rlock__(rwmtx); -} - -int -ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_runlock__(rwmtx); -} - -int -ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_tryrwlock__(rwmtx); -} - -int -ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_rwlock__(rwmtx); -} - -int -ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwmutex_rwunlock__(rwmtx); -} - -#endif /* #ifdef ETHR_HAVE_PTHREAD_RWLOCK_INIT */ - -/* - * Current time - */ - -int -ethr_time_now(ethr_timeval *time) -{ - int res; - struct timeval tv; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!time) { - ASSERT(0); - return EINVAL; - } -#endif - - res = gettimeofday(&tv, NULL); - time->tv_sec = (long) tv.tv_sec; - time->tv_nsec = ((long) tv.tv_usec)*1000; - return res; -} - -/* - * Thread specific data - */ - -int -ethr_tsd_key_create(ethr_tsd_key *keyp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!keyp) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_key_create((pthread_key_t *) keyp, NULL); -} - -int -ethr_tsd_key_delete(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_key_delete((pthread_key_t) key); -} - -int -ethr_tsd_set(ethr_tsd_key key, void *value) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return pthread_setspecific((pthread_key_t) key, value); -} - -void * -ethr_tsd_get(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return NULL; - } -#endif - return pthread_getspecific((pthread_key_t) key); -} - -/* - * Signal functions - */ - -#if ETHR_HAVE_ETHR_SIG_FUNCS - -int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!set && !oset) { - ASSERT(0); - return EINVAL; - } -#endif - return pthread_sigmask(how, set, oset); -} - -int ethr_sigwait(const sigset_t *set, int *sig) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!set || !sig) { - ASSERT(0); - return EINVAL; - } -#endif - if (sigwait(set, sig) < 0) - return errno; - return 0; -} - -#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */ - -#elif defined(ETHR_WIN32_THREADS) -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ - * Native win32 threads implementation * -\* */ - -#define INVALID_TID -1 - -/* The spin count values are more or less taken out of the blue */ -#define ETHR_MUTEX_SPIN_COUNT 5000 -#define ETHR_COND_SPIN_COUNT 1000 - -ethr_tid serial_shift; /* Bits to shift serial when constructing a tid */ -ethr_tid last_serial; /* Last thread table serial used */ -ethr_tid last_ix; /* Last thread table index used */ -ethr_tid thr_ix_mask; /* Mask used to mask out thread table index from a tid */ - -/* Event used for conditional variables. On per thread. */ -/*typedef struct cnd_wait_event__ cnd_wait_event_;*/ -struct cnd_wait_event__ { - HANDLE handle; - cnd_wait_event_ *prev; - cnd_wait_event_ *next; - int in_queue; -}; - -/* Thread specific data. Stored in the thread table */ -typedef struct { - ethr_tid thr_id; - HANDLE thr_handle; - ethr_tid joiner; - void *result; - cnd_wait_event_ wait_event; -} thr_data_; - -/* Argument passed to thr_wrapper() */ -typedef struct { - void * (*func)(void *); - void * arg; - thr_data_ *ptd; - thr_data_ *td; - int res; - void *prep_func_res; -} thr_wrap_data_; - - -static CRITICAL_SECTION thr_table_cs; /* Critical section used to protect - the thread table from concurrent - accesses. */ -static CRITICAL_SECTION fake_static_init_cs; /* Critical section used to protect - initialazition of 'statically - initialized' mutexes */ -static thr_data_ * thr_table[ETHR_MAX_THREADS]; /* The thread table */ - -static DWORD tls_own_thr_data; - -static thr_data_ main_thr_data; - -#define THR_IX(TID) ((TID) & thr_ix_mask) -#define OWN_THR_DATA ((thr_data_ *) TlsGetValue(tls_own_thr_data)) - -/* - * ---------------------------------------------------------------------------- - * Static functions - * ---------------------------------------------------------------------------- - */ - -static int -get_errno(void) -{ - switch (GetLastError()) { - case ERROR_INVALID_FUNCTION: return EINVAL; /* 1 */ - case ERROR_FILE_NOT_FOUND: return ENOENT; /* 2 */ - case ERROR_PATH_NOT_FOUND: return ENOENT; /* 3 */ - case ERROR_TOO_MANY_OPEN_FILES: return EMFILE; /* 4 */ - case ERROR_ACCESS_DENIED: return EACCES; /* 5 */ - case ERROR_INVALID_HANDLE: return EBADF; /* 6 */ - case ERROR_ARENA_TRASHED: return ENOMEM; /* 7 */ - case ERROR_NOT_ENOUGH_MEMORY: return ENOMEM; /* 8 */ - case ERROR_INVALID_BLOCK: return ENOMEM; /* 9 */ - case ERROR_BAD_ENVIRONMENT: return E2BIG; /* 10 */ - case ERROR_BAD_FORMAT: return ENOEXEC; /* 11 */ - case ERROR_INVALID_ACCESS: return EINVAL; /* 12 */ - case ERROR_INVALID_DATA: return EINVAL; /* 13 */ - case ERROR_OUTOFMEMORY: return ENOMEM; /* 14 */ - case ERROR_INVALID_DRIVE: return ENOENT; /* 15 */ - case ERROR_CURRENT_DIRECTORY: return EACCES; /* 16 */ - case ERROR_NOT_SAME_DEVICE: return EXDEV; /* 17 */ - case ERROR_NO_MORE_FILES: return ENOENT; /* 18 */ - case ERROR_WRITE_PROTECT: return EACCES; /* 19 */ - case ERROR_BAD_UNIT: return EACCES; /* 20 */ - case ERROR_NOT_READY: return EACCES; /* 21 */ - case ERROR_BAD_COMMAND: return EACCES; /* 22 */ - case ERROR_CRC: return EACCES; /* 23 */ - case ERROR_BAD_LENGTH: return EACCES; /* 24 */ - case ERROR_SEEK: return EACCES; /* 25 */ - case ERROR_NOT_DOS_DISK: return EACCES; /* 26 */ - case ERROR_SECTOR_NOT_FOUND: return EACCES; /* 27 */ - case ERROR_OUT_OF_PAPER: return EACCES; /* 28 */ - case ERROR_WRITE_FAULT: return EACCES; /* 29 */ - case ERROR_READ_FAULT: return EACCES; /* 30 */ - case ERROR_GEN_FAILURE: return EACCES; /* 31 */ - case ERROR_SHARING_VIOLATION: return EACCES; /* 32 */ - case ERROR_LOCK_VIOLATION: return EACCES; /* 33 */ - case ERROR_WRONG_DISK: return EACCES; /* 34 */ - case ERROR_SHARING_BUFFER_EXCEEDED: return EACCES; /* 36 */ - case ERROR_BAD_NETPATH: return ENOENT; /* 53 */ - case ERROR_NETWORK_ACCESS_DENIED: return EACCES; /* 65 */ - case ERROR_BAD_NET_NAME: return ENOENT; /* 67 */ - case ERROR_FILE_EXISTS: return EEXIST; /* 80 */ - case ERROR_CANNOT_MAKE: return EACCES; /* 82 */ - case ERROR_FAIL_I24: return EACCES; /* 83 */ - case ERROR_INVALID_PARAMETER: return EINVAL; /* 87 */ - case ERROR_NO_PROC_SLOTS: return EAGAIN; /* 89 */ - case ERROR_DRIVE_LOCKED: return EACCES; /* 108 */ - case ERROR_BROKEN_PIPE: return EPIPE; /* 109 */ - case ERROR_DISK_FULL: return ENOSPC; /* 112 */ - case ERROR_INVALID_TARGET_HANDLE: return EBADF; /* 114 */ - case ERROR_WAIT_NO_CHILDREN: return ECHILD; /* 128 */ - case ERROR_CHILD_NOT_COMPLETE: return ECHILD; /* 129 */ - case ERROR_DIRECT_ACCESS_HANDLE: return EBADF; /* 130 */ - case ERROR_NEGATIVE_SEEK: return EINVAL; /* 131 */ - case ERROR_SEEK_ON_DEVICE: return EACCES; /* 132 */ - case ERROR_DIR_NOT_EMPTY: return ENOTEMPTY;/* 145 */ - case ERROR_NOT_LOCKED: return EACCES; /* 158 */ - case ERROR_BAD_PATHNAME: return ENOENT; /* 161 */ - case ERROR_MAX_THRDS_REACHED: return EAGAIN; /* 164 */ - case ERROR_LOCK_FAILED: return EACCES; /* 167 */ - case ERROR_ALREADY_EXISTS: return EEXIST; /* 183 */ - case ERROR_INVALID_STARTING_CODESEG: return ENOEXEC; /* 188 */ - case ERROR_INVALID_STACKSEG: return ENOEXEC; /* 189 */ - case ERROR_INVALID_MODULETYPE: return ENOEXEC; /* 190 */ - case ERROR_INVALID_EXE_SIGNATURE: return ENOEXEC; /* 191 */ - case ERROR_EXE_MARKED_INVALID: return ENOEXEC; /* 192 */ - case ERROR_BAD_EXE_FORMAT: return ENOEXEC; /* 193 */ - case ERROR_ITERATED_DATA_EXCEEDS_64k: return ENOEXEC; /* 194 */ - case ERROR_INVALID_MINALLOCSIZE: return ENOEXEC; /* 195 */ - case ERROR_DYNLINK_FROM_INVALID_RING: return ENOEXEC; /* 196 */ - case ERROR_IOPL_NOT_ENABLED: return ENOEXEC; /* 197 */ - case ERROR_INVALID_SEGDPL: return ENOEXEC; /* 198 */ - case ERROR_AUTODATASEG_EXCEEDS_64k: return ENOEXEC; /* 199 */ - case ERROR_RING2SEG_MUST_BE_MOVABLE: return ENOEXEC; /* 200 */ - case ERROR_RELOC_CHAIN_XEEDS_SEGLIM: return ENOEXEC; /* 201 */ - case ERROR_INFLOOP_IN_RELOC_CHAIN: return ENOEXEC; /* 202 */ - case ERROR_FILENAME_EXCED_RANGE: return ENOENT; /* 206 */ - case ERROR_NESTING_NOT_ALLOWED: return EAGAIN; /* 215 */ - case ERROR_NOT_ENOUGH_QUOTA: return ENOMEM; /* 1816 */ - default: return EINVAL; - } -} - -static ETHR_INLINE thr_data_ * -tid2thr(ethr_tid tid) -{ - ethr_tid ix; - thr_data_ *td; - - if (tid < 0) - return NULL; - ix = THR_IX(tid); - if (ix >= ETHR_MAX_THREADS) - return NULL; - td = thr_table[ix]; - if (!td) - return NULL; - if (td->thr_id != tid) - return NULL; - return td; -} - -static ETHR_INLINE void -new_tid(ethr_tid *new_tid, ethr_tid *new_serial, ethr_tid *new_ix) -{ - ethr_tid tmp_serial = last_serial; - ethr_tid tmp_ix = last_ix + 1; - ethr_tid start_ix = tmp_ix; - - - do { - if (tmp_ix >= ETHR_MAX_THREADS) { - tmp_serial++; - if ((tmp_serial << serial_shift) < 0) - tmp_serial = 0; - tmp_ix = 0; - } - if (!thr_table[tmp_ix]) { - *new_tid = (tmp_serial << serial_shift) | tmp_ix; - *new_serial = tmp_serial; - *new_ix = tmp_ix; - return; - } - tmp_ix++; - } while (tmp_ix != start_ix); - - *new_tid = INVALID_TID; - *new_serial = INVALID_TID; - *new_ix = INVALID_TID; - -} - - -static void thr_exit_cleanup(thr_data_ *td, void *res) -{ - - ASSERT(td == OWN_THR_DATA); - - run_exit_handlers(); - - EnterCriticalSection(&thr_table_cs); - CloseHandle(td->wait_event.handle); - if (td->thr_handle == INVALID_HANDLE_VALUE) { - /* We are detached; cleanup thread table */ - ASSERT(td->joiner == INVALID_TID); - ASSERT(td == thr_table[THR_IX(td->thr_id)]); - thr_table[THR_IX(td->thr_id)] = NULL; - if (td != &main_thr_data) - (*freep)((void *) td); - } - else { - /* Save result and let joining thread cleanup */ - td->result = res; - } - LeaveCriticalSection(&thr_table_cs); -} - -static unsigned __stdcall thr_wrapper(LPVOID args) -{ - void *(*func)(void*) = ((thr_wrap_data_ *) args)->func; - void *arg = ((thr_wrap_data_ *) args)->arg; - thr_data_ *td = ((thr_wrap_data_ *) args)->td; - - td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL); - if (td->wait_event.handle == INVALID_HANDLE_VALUE - || !TlsSetValue(tls_own_thr_data, (LPVOID) td)) { - ((thr_wrap_data_ *) args)->res = get_errno(); - if (td->wait_event.handle != INVALID_HANDLE_VALUE) - CloseHandle(td->wait_event.handle); - SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle); - _endthreadex((unsigned) 0); - ASSERT(0); - } - - td->wait_event.prev = NULL; - td->wait_event.next = NULL; - td->wait_event.in_queue = 0; - - if (thread_create_child_func) - (*thread_create_child_func)(((thr_wrap_data_ *) args)->prep_func_res); - - ASSERT(td == OWN_THR_DATA); - - ((thr_wrap_data_ *) args)->res = 0; - SetEvent(((thr_wrap_data_ *) args)->ptd->wait_event.handle); - - thr_exit_cleanup(td, (*func)(arg)); - return 0; -} - -int -ethr_fake_static_mutex_init(ethr_mutex *mtx) -{ - EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - /* Got here under race conditions; check again... */ - if (!mtx->initialized) { - if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, - ETHR_MUTEX_SPIN_COUNT)) - return get_errno(); - mtx->initialized = ETHR_MUTEX_INITIALIZED; - } - LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - return 0; -} - -static int -fake_static_cond_init(ethr_cond *cnd) -{ - EnterCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - /* Got here under race conditions; check again... */ - if (!cnd->initialized) { - if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, - ETHR_COND_SPIN_COUNT)) - return get_errno(); - cnd->queue = NULL; - cnd->queue_end = NULL; - cnd->initialized = ETHR_COND_INITIALIZED; - } - LeaveCriticalSection((CRITICAL_SECTION *) &fake_static_init_cs); - return 0; -} - -#ifdef __GNUC__ -#define LL_LITERAL(X) X##LL -#else -#define LL_LITERAL(X) X##i64 -#endif - -#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600) - -static ETHR_INLINE void -get_curr_time(long *sec, long *nsec) -{ - SYSTEMTIME t; - FILETIME ft; - LONGLONG lft; - - GetSystemTime(&t); - SystemTimeToFileTime(&t, &ft); - memcpy(&lft, &ft, sizeof(lft)); - *nsec = ((long) (lft % LL_LITERAL(10000000)))*100; - *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF); -} - -static cnd_wait_event_ *cwe_freelist; -static CRITICAL_SECTION cwe_cs; - -static int -alloc_cwe(cnd_wait_event_ **cwe_res) -{ - cnd_wait_event_ *cwe; - EnterCriticalSection(&cwe_cs); - cwe = cwe_freelist; - if (cwe) { - cwe_freelist = cwe->next; - LeaveCriticalSection(&cwe_cs); - } - else { - LeaveCriticalSection(&cwe_cs); - cwe = (*allocp)(sizeof(cnd_wait_event_)); - if (!cwe) - return ENOMEM; - cwe->handle = CreateEvent(NULL, FALSE, FALSE, NULL); - if (cwe->handle == INVALID_HANDLE_VALUE) { - int res = get_errno(); - (*freep)(cwe); - return res; - } - } - *cwe_res = cwe; - return 0; -} - -static -free_cwe(cnd_wait_event_ *cwe) -{ - EnterCriticalSection(&cwe_cs); - cwe->next = cwe_freelist; - cwe_freelist = cwe; - LeaveCriticalSection(&cwe_cs); -} - -static ETHR_INLINE int -condwait(ethr_cond *cnd, - ethr_mutex *mtx, - int with_timeout, - ethr_timeval *timeout) -{ - int res; - thr_data_ *td; - cnd_wait_event_ *cwe; - DWORD code; - long time; /* time until timeout in milli seconds */ - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - - if (!mtx - || mtx->initialized != ETHR_MUTEX_INITIALIZED - || !cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED) - || (with_timeout && !timeout)) { - ASSERT(0); - return EINVAL; - } -#endif - - td = OWN_THR_DATA; - if (td) - cwe = &td->wait_event; - else { /* A non-ethread thread */ - res = alloc_cwe(&cwe); - if (res != 0) - return res; - } - - if (!cnd->initialized) - fake_static_cond_init(cnd); - EnterCriticalSection(&cnd->cs); - - ASSERT(!cwe->in_queue); - if (cnd->queue_end) { - ASSERT(cnd->queue); - cwe->prev = cnd->queue_end; - cwe->next = NULL; - cnd->queue_end->next = cwe; - cnd->queue_end = cwe; - } - else { - ASSERT(!cnd->queue); - cwe->prev = NULL; - cwe->next = NULL; - cnd->queue = cwe; - cnd->queue_end = cwe; - } - cwe->in_queue = 1; - - LeaveCriticalSection(&cnd->cs); - - LeaveCriticalSection(&mtx->cs); - - if (!with_timeout) - time = INFINITE; - else { - long sec, nsec; - ASSERT(timeout); - get_curr_time(&sec, &nsec); - time = (timeout->tv_sec - sec)*1000; - time += (timeout->tv_nsec - nsec + 500)/1000000; - if (time < 0) - time = 0; - } - - /* wait for event to signal */ - code = WaitForSingleObject(cwe->handle, time); - - EnterCriticalSection(&mtx->cs); - - if (code == WAIT_OBJECT_0) { - /* We were woken by a signal or a broadcast ... */ - res = 0; - - /* ... no need to remove event from wait queue since this was - taken care of by the signal or broadcast */ -#ifdef DEBUG - EnterCriticalSection(&cnd->cs); - ASSERT(!cwe->in_queue); - LeaveCriticalSection(&cnd->cs); -#endif - - } - else { - /* We timed out... */ - res = ETIMEDOUT; - - /* ... probably have to remove event from wait queue ... */ - EnterCriticalSection(&cnd->cs); - - if (cwe->in_queue) { /* ... but we must check that we are in queue - since a signal or broadcast after timeout - may have removed us from the queue */ - if (cwe->prev) { - cwe->prev->next = cwe->next; - } - else { - ASSERT(cnd->queue == cwe); - cnd->queue = cwe->next; - } - - if (cwe->next) { - cwe->next->prev = cwe->prev; - } - else { - ASSERT(cnd->queue_end == cwe); - cnd->queue_end = cwe->prev; - } - cwe->in_queue = 0; - } - - LeaveCriticalSection(&cnd->cs); - - } - - if (!td) - free_cwe(cwe); - - return res; - -} - - -/* - * ---------------------------------------------------------------------------- - * Exported functions - * ---------------------------------------------------------------------------- - */ - -int -ethr_init(ethr_init_data *id) -{ -#ifdef _WIN32_WINNT - DWORD major = (_WIN32_WINNT >> 8) & 0xff; - DWORD minor = _WIN32_WINNT & 0xff; - OSVERSIONINFO os_version; -#endif - int err = 0; - thr_data_ *td = &main_thr_data; - unsigned long i; - - if (!ethr_not_inited) - return EINVAL; - -#ifdef _WIN32_WINNT - os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); - GetVersionEx(&os_version); - if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT - || os_version.dwMajorVersion < major - || (os_version.dwMajorVersion == major - && os_version.dwMinorVersion < minor)) - return ENOTSUP; -#endif - - ASSERT(ETHR_MAX_THREADS > 0); - for (i = ETHR_MAX_THREADS - 1, serial_shift = 0; - i; - serial_shift++, i >>= 1); - thr_ix_mask = ~(~((ethr_tid) 0) << serial_shift); - - tls_own_thr_data = TlsAlloc(); - if (tls_own_thr_data == TLS_OUT_OF_INDEXES) - goto error; - - last_serial = 0; - last_ix = 0; - - td->thr_id = 0; - td->thr_handle = GetCurrentThread(); - td->joiner = INVALID_TID; - td->result = NULL; - td->wait_event.handle = CreateEvent(NULL, FALSE, FALSE, NULL); - if (td->wait_event.handle == INVALID_HANDLE_VALUE) - goto error; - td->wait_event.prev = NULL; - td->wait_event.next = NULL; - td->wait_event.in_queue = 0; - thr_table[0] = td; - - if (!TlsSetValue(tls_own_thr_data, (LPVOID) td)) - goto error; - - ASSERT(td == OWN_THR_DATA); - - - cwe_freelist = NULL; - if (!InitializeCriticalSectionAndSpinCount(&cwe_cs, - ETHR_MUTEX_SPIN_COUNT)) - goto error; - - for (i = 1; i < ETHR_MAX_THREADS; i++) - thr_table[i] = NULL; - - if (!InitializeCriticalSectionAndSpinCount(&thr_table_cs, - ETHR_MUTEX_SPIN_COUNT)) - goto error; - if (!InitializeCriticalSectionAndSpinCount(&fake_static_init_cs, - ETHR_MUTEX_SPIN_COUNT)) - goto error; - ethr_not_inited = 0; - - err = init_common(id); - if (err) - goto error; - - return 0; - - error: - ethr_not_inited = 1; - if (err == 0) - err = get_errno(); - ASSERT(err != 0); - if (td->thr_handle != INVALID_HANDLE_VALUE) - CloseHandle(td->thr_handle); - if (td->wait_event.handle != INVALID_HANDLE_VALUE) - CloseHandle(td->wait_event.handle); - return err; -} - -/* - * Thread functions. - */ - -int -ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, - ethr_thr_opts *opts) -{ - int err = 0; - thr_wrap_data_ twd; - thr_data_ *my_td, *child_td = NULL; - ethr_tid child_tid, child_serial, child_ix; - DWORD code; - unsigned ID; - unsigned stack_size = 0; /* 0 = system default */ - int use_stack_size = (opts && opts->suggested_stack_size >= 0 - ? opts->suggested_stack_size - : -1 /* Use system default */); - -#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE - if (use_stack_size < 0) - use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; -#endif - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!tid || !func) { - ASSERT(0); - return EINVAL; - } -#endif - - my_td = OWN_THR_DATA; - if (!my_td) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return EACCES; - } - - if (use_stack_size >= 0) { - size_t suggested_stack_size = (size_t) use_stack_size; -#ifdef DEBUG - suggested_stack_size /= 2; /* Make sure we got margin */ -#endif - if (suggested_stack_size < min_stack_size) - stack_size = (unsigned) ETHR_KW2B(min_stack_size); - else if (suggested_stack_size > max_stack_size) - stack_size = (unsigned) ETHR_KW2B(max_stack_size); - else - stack_size = - (unsigned) ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); - } - - EnterCriticalSection(&thr_table_cs); - - /* Call prepare func if it exist */ - if (thread_create_prepare_func) - twd.prep_func_res = (*thread_create_prepare_func)(); - else - twd.prep_func_res = NULL; - - /* Find a new thread id to use */ - new_tid(&child_tid, &child_serial, &child_ix); - if (child_tid == INVALID_TID) { - err = EAGAIN; - goto error; - } - - ASSERT(child_ix == THR_IX(child_tid)); - - *tid = child_tid; - - ASSERT(!thr_table[child_ix]); - - /* Alloc thread data */ - thr_table[child_ix] = child_td = (thr_data_ *) (*allocp)(sizeof(thr_data_)); - if (!child_td) { - err = ENOMEM; - goto error; - } - - /* Init thread data */ - - child_td->thr_id = child_tid; - child_td->thr_handle = INVALID_HANDLE_VALUE; - child_td->joiner = INVALID_TID; - child_td->result = NULL; - /* 'child_td->wait_event' is initialized by child thread */ - - - /* Init thread wrapper data */ - - twd.func = func; - twd.arg = arg; - twd.ptd = my_td; - twd.td = child_td; - twd.res = 0; - - ASSERT(!my_td->wait_event.in_queue); - - /* spawn the thr_wrapper function */ - child_td->thr_handle = (HANDLE) _beginthreadex(NULL, - stack_size, - thr_wrapper, - (LPVOID) &twd, - 0, - &ID); - if (child_td->thr_handle == (HANDLE) 0) { - child_td->thr_handle = INVALID_HANDLE_VALUE; - goto error; - } - - ASSERT(child_td->thr_handle != INVALID_HANDLE_VALUE); - - /* Wait for child to finish initialization */ - code = WaitForSingleObject(my_td->wait_event.handle, INFINITE); - if (twd.res || code != WAIT_OBJECT_0) { - err = twd.res; - goto error; - } - - if (opts && opts->detached) { - CloseHandle(child_td->thr_handle); - child_td->thr_handle = INVALID_HANDLE_VALUE; - } - - last_serial = child_serial; - last_ix = child_ix; - - ASSERT(thr_table[child_ix] == child_td); - - if (thread_create_parent_func) - (*thread_create_parent_func)(twd.prep_func_res); - - LeaveCriticalSection(&thr_table_cs); - - return 0; - - error: - - if (err == 0) - err = get_errno(); - ASSERT(err != 0); - - if (thread_create_parent_func) - (*thread_create_parent_func)(twd.prep_func_res); - - if (child_ix != INVALID_TID) { - - if (child_td) { - ASSERT(thr_table[child_ix] == child_td); - - if (child_td->thr_handle != INVALID_HANDLE_VALUE) { - WaitForSingleObject(child_td->thr_handle, INFINITE); - CloseHandle(child_td->thr_handle); - } - - (*freep)((void *) child_td); - thr_table[child_ix] = NULL; - } - } - - *tid = INVALID_TID; - - LeaveCriticalSection(&thr_table_cs); - return err; -} - -int ethr_thr_join(ethr_tid tid, void **res) -{ - int err = 0; - DWORD code; - thr_data_ *td; - thr_data_ *my_td; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - - my_td = OWN_THR_DATA; - - if (!my_td) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return EACCES; - } - - EnterCriticalSection(&thr_table_cs); - - td = tid2thr(tid); - if (!td) - err = ESRCH; - else if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */ - || td->joiner != INVALID_TID) /* i.e. someone else is joining */ - err = EINVAL; - else if (my_td == td) - err = EDEADLK; - else - td->joiner = my_td->thr_id; - - LeaveCriticalSection(&thr_table_cs); - - if (err) - goto error; - - /* Wait for thread to terminate */ - code = WaitForSingleObject(td->thr_handle, INFINITE); - if (code != WAIT_OBJECT_0) - goto error; - - EnterCriticalSection(&thr_table_cs); - - ASSERT(td == tid2thr(tid)); - ASSERT(td->thr_handle != INVALID_HANDLE_VALUE); - ASSERT(td->joiner == my_td->thr_id); - - if (res) - *res = td->result; - - CloseHandle(td->thr_handle); - ASSERT(td == thr_table[THR_IX(td->thr_id)]); - thr_table[THR_IX(td->thr_id)] = NULL; - if (td != &main_thr_data) - (*freep)((void *) td); - - LeaveCriticalSection(&thr_table_cs); - - return 0; - - error: - if (err == 0) - err = get_errno(); - ASSERT(err != 0); - return err; -} - - -int -ethr_thr_detach(ethr_tid tid) -{ - int res; - DWORD code; - thr_data_ *td; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - - if (!OWN_THR_DATA) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return EACCES; - } - - EnterCriticalSection(&thr_table_cs); - - td = tid2thr(tid); - if (!td) - res = ESRCH; - if (td->thr_handle == INVALID_HANDLE_VALUE /* i.e. detached */ - || td->joiner != INVALID_TID) /* i.e. someone is joining */ - res = EINVAL; - else { - res = 0; - CloseHandle(td->thr_handle); - td->thr_handle = INVALID_HANDLE_VALUE; - } - - LeaveCriticalSection(&thr_table_cs); - - return res; -} - - -void -ethr_thr_exit(void *res) -{ - thr_data_ *td; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return; - } -#endif - td = OWN_THR_DATA; - if (!td) { - /* Only ethreads are allowed to call this function */ - ASSERT(0); - return; - } - thr_exit_cleanup(td, res); - _endthreadex((unsigned) 0); -} - -ethr_tid -ethr_self(void) -{ - thr_data_ *td; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return INVALID_TID; - } -#endif - /* It is okay for non-ethreads (i.e. native win32 threads) to call - ethr_self(). They will however be returned the INVALID_TID. */ - td = OWN_THR_DATA; - if (!td) - return INVALID_TID; - return td->thr_id; -} - -int -ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) -{ - /* INVALID_TID does not equal any tid, not even the INVALID_TID */ - return tid1 == tid2 && tid1 != INVALID_TID; -} - -/* - * Mutex functions. - */ - -int -ethr_mutex_init(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx) { - ASSERT(0); - return EINVAL; - } -#endif - if (!InitializeCriticalSectionAndSpinCount(&mtx->cs, ETHR_MUTEX_SPIN_COUNT)) - return get_errno(); - mtx->initialized = ETHR_MUTEX_INITIALIZED; -#if ETHR_XCHK - mtx->is_rec_mtx = 0; -#endif - return 0; -} - -int -ethr_rec_mutex_init(ethr_mutex *mtx) -{ - int res; - res = ethr_mutex_init(mtx); -#if ETHR_XCHK - mtx->is_rec_mtx = 1; -#endif - return res; -} - -int -ethr_mutex_destroy(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - DeleteCriticalSection(&mtx->cs); - mtx->initialized = 0; - return 0; -} - -int ethr_mutex_set_forksafe(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return 0; /* No fork() */ -} - -int ethr_mutex_unset_forksafe(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - return 0; /* No fork() */ -} - -int -ethr_mutex_trylock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx - || (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - if (!mtx->initialized) { - int res = ethr_fake_static_mutex_init(mtx); - if (res != 0) - return res; - } - return ethr_mutex_trylock__(mtx); -} - -int -ethr_mutex_lock(ethr_mutex *mtx) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx - || (mtx->initialized && mtx->initialized != ETHR_MUTEX_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_lock__(mtx); -} - -int -ethr_mutex_unlock(ethr_mutex *mtx) -{ -#if ETHR_XCHK - int res; - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!mtx || mtx->initialized != ETHR_MUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_mutex_unlock__(mtx); -} - -/* - * Condition variable functions. - */ - -int -ethr_cond_init(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd) { - ASSERT(0); - return EINVAL; - } -#endif - if (!InitializeCriticalSectionAndSpinCount(&cnd->cs, ETHR_COND_SPIN_COUNT)) - return get_errno(); - cnd->queue = NULL; - cnd->queue_end = NULL; - cnd->initialized = ETHR_COND_INITIALIZED; - return 0; -} - -int -ethr_cond_destroy(ethr_cond *cnd) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED) - || cnd->queue) { - ASSERT(0); - return EINVAL; - } -#endif - DeleteCriticalSection(&cnd->cs); - cnd->initialized = 0; - return 0; -} - -int -ethr_cond_signal(ethr_cond *cnd) -{ - cnd_wait_event_ *cwe; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - if (!cnd->initialized) { - int res = fake_static_cond_init(cnd); - if (res != 0) - return res; - } - EnterCriticalSection(&cnd->cs); - cwe = cnd->queue; - if (cwe) { - ASSERT(cwe->in_queue); - SetEvent(cnd->queue->handle); - if (cwe->next) - cwe->next->prev = NULL; - else { - ASSERT(cnd->queue_end == cnd->queue); - cnd->queue_end = NULL; - } - cnd->queue = cwe->next; - cwe->in_queue = 0; - } - LeaveCriticalSection(&cnd->cs); - return 0; -} - -int -ethr_cond_broadcast(ethr_cond *cnd) -{ - cnd_wait_event_ *cwe; - -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!cnd - || (cnd->initialized && cnd->initialized != ETHR_COND_INITIALIZED)) { - ASSERT(0); - return EINVAL; - } -#endif - if (!cnd->initialized) { - int res = fake_static_cond_init(cnd); - if (res != 0) - return res; - } - EnterCriticalSection(&cnd->cs); - for (cwe = cnd->queue; cwe; cwe = cwe->next) { - ASSERT(cwe->in_queue); - SetEvent(cwe->handle); - cwe->in_queue = 0; - } - cnd->queue = NULL; - cnd->queue_end = NULL; - LeaveCriticalSection(&cnd->cs); - return 0; - -} - -int -ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) -{ - return condwait(cnd, mtx, 0, NULL); -} - -int -ethr_cond_timedwait(ethr_cond *cnd, ethr_mutex *mtx, ethr_timeval *timeout) -{ - return condwait(cnd, mtx, 1, timeout); -} - -int -ethr_time_now(ethr_timeval *time) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!time) { - ASSERT(0); - return EINVAL; - } -#endif - get_curr_time(&time->tv_sec, &time->tv_nsec); - return 0; -} - -/* - * Thread specific data - */ - -int -ethr_tsd_key_create(ethr_tsd_key *keyp) -{ - DWORD key; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!keyp) { - ASSERT(0); - return EINVAL; - } -#endif - key = TlsAlloc(); - if (key == TLS_OUT_OF_INDEXES) - return get_errno(); - *keyp = (ethr_tsd_key) key; - return 0; -} - -int -ethr_tsd_key_delete(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - if (!TlsFree((DWORD) key)) - return get_errno(); - return 0; -} - -int -ethr_tsd_set(ethr_tsd_key key, void *value) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } -#endif - if (!TlsSetValue((DWORD) key, (LPVOID) value)) - return get_errno(); - return 0; -} - -void * -ethr_tsd_get(ethr_tsd_key key) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return NULL; - } -#endif - return (void *) TlsGetValue((DWORD) key); -} - -/* Misc */ - -#ifndef ETHR_HAVE_OPTIMIZED_LOCKS - -int -ethr_do_spinlock_init(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX)) - return 0; - else - return get_errno(); -} - -int -ethr_do_rwlock_init(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - lock->counter = 0; - if (InitializeCriticalSectionAndSpinCount(&lock->cs, INT_MAX)) - return 0; - else - return get_errno(); -} - -#endif /* #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS */ - -#else -#error "Missing thread implementation" -#endif - -/* Atomics */ - -int -ethr_atomic_init(ethr_atomic_t *var, long i) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_init__(var, i); -} - -int -ethr_atomic_set(ethr_atomic_t *var, long i) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_set__(var, i); -} - -int -ethr_atomic_read(ethr_atomic_t *var, long *i) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !i) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_read__(var, i); -} - - -int -ethr_atomic_addtest(ethr_atomic_t *var, long incr, long *testp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !testp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_addtest__(var, incr, testp); -} - -int -ethr_atomic_inctest(ethr_atomic_t *incp, long *testp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!incp || !testp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_inctest__(incp, testp); -} - -int -ethr_atomic_dectest(ethr_atomic_t *decp, long *testp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!decp || !testp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_dectest__(decp, testp); -} - -int -ethr_atomic_add(ethr_atomic_t *var, long incr) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_add__(var, incr); -} - -int -ethr_atomic_inc(ethr_atomic_t *incp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!incp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_inc__(incp); -} - -int -ethr_atomic_dec(ethr_atomic_t *decp) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!decp) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_dec__(decp); -} - -int -ethr_atomic_and_old(ethr_atomic_t *var, long mask, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_and_old__(var, mask, old); -} - -int -ethr_atomic_or_old(ethr_atomic_t *var, long mask, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_or_old__(var, mask, old); -} - -int -ethr_atomic_xchg(ethr_atomic_t *var, long new, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_xchg__(var, new, old); -} - -int -ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected, long *old) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!var || !old) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_atomic_cmpxchg__(var, new, expected, old); -} - -/* Spinlocks and rwspinlocks */ - -int -ethr_spinlock_init(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spinlock_init__(lock); -} - -int -ethr_spinlock_destroy(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spinlock_destroy__(lock); -} - - -int -ethr_spin_unlock(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spin_unlock__(lock); -} - -int -ethr_spin_lock(ethr_spinlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_spin_lock__(lock); -} - -int -ethr_rwlock_init(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwlock_init__(lock); -} - -int -ethr_rwlock_destroy(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_rwlock_destroy__(lock); -} - -int -ethr_read_unlock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_read_unlock__(lock); -} - -int -ethr_read_lock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_read_lock__(lock); -} - -int -ethr_write_unlock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_write_unlock__(lock); -} - -int -ethr_write_lock(ethr_rwlock_t *lock) -{ -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!lock) { - ASSERT(0); - return EINVAL; - } -#endif - return ethr_write_lock__(lock); -} - - -int -ethr_gate_init(ethr_gate *gp) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_init(&gp->mtx); - if (res != 0) - return res; - res = ethr_cond_init(&gp->cnd); - if (res != 0) { - ethr_mutex_destroy(&gp->mtx); - return res; - } - gp->open = 0; - return 0; -} - -int -ethr_gate_destroy(ethr_gate *gp) -{ - int res, dres; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_destroy(&gp->mtx); - dres = ethr_cond_destroy(&gp->cnd); - if (res == 0) - res = dres; - gp->open = 0; - return res; -} - -int -ethr_gate_close(ethr_gate *gp) -{ - int res; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - gp->open = 0; - res = ethr_mutex_unlock__(&gp->mtx); - return res; -} - -int -ethr_gate_let_through(ethr_gate *gp, unsigned no) -{ - int res, ures; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - gp->open += no; - res = (gp->open == 1 - ? ethr_cond_signal(&gp->cnd) - : ethr_cond_broadcast(&gp->cnd)); - ures = ethr_mutex_unlock__(&gp->mtx); - if (res != 0) - res = ures; - return res; -} - -int -ethr_gate_swait(ethr_gate *gp, int spincount) -{ - int res, ures, n; -#if ETHR_XCHK - if (ethr_not_inited) { - ASSERT(0); - return EACCES; - } - if (!gp) { - ASSERT(0); - return EINVAL; - } -#endif - n = spincount; - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - while (n >= 0 && !gp->open) { - res = ethr_mutex_unlock__(&gp->mtx); - if (res != 0) - return res; - res = ethr_mutex_lock__(&gp->mtx); - if (res != 0) - return res; - n--; - } - while (!gp->open) { - res = ethr_cond_wait(&gp->cnd, &gp->mtx); - if (res != 0 && res != EINTR) - goto done; - } - gp->open--; - done: - ures = ethr_mutex_unlock__(&gp->mtx); - if (res == 0) - res = ures; - return res; -} - - -int -ethr_gate_wait(ethr_gate *gp) -{ - return ethr_gate_swait(gp, 0); -} - - -/* rwmutex fallback */ -#ifdef ETHR_USE_RWMTX_FALLBACK - -int -ethr_rwmutex_init(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_init(&rwmtx->mtx); - if (res != 0) - return res; - ethr_cond_init(&rwmtx->rcnd); - if (res != 0) - goto error_cleanup1; - res = ethr_cond_init(&rwmtx->wcnd); - if (res != 0) - goto error_cleanup2; - rwmtx->readers = 0; - rwmtx->waiting_readers = 0; - rwmtx->waiting_writers = 0; -#if ETHR_XCHK - rwmtx->initialized = ETHR_RWMUTEX_INITIALIZED; -#endif - return 0; - error_cleanup2: - ethr_cond_destroy(&rwmtx->rcnd); - error_cleanup1: - ethr_mutex_destroy(&rwmtx->mtx); - return res; -} - -int -ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) -{ - int res, pres; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } - rwmtx->initialized = 0; -#endif - res = ethr_mutex_destroy(&rwmtx->mtx); - pres = ethr_cond_destroy(&rwmtx->rcnd); - if (res == 0) - res = pres; - pres = ethr_cond_destroy(&rwmtx->wcnd); - if (res == 0) - res = pres; - return res; -} - -int -ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_trylock__(&rwmtx->mtx); - if (res != 0) - return res; - if (!rwmtx->waiting_writers) { - res = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - return EBUSY; - return res; - } - rwmtx->readers++; - return ethr_mutex_unlock__(&rwmtx->mtx); -} - -int -ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&rwmtx->mtx); - if (res != 0) - return res; - while (rwmtx->waiting_writers) { - rwmtx->waiting_readers++; - res = ethr_cond_wait(&rwmtx->rcnd, &rwmtx->mtx); - rwmtx->waiting_readers--; - if (res != 0 && res != EINTR) { - (void) ethr_mutex_unlock__(&rwmtx->mtx); - return res; - } - } - rwmtx->readers++; - return ethr_mutex_unlock__(&rwmtx->mtx); -} - -int -ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) -{ - int res, ures; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&rwmtx->mtx); - if (res != 0) - return res; - rwmtx->readers--; - if (!rwmtx->readers && rwmtx->waiting_writers) - res = ethr_cond_signal(&rwmtx->wcnd); - ures = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - res = ures; - return res; -} - -int -ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_trylock__(&rwmtx->mtx); - if (res != 0) - return res; - if (!rwmtx->readers && !rwmtx->waiting_writers) - return 0; - else { - res = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - return EBUSY; - return res; - } -} - -int -ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) -{ - int res; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = ethr_mutex_lock__(&rwmtx->mtx); - if (res != 0) - return res; - if (!rwmtx->readers && !rwmtx->waiting_writers) - return 0; - - while (rwmtx->readers) { - rwmtx->waiting_writers++; - res = ethr_cond_wait(&rwmtx->wcnd, &rwmtx->mtx); - rwmtx->waiting_writers--; - if (res != 0 && res != EINTR) { - (void) ethr_rwmutex_rwunlock(rwmtx); - return res; - } - } - return 0; -} - -int -ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) -{ - int res, ures; -#if ETHR_XCHK - if (!rwmtx || rwmtx->initialized != ETHR_RWMUTEX_INITIALIZED) { - ASSERT(0); - return EINVAL; - } -#endif - res = 0; - if (rwmtx->waiting_writers) - res = ethr_cond_signal(&rwmtx->wcnd); - else if (rwmtx->waiting_readers) - res = ethr_cond_broadcast(&rwmtx->rcnd); - ures = ethr_mutex_unlock__(&rwmtx->mtx); - if (res == 0) - res = ures; - return res; -} - -#endif /* #ifdef ETHR_USE_RWMTX_FALLBACK */ - -void -ethr_compiler_barrier(void) -{ - -} - -#ifdef DEBUG - -#include <stdio.h> -int ethr_assert_failed(char *f, int l, char *a) -{ - fprintf(stderr, "%s:%d: Assertion failed: %s\n", f, l, a); - abort(); - return 0; -} - -#endif - - diff --git a/erts/lib_src/pthread/ethr_event.c b/erts/lib_src/pthread/ethr_event.c new file mode 100644 index 0000000000..9434d60d0a --- /dev/null +++ b/erts/lib_src/pthread/ethr_event.c @@ -0,0 +1,225 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_EVENT_IMPL__ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "ethread.h" + +#if defined(ETHR_LINUX_FUTEX_IMPL__) +/* --- Linux futex implementation of ethread events ------------------------- */ + +#include <sched.h> +#include <errno.h> + +#define ETHR_YIELD_AFTER_BUSY_LOOPS 50 + +int +ethr_event_init(ethr_event *e) +{ + ethr_atomic32_init(&e->futex, ETHR_EVENT_OFF__); + return 0; +} + +int +ethr_event_destroy(ethr_event *e) +{ + return 0; +} + +static ETHR_INLINE int +wait__(ethr_event *e, int spincount) +{ + unsigned sc = spincount; + int res; + ethr_sint32_t val; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + if (spincount < 0) + ETHR_FATAL_ERROR__(EINVAL); + + while (1) { + while (1) { + val = ethr_atomic32_read(&e->futex); + if (val == ETHR_EVENT_ON__) + return 0; + if (sc == 0) + break; + sc--; + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + res = ETHR_YIELD(); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + } + + if (val != ETHR_EVENT_OFF_WAITER__) { + val = ethr_atomic32_cmpxchg(&e->futex, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); + + if (val == ETHR_EVENT_ON__) + return 0; + ETHR_ASSERT(val == ETHR_EVENT_OFF__); + } + + res = ETHR_FUTEX__(&e->futex, + ETHR_FUTEX_WAIT__, + ETHR_EVENT_OFF_WAITER__); + if (res == EINTR) + break; + if (res != 0 && res != EWOULDBLOCK) + ETHR_FATAL_ERROR__(res); + } + + return res; +} + +#elif defined(ETHR_PTHREADS) +/* --- Posix mutex/cond implementation of events ---------------------------- */ + +int +ethr_event_init(ethr_event *e) +{ + int res; + ethr_atomic32_init(&e->state, ETHR_EVENT_OFF__); + res = pthread_mutex_init(&e->mtx, NULL); + if (res != 0) + return res; + res = pthread_cond_init(&e->cnd, NULL); + if (res != 0) { + pthread_mutex_destroy(&e->mtx); + return res; + } + return 0; +} + +int +ethr_event_destroy(ethr_event *e) +{ + int res; + res = pthread_mutex_destroy(&e->mtx); + if (res != 0) + return res; + res = pthread_cond_destroy(&e->cnd); + if (res != 0) + return res; + return 0; +} + +static ETHR_INLINE int +wait__(ethr_event *e, int spincount) +{ + int sc = spincount; + ethr_sint32_t val; + int res, ulres; + int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + if (spincount < 0) + ETHR_FATAL_ERROR__(EINVAL); + + while (1) { + val = ethr_atomic32_read(&e->state); + if (val == ETHR_EVENT_ON__) + return 0; + if (sc == 0) + break; + sc--; + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + res = ETHR_YIELD(); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + } + + if (val != ETHR_EVENT_OFF_WAITER__) { + val = ethr_atomic32_cmpxchg(&e->state, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); + if (val == ETHR_EVENT_ON__) + return 0; + ETHR_ASSERT(val == ETHR_EVENT_OFF__); + } + + ETHR_ASSERT(val == ETHR_EVENT_OFF_WAITER__ + || val == ETHR_EVENT_OFF__); + + res = pthread_mutex_lock(&e->mtx); + if (res != 0) + ETHR_FATAL_ERROR__(res); + + while (1) { + + val = ethr_atomic32_read(&e->state); + if (val == ETHR_EVENT_ON__) + break; + + res = pthread_cond_wait(&e->cnd, &e->mtx); + if (res == EINTR) + break; + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + + ulres = pthread_mutex_unlock(&e->mtx); + if (ulres != 0) + ETHR_FATAL_ERROR__(ulres); + + return res; /* 0 || EINTR */ +} + +#else +#error No ethread event implementation +#endif + +void +ethr_event_reset(ethr_event *e) +{ + ethr_event_reset__(e); +} + +void +ethr_event_set(ethr_event *e) +{ + ethr_event_set__(e); +} + +int +ethr_event_wait(ethr_event *e) +{ + return wait__(e, 0); +} + +int +ethr_event_swait(ethr_event *e, int spincount) +{ + return wait__(e, spincount); +} diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c new file mode 100644 index 0000000000..f047104103 --- /dev/null +++ b/erts/lib_src/pthread/ethread.c @@ -0,0 +1,451 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Pthread implementation of the ethread library + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_CHILD_WAIT_SPIN_COUNT 4000 + +#include <stdio.h> +#ifdef ETHR_TIME_WITH_SYS_TIME +# include <time.h> +# include <sys/time.h> +#else +# ifdef ETHR_HAVE_SYS_TIME_H +# include <sys/time.h> +# else +# include <time.h> +# endif +#endif +#include <sys/types.h> +#include <unistd.h> +#include <signal.h> + +#include <limits.h> + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHREAD_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" + +#ifndef ETHR_HAVE_ETHREAD_DEFINES +#error Missing configure defines +#endif + +pthread_key_t ethr_ts_event_key__; +static int child_wait_spin_count; + +/* + * -------------------------------------------------------------------------- + * Static functions + * -------------------------------------------------------------------------- + */ + +static void thr_exit_cleanup(void) +{ + ethr_run_exit_handlers__(); +} + + +/* Argument passed to thr_wrapper() */ +typedef struct { + ethr_atomic32_t result; + ethr_ts_event *tse; + void *(*thr_func)(void *); + void *arg; + void *prep_func_res; +} ethr_thr_wrap_data__; + +static void *thr_wrapper(void *vtwd) +{ + ethr_sint32_t result; + void *res; + ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd; + void *(*thr_func)(void *) = twd->thr_func; + void *arg = twd->arg; + ethr_ts_event *tsep = NULL; + + result = (ethr_sint32_t) ethr_make_ts_event__(&tsep); + + if (result == 0) { + tsep->iflgs |= ETHR_TS_EV_ETHREAD; + if (ethr_thr_child_func__) + ethr_thr_child_func__(twd->prep_func_res); + } + + tsep = twd->tse; /* We aren't allowed to follow twd after + result has been set! */ + + ethr_atomic32_set(&twd->result, result); + + ethr_event_set(&tsep->event); + + res = result == 0 ? (*thr_func)(arg) : NULL; + + thr_exit_cleanup(); + return res; +} + +/* internal exports */ + +int ethr_set_tse__(ethr_ts_event *tsep) +{ + return pthread_setspecific(ethr_ts_event_key__, (void *) tsep); +} + +ethr_ts_event *ethr_get_tse__(void) +{ + return pthread_getspecific(ethr_ts_event_key__); +} + +/* + * -------------------------------------------------------------------------- + * Exported functions + * -------------------------------------------------------------------------- + */ + +int +ethr_init(ethr_init_data *id) +{ + int res; + + if (!ethr_not_inited__) + return EINVAL; + + ethr_not_inited__ = 0; + + res = ethr_init_common__(id); + if (res != 0) + goto error; + + child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT; + if (erts_get_cpu_configured(ethr_cpu_info__) == 1) + child_wait_spin_count = 0; + + res = pthread_key_create(ðr_ts_event_key__, ethr_ts_event_destructor__); + + return 0; + error: + ethr_not_inited__ = 1; + return res; + +} + +int +ethr_late_init(ethr_late_init_data *id) +{ + int res = ethr_late_init_common__(id); + if (res != 0) + return res; + ethr_not_completely_inited__ = 0; + return res; +} + +int +ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, + ethr_thr_opts *opts) +{ + ethr_thr_wrap_data__ twd; + pthread_attr_t attr; + int res, dres; + int use_stack_size = (opts && opts->suggested_stack_size >= 0 + ? opts->suggested_stack_size + : -1 /* Use system default */); + +#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE + if (use_stack_size < 0) + use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; +#endif + +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!tid || !func) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + + ethr_atomic32_init(&twd.result, (ethr_sint32_t) -1); + twd.tse = ethr_get_ts_event(); + twd.thr_func = func; + twd.arg = arg; + + res = pthread_attr_init(&attr); + if (res != 0) + return res; + + /* Error cleanup needed after this point */ + + /* Schedule child thread in system scope (if possible) ... */ + res = pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); + if (res != 0 && res != ENOTSUP) + goto error; + + if (use_stack_size >= 0) { + size_t suggested_stack_size = (size_t) use_stack_size; + size_t stack_size; +#ifdef ETHR_DEBUG + suggested_stack_size /= 2; /* Make sure we got margin */ +#endif +#ifdef ETHR_STACK_GUARD_SIZE + /* The guard is at least on some platforms included in the stack size + passed when creating threads */ + suggested_stack_size += ETHR_B2KW(ETHR_STACK_GUARD_SIZE); +#endif + if (suggested_stack_size < ethr_min_stack_size__) + stack_size = ETHR_KW2B(ethr_min_stack_size__); + else if (suggested_stack_size > ethr_max_stack_size__) + stack_size = ETHR_KW2B(ethr_max_stack_size__); + else + stack_size = ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); + (void) pthread_attr_setstacksize(&attr, stack_size); + } + +#ifdef ETHR_STACK_GUARD_SIZE + (void) pthread_attr_setguardsize(&attr, ETHR_STACK_GUARD_SIZE); +#endif + + /* Detached or joinable... */ + res = pthread_attr_setdetachstate(&attr, + (opts && opts->detached + ? PTHREAD_CREATE_DETACHED + : PTHREAD_CREATE_JOINABLE)); + if (res != 0) + goto error; + + /* Call prepare func if it exist */ + if (ethr_thr_prepare_func__) + twd.prep_func_res = ethr_thr_prepare_func__(); + else + twd.prep_func_res = NULL; + + res = pthread_create((pthread_t *) tid, &attr, thr_wrapper, (void*) &twd); + + if (res == 0) { + int spin_count = child_wait_spin_count; + + /* Wait for child to initialize... */ + while (1) { + ethr_sint32_t result; + ethr_event_reset(&twd.tse->event); + + result = ethr_atomic32_read(&twd.result); + if (result == 0) + break; + + if (result > 0) { + res = (int) result; + goto error; + } + + res = ethr_event_swait(&twd.tse->event, spin_count); + if (res != 0 && res != EINTR) + goto error; + spin_count = 0; + } + } + + /* Cleanup... */ + + error: + dres = pthread_attr_destroy(&attr); + if (res == 0) + res = dres; + if (ethr_thr_parent_func__) + ethr_thr_parent_func__(twd.prep_func_res); + return res; +} + +int +ethr_thr_join(ethr_tid tid, void **res) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_join((pthread_t) tid, res); +} + +int +ethr_thr_detach(ethr_tid tid) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_detach((pthread_t) tid); +} + +void +ethr_thr_exit(void *res) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return; + } +#endif + thr_exit_cleanup(); + pthread_exit(res); +} + +ethr_tid +ethr_self(void) +{ + return (ethr_tid) pthread_self(); +} + +int +ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) +{ + return pthread_equal((pthread_t) tid1, (pthread_t) tid2); +} + + +/* + * Thread specific events + */ + +ethr_ts_event * +ethr_get_ts_event(void) +{ + return ethr_get_ts_event__(); +} + +void +ethr_leave_ts_event(ethr_ts_event *tsep) +{ + ethr_leave_ts_event__(tsep); +} + +/* + * Thread specific data + */ + +int +ethr_tsd_key_create(ethr_tsd_key *keyp) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!keyp) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return pthread_key_create((pthread_key_t *) keyp, NULL); +} + +int +ethr_tsd_key_delete(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_key_delete((pthread_key_t) key); +} + +int +ethr_tsd_set(ethr_tsd_key key, void *value) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + return pthread_setspecific((pthread_key_t) key, value); +} + +void * +ethr_tsd_get(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return NULL; + } +#endif + return pthread_getspecific((pthread_key_t) key); +} + +/* + * Signal functions + */ + +#if ETHR_HAVE_ETHR_SIG_FUNCS + +int ethr_sigmask(int how, const sigset_t *set, sigset_t *oset) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!set && !oset) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + return pthread_sigmask(how, set, oset); +} + +int ethr_sigwait(const sigset_t *set, int *sig) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!set || !sig) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + if (sigwait(set, sig) < 0) + return errno; + return 0; +} + +#endif /* #if ETHR_HAVE_ETHR_SIG_FUNCS */ + +ETHR_IMPL_NORETURN__ +ethr_abort__(void) +{ + abort(); +} diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c new file mode 100644 index 0000000000..68f093f49c --- /dev/null +++ b/erts/lib_src/win/ethr_event.c @@ -0,0 +1,123 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Author: Rickard Green + */ + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_EVENT_IMPL__ + +#include "ethread.h" + +/* --- Windows implementation of thread events ------------------------------ */ + +#pragma intrinsic(_InterlockedExchangeAdd) +#pragma intrinsic(_InterlockedCompareExchange) + +int +ethr_event_init(ethr_event *e) +{ + e->state = ETHR_EVENT_OFF__; + e->handle = CreateEvent(NULL, FALSE, FALSE, NULL); + if (e->handle == INVALID_HANDLE_VALUE) + return ethr_win_get_errno__(); + return 0; +} + +int +ethr_event_destroy(ethr_event *e) +{ + BOOL res = CloseHandle(e->handle); + return res == 0 ? ethr_win_get_errno__() : 0; +} + +void +ethr_event_set(ethr_event *e) +{ + ethr_event_set__(e); +} + +void +ethr_event_reset(ethr_event *e) +{ + ethr_event_reset__(e); +} + +static ETHR_INLINE int +wait(ethr_event *e, int spincount) +{ + LONG state; + DWORD code; + int sc, res, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + + if (spincount < 0) + ETHR_FATAL_ERROR__(EINVAL); + + sc = spincount; + + while (1) { + long on; + while (1) { +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ + state = e->state; +#else + state = _InterlockedExchangeAdd(&e->state, (LONG) 0); +#endif + if (state == ETHR_EVENT_ON__) + return 0; + if (sc == 0) + break; + sc--; + ETHR_SPIN_BODY; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + res = ETHR_YIELD(); + if (res != 0) + ETHR_FATAL_ERROR__(res); + } + } + + if (state != ETHR_EVENT_OFF_WAITER__) { + state = _InterlockedCompareExchange(&e->state, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); + if (state == ETHR_EVENT_ON__) + return 0; + ETHR_ASSERT(state == ETHR_EVENT_OFF__); + } + + code = WaitForSingleObject(e->handle, INFINITE); + if (code != WAIT_OBJECT_0) + ETHR_FATAL_ERROR__(ethr_win_get_errno__()); + } + +} + +int +ethr_event_wait(ethr_event *e) +{ + return wait(e, 0); +} + +int +ethr_event_swait(ethr_event *e, int spincount) +{ + return wait(e, spincount); +} diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c new file mode 100644 index 0000000000..789a360b11 --- /dev/null +++ b/erts/lib_src/win/ethread.c @@ -0,0 +1,586 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Windows native threads implementation of the ethread library + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_CHILD_WAIT_SPIN_COUNT 4000 + +#undef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <process.h> +#include <winerror.h> +#include <stdio.h> +#include <limits.h> + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHREAD_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" + +#ifndef ETHR_HAVE_ETHREAD_DEFINES +#error Missing configure defines +#endif + +/* Argument passed to thr_wrapper() */ +typedef struct { + ethr_tid *tid; + ethr_atomic32_t result; + ethr_ts_event *tse; + void *(*thr_func)(void *); + void *arg; + void *prep_func_res; +} ethr_thr_wrap_data__; + +#define ETHR_INVALID_TID_ID -1 + +struct ethr_join_data_ { + HANDLE handle; + void *res; +}; + +static ethr_atomic_t thread_id_counter; +static DWORD own_tid_key; +static ethr_tid main_thr_tid; +static int child_wait_spin_count; + +DWORD ethr_ts_event_key__; + +#define ETHR_GET_OWN_TID__ ((ethr_tid *) TlsGetValue(own_tid_key)) + +/* + * -------------------------------------------------------------------------- + * Static functions + * -------------------------------------------------------------------------- + */ + +static void thr_exit_cleanup(ethr_tid *tid, void *res) +{ + + ETHR_ASSERT(tid == ETHR_GET_OWN_TID__); + + if (tid->jdata) + tid->jdata->res = res; + + ethr_run_exit_handlers__(); + ethr_ts_event_destructor__((void *) ethr_get_tse__()); +} + +static unsigned __stdcall thr_wrapper(LPVOID vtwd) +{ + ethr_tid my_tid; + ethr_sint32_t result; + void *res; + ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd; + void *(*thr_func)(void *) = twd->thr_func; + void *arg = twd->arg; + ethr_ts_event *tsep = NULL; + + result = (ethr_sint32_t) ethr_make_ts_event__(&tsep); + + if (result == 0) { + tsep->iflgs |= ETHR_TS_EV_ETHREAD; + my_tid = *twd->tid; + if (!TlsSetValue(own_tid_key, (LPVOID) &my_tid)) { + result = (ethr_sint32_t) ethr_win_get_errno__(); + ethr_free_ts_event__(tsep); + } + else { + if (ethr_thr_child_func__) + ethr_thr_child_func__(twd->prep_func_res); + } + } + + tsep = twd->tse; /* We aren't allowed to follow twd after + result has been set! */ + + ethr_atomic32_set(&twd->result, result); + + ethr_event_set(&tsep->event); + + res = result == 0 ? (*thr_func)(arg) : NULL; + + thr_exit_cleanup(&my_tid, res); + return 0; +} + +/* internal exports */ + +int +ethr_win_get_errno__(void) +{ + return erts_get_last_win_errno(); +} + +int ethr_set_tse__(ethr_ts_event *tsep) +{ + return (TlsSetValue(ethr_ts_event_key__, (LPVOID) tsep) + ? 0 + : ethr_win_get_errno__()); +} + +ethr_ts_event *ethr_get_tse__(void) +{ + return (ethr_ts_event *) TlsGetValue(ethr_ts_event_key__); +} + +ETHR_IMPL_NORETURN__ +ethr_abort__(void) +{ +#if 1 + DebugBreak(); +#else + abort(); +#endif +} + +/* + * ---------------------------------------------------------------------------- + * Exported functions + * ---------------------------------------------------------------------------- + */ + +int +ethr_init(ethr_init_data *id) +{ +#ifdef _WIN32_WINNT + DWORD major = (_WIN32_WINNT >> 8) & 0xff; + DWORD minor = _WIN32_WINNT & 0xff; + OSVERSIONINFO os_version; +#endif + int err = 0; + unsigned long i; + + if (!ethr_not_inited__) + return EINVAL; + +#ifdef _WIN32_WINNT + os_version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); + GetVersionEx(&os_version); + if (os_version.dwPlatformId != VER_PLATFORM_WIN32_NT + || os_version.dwMajorVersion < major + || (os_version.dwMajorVersion == major + && os_version.dwMinorVersion < minor)) + return ENOTSUP; +#endif + err = ethr_init_common__(id); + if (err) + goto error; + + own_tid_key = TlsAlloc(); + if (own_tid_key == TLS_OUT_OF_INDEXES) + goto error; + + ethr_atomic_init(&thread_id_counter, 0); + + main_thr_tid.id = 0; + main_thr_tid.jdata = NULL; + + if (!TlsSetValue(own_tid_key, (LPVOID) &main_thr_tid)) + goto error; + + ETHR_ASSERT(&main_thr_tid == ETHR_GET_OWN_TID__); + + ethr_ts_event_key__ = TlsAlloc(); + if (ethr_ts_event_key__ == TLS_OUT_OF_INDEXES) + goto error; + + child_wait_spin_count = ETHR_CHILD_WAIT_SPIN_COUNT; + if (erts_get_cpu_configured(ethr_cpu_info__) == 1) + child_wait_spin_count = 0; + + ethr_not_inited__ = 0; + + return 0; + + error: + ethr_not_inited__ = 1; + if (err == 0) + err = ethr_win_get_errno__(); + ETHR_ASSERT(err != 0); + return err; +} + +int +ethr_late_init(ethr_late_init_data *id) +{ + int res = ethr_late_init_common__(id); + if (res != 0) + return res; + ethr_not_completely_inited__ = 0; + return res; +} + + +/* + * Thread functions. + */ + +int +ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, + ethr_thr_opts *opts) +{ + HANDLE handle = INVALID_HANDLE_VALUE; + int err = 0; + ethr_thr_wrap_data__ twd; + DWORD code; + unsigned ID; + unsigned stack_size = 0; /* 0 = system default */ + int use_stack_size = (opts && opts->suggested_stack_size >= 0 + ? opts->suggested_stack_size + : -1 /* Use system default */); + +#ifdef ETHR_MODIFIED_DEFAULT_STACK_SIZE + if (use_stack_size < 0) + use_stack_size = ETHR_MODIFIED_DEFAULT_STACK_SIZE; +#endif + +#if ETHR_XCHK + if (ethr_not_completely_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!tid || !func) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + + do { + tid->id = ethr_atomic_inc_read(&thread_id_counter); + } while (tid->id == ETHR_INVALID_TID_ID); + + if (opts && opts->detached) + tid->jdata = NULL; + else { + tid->jdata = ethr_mem__.std.alloc(sizeof(struct ethr_join_data_)); + if (!tid->jdata) + return ENOMEM; + tid->jdata->handle = INVALID_HANDLE_VALUE; + tid->jdata->res = NULL; + } + + if (use_stack_size >= 0) { + size_t suggested_stack_size = (size_t) use_stack_size; +#ifdef ETHR_DEBUG + suggested_stack_size /= 2; /* Make sure we got margin */ +#endif + if (suggested_stack_size < ethr_min_stack_size__) + stack_size = (unsigned) ETHR_KW2B(ethr_min_stack_size__); + else if (suggested_stack_size > ethr_max_stack_size__) + stack_size = (unsigned) ETHR_KW2B(ethr_max_stack_size__); + else + stack_size = (unsigned) + ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); + } + + ethr_atomic32_init(&twd.result, -1); + + twd.tid = tid; + twd.thr_func = func; + twd.arg = arg; + twd.tse = ethr_get_ts_event(); + + /* Call prepare func if it exist */ + if (ethr_thr_prepare_func__) + twd.prep_func_res = ethr_thr_prepare_func__(); + else + twd.prep_func_res = NULL; + + /* spawn the thr_wrapper function */ + handle = (HANDLE) _beginthreadex(NULL, stack_size, thr_wrapper, + (LPVOID) &twd, 0, &ID); + if (handle == (HANDLE) 0) { + handle = INVALID_HANDLE_VALUE; + goto error; + } + else { + int spin_count = child_wait_spin_count; + + ETHR_ASSERT(handle != INVALID_HANDLE_VALUE); + + if (!tid->jdata) + CloseHandle(handle); + else + tid->jdata->handle = handle; + + /* Wait for child to initialize... */ + while (1) { + ethr_sint32_t result; + int err; + ethr_event_reset(&twd.tse->event); + + result = ethr_atomic32_read(&twd.result); + if (result == 0) + break; + + if (result > 0) { + err = (int) result; + goto error; + } + + err = ethr_event_swait(&twd.tse->event, spin_count); + if (err && err != EINTR) + goto error; + spin_count = 0; + } + } + + if (ethr_thr_parent_func__) + ethr_thr_parent_func__(twd.prep_func_res); + + if (twd.tse) + ethr_leave_ts_event(twd.tse); + + return 0; + + error: + + if (err == 0) + err = ethr_win_get_errno__(); + ETHR_ASSERT(err != 0); + + if (ethr_thr_parent_func__) + ethr_thr_parent_func__(twd.prep_func_res); + + if (handle != INVALID_HANDLE_VALUE) { + WaitForSingleObject(handle, INFINITE); + CloseHandle(handle); + } + + if (tid->jdata) { + ethr_mem__.std.free(tid->jdata); + tid->jdata = NULL; + } + + tid->id = ETHR_INVALID_TID_ID; + + if (twd.tse) + ethr_leave_ts_event(twd.tse); + + return err; +} + +int ethr_thr_join(ethr_tid tid, void **res) +{ + DWORD code; + +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + + if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata) + return EINVAL; + + /* Wait for thread to terminate */ + code = WaitForSingleObject(tid.jdata->handle, INFINITE); + if (code != WAIT_OBJECT_0) + return ethr_win_get_errno__(); + + CloseHandle(tid.jdata->handle); + tid.jdata->handle = INVALID_HANDLE_VALUE; + + if (res) + *res = tid.jdata->res; + + /* + * User better not try to join or detach again; or + * bad things will happen... (users responsibility) + */ + + ethr_mem__.std.free(tid.jdata); + + return 0; +} + + +int +ethr_thr_detach(ethr_tid tid) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + + if (tid.id == ETHR_INVALID_TID_ID || !tid.jdata) + return EINVAL; + + CloseHandle(tid.jdata->handle); + tid.jdata->handle = INVALID_HANDLE_VALUE; + + /* + * User better not try to join or detach again; or + * bad things will happen... (users responsibility) + */ + + ethr_mem__.std.free(tid.jdata); + + return 0; +} + + +void +ethr_thr_exit(void *res) +{ + ethr_tid *tid; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return; + } +#endif + tid = ETHR_GET_OWN_TID__; + if (!tid) { + ETHR_ASSERT(0); + _endthreadex((unsigned) 0); + } + thr_exit_cleanup(tid, res); + _endthreadex((unsigned) 0); +} + +ethr_tid +ethr_self(void) +{ + ethr_tid *tid; +#if ETHR_XCHK + if (ethr_not_inited__) { + ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL}; + ETHR_ASSERT(0); + return dummy_tid; + } +#endif + /* It is okay for non-ethreads (i.e. native win32 threads) to call + ethr_self(). They will however be returned an invalid tid. */ + tid = ETHR_GET_OWN_TID__; + if (!tid) { + ethr_tid dummy_tid = {ETHR_INVALID_TID_ID, NULL}; + return dummy_tid; + } + return *tid; +} + +int +ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) +{ + /* An invalid tid does not equal any tid, not even an invalid tid */ + return tid1.id == tid2.id && tid1.id != ETHR_INVALID_TID_ID; +} + +/* + * Thread specific data + */ + +int +ethr_tsd_key_create(ethr_tsd_key *keyp) +{ + DWORD key; +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } + if (!keyp) { + ETHR_ASSERT(0); + return EINVAL; + } +#endif + key = TlsAlloc(); + if (key == TLS_OUT_OF_INDEXES) + return ethr_win_get_errno__(); + *keyp = (ethr_tsd_key) key; + return 0; +} + +int +ethr_tsd_key_delete(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + if (!TlsFree((DWORD) key)) + return ethr_win_get_errno__(); + return 0; +} + +int +ethr_tsd_set(ethr_tsd_key key, void *value) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return EACCES; + } +#endif + if (!TlsSetValue((DWORD) key, (LPVOID) value)) + return ethr_win_get_errno__(); + return 0; +} + +void * +ethr_tsd_get(ethr_tsd_key key) +{ +#if ETHR_XCHK + if (ethr_not_inited__) { + ETHR_ASSERT(0); + return NULL; + } +#endif + return (void *) TlsGetValue((DWORD) key); +} + + +/* + * Thread specific events + */ + +ethr_ts_event * +ethr_get_ts_event(void) +{ + return ethr_get_ts_event__(); +} + +void +ethr_leave_ts_event(ethr_ts_event *tsep) +{ + ethr_leave_ts_event__(tsep); +} + +ethr_ts_event * +ethr_create_ts_event__(void) +{ + ethr_ts_event *tsep; + ethr_make_ts_event__(&tsep); + return tsep; +} diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex afd8a90b3f..87ff5119fd 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 4ec84948d8..6b0d96ff8e 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex c3e746f3ee..8a7a9a1314 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam Binary files differindex 4b2d8bb2de..5d544ff4aa 100644 --- a/erts/preloaded/ebin/otp_ring0.beam +++ b/erts/preloaded/ebin/otp_ring0.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex 2916baaa77..3ed02ecd44 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex 46912e2bea..79a8d22366 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam Binary files differindex ccf8aff6f6..3cc8c6b8be 100644 --- a/erts/preloaded/ebin/prim_zip.beam +++ b/erts/preloaded/ebin/prim_zip.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex ccd597ba68..3f9e867542 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index 785ad531f3..145638802f 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2009. All Rights Reserved. +# Copyright Ericsson AB 2008-2010. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -20,8 +20,7 @@ # be used when the preloaded modules actually are to be updated (i.e. the # beam files are to be recompiled, which is normally not done). # The beam files are placed in the current directory and should be copied -# to the ../ebin directory by using the commit target (only works in -# clearcase). +# to the ../ebin directory by using the copy target. include $(ERL_TOP)/make/target.mk include $(ERL_TOP)/make/$(TARGET)/otp.mk @@ -62,26 +61,9 @@ debug opt: $(TARGET_FILES) clean: rm -f $(TARGET_FILES) -prepare: - cleartool co -nc $(STATIC_EBIN)/* - cleartool co -nc $(STATIC_EBIN) - copy: - for x in *.beam; do\ - if test '!' -f $(STATIC_EBIN)/$$x; then\ - cleartool mkelem -nc $$x;\ - fi;\ - done cp *.beam $(STATIC_EBIN) -commit: - cleartool ci -ident -nc $(STATIC_EBIN)/*.beam - cleartool ci -ident -nc $(STATIC_EBIN) - -cancel: - -cleartool unco -rm $(STATIC_EBIN) - -cleartool unco -rm $(STATIC_EBIN)/*.beam - include $(ERL_TOP)/make/otp_release_targets.mk diff --git a/erts/preloaded/src/erl_prim_loader.erl b/erts/preloaded/src/erl_prim_loader.erl index a13292d5ab..024b20eadb 100644 --- a/erts/preloaded/src/erl_prim_loader.erl +++ b/erts/preloaded/src/erl_prim_loader.erl @@ -405,7 +405,7 @@ handle_timeout(State = #state{loader = inet}, Parent) -> efile_multi_get_file_from_port(State, ModFiles, Paths, Fun) -> Ref = make_ref(), %% More than 200 processes is no gain. - Max = min(200, erlang:system_info(thread_pool_size)), + Max = erlang:min(200, erlang:system_info(thread_pool_size)), efile_multi_get_file_from_port2(ModFiles, 0, Max, State, Paths, Fun, Ref, ok). efile_multi_get_file_from_port2([MF | MFs], Out, Max, State, Paths, Fun, Ref, Ret) when Out < Max -> @@ -1189,9 +1189,6 @@ keyins(X, I, [Y | T]) when X < element(I,Y) -> [X,Y|T]; keyins(X, I, [Y | T]) -> [Y | keyins(X, I, T)]; keyins(X, _I, []) -> [X]. -min(X, Y) when X < Y -> X; -min(_X, Y) -> Y. - to_strs([P|Paths]) when is_atom(P) -> [atom_to_list(P)|to_strs(Paths)]; to_strs([P|Paths]) when is_list(P) -> diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 1edb5e72db..935c2de253 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -44,7 +44,14 @@ -deprecated([hash/2]). -deprecated([concat_binary/1]). --compile(nowarn_bif_clash). +% Get rid of autoimports of spawn to avoid clashes with ourselves. +-compile({no_auto_import,[spawn/1]}). +-compile({no_auto_import,[spawn/4]}). +-compile({no_auto_import,[spawn_link/1]}). +-compile({no_auto_import,[spawn_link/4]}). +-compile({no_auto_import,[spawn_opt/2]}). +-compile({no_auto_import,[spawn_opt/4]}). +-compile({no_auto_import,[spawn_opt/5]}). %%-------------------------------------------------------------------------- @@ -55,10 +62,10 @@ %%-------------------------------------------------------------------------- apply(Fun, Args) -> - apply(Fun, Args). + erlang:apply(Fun, Args). apply(Mod, Name, Args) -> - apply(Mod, Name, Args). + erlang:apply(Mod, Name, Args). %% Spawns with a fun diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 3b98b9cddc..24430a3d40 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -51,6 +51,9 @@ get_status/0,boot/1,get_arguments/0,get_plain_arguments/0, get_argument/1,script_id/0]). +%% for the on_load functionality; not for general use +-export([run_on_load_handlers/0]). + %% internal exports -export([fetch_loaded/0,ensure_loaded/1,make_permanent/2, notify_when_started/1,wait_until_started/0, @@ -69,6 +72,7 @@ script_id = [], loaded = [], subscribed = []}). +-type state() :: #state{}. -define(ON_LOAD_HANDLER, init__boot__on_load_handler). @@ -143,10 +147,10 @@ restart() -> init ! {stop,restart}, ok. -spec reboot() -> 'ok'. reboot() -> init ! {stop,reboot}, ok. --spec stop() -> no_return(). +-spec stop() -> 'ok'. stop() -> init ! {stop,stop}, ok. --spec stop(non_neg_integer() | string()) -> no_return(). +-spec stop(non_neg_integer() | string()) -> 'ok'. stop(Status) -> init ! {stop,{stop,Status}}, ok. -spec boot([binary()]) -> no_return(). @@ -275,7 +279,7 @@ crash(String, List) -> halt(halt_string(String, List)). %% Status is {InternalStatus,ProvidedStatus} --spec boot_loop(pid(), #state{}) -> no_return(). +-spec boot_loop(pid(), state()) -> no_return(). boot_loop(BootPid, State) -> receive {BootPid,loaded,ModLoaded} -> @@ -308,24 +312,6 @@ boot_loop(BootPid, State) -> {stop,Reason} -> stop(Reason,State); {From,fetch_loaded} -> %% Fetch and reset initially loaded modules. - case whereis(?ON_LOAD_HANDLER) of - undefined -> - %% There is no on_load handler process, - %% probably because init:restart/0 has been - %% called and it is not the first time we - %% pass through here. - ok; - Pid when is_pid(Pid) -> - Pid ! run_on_load, - receive - {'EXIT',Pid,on_load_done} -> - ok; - {'EXIT',Pid,Res} -> - %% Failure to run an on_load handler. - %% This is fatal during start-up. - exit(Res) - end - end, From ! {init,State#state.loaded}, garb_boot_loop(BootPid,State#state{loaded = []}); {From,{ensure_loaded,Module}} -> @@ -736,6 +722,7 @@ do_boot(Init,Flags,Start) -> BootList = get_boot(BootFile,Root), LoadMode = b2a(get_flag('-mode',Flags,false)), Deb = b2a(get_flag('-init_debug',Flags,false)), + catch ?ON_LOAD_HANDLER ! {init_debug_flag,Deb}, BootVars = get_flag_args('-boot_var',Flags), ParallelLoad = (Pgm =:= "efile") and (erlang:system_info(thread_pool_size) > 0), @@ -1335,23 +1322,44 @@ archive_extension() -> %%% Support for handling of on_load functions. %%% +run_on_load_handlers() -> + Ref = monitor(process, ?ON_LOAD_HANDLER), + catch ?ON_LOAD_HANDLER ! run_on_load, + receive + {'DOWN',Ref,process,_,noproc} -> + %% There is no on_load handler process, + %% probably because init:restart/0 has been + %% called and it is not the first time we + %% pass through here. + ok; + {'DOWN',Ref,process,_,on_load_done} -> + ok; + {'DOWN',Ref,process,_,Res} -> + %% Failure to run an on_load handler. + %% This is fatal during start-up. + exit(Res) + end. + start_on_load_handler_process() -> register(?ON_LOAD_HANDLER, - spawn_link(fun on_load_handler_init/0)). + spawn(fun on_load_handler_init/0)). on_load_handler_init() -> - on_load_loop([]). + on_load_loop([], false). -on_load_loop(Mods) -> +on_load_loop(Mods, Debug0) -> receive + {init_debug_flag,Debug} -> + on_load_loop(Mods, Debug); {loaded,Mod} -> - on_load_loop([Mod|Mods]); + on_load_loop([Mod|Mods], Debug0); run_on_load -> - run_on_load_handlers(Mods), + run_on_load_handlers(Mods, Debug0), exit(on_load_done) end. -run_on_load_handlers([M|Ms]) -> +run_on_load_handlers([M|Ms], Debug) -> + debug(Debug, {running_on_load_handler,M}), Fun = fun() -> Res = erlang:call_on_load_function(M), exit(Res) @@ -1363,9 +1371,12 @@ run_on_load_handlers([M|Ms]) -> erlang:finish_after_on_load(M, Keep), case Keep of false -> - exit({on_load_function_failed,M}); + Error = {on_load_function_failed,M}, + debug(Debug, Error), + exit(Error); true -> - run_on_load_handlers(Ms) + debug(Debug, {on_load_handler_returned_ok,M}), + run_on_load_handlers(Ms, Debug) end end; -run_on_load_handlers([]) -> ok. +run_on_load_handlers([], _) -> ok. diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index 43e6f6cd88..10be852e92 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -1,19 +1,19 @@ %% %% %CopyrightBegin% -%% -%% Copyright Ericsson AB 2000-2009. All Rights Reserved. -%% +%% +%% Copyright Ericsson AB 2000-2010. All Rights Reserved. +%% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in %% compliance with the License. You should have received a copy of the %% Erlang Public License along with this software. If not, it can be %% retrieved online at http://www.erlang.org/. -%% +%% %% Software distributed under the License is distributed on an "AS IS" %% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See %% the License for the specific language governing rights and limitations %% under the License. -%% +%% %% %CopyrightEnd% %% -module(prim_file). @@ -25,7 +25,7 @@ %%% Interface towards a single file's contents. Uses ?FD_DRV. %% Generic file contents operations --export([open/2, close/1, sync/1, position/2, truncate/1, +-export([open/2, close/1, datasync/1, sync/1, advise/4, position/2, truncate/1, write/2, pwrite/2, pwrite/3, read/2, read_line/1, pread/2, pread/3, copy/3]). %% Specialized file operations @@ -96,6 +96,8 @@ -define(FILE_IPREAD, 27). -define(FILE_ALTNAME, 28). -define(FILE_READ_LINE, 29). +-define(FILE_FDATASYNC, 30). +-define(FILE_ADVISE, 31). %% Driver responses -define(FILE_RESP_OK, 0). @@ -107,6 +109,8 @@ -define(FILE_RESP_LDATA, 6). -define(FILE_RESP_N2DATA, 7). -define(FILE_RESP_EOF, 8). +-define(FILE_RESP_FNAME, 9). +-define(FILE_RESP_ALL_DATA, 10). %% Open modes for the driver's open function. -define(EFILE_MODE_READ, 1). @@ -114,9 +118,10 @@ -define(EFILE_MODE_READ_WRITE, 3). -define(EFILE_MODE_APPEND, 4). -define(EFILE_COMPRESSED, 8). +-define(EFILE_MODE_EXCL, 16). %% Use this mask to get just the mode bits to be passed to the driver. --define(EFILE_MODE_MASK, 15). +-define(EFILE_MODE_MASK, 31). %% Seek modes for the driver's seek function. -define(EFILE_SEEK_SET, 0). @@ -130,6 +135,13 @@ %% IPREAD variants -define(IPREAD_S32BU_P32BU, 0). +%% POSIX file advises +-define(POSIX_FADV_NORMAL, 0). +-define(POSIX_FADV_RANDOM, 1). +-define(POSIX_FADV_SEQUENTIAL, 2). +-define(POSIX_FADV_WILLNEED, 3). +-define(POSIX_FADV_DONTNEED, 4). +-define(POSIX_FADV_NOREUSE, 5). %%%----------------------------------------------------------------- @@ -143,7 +155,7 @@ %% Opens a file using the driver port Port. Returns {error, Reason} %% | {ok, FileDescriptor} open(Port, File, ModeList) when is_port(Port), - is_list(File), + (is_list(File) orelse is_binary(File)), is_list(ModeList) -> case open_mode(ModeList) of {Mode, _Portopts, _Setopts} -> @@ -155,10 +167,11 @@ open(_,_,_) -> {error, badarg}. %% Opens a file. Returns {error, Reason} | {ok, FileDescriptor}. -open(File, ModeList) when is_list(File), is_list(ModeList) -> +open(File, ModeList) when (is_list(File) orelse is_binary(File)), + is_list(ModeList) -> case open_mode(ModeList) of {Mode, Portopts, Setopts} -> - open_int({?FD_DRV, Portopts}, File, Mode, Setopts); + open_int({?FD_DRV, Portopts},File, Mode, Setopts); Reason -> {error, Reason} end; @@ -186,7 +199,7 @@ open_int({Driver, Portopts}, File, Mode, Setopts) -> end; open_int(Port, File, Mode, Setopts) -> M = Mode band ?EFILE_MODE_MASK, - case drv_command(Port, [<<?FILE_OPEN, M:32>>, File, 0]) of + case drv_command(Port, [<<?FILE_OPEN, M:32>>, pathname(File)]) of {ok, Number} -> open_int_setopts(Port, Number, Setopts); Error -> @@ -220,7 +233,35 @@ close(#file_descriptor{module = ?MODULE, data = {Port, _}}) -> close(Port) when is_port(Port) -> drv_close(Port). +-define(ADVISE(Offs, Len, Adv), + <<?FILE_ADVISE, Offs:64/signed, Len:64/signed, + Adv:32/signed>>). +%% Returns {error, Reason} | ok. +advise(#file_descriptor{module = ?MODULE, data = {Port, _}}, + Offset, Length, Advise) -> + case Advise of + normal -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_NORMAL), + drv_command(Port, Cmd); + random -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_RANDOM), + drv_command(Port, Cmd); + sequential -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_SEQUENTIAL), + drv_command(Port, Cmd); + will_need -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_WILLNEED), + drv_command(Port, Cmd); + dont_need -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_DONTNEED), + drv_command(Port, Cmd); + no_reuse -> + Cmd = ?ADVISE(Offset, Length, ?POSIX_FADV_NOREUSE), + drv_command(Port, Cmd); + _ -> + {error, einval} + end. %% Returns {error, Reason} | ok. write(#file_descriptor{module = ?MODULE, data = {Port, _}}, Bytes) -> @@ -292,6 +333,9 @@ pwrite(#file_descriptor{module = ?MODULE}, _, _) -> {error, badarg}. +%% Returns {error, Reason} | ok. +datasync(#file_descriptor{module = ?MODULE, data = {Port, _}}) -> + drv_command(Port, [?FILE_FDATASYNC]). %% Returns {error, Reason} | ok. sync(#file_descriptor{module = ?MODULE, data = {Port, _}}) -> @@ -448,7 +492,7 @@ ipread_s32bu_p32bu(#file_descriptor{module = ?MODULE, data = {_, _}}, %% Returns {ok, Contents} | {error, Reason} -read_file(File) -> +read_file(File) when (is_list(File) orelse is_binary(File)) -> case drv_open(?FD_DRV, [binary]) of {ok, Port} -> Result = read_file(Port, File), @@ -456,11 +500,14 @@ read_file(File) -> Result; {error, _} = Error -> Error - end. + end; +read_file(_) -> + {error, badarg}. %% Takes a Port opened with open/1. -read_file(Port, File) when is_port(Port) -> - Cmd = [?FILE_READ_FILE | File], +read_file(Port, File) when is_port(Port), + (is_list(File) orelse is_binary(File))-> + Cmd = [?FILE_READ_FILE | pathname(File)], case drv_command(Port, Cmd) of {error, enomem} -> %% It could possibly help to do a @@ -471,12 +518,14 @@ read_file(Port, File) when is_port(Port) -> drv_command(Port, Cmd); Result -> Result - end. + end; +read_file(_,_) -> + {error, badarg}. %% Returns {error, Reason} | ok. -write_file(File, Bin) -> +write_file(File, Bin) when (is_list(File) orelse is_binary(File)) -> case open(File, [binary, write]) of {ok, Handle} -> Result = write(Handle, Bin), @@ -484,8 +533,10 @@ write_file(File, Bin) -> Result; Error -> Error - end. - + end; +write_file(_, _) -> + {error, badarg}. + %%%----------------------------------------------------------------- @@ -498,7 +549,7 @@ write_file(File, Bin) -> %% Returns {ok, Port}, the Port should be used as first argument in all %% the following functions. Returns {error, Reason} upon failure. start() -> - try erlang:open_port({spawn, atom_to_list(?DRV)}, []) of + try erlang:open_port({spawn, atom_to_list(?DRV)}, [binary]) of Port -> {ok, Port} catch @@ -555,7 +606,7 @@ get_cwd(_, _) -> {error, badarg}. get_cwd_int(Drive) -> - get_cwd_int({?DRV, []}, Drive). + get_cwd_int({?DRV, [binary]}, Drive). get_cwd_int(Port, Drive) -> drv_command(Port, <<?FILE_PWD, Drive>>). @@ -565,7 +616,7 @@ get_cwd_int(Port, Drive) -> %% set_cwd/{1,2} set_cwd(Dir) -> - set_cwd_int({?DRV, []}, Dir). + set_cwd_int({?DRV, [binary]}, Dir). set_cwd(Port, Dir) when is_port(Port) -> set_cwd_int(Port, Dir). @@ -591,89 +642,88 @@ set_cwd_int(Port, Dir0) -> end), %% Dir is now either a string or an EXIT tuple. %% An EXIT tuple will fail in the following catch. - drv_command(Port, [?FILE_CHDIR, Dir, 0]). + drv_command(Port, [?FILE_CHDIR, pathname(Dir)]). %% delete/{1,2} delete(File) -> - delete_int({?DRV, []}, File). + delete_int({?DRV, [binary]}, File). delete(Port, File) when is_port(Port) -> delete_int(Port, File). delete_int(Port, File) -> - drv_command(Port, [?FILE_DELETE, File, 0]). + drv_command(Port, [?FILE_DELETE, pathname(File)]). %% rename/{2,3} rename(From, To) -> - rename_int({?DRV, []}, From, To). + rename_int({?DRV, [binary]}, From, To). rename(Port, From, To) when is_port(Port) -> rename_int(Port, From, To). rename_int(Port, From, To) -> - drv_command(Port, [?FILE_RENAME, From, 0, To, 0]). + drv_command(Port, [?FILE_RENAME, pathname(From), pathname(To)]). %% make_dir/{1,2} make_dir(Dir) -> - make_dir_int({?DRV, []}, Dir). + make_dir_int({?DRV, [binary]}, Dir). make_dir(Port, Dir) when is_port(Port) -> make_dir_int(Port, Dir). make_dir_int(Port, Dir) -> - drv_command(Port, [?FILE_MKDIR, Dir, 0]). + drv_command(Port, [?FILE_MKDIR, pathname(Dir)]). %% del_dir/{1,2} del_dir(Dir) -> - del_dir_int({?DRV, []}, Dir). + del_dir_int({?DRV, [binary]}, Dir). del_dir(Port, Dir) when is_port(Port) -> del_dir_int(Port, Dir). del_dir_int(Port, Dir) -> - drv_command(Port, [?FILE_RMDIR, Dir, 0]). + drv_command(Port, [?FILE_RMDIR, pathname(Dir)]). %% read_file_info/{1,2} read_file_info(File) -> - read_file_info_int({?DRV, []}, File). + read_file_info_int({?DRV, [binary]}, File). read_file_info(Port, File) when is_port(Port) -> read_file_info_int(Port, File). read_file_info_int(Port, File) -> - drv_command(Port, [?FILE_FSTAT, File, 0]). + drv_command(Port, [?FILE_FSTAT, pathname(File)]). %% altname/{1,2} altname(File) -> - altname_int({?DRV, []}, File). + altname_int({?DRV, [binary]}, File). altname(Port, File) when is_port(Port) -> altname_int(Port, File). altname_int(Port, File) -> - drv_command(Port, [?FILE_ALTNAME, File, 0]). - + drv_command(Port, [?FILE_ALTNAME, pathname(File)]). %% write_file_info/{2,3} write_file_info(File, Info) -> - write_file_info_int({?DRV, []}, File, Info). + write_file_info_int({?DRV, [binary]}, File, Info). write_file_info(Port, File, Info) when is_port(Port) -> write_file_info_int(Port, File, Info). @@ -699,72 +749,72 @@ write_file_info_int(Port, date_to_bytes(Atime), date_to_bytes(Mtime), date_to_bytes(Ctime), - File, 0]). + pathname(File)]). %% make_link/{2,3} make_link(Old, New) -> - make_link_int({?DRV, []}, Old, New). + make_link_int({?DRV, [binary]}, Old, New). make_link(Port, Old, New) when is_port(Port) -> make_link_int(Port, Old, New). make_link_int(Port, Old, New) -> - drv_command(Port, [?FILE_LINK, Old, 0, New, 0]). + drv_command(Port, [?FILE_LINK, pathname(Old), pathname(New)]). %% make_symlink/{2,3} make_symlink(Old, New) -> - make_symlink_int({?DRV, []}, Old, New). + make_symlink_int({?DRV, [binary]}, Old, New). make_symlink(Port, Old, New) when is_port(Port) -> make_symlink_int(Port, Old, New). make_symlink_int(Port, Old, New) -> - drv_command(Port, [?FILE_SYMLINK, Old, 0, New, 0]). + drv_command(Port, [?FILE_SYMLINK, pathname(Old), pathname(New)]). %% read_link/{2,3} read_link(Link) -> - read_link_int({?DRV, []}, Link). + read_link_int({?DRV, [binary]}, Link). read_link(Port, Link) when is_port(Port) -> read_link_int(Port, Link). read_link_int(Port, Link) -> - drv_command(Port, [?FILE_READLINK, Link, 0]). + drv_command(Port, [?FILE_READLINK, pathname(Link)]). %% read_link_info/{2,3} read_link_info(Link) -> - read_link_info_int({?DRV, []}, Link). + read_link_info_int({?DRV, [binary]}, Link). read_link_info(Port, Link) when is_port(Port) -> read_link_info_int(Port, Link). read_link_info_int(Port, Link) -> - drv_command(Port, [?FILE_LSTAT, Link, 0]). + drv_command(Port, [?FILE_LSTAT, pathname(Link)]). %% list_dir/{1,2} list_dir(Dir) -> - list_dir_int({?DRV, []}, Dir). + list_dir_int({?DRV, [binary]}, Dir). list_dir(Port, Dir) when is_port(Port) -> list_dir_int(Port, Dir). list_dir_int(Port, Dir) -> - drv_command(Port, [?FILE_READDIR, Dir, 0], []). + drv_command(Port, [?FILE_READDIR, pathname(Dir)], []). @@ -918,6 +968,8 @@ open_mode([compressed|Rest], Mode, Portopts, Setopts) -> open_mode([append|Rest], Mode, Portopts, Setopts) -> open_mode(Rest, Mode bor ?EFILE_MODE_APPEND bor ?EFILE_MODE_WRITE, Portopts, Setopts); +open_mode([exclusive|Rest], Mode, Portopts, Setopts) -> + open_mode(Rest, Mode bor ?EFILE_MODE_EXCL, Portopts, Setopts); open_mode([delayed_write|Rest], Mode, Portopts, Setopts) -> open_mode([{delayed_write, 64*1024, 2000}|Rest], Mode, Portopts, Setopts); @@ -983,8 +1035,6 @@ lseek_position(_) -> translate_response(?FILE_RESP_OK, []) -> ok; -translate_response(?FILE_RESP_OK, Data) -> - {ok, Data}; translate_response(?FILE_RESP_ERROR, List) when is_list(List) -> {error, list_to_atom(List)}; translate_response(?FILE_RESP_NUMBER, List) -> @@ -1031,6 +1081,16 @@ translate_response(?FILE_RESP_N2DATA = X, L0) when is_list(L0) -> end; translate_response(?FILE_RESP_EOF, []) -> eof; +translate_response(?FILE_RESP_FNAME, []) -> + ok; +translate_response(?FILE_RESP_FNAME, Data) when is_binary(Data) -> + {ok, prim_file:internal_native2name(Data)}; +translate_response(?FILE_RESP_FNAME, Data) -> + {ok, Data}; + +translate_response(?FILE_RESP_ALL_DATA, Data) -> + {ok, Data}; + translate_response(X, Data) -> {error, {bad_response_from_port, [X | Data]}}. @@ -1166,3 +1226,9 @@ lists_split([Hd | Tl], N, Rev) -> reverse(X) -> lists:reverse(X, []). reverse(L, T) -> lists:reverse(L, T). + +% Will add zero termination too +% The 'EXIT' tuple from a bad argument will eventually generate an error +% in list_to_binary, which is caught and generates the {error,badarg} return +pathname(File) -> + (catch prim_file:internal_name2native(File)). diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl index 0feb591efb..446656e45f 100644 --- a/erts/preloaded/src/prim_inet.erl +++ b/erts/preloaded/src/prim_inet.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2009. All Rights Reserved. +%% Copyright Ericsson AB 2000-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -37,7 +37,7 @@ -export([setopt/3, setopts/2, getopt/2, getopts/2, is_sockopt_val/2]). -export([chgopt/3, chgopts/2]). -export([getstat/2, getfd/1, getindex/1, getstatus/1, gettype/1, - getiflist/1, ifget/3, ifset/3, + getifaddrs/1, getiflist/1, ifget/3, ifset/3, gethostname/1]). -export([getservbyname/3, getservbyport/3]). -export([peername/1, setpeername/2]). @@ -216,9 +216,10 @@ bindx(S, AddFlag, Addrs) -> sctp -> %% Really multi-homed "bindx". Stringified args: %% [AddFlag, (Port, IP)+]: - Args = ?int8(AddFlag) ++ - lists:concat([?int16(Port)++ip_to_bytes(IP) || - {IP, Port} <- Addrs]), + Args = + [?int8(AddFlag)| + [[?int16(Port)|ip_to_bytes(IP)] || + {IP, Port} <- Addrs]], case ctl_cmd(S, ?SCTP_REQ_BINDX, Args) of {ok,_} -> {ok, S}; Error -> Error @@ -623,7 +624,7 @@ chgopt(S, Opt, Value) when is_port(S) -> chgopts(S, [{Opt,Value}]). chgopts(S, Opts) when is_port(S), is_list(Opts) -> - case inet:getopts(S, need_template(Opts)) of + case getopts(S, need_template(Opts)) of {ok,Templates} -> try merge_options(Opts, Templates) of NewOpts -> @@ -636,7 +637,94 @@ chgopts(S, Opts) when is_port(S), is_list(Opts) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% -%% IFLIST(insock()) -> {ok,IfNameList} | {error, Reason} +%% getifaddrs(insock()) -> {ok,IfAddrsList} | {error, Reason} +%% +%% IfAddrsList = [{Name,[Opts]}] +%% Name = string() +%% Opts = {flags,[Flag]} | {addr,Addr} | {netmask,Addr} | {broadaddr,Addr} +%% | {dstaddr,Addr} | {hwaddr,HwAddr} | {mtu,integer()} +%% Flag = up | broadcast | loopback | running | multicast +%% Addr = ipv4addr() | ipv6addr() +%% HwAddr = ethernet_addr() +%% +%% get interface name and addresses list +%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +getifaddrs(S) when is_port(S) -> + case ctl_cmd(S, ?INET_REQ_GETIFADDRS, []) of + {ok, Data} -> + {ok, comp_ifaddrs(build_ifaddrs(Data), ktree_empty())}; + {error,enotsup} -> + case getiflist(S) of + {ok, IFs} -> + {ok, getifaddrs_ifget(S, IFs)}; + Err1 -> Err1 + end; + Err2 -> Err2 + end. + +%% Restructure interface properties per interface and remove duplicates + +comp_ifaddrs([{If,Opts}|IfOpts], T) -> + case ktree_is_defined(If, T) of + true -> + OptSet = comp_ifaddrs_add(ktree_get(If, T), Opts), + comp_ifaddrs(IfOpts, ktree_update(If, OptSet, T)); + false -> + OptSet = comp_ifaddrs_add(ktree_empty(), Opts), + comp_ifaddrs(IfOpts, ktree_insert(If, OptSet, T)) + end; +comp_ifaddrs([], T) -> + [{If,ktree_keys(ktree_get(If, T))} || If <- ktree_keys(T)]. + +comp_ifaddrs_add(OptSet, [Opt|Opts]) -> + case ktree_is_defined(Opt, OptSet) of + true + when element(1, Opt) =:= flags; + element(1, Opt) =:= hwaddr -> + comp_ifaddrs_add(OptSet, Opts); + _ -> + comp_ifaddrs_add(ktree_insert(Opt, undefined, OptSet), Opts) + end; +comp_ifaddrs_add(OptSet, []) -> OptSet. + +%% Legacy emulation of getifaddrs + +getifaddrs_ifget(_, []) -> []; +getifaddrs_ifget(S, [IF|IFs]) -> + case ifget(S, IF, [flags]) of + {ok,[{flags,Flags}]=FlagsVals} -> + BroadOpts = + case member(broadcast, Flags) of + true -> + [broadaddr,hwaddr]; + false -> + [hwaddr] + end, + P2POpts = + case member(pointtopoint, Flags) of + true -> + [dstaddr|BroadOpts]; + false -> + BroadOpts + end, + getifaddrs_ifget(S, IFs, IF, FlagsVals, [addr,netmask|P2POpts]); + _ -> + getifaddrs_ifget(S, IFs, IF, [], [addr,netmask,hwaddr]) + end. + +getifaddrs_ifget(S, IFs, IF, FlagsVals, Opts) -> + OptVals = + case ifget(S, IF, Opts) of + {ok,OVs} -> OVs; + _ -> [] + end, + [{IF,FlagsVals++OptVals}|getifaddrs_ifget(S, IFs)]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% +%% getiflist(insock()) -> {ok,IfNameList} | {error, Reason} %% %% get interface name list %% @@ -1206,7 +1294,7 @@ type_opt_1(sctp_default_send_param) -> timetolive = [uint32,0], tsn = [], cumtsn = [], - assoc_id = [sctp_assoc_id,0]}}]; + assoc_id = [[sctp_assoc_id,0]]}}]; %% for SCTP_OPT_EVENTS type_opt_1(sctp_events) -> [{record,#sctp_event_subscribe{ @@ -1325,6 +1413,19 @@ type_value_2({enum,List}, Enum) -> {value,_} -> true; false -> false end; +type_value_2(sockaddr, Addr) -> + case Addr of + any -> true; + loopback -> true; + {A,B,C,D} when ?ip(A,B,C,D) -> true; + {A,B,C,D,E,F,G,H} when ?ip6(A,B,C,D,E,F,G,H) -> true; + _ -> false + end; +type_value_2(linkaddr, Addr) when is_list(Addr) -> + case len(Addr, 32768) of + undefined -> false; + _ -> true + end; type_value_2({bitenumlist,List}, EnumList) -> case enum_vals(EnumList, List) of Ls when is_list(Ls) -> true; @@ -1413,14 +1514,21 @@ enc_value_2(addr, {any,Port}) -> [?INET_AF_ANY|?int16(Port)]; enc_value_2(addr, {loopback,Port}) -> [?INET_AF_LOOPBACK|?int16(Port)]; -enc_value_2(addr, {IP,Port}) -> - case tuple_size(IP) of - 4 -> - [?INET_AF_INET,?int16(Port)|ip4_to_bytes(IP)]; - 8 -> - [?INET_AF_INET6,?int16(Port)|ip6_to_bytes(IP)] - end; +enc_value_2(addr, {IP,Port}) when tuple_size(IP) =:= 4 -> + [?INET_AF_INET,?int16(Port)|ip4_to_bytes(IP)]; +enc_value_2(addr, {IP,Port}) when tuple_size(IP) =:= 8 -> + [?INET_AF_INET6,?int16(Port)|ip6_to_bytes(IP)]; enc_value_2(ether, [X1,X2,X3,X4,X5,X6]) -> [X1,X2,X3,X4,X5,X6]; +enc_value_2(sockaddr, any) -> + [?INET_AF_ANY]; +enc_value_2(sockaddr, loopback) -> + [?INET_AF_LOOPBACK]; +enc_value_2(sockaddr, IP) when tuple_size(IP) =:= 4 -> + [?INET_AF_INET|ip4_to_bytes(IP)]; +enc_value_2(sockaddr, IP) when tuple_size(IP) =:= 8 -> + [?INET_AF_INET6|ip6_to_bytes(IP)]; +enc_value_2(linkaddr, Linkaddr) -> + [?int16(length(Linkaddr)),Linkaddr]; enc_value_2(sctp_assoc_id, Val) -> ?int32(Val); %% enc_value_2(sctp_assoc_id, Bin) -> [byte_size(Bin),Bin]; enc_value_2({enum,List}, Enum) -> @@ -1465,6 +1573,10 @@ dec_value(time, [X3,X2,X1,X0|T]) -> end; dec_value(ip, [A,B,C,D|T]) -> {{A,B,C,D}, T}; dec_value(ether,[X1,X2,X3,X4,X5,X6|T]) -> {[X1,X2,X3,X4,X5,X6],T}; +dec_value(sockaddr, [X|T]) -> + get_ip(X, T); +dec_value(linkaddr, [X1,X0|T]) -> + split(?i16(X1,X0), T); dec_value({enum,List}, [X3,X2,X1,X0|T]) -> Val = ?i32(X3,X2,X1,X0), case enum_name(Val, List) of @@ -1480,7 +1592,7 @@ dec_value({bitenumlist,List}, [X3,X2,X1,X0|T]) -> %% {enum_names(Val, List), T}; dec_value(binary,[L0,L1,L2,L3|List]) -> Len = ?i32(L0,L1,L2,L3), - {X,T}=lists:split(Len,List), + {X,T}=split(Len,List), {list_to_binary(X),T}; dec_value(Types, List) when is_tuple(Types) -> {L,T} = dec_value_tuple(Types, List, 1, []), @@ -1495,7 +1607,7 @@ dec_value_tuple(Types, List, N, Acc) {Term,Tail} = dec_value(element(N, Types), List), dec_value_tuple(Types, Tail, N+1, [Term|Acc]); dec_value_tuple(_, List, _, Acc) -> - {lists:reverse(Acc),List}. + {rev(Acc),List}. borlist([V|Vs], Value) -> borlist(Vs, V bor Value); @@ -1702,11 +1814,11 @@ merge_fields(_, _, _) -> []. %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -type_ifopt(addr) -> ip; -type_ifopt(broadaddr) -> ip; -type_ifopt(dstaddr) -> ip; +type_ifopt(addr) -> sockaddr; +type_ifopt(broadaddr) -> sockaddr; +type_ifopt(dstaddr) -> sockaddr; type_ifopt(mtu) -> int; -type_ifopt(netmask) -> ip; +type_ifopt(netmask) -> sockaddr; type_ifopt(flags) -> {bitenumlist, [{up, ?INET_IFF_UP}, @@ -1718,7 +1830,7 @@ type_ifopt(flags) -> {no_pointtopoint, ?INET_IFF_NPOINTTOPOINT}, {running, ?INET_IFF_RUNNING}, {multicast, ?INET_IFF_MULTICAST}]}; -type_ifopt(hwaddr) -> ether; +type_ifopt(hwaddr) -> linkaddr; type_ifopt(Opt) when is_atom(Opt) -> undefined. enc_ifopt(addr) -> ?INET_IFOPT_ADDR; @@ -1903,6 +2015,30 @@ encode_ifname(Name) -> if N > 255 -> {error, einval}; true -> {ok,[N | Name]} end. + +build_ifaddrs(Cs) -> + build_ifaddrs(Cs, []). +%% +build_ifaddrs([], []) -> + []; +build_ifaddrs([0|Cs], Acc) -> + Name = utf8_to_characters(rev(Acc)), + {Opts,Rest} = build_ifaddrs_opts(Cs, []), + [{Name,Opts}|build_ifaddrs(Rest)]; +build_ifaddrs([C|Cs], Acc) -> + build_ifaddrs(Cs, [C|Acc]). + +build_ifaddrs_opts([0|Cs], Acc) -> + {rev(Acc),Cs}; +build_ifaddrs_opts([C|Cs]=CCs, Acc) -> + case dec_ifopt(C) of + undefined -> + erlang:error(badarg, [CCs,Acc]); + Opt -> + Type = type_ifopt(Opt), + {Val,Rest} = dec_value(Type, Cs), + build_ifaddrs_opts(Rest, [{Opt,Val}|Acc]) + end. build_iflist(Cs) -> build_iflist(Cs, [], []). @@ -1927,6 +2063,80 @@ rev(L) -> rev(L,[]). rev([C|L],Acc) -> rev(L,[C|Acc]); rev([],Acc) -> Acc. +split(N, L) -> split(N, L, []). +split(0, L, R) when is_list(L) -> {rev(R),L}; +split(N, [H|T], R) when is_integer(N), N > 0 -> split(N-1, T, [H|R]). + +len(L, N) -> len(L, N, 0). +len([], N, C) when is_integer(N), N >= 0 -> C; +len(L, 0, _) when is_list(L) -> undefined; +len([_|L], N, C) when is_integer(N), N >= 0 -> len(L, N-1, C+1). + +member(X, [X|_]) -> true; +member(X, [_|Xs]) -> member(X, Xs); +member(_, []) -> false. + + + +%% Lookup tree that keeps key insert order + +ktree_empty() -> {[],tree()}. +ktree_is_defined(Key, {_,T}) -> tree(T, Key, is_defined). +ktree_get(Key, {_,T}) -> tree(T, Key, get). +ktree_insert(Key, V, {Keys,T}) -> {[Key|Keys],tree(T, Key, {insert,V})}. +ktree_update(Key, V, {Keys,T}) -> {Keys,tree(T, Key, {update,V})}. +ktree_keys({Keys,_}) -> rev(Keys). + +%% Simple lookup tree. Hash the key to get statistical balance. +%% Key is matched equal, not compared equal. + +tree() -> nil. +tree(T, Key, Op) -> tree(T, Key, Op, erlang:phash2(Key)). + +tree(nil, _, is_defined, _) -> false; +tree(nil, K, {insert,V}, _) -> {K,V,nil,nil}; +tree({K,_,_,_}, K, is_defined, _) -> true; +tree({K,V,_,_}, K, get, _) -> V; +tree({K,_,L,R}, K, {update,V}, _) -> {K,V,L,R}; +tree({K0,V0,L,R}, K, Op, H) -> + H0 = erlang:phash2(K0), + if H0 < H; H0 =:= H, K0 < K -> + if is_tuple(Op) -> + {K0,V0,tree(L, K, Op, H),R}; + true -> + tree(L, K, Op, H) + end; + true -> + if is_tuple(Op) -> + {K0,V0,L,tree(R, K, Op, H)}; + true -> + tree(R, K, Op, H) + end + end. + + + +utf8_to_characters([]) -> []; +utf8_to_characters([B|Bs]=Arg) when (B band 16#FF) =:= B -> + if 16#F8 =< B -> + erlang:error(badarg, [Arg]); + 16#F0 =< B -> + utf8_to_characters(Bs, B band 16#07, 3); + 16#E0 =< B -> + utf8_to_characters(Bs, B band 16#0F, 2); + 16#C0 =< B -> + utf8_to_characters(Bs, B band 16#1F, 1); + 16#80 =< B -> + erlang:error(badarg, [Arg]); + true -> + [B|utf8_to_characters(Bs)] + end. +%% +utf8_to_characters(Bs, U, 0) -> + [U|utf8_to_characters(Bs)]; +utf8_to_characters([B|Bs], U, N) when ((B band 16#3F) bor 16#80) =:= B -> + utf8_to_characters(Bs, (U bsl 6) bor (B band 16#3F), N-1). + ip_to_bytes(IP) when tuple_size(IP) =:= 4 -> ip4_to_bytes(IP); ip_to_bytes(IP) when tuple_size(IP) =:= 8 -> ip6_to_bytes(IP). diff --git a/erts/test/Makefile b/erts/test/Makefile index 796403e182..94458da019 100644 --- a/erts/test/Makefile +++ b/erts/test/Makefile @@ -28,6 +28,7 @@ EBIN = . # ---------------------------------------------------- MODULES= \ + autoimport_SUITE \ erlc_SUITE \ install_SUITE \ nt_SUITE \ @@ -38,12 +39,14 @@ MODULES= \ erlexec_SUITE \ z_SUITE +ERL_XML = $(ERL_TOP)/erts/doc/src/erlang.xml +ERL_XML_TARGET = autoimport_SUITE_data/erlang.xml ERL_FILES= $(MODULES:%=%.erl) TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) -EXTRA_FILES = install_SUITE_data/install_bin +EXTRA_FILES = install_SUITE_data/install_bin $(ERL_XML_TARGET) # ---------------------------------------------------- # Release directory specification @@ -65,6 +68,10 @@ install_SUITE_data/install_bin: ../../make/install_bin rm -f $@ cp -p $< $@ +$(ERL_XML_TARGET): $(ERL_XML) + rm -f $@ + cp -p $< $@ + clean: rm -f $(TARGET_FILES) $(EXTRA_FILES) rm -f core *~ diff --git a/erts/test/autoimport_SUITE.erl b/erts/test/autoimport_SUITE.erl new file mode 100644 index 0000000000..2430dac78d --- /dev/null +++ b/erts/test/autoimport_SUITE.erl @@ -0,0 +1,163 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%%% Purpose: Test erlang.xml re autoimports +-module(autoimport_SUITE). + +-include_lib("test_server/include/test_server.hrl"). +-export([all/1,init_per_testcase/2,fin_per_testcase/2,autoimports/1]). +-define(TEST_TIMEOUT, ?t:seconds(180)). + +all(suite) -> [autoimports]. + +init_per_testcase(_Func, Config) -> + Dog = test_server:timetrap(?TEST_TIMEOUT), + [{watchdog, Dog} | Config]. + +fin_per_testcase(_Func, Config) -> + Dog = ?config(watchdog, Config), + catch test_server:timetrap_cancel(Dog), + ok. + + +autoimports(suite) -> + []; +autoimports(doc) -> + ["Check that erlang.xml documents autoimports correctly"]; +autoimports(Config) when is_list(Config) -> + ?line XML = filename:join([?config(data_dir,Config),"erlang.xml"]), + ?line case xml(XML) of + [] -> + ok; + List -> + lists:foreach(fun({[],F,A}) -> + io:format("erlang:~s/~p is wrongly " + "documented as ~s/~p~n", + [F,A,F,A]); + ({"erlang",F,A}) -> + io:format("~s/~p is wrongly " + "documented as " + "erlang:~s/~p~n", + [F,A,F,A]) + end,List), + ?t:fail({wrong_autoimports,List}) + end. + +%% +%% Ugly chunk of code to heuristically analyze the erlang.xml +%% documentation file. Don't view this as an example... +%% + +xml(XMLFile) -> + {ok,File} = file:open(XMLFile,[read]), + DocData = xloop(file:read_line(File),File), + file:close(File), + analyze(DocData). + +xloop({ok,Line},File) -> + case re:run(Line,"\\<name\\>",[{capture,none}]) of + nomatch -> + xloop(file:read_line(File),File); + match -> + X = re:replace(Line,"\\).*",")",[{return,list}]), + Y = re:replace(X,".*\\>","",[{return,list}]), + Mod = get_module(Y), + Rest1 = fstrip(Mod++":",Y), + Func = get_function(Rest1), + Rest2 = fstrip(Func++"(", Rest1), + Argc = count_args(Rest2,1,0,false), + [{Mod,Func,Argc} | + xloop(file:read_line(File),File)] + end; +xloop(_,_) -> + []. + +analyze([{[],Func,Arity}|T]) -> + case erl_internal:bif(list_to_atom(Func),Arity) of + true -> + analyze(T); + false -> + [{[],Func,Arity} | + analyze(T)] + end; +analyze([{"erlang",Func,Arity}|T]) -> + case erl_internal:bif(list_to_atom(Func),Arity) of + true -> + [{"erlang",Func,Arity}|analyze(T)]; + false -> + analyze(T) + end; +analyze([_|T]) -> + analyze(T); +analyze([]) -> + []. + + +count_args([],_,N,false) -> + N; +count_args([],_,N,true) -> + N+1; +count_args(_,0,N,true) -> + N+1; +count_args(_,0,N,false) -> + N; +count_args([Par|T],Level,N,Got) when (Par =:= 40) or + (Par =:= 123) or (Par =:= 91) -> + count_args(T,Level+1,N,(Level =:= 1) or Got); +count_args([41|T],1,N,true) -> + count_args(T,0,N+1,false); +count_args([Par|T],Level,N, Got) when (Par =:= 41) or + (Par =:= 125) or (Par =:= 93) -> + count_args(T,Level-1,N,Got); +count_args([$,|T],1,N,true) -> + count_args(T,1,N+1,false); +count_args([$ |T],A,B,C) -> + count_args(T,A,B,C); +count_args([_|T],1,N,_) -> + count_args(T,1,N,true); +count_args([_|T],A,B,C) -> + count_args(T,A,B,C). + +fstrip([],X) -> + X; +fstrip(_,[]) -> + []; +fstrip([H|T1],[H|T2]) -> + fstrip(T1,T2); +fstrip(_,L) -> + L. + +get_module(X) -> + get_module(X,[]). +get_module([],_) -> + []; +get_module([$:|_],Acc) -> + lists:reverse(Acc); +get_module([40|_],_) -> %( + []; +get_module([H|T],Acc) -> + get_module(T,[H|Acc]). + +get_function(X) -> + get_function(X,[]). +get_function([],_) -> + []; +get_function([40|_],Acc) -> %( + lists:reverse(Acc); +get_function([H|T],Acc) -> + get_function(T,[H|Acc]). diff --git a/erts/test/autoimport_SUITE_data/dummy.txt b/erts/test/autoimport_SUITE_data/dummy.txt new file mode 100644 index 0000000000..972643e527 --- /dev/null +++ b/erts/test/autoimport_SUITE_data/dummy.txt @@ -0,0 +1,19 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +%% Purpouse: Dummy diff --git a/erts/test/erlc_SUITE.erl b/erts/test/erlc_SUITE.erl index 437f020f99..4797f78be2 100644 --- a/erts/test/erlc_SUITE.erl +++ b/erts/test/erlc_SUITE.erl @@ -21,13 +21,13 @@ %% Tests the erlc command by compiling various types of files. -export([all/1, compile_erl/1, compile_yecc/1, compile_script/1, - compile_mib/1, good_citizen/1, deep_cwd/1]). + compile_mib/1, good_citizen/1, deep_cwd/1, arg_overflow/1]). -include_lib("test_server/include/test_server.hrl"). all(suite) -> [compile_erl, compile_yecc, compile_script, compile_mib, - good_citizen, deep_cwd]. + good_citizen, deep_cwd, arg_overflow]. %% Copy from erlc_SUITE_data/include/erl_test.hrl. @@ -189,6 +189,18 @@ deep_cwd_1(PrivDir) -> ?line true = filelib:is_file("test.beam"), ok. +%% Test that a large number of command line switches does not +%% overflow the argument buffer +arg_overflow(Config) when is_list(Config) -> + ?line {SrcDir, _OutDir, Cmd} = get_cmd(Config), + ?line FileName = filename:join(SrcDir, "erl_test_ok.erl"), + ?line Args = lists:flatten([ ["-D", integer_to_list(N), "=1 "] || + N <- lists:seq(1,10000) ]), + ?line run(Config, Cmd, FileName, Args, + ["Warning: function foo/0 is unused\$", + "_OK_"]), + ok. + erlc() -> case os:find_executable("erlc") of false -> diff --git a/erts/test/erlexec_SUITE.erl b/erts/test/erlexec_SUITE.erl index 164ce9faaf..6adb865f6d 100644 --- a/erts/test/erlexec_SUITE.erl +++ b/erts/test/erlexec_SUITE.erl @@ -33,7 +33,7 @@ -export([all/1, init_per_testcase/2, fin_per_testcase/2]). --export([args_file/1, evil_args_file/1, env/1, args_file_env/1, otp_7461/1, otp_7461_remote/1, otp_8209/1]). +-export([args_file/1, evil_args_file/1, env/1, args_file_env/1, otp_7461/1, otp_7461_remote/1, otp_8209/1, zdbbl_dist_buf_busy_limit/1]). -include_lib("test_server/include/test_server.hrl"). @@ -53,7 +53,8 @@ fin_per_testcase(_Case, Config) -> all(doc) -> []; all(suite) -> - [args_file, evil_args_file, env, args_file_env, otp_7461, otp_8209]. + [args_file, evil_args_file, env, args_file_env, otp_7461, otp_8209, + zdbbl_dist_buf_busy_limit]. otp_8209(doc) -> @@ -330,6 +331,25 @@ otp_7461_remote([halt, Pid]) -> io:format("halt order from ~p to node ~p\n",[Pid,node()]), halt(). +zdbbl_dist_buf_busy_limit(doc) -> + ["Check +zdbbl flag"]; +zdbbl_dist_buf_busy_limit(suite) -> + []; +zdbbl_dist_buf_busy_limit(Config) when is_list(Config) -> + LimKB = 1122233, + LimB = LimKB*1024, + ?line {ok,[[PName]]} = init:get_argument(progname), + ?line SNameS = "erlexec_test_02", + ?line SName = list_to_atom(SNameS++"@"++ + hd(tl(string:tokens(atom_to_list(node()),"@")))), + ?line Cmd = PName ++ " -sname "++SNameS++" -setcookie "++ + atom_to_list(erlang:get_cookie()) ++ + " +zdbbl " ++ integer_to_list(LimKB), + ?line open_port({spawn,Cmd},[]), + ?line pong = loop_ping(SName,40), + ?line LimB = rpc:call(SName,erlang,system_info,[dist_buf_busy_limit]), + ?line ok = cleanup_node(SNameS, 10), + ok. %% diff --git a/erts/test/ethread_SUITE.erl b/erts/test/ethread_SUITE.erl index bbc79e9381..69e5af802f 100644 --- a/erts/test/ethread_SUITE.erl +++ b/erts/test/ethread_SUITE.erl @@ -37,21 +37,15 @@ equal_tids/1, mutex/1, try_lock_mutex/1, - recursive_mutex/1, - time_now/1, cond_wait/1, - cond_timedwait/1, broadcast/1, detached_thread/1, max_threads/1, - forksafety/1, - vfork/1, tsd/1, spinlock/1, rwspinlock/1, rwmutex/1, - atomic/1, - gate/1]). + atomic/1]). -include_lib("test_server/include/test_server.hrl"). @@ -60,21 +54,15 @@ tests() -> equal_tids, mutex, try_lock_mutex, - recursive_mutex, - time_now, cond_wait, - cond_timedwait, broadcast, detached_thread, max_threads, - forksafety, - vfork, tsd, spinlock, rwspinlock, rwmutex, - atomic, - gate]. + atomic]. all(doc) -> []; all(suite) -> tests(). @@ -114,24 +102,6 @@ try_lock_mutex(suite) -> try_lock_mutex(Config) -> run_case(Config, "try_lock_mutex", ""). -recursive_mutex(doc) -> - ["Tests recursive mutexes."]; -recursive_mutex(suite) -> - []; -recursive_mutex(Config) -> - run_case(Config, "recursive_mutex", ""). - -time_now(doc) -> - ["Tests ethr_time_now by comparing time values with Erlang."]; -time_now(suite) -> - []; -time_now(Config) -> - run_case(Config, "time_now", "", fun (P) -> - spawn_link(fun () -> - watchdog(P) - end) - end). - wd_dispatch(P) -> receive bye -> @@ -169,13 +139,6 @@ cond_wait(suite) -> cond_wait(Config) -> run_case(Config, "cond_wait", ""). -cond_timedwait(doc) -> - ["Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast."]; -cond_timedwait(suite) -> - []; -cond_timedwait(Config) -> - run_case(Config, "cond_timedwait", ""). - broadcast(doc) -> ["Tests that a ethr_cond_broadcast really wakes up all waiting threads"]; broadcast(suite) -> @@ -195,26 +158,15 @@ max_threads(doc) -> max_threads(suite) -> []; max_threads(Config) -> - run_case(Config, "max_threads", ""). - -forksafety(doc) -> - ["Tests forksafety."]; -forksafety(suite) -> - []; -forksafety(Config) -> - run_case(Config, "forksafety", ""). - -vfork(doc) -> - ["Tests vfork with threads."]; -vfork(suite) -> - case ?t:os_type() of - {unix, osf1} -> - {skip, "vfork() known to hang multi-threaded applications on osf1"}; + case {os:type(), os:version()} of + {{unix,darwin}, {9, _, _}} -> + %% For some reason pthread_create() crashes when more + %% threads cannot be created, instead of returning an + %% error code on our MacOS X Leopard machine... + {skipped, "MacOS X Leopard cannot cope with this test..."}; _ -> - [] - end; -vfork(Config) -> - run_case(Config, "vfork", ""). + run_case(Config, "max_threads", "") + end. tsd(doc) -> ["Tests thread specific data."]; @@ -251,13 +203,6 @@ atomic(suite) -> atomic(Config) -> run_case(Config, "atomic", ""). -gate(doc) -> - ["Tests gates."]; -gate(suite) -> - []; -gate(Config) -> - run_case(Config, "gate", ""). - %% %% %% Auxiliary functions diff --git a/erts/test/ethread_SUITE_data/ethread_tests.c b/erts/test/ethread_SUITE_data/ethread_tests.c index f779f13c51..0b59ff5aa6 100644 --- a/erts/test/ethread_SUITE_data/ethread_tests.c +++ b/erts/test/ethread_SUITE_data/ethread_tests.c @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -90,10 +90,12 @@ static void print_line(char *frmt,...) print_eol(); } +#if 0 /* Currently not used; silence annoying warning... */ static void print(char *frmt,...) { PRINT_VA_LIST(frmt); } +#endif static void fail(char *frmt,...) { @@ -197,8 +199,8 @@ create_join_thread_test(void) } for (i = 1; i <= CJTT_NO_THREADS; i++) { - int *tres; - res = ethr_thr_join(cjtt_tids[i], (void **) &tres); + void *tres; + res = ethr_thr_join(cjtt_tids[i], &tres); ASSERT(res == 0); ASSERT(tres == &cjtt_res[i]); ASSERT(cjtt_res[i] == i); @@ -216,8 +218,8 @@ create_join_thread_test(void) #define ETT_THREADS 100000 static ethr_tid ett_tids[3]; -static ethr_mutex ett_mutex = ETHR_MUTEX_INITER; -static ethr_cond ett_cond = ETHR_COND_INITER; +static ethr_mutex ett_mutex; +static ethr_cond ett_cond; static int ett_terminate; static void * @@ -234,14 +236,12 @@ static void * ett_thread2(void *unused) { int res; - res = ethr_mutex_lock(&ett_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&ett_mutex); while (!ett_terminate) { res = ethr_cond_wait(&ett_cond, &ett_mutex); ASSERT(res == 0); } - res = ethr_mutex_unlock(&ett_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&ett_mutex); return NULL; } @@ -250,6 +250,10 @@ equal_tids_test(void) { int res, i; + res = ethr_mutex_init(&ett_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&ett_cond); + ASSERT(res == 0); ett_tids[0] = ethr_self(); res = ethr_thr_create(&ett_tids[1], ett_thread, (void *) &ett_tids[1], NULL); @@ -298,13 +302,10 @@ equal_tids_test(void) ASSERT(res == 0); } - res = ethr_mutex_lock(&ett_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&ett_mutex); ett_terminate = 1; - res = ethr_cond_signal(&ett_cond); - ASSERT(res == 0); - res = ethr_mutex_unlock(&ett_mutex); - ASSERT(res == 0); + ethr_cond_signal(&ett_cond); + ethr_mutex_unlock(&ett_mutex); res = ethr_thr_join(ett_tids[1], NULL); ASSERT(res == 0); @@ -321,17 +322,14 @@ equal_tids_test(void) * Tests mutexes. */ -static ethr_mutex mt_mutex = ETHR_MUTEX_INITER; +static ethr_mutex mt_mutex; static int mt_data; void * mt_thread(void *unused) { - int res; - print_line("Aux thread tries to lock mutex"); - res = ethr_mutex_lock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mt_mutex); print_line("Aux thread locked mutex"); ASSERT(mt_data == 0); @@ -345,8 +343,7 @@ mt_thread(void *unused) ASSERT(mt_data == 1); - res = ethr_mutex_unlock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mt_mutex); print_line("Aux thread unlocked mutex"); return NULL; @@ -356,18 +353,18 @@ mt_thread(void *unused) static void mutex_test(void) { - int do_restart = 1; int res; ethr_tid tid; - print_line("Running test with statically initialized mutex"); + print_line("Trying to initialize mutex"); + res = ethr_mutex_init(&mt_mutex); + ASSERT(res == 0); + print_line("Initialized mutex"); - restart: mt_data = 0; print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mt_mutex); print_line("Main thread locked mutex"); ASSERT(mt_data == 0); @@ -383,8 +380,7 @@ mutex_test(void) ASSERT(mt_data == 0); - res = ethr_mutex_unlock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mt_mutex); print_line("Main thread unlocked mutex"); print_line("Main thread goes to sleep for 1 second"); @@ -392,8 +388,7 @@ mutex_test(void) print_line("Main thread woke up"); print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mt_mutex); print_line("Main thread locked mutex"); ASSERT(mt_data == 1); @@ -404,8 +399,7 @@ mutex_test(void) ASSERT(mt_data == 1); - res = ethr_mutex_unlock(&mt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mt_mutex); print_line("Main thread unlocked mutex"); res = ethr_thr_join(tid, NULL); @@ -416,20 +410,6 @@ mutex_test(void) ASSERT(res == 0); print_line("Main thread destroyed mutex"); - if (do_restart) { - do_restart = 0; - - print_line("Running test with dynamically initialized mutex"); - - print_line("Trying to initialize mutex"); - res = ethr_mutex_init(&mt_mutex); - ASSERT(res == 0); - print_line("Initialized mutex"); - - goto restart; - - } - } /* @@ -438,9 +418,9 @@ mutex_test(void) * Tests try lock mutex operation. */ -static ethr_mutex tlmt_mtx1 = ETHR_MUTEX_INITER; -static ethr_mutex tlmt_mtx2 = ETHR_MUTEX_INITER; -static ethr_cond tlmt_cnd2 = ETHR_COND_INITER; +static ethr_mutex tlmt_mtx1; +static ethr_mutex tlmt_mtx2; +static ethr_cond tlmt_cnd2; static int tlmt_mtx1_locked; static int tlmt_mtx1_do_unlock; @@ -450,32 +430,24 @@ tlmt_thread(void *unused) { int res; - res = ethr_mutex_lock(&tlmt_mtx1); - ASSERT(res == 0); - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx1); + ethr_mutex_lock(&tlmt_mtx2); tlmt_mtx1_locked = 1; - res = ethr_cond_signal(&tlmt_cnd2); - ASSERT(res == 0); + ethr_cond_signal(&tlmt_cnd2); while (!tlmt_mtx1_do_unlock) { res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); - res = ethr_mutex_unlock(&tlmt_mtx1); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx2); + ethr_mutex_unlock(&tlmt_mtx1); - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx2); tlmt_mtx1_locked = 0; - res = ethr_cond_signal(&tlmt_cnd2); - ASSERT(res == 0); - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_cond_signal(&tlmt_cnd2); + ethr_mutex_unlock(&tlmt_mtx2); return NULL; } @@ -486,48 +458,49 @@ try_lock_mutex_test(void) int i, res; ethr_tid tid; + res = ethr_mutex_init(&tlmt_mtx1); + ASSERT(res == 0); + res = ethr_mutex_init(&tlmt_mtx2); + ASSERT(res == 0); + res = ethr_cond_init(&tlmt_cnd2); + ASSERT(res == 0); + tlmt_mtx1_locked = 0; tlmt_mtx1_do_unlock = 0; res = ethr_thr_create(&tid, tlmt_thread, NULL, NULL); ASSERT(res == 0); - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx2); while (!tlmt_mtx1_locked) { res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx2); for (i = 0; i < 10; i++) { res = ethr_mutex_trylock(&tlmt_mtx1); ASSERT(res == EBUSY); } - res = ethr_mutex_lock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_lock(&tlmt_mtx2); tlmt_mtx1_do_unlock = 1; - res = ethr_cond_signal(&tlmt_cnd2); - ASSERT(res == 0); + ethr_cond_signal(&tlmt_cnd2); while (tlmt_mtx1_locked) { res = ethr_cond_wait(&tlmt_cnd2, &tlmt_mtx2); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&tlmt_mtx2); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx2); res = ethr_mutex_trylock(&tlmt_mtx1); ASSERT(res == 0); - res = ethr_mutex_unlock(&tlmt_mtx1); - ASSERT(res == 0); + ethr_mutex_unlock(&tlmt_mtx1); res = ethr_thr_join(tid, NULL); ASSERT(res == 0); @@ -541,328 +514,88 @@ try_lock_mutex_test(void) } /* - * The recursive mutex test case. - * - * Tests recursive mutexes. - */ - -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - -static ethr_mutex rmt_mutex -#ifdef ETHR_REC_MUTEX_INITER - = ETHR_REC_MUTEX_INITER -#endif - ; -static int rmt_data; - -void * -rmt_thread(void *unused) -{ - int res; - - print_line("Aux thread tries to lock mutex"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Aux thread locked mutex"); - - ASSERT(rmt_data == 0); - - rmt_data = 1; - print_line("Aux thread wrote"); - - print_line("Aux thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Aux thread woke up"); - - ASSERT(rmt_data == 1); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Aux thread unlocked mutex"); - - return NULL; -} - -#endif - -static void -recursive_mutex_test(void) -{ -#ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT - int do_restart = 1; - int res; - ethr_tid tid; - -#ifdef ETHR_REC_MUTEX_INITER - print_line("Running test with statically initialized mutex"); -#else - goto dynamic_init; -#endif - - restart: - rmt_data = 0; - - print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread locked mutex"); - - print_line("Main thread tries to lock mutex again"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread locked mutex again"); - - ASSERT(rmt_data == 0); - - print_line("Main thread about to create aux thread"); - res = ethr_thr_create(&tid, rmt_thread, NULL, NULL); - ASSERT(res == 0); - print_line("Main thread created aux thread"); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - ASSERT(rmt_data == 0); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread unlocked mutex"); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - ASSERT(rmt_data == 0); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread unlocked mutex again"); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - print_line("Main thread tries to lock mutex"); - res = ethr_mutex_lock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread locked mutex"); - - ASSERT(rmt_data == 1); - - print_line("Main thread goes to sleep for 1 second"); - do_sleep(1); - print_line("Main thread woke up"); - - ASSERT(rmt_data == 1); - - res = ethr_mutex_unlock(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread unlocked mutex"); - - res = ethr_thr_join(tid, NULL); - ASSERT(res == 0); - print_line("Main thread joined aux thread"); - - res = ethr_mutex_destroy(&rmt_mutex); - ASSERT(res == 0); - print_line("Main thread destroyed mutex"); - - if (do_restart) { -#ifndef ETHR_REC_MUTEX_INITER - dynamic_init: -#endif - do_restart = 0; - - print_line("Running test with dynamically initialized mutex"); - - print_line("Trying to initialize mutex"); - res = ethr_rec_mutex_init(&rmt_mutex); - ASSERT(res == 0); - print_line("Initialized mutex"); - - goto restart; - } - -#ifndef ETHR_REC_MUTEX_INITER - succeed("Static initializer for recursive mutexes not supported"); -#endif - -#else /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */ - skip("Recursive mutexes not supported"); -#endif /* #ifdef ETHR_HAVE_ETHR_REC_MUTEX_INIT */ -} - -/* - * The time now test. - * - * Tests ethr_time_now by comparing time values with Erlang. - */ -#define TNT_MAX_TIME_DIFF 200000 -#define TNT_MAX_TIME_VALUES 52 - -static void -time_now_test(void) -{ - int scanf_res, time_now_res, i, no_values, max_abs_diff; - static ethr_timeval tv[TNT_MAX_TIME_VALUES]; - static int ms[TNT_MAX_TIME_VALUES]; - - i = 0; - do { - ASSERT(i < TNT_MAX_TIME_VALUES); - scanf_res = scanf("%d", &ms[i]); - time_now_res = ethr_time_now(&tv[i]); - ASSERT(scanf_res == 1); - ASSERT(time_now_res == 0); -#if 0 - print_line("Got %d; %ld:%ld", ms[i], tv[i].tv_sec, tv[i].tv_nsec); -#endif - i++; - } while (ms[i-1] >= 0); - - no_values = i-1; - - ASSERT(ms[0] == 0); - - print_line("TNT_MAX_TIME_DIFF = %d (us)", TNT_MAX_TIME_DIFF); - - max_abs_diff = 0; - - for (i = 1; i < no_values; i++) { - long diff; - long tn_us; - long e_us; - - tn_us = (tv[i].tv_sec - tv[0].tv_sec) * 1000000; - tn_us += (tv[i].tv_nsec - tv[0].tv_nsec)/1000; - - e_us = ms[i]*1000; - - diff = e_us - tn_us; - - print_line("Erlang time = %ld us; ethr_time_now = %ld us; diff %ld us", - e_us, tn_us, diff); - - if (max_abs_diff < abs((int) diff)) { - max_abs_diff = abs((int) diff); - } - - ASSERT(e_us - TNT_MAX_TIME_DIFF <= tn_us); - ASSERT(tn_us <= e_us + TNT_MAX_TIME_DIFF); - } - - print_line("Max absolute diff = %d us", max_abs_diff); - succeed("Max absolute diff = %d us", max_abs_diff); -} - -/* * The cond wait test case. * * Tests ethr_cond_wait with ethr_cond_signal and ethr_cond_broadcast. */ -static ethr_mutex cwt_mutex = ETHR_MUTEX_INITER; -static ethr_cond cwt_cond = ETHR_COND_INITER; +static ethr_mutex cwt_mutex; +static ethr_cond cwt_cond; static int cwt_counter; void * -cwt_thread(void *is_timedwait_test_ptr) +cwt_thread(void *unused) { - int use_timedwait = *((int *) is_timedwait_test_ptr); int res; - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); - if (use_timedwait) { - ethr_timeval tv; - res = ethr_time_now(&tv); - ASSERT(res == 0); - tv.tv_sec += 3600; /* Make sure we won't time out */ - - do { - res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tv); - } while (res == EINTR); - ASSERT(res == 0); - } - else { - do { - res = ethr_cond_wait(&cwt_cond, &cwt_mutex); - } while (res == EINTR); - ASSERT(res == 0); - } + do { + res = ethr_cond_wait(&cwt_cond, &cwt_mutex); + } while (res == EINTR); + ASSERT(res == 0); cwt_counter++; - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); return NULL; } static void -cond_wait_test(int is_timedwait_test) +cond_wait_test(void) { - int do_restart = !is_timedwait_test; ethr_tid tid1, tid2; int res; - if (!is_timedwait_test) - print_line("Running test with statically initialized mutex and cond"); + res = ethr_mutex_init(&cwt_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&cwt_cond); + ASSERT(res == 0); - restart: /* Wake with signal */ cwt_counter = 0; - res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL); ASSERT(res == 0); - res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL); ASSERT(res == 0); do_sleep(1); /* Make sure threads waits on cond var */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); - res = ethr_cond_signal(&cwt_cond); /* Wake one thread */ - ASSERT(res == 0); + ethr_cond_signal(&cwt_cond); /* Wake one thread */ do_sleep(1); /* Make sure awakened thread waits on mutex */ ASSERT(cwt_counter == 0); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); do_sleep(1); /* Let awakened thread proceed */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); ASSERT(cwt_counter == 1); - res = ethr_cond_signal(&cwt_cond); /* Wake the other thread */ - ASSERT(res == 0); + ethr_cond_signal(&cwt_cond); /* Wake the other thread */ do_sleep(1); /* Make sure awakened thread waits on mutex */ ASSERT(cwt_counter == 1); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); do_sleep(1); /* Let awakened thread proceed */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); ASSERT(cwt_counter == 2); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); res = ethr_thr_join(tid1, NULL); ASSERT(res == 0); @@ -875,35 +608,30 @@ cond_wait_test(int is_timedwait_test) cwt_counter = 0; - res = ethr_thr_create(&tid1, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid1, cwt_thread, NULL, NULL); ASSERT(res == 0); - res = ethr_thr_create(&tid2, cwt_thread, (void *) &is_timedwait_test, NULL); + res = ethr_thr_create(&tid2, cwt_thread, NULL, NULL); ASSERT(res == 0); do_sleep(1); /* Make sure threads waits on cond var */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); - res = ethr_cond_broadcast(&cwt_cond); /* Wake the threads */ - ASSERT(res == 0); + ethr_cond_broadcast(&cwt_cond); /* Wake the threads */ do_sleep(1); /* Make sure awakened threads wait on mutex */ ASSERT(cwt_counter == 0); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); do_sleep(1); /* Let awakened threads proceed */ - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&cwt_mutex); ASSERT(cwt_counter == 2); - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&cwt_mutex); res = ethr_thr_join(tid1, NULL); ASSERT(res == 0); @@ -916,113 +644,6 @@ cond_wait_test(int is_timedwait_test) res = ethr_cond_destroy(&cwt_cond); ASSERT(res == 0); - if (do_restart) { - do_restart = 0; - res = ethr_mutex_init(&cwt_mutex); - ASSERT(res == 0); - res = ethr_cond_init(&cwt_cond); - ASSERT(res == 0); - print_line("Running test with dynamically initialized mutex and cond"); - goto restart; - } -} - -/* - * The cond timedwait test case. - * - * Tests ethr_cond_timedwait with ethr_cond_signal and ethr_cond_broadcast. - */ - -#define CTWT_MAX_TIME_DIFF 100000 - -static long -ctwt_check_timeout(long to) -{ - int res; - ethr_timeval tva, tvb; - long diff, abs_diff; - - res = ethr_time_now(&tva); - ASSERT(res == 0); - - tva.tv_sec += to / 1000; - tva.tv_nsec += (to % 1000) * 1000000; - if (tva.tv_nsec >= 1000000000) { - tva.tv_sec++; - tva.tv_nsec -= 1000000000; - ASSERT(tva.tv_nsec < 1000000000); - } - - do { - res = ethr_cond_timedwait(&cwt_cond, &cwt_mutex, &tva); - } while (res == EINTR); - ASSERT(res == ETIMEDOUT); - - res = ethr_time_now(&tvb); - ASSERT(res == 0); - - diff = (tvb.tv_sec - tva.tv_sec) * 1000000; - diff += (tvb.tv_nsec - tva.tv_nsec)/1000; - - print("Timeout=%ld; ", to); - print("tva.tv_sec=%ld tva.tv_nsec=%ld; ", tva.tv_sec, tva.tv_nsec); - print("tvb.tv_sec=%ld tvb.tv_nsec=%ld; ", tvb.tv_sec, tvb.tv_nsec); - print_line("diff (tvb - tva) = %ld us", diff); - - abs_diff = (long) abs((int) diff); - - ASSERT(CTWT_MAX_TIME_DIFF >= abs_diff); - return abs_diff; -} - -static void -cond_timedwait_test(void) -{ - int do_restart = 1; - long abs_diff, max_abs_diff = 0; - int res; - -#define CTWT_UPD_MAX_DIFF if (abs_diff > max_abs_diff) max_abs_diff = abs_diff; - - print_line("Running test with statically initialized mutex and cond"); - - print_line("CTWT_MAX_TIME_DIFF=%d", CTWT_MAX_TIME_DIFF); - - restart: - - res = ethr_mutex_lock(&cwt_mutex); - ASSERT(res == 0); - - abs_diff = ctwt_check_timeout(300); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(700); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(1000); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(2300); - CTWT_UPD_MAX_DIFF; - abs_diff = ctwt_check_timeout(5100); - CTWT_UPD_MAX_DIFF; - - res = ethr_mutex_unlock(&cwt_mutex); - ASSERT(res == 0); - - cond_wait_test(1); - - if (do_restart) { - do_restart = 0; - res = ethr_mutex_init(&cwt_mutex); - ASSERT(res == 0); - res = ethr_cond_init(&cwt_cond); - ASSERT(res == 0); - print_line("Running test with dynamically initialized mutex and cond"); - goto restart; - } - - print_line("Max absolute diff = %d us", max_abs_diff); - succeed("Max absolute diff = %d us", max_abs_diff); - -#undef CTWT_UPD_MAX_DIFF } /* @@ -1037,9 +658,9 @@ cond_timedwait_test(void) static int bct_woken = 0; static int bct_waiting = 0; static int bct_done = 0; -static ethr_mutex bct_mutex = ETHR_MUTEX_INITER; -static ethr_cond bct_cond = ETHR_COND_INITER; -static ethr_cond bct_cntrl_cond = ETHR_COND_INITER; +static ethr_mutex bct_mutex; +static ethr_cond bct_cond; +static ethr_cond bct_cntrl_cond; static void * @@ -1047,30 +668,24 @@ bct_thread(void *unused) { int res; - res = ethr_mutex_lock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&bct_mutex); while (!bct_done) { bct_waiting++; - if (bct_waiting == BCT_THREADS) { - res = ethr_cond_signal(&bct_cntrl_cond); - ASSERT(res == 0); - } + if (bct_waiting == BCT_THREADS) + ethr_cond_signal(&bct_cntrl_cond); do { res = ethr_cond_wait(&bct_cond, &bct_mutex); } while (res == EINTR); ASSERT(res == 0); bct_woken++; - if (bct_woken == BCT_THREADS) { - res = ethr_cond_signal(&bct_cntrl_cond); - ASSERT(res == 0); - } + if (bct_woken == BCT_THREADS) + ethr_cond_signal(&bct_cntrl_cond); } - res = ethr_mutex_unlock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&bct_mutex); return NULL; } @@ -1081,14 +696,20 @@ broadcast_test(void) int res, i; ethr_tid tid[BCT_THREADS]; + res = ethr_mutex_init(&bct_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&bct_cntrl_cond); + ASSERT(res == 0); + res = ethr_cond_init(&bct_cond); + ASSERT(res == 0); + for (i = 0; i < BCT_THREADS; i++) { res = ethr_thr_create(&tid[i], bct_thread, NULL, NULL); ASSERT(res == 0); } - res = ethr_mutex_lock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&bct_mutex); for (i = 0; i < BCT_NO_OF_WAITS; i++) { @@ -1101,8 +722,7 @@ broadcast_test(void) bct_woken = 0; /* Wake all threads */ - res = ethr_cond_broadcast(&bct_cond); - ASSERT(res == 0); + ethr_cond_broadcast(&bct_cond); while (bct_woken != BCT_THREADS) { res = ethr_cond_wait(&bct_cntrl_cond, &bct_mutex); @@ -1114,13 +734,11 @@ broadcast_test(void) bct_done = 1; /* Wake all threads */ - res = ethr_cond_broadcast(&bct_cond); - ASSERT(res == 0); + ethr_cond_broadcast(&bct_cond); - res = ethr_mutex_unlock(&bct_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&bct_mutex); - for (i = 0; i < BCT_THREADS - 1; i++) { + for (i = 0; i < BCT_THREADS; i++) { res = ethr_thr_join(tid[i], NULL); ASSERT(res == 0); } @@ -1143,26 +761,22 @@ broadcast_test(void) #define DT_THREADS (50*1024) #define DT_BATCH_SIZE 64 -static ethr_mutex dt_mutex = ETHR_MUTEX_INITER; -static ethr_cond dt_cond = ETHR_COND_INITER; +static ethr_mutex dt_mutex; +static ethr_cond dt_cond; static int dt_count; static int dt_limit; static void * dt_thread(void *unused) { - int res; - - res = ethr_mutex_lock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&dt_mutex); dt_count++; if (dt_count >= dt_limit) ethr_cond_signal(&dt_cond); - res = ethr_mutex_unlock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&dt_mutex); return NULL; } @@ -1174,6 +788,11 @@ detached_thread_test(void) ethr_tid tid[DT_BATCH_SIZE]; int i, j, res; + res = ethr_mutex_init(&dt_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&dt_cond); + ASSERT(res == 0); + thr_opts.detached = 1; dt_count = 0; dt_limit = 0; @@ -1187,14 +806,12 @@ detached_thread_test(void) ASSERT(res == 0); } - res = ethr_mutex_lock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&dt_mutex); while (dt_count < dt_limit) { res = ethr_cond_wait(&dt_cond, &dt_mutex); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&dt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&dt_mutex); print_line("dt_count = %d", dt_count); } @@ -1211,8 +828,8 @@ detached_thread_test(void) #define MTT_TIMES 10 static int mtt_terminate; -static ethr_mutex mtt_mutex = ETHR_MUTEX_INITER; -static ethr_cond mtt_cond = ETHR_COND_INITER; +static ethr_mutex mtt_mutex; +static ethr_cond mtt_cond; static char mtt_string[22*MTT_TIMES]; /* 22 is enough for ", %d" */ @@ -1220,16 +837,14 @@ void *mtt_thread(void *unused) { int res; - res = ethr_mutex_lock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mtt_mutex); while (!mtt_terminate) { res = ethr_cond_wait(&mtt_cond, &mtt_mutex); ASSERT(res == 0 || res == EINTR); } - res = ethr_mutex_unlock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mtt_mutex); return NULL; } @@ -1265,16 +880,13 @@ mtt_create_join_threads(void) print_line("%d = ethr_thr_create()", res); print_line("Number of created threads: %d", no_threads); - res = ethr_mutex_lock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_lock(&mtt_mutex); mtt_terminate = 1; - res = ethr_cond_broadcast(&mtt_cond); - ASSERT(res == 0); + ethr_cond_broadcast(&mtt_cond); - res = ethr_mutex_unlock(&mtt_mutex); - ASSERT(res == 0); + ethr_mutex_unlock(&mtt_mutex); while (ix) { res = ethr_thr_join(tids[--ix], NULL); @@ -1292,9 +904,14 @@ mtt_create_join_threads(void) static void max_threads_test(void) { - int no_threads[MTT_TIMES], i, up, down, eq; + int no_threads[MTT_TIMES], i, up, down, eq, res; char *str; + res = ethr_mutex_init(&mtt_mutex); + ASSERT(res == 0); + res = ethr_cond_init(&mtt_cond); + ASSERT(res == 0); + for (i = 0; i < MTT_TIMES; i++) { no_threads[i] = mtt_create_join_threads(); } @@ -1326,272 +943,6 @@ max_threads_test(void) } - -/* - * The forksafety test case. - * - * Tests forksafety. - */ -#ifdef __WIN32__ -#define NO_FORK_PRESENT -#endif - -#ifndef NO_FORK_PRESENT - -static ethr_mutex ft_test_inner_mutex = ETHR_MUTEX_INITER; -static ethr_mutex ft_test_outer_mutex = ETHR_MUTEX_INITER; -static ethr_mutex ft_go_mutex = ETHR_MUTEX_INITER; -static ethr_cond ft_go_cond = ETHR_COND_INITER; -static int ft_go; -static int ft_have_forked; - -static void * -ft_thread(void *unused) -{ - int res; - - res = ethr_mutex_lock(&ft_test_outer_mutex); - ASSERT(res == 0); - - res = ethr_mutex_lock(&ft_go_mutex); - ASSERT(res == 0); - - ft_go = 1; - res = ethr_cond_signal(&ft_go_cond); - ASSERT(res == 0); - res = ethr_mutex_unlock(&ft_go_mutex); - ASSERT(res == 0); - - do_sleep(1); - ASSERT(!ft_have_forked); - - res = ethr_mutex_lock(&ft_test_inner_mutex); - ASSERT(res == 0); - - res = ethr_mutex_unlock(&ft_test_inner_mutex); - ASSERT(res == 0); - - do_sleep(1); - ASSERT(!ft_have_forked); - - res = ethr_mutex_unlock(&ft_test_outer_mutex); - ASSERT(res == 0); - - do_sleep(1); - ASSERT(ft_have_forked); - - - return NULL; -} - -#endif /* #ifndef NO_FORK_PRESENT */ - -static void -forksafety_test(void) -{ -#ifdef NO_FORK_PRESENT - skip("No fork() present; nothing to test"); -#elif defined(DEBUG) - skip("Doesn't work in debug build"); -#else - char snd_msg[] = "ok, bye!"; - char rec_msg[sizeof(snd_msg)*2]; - ethr_tid tid; - int res; - int fds[2]; - - - res = ethr_mutex_set_forksafe(&ft_test_inner_mutex); - if (res == ENOTSUP) { - skip("Forksafety not supported on this platform!"); - } - ASSERT(res == 0); - res = ethr_mutex_set_forksafe(&ft_test_outer_mutex); - ASSERT(res == 0); - - - res = pipe(fds); - ASSERT(res == 0); - - ft_go = 0; - ft_have_forked = 0; - - res = ethr_mutex_lock(&ft_go_mutex); - ASSERT(res == 0); - - res = ethr_thr_create(&tid, ft_thread, NULL, NULL); - ASSERT(res == 0); - - do { - res = ethr_cond_wait(&ft_go_cond, &ft_go_mutex); - } while (res == EINTR || !ft_go); - ASSERT(res == 0); - - res = ethr_mutex_unlock(&ft_go_mutex); - ASSERT(res == 0); - - res = fork(); - ft_have_forked = 1; - if (res == 0) { - close(fds[0]); - res = ethr_mutex_lock(&ft_test_outer_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_lock(&ft_test_inner_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_unlock(&ft_test_inner_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_unlock(&ft_test_outer_mutex); - if (res != 0) - _exit(1); - - res = ethr_mutex_destroy(&ft_test_inner_mutex); - if (res != 0) - _exit(1); - res = ethr_mutex_destroy(&ft_test_outer_mutex); - if (res != 0) - _exit(1); - - res = (int) write(fds[1], (void *) snd_msg, sizeof(snd_msg)); - if (res != sizeof(snd_msg)) - _exit(1); - close(fds[1]); - _exit(0); - } - ASSERT(res > 0); - close(fds[1]); - - res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg)); - ASSERT(res == (int) sizeof(snd_msg)); - ASSERT(strcmp(snd_msg, rec_msg) == 0); - - close(fds[0]); -#endif -} - - -/* - * The vfork test case. - * - * Tests vfork with threads. - */ - -#ifdef __WIN32__ -#define NO_VFORK_PRESENT -#endif - -#ifndef NO_VFORK_PRESENT - -#undef vfork - -static ethr_mutex vt_mutex = ETHR_MUTEX_INITER; - -static void * -vt_thread(void *vprog) -{ - char *prog = (char *) vprog; - int res; - char snd_msg[] = "ok, bye!"; - char rec_msg[sizeof(snd_msg)*2]; - int fds[2]; - char closefd[20]; - char writefd[20]; - - res = pipe(fds); - ASSERT(res == 0); - - res = sprintf(closefd, "%d", fds[0]); - ASSERT(res <= 20); - res = sprintf(writefd, "%d", fds[1]); - ASSERT(res <= 20); - - print("parent: About to vfork and execute "); - print("execlp(\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\", NULL)", - prog, prog, "vfork", "exec", snd_msg, closefd, writefd); - print_line(" in child"); - res = vfork(); - if (res == 0) { - execlp(prog, prog, "vfork", "exec", snd_msg, closefd, writefd, NULL); - _exit(1); - } - ASSERT(res > 0); - - print_line("parent: I'm back"); - - close(fds[1]); - - res = (int) read(fds[0], (void *) rec_msg, sizeof(rec_msg)); - print_line("parent: %d = read()", res); - print_line("parent: rec_msg=\"%s\"", rec_msg); - ASSERT(res == (int) sizeof(snd_msg)); - ASSERT(strcmp(snd_msg, rec_msg) == 0); - - close(fds[0]); - - return NULL; -} - -#endif /* #ifndef NO_VFORK_PRESENT */ - -static void -vfork_test(int argc, char *argv[]) -{ -#ifdef NO_VFORK_PRESENT - skip("No vfork() present; nothing to test"); -#else - int res; - ethr_tid tid; - - if (argc == 6 && strcmp("exec", argv[2]) == 0) { - /* We are child after vfork() and execlp() ... */ - - char *snd_msg; - int closefd; - int writefd; - - snd_msg = argv[3]; - closefd = atoi(argv[4]); - writefd = atoi(argv[5]); - - print_line("child: snd_msg=\"%s\"; closefd=%d writefd=%d", - snd_msg, closefd, writefd); - - close(closefd); - - res = (int) write(writefd, (void *) snd_msg, strlen(snd_msg)+1); - print_line("child: %d = write()", res); - if (res != strlen(snd_msg)+1) - exit(1); - close(writefd); - print_line("child: bye"); - exit(0); - } - ASSERT(argc == 2); - - res = ethr_mutex_set_forksafe(&vt_mutex); - ASSERT(res == 0 || res == ENOTSUP); - res = ethr_mutex_lock(&vt_mutex); - ASSERT(res == 0); - - res = ethr_thr_create(&tid, vt_thread, (void *) argv[0], NULL); - ASSERT(res == 0); - - do_sleep(1); - - res = ethr_mutex_unlock(&vt_mutex); - ASSERT(res == 0); - - res = ethr_thr_join(tid, NULL); - ASSERT(res == 0); - - res = ethr_mutex_destroy(&vt_mutex); - ASSERT(res == 0); -#endif -} - - /* * The tsd test case. * @@ -1651,11 +1002,8 @@ static int st_data; void * st_thread(void *unused) { - int res; - print_line("Aux thread tries to lock spinlock"); - res = ethr_spin_lock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_lock(&st_spinlock); print_line("Aux thread locked spinlock"); ASSERT(st_data == 0); @@ -1669,8 +1017,7 @@ st_thread(void *unused) ASSERT(st_data == 1); - res = ethr_spin_unlock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_unlock(&st_spinlock); print_line("Aux thread unlocked spinlock"); return NULL; @@ -1691,8 +1038,7 @@ spinlock_test(void) st_data = 0; print_line("Main thread tries to lock spinlock"); - res = ethr_spin_lock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_lock(&st_spinlock); print_line("Main thread locked spinlock"); ASSERT(st_data == 0); @@ -1708,8 +1054,7 @@ spinlock_test(void) ASSERT(st_data == 0); - res = ethr_spin_unlock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_unlock(&st_spinlock); print_line("Main thread unlocked spinlock"); print_line("Main thread goes to sleep for 1 second"); @@ -1717,8 +1062,7 @@ spinlock_test(void) print_line("Main thread woke up"); print_line("Main thread tries to lock spinlock"); - res = ethr_spin_lock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_lock(&st_spinlock); print_line("Main thread locked spinlock"); ASSERT(st_data == 1); @@ -1729,8 +1073,7 @@ spinlock_test(void) ASSERT(st_data == 1); - res = ethr_spin_unlock(&st_spinlock); - ASSERT(res == 0); + ethr_spin_unlock(&st_spinlock); print_line("Main thread unlocked spinlock"); res = ethr_thr_join(tid, NULL); @@ -1757,23 +1100,19 @@ void * rwst_thread(void *unused) { int data; - int res; print_line("Aux thread tries to read lock rwspinlock"); - res = ethr_read_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_lock(&rwst_rwspinlock); print_line("Aux thread read locked rwspinlock"); ASSERT(rwst_data == 4711); print_line("Aux thread tries to read unlock rwspinlock"); - res = ethr_read_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_unlock(&rwst_rwspinlock); print_line("Aux thread read unlocked rwspinlock"); print_line("Aux thread tries to write lock rwspinlock"); - res = ethr_write_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_lock(&rwst_rwspinlock); print_line("Aux thread write locked rwspinlock"); data = ++rwst_data; @@ -1787,8 +1126,7 @@ rwst_thread(void *unused) ++rwst_data; print_line("Aux thread tries to write unlock rwspinlock"); - res = ethr_write_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_unlock(&rwst_rwspinlock); print_line("Aux thread write unlocked rwspinlock"); return NULL; @@ -1810,8 +1148,7 @@ rwspinlock_test(void) rwst_data = 4711; print_line("Main thread tries to read lock rwspinlock"); - res = ethr_read_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_lock(&rwst_rwspinlock); print_line("Main thread read locked rwspinlock"); ASSERT(rwst_data == 4711); @@ -1828,13 +1165,11 @@ rwspinlock_test(void) ASSERT(rwst_data == 4711); print_line("Main thread tries to read unlock rwspinlock"); - res = ethr_read_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_read_unlock(&rwst_rwspinlock); print_line("Main thread read unlocked rwspinlock"); print_line("Main thread tries to write lock rwspinlock"); - res = ethr_write_lock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_lock(&rwst_rwspinlock); print_line("Main thread write locked rwspinlock"); data = ++rwst_data; @@ -1847,8 +1182,7 @@ rwspinlock_test(void) ++rwst_data; print_line("Main thread tries to write unlock rwspinlock"); - res = ethr_write_unlock(&rwst_rwspinlock); - ASSERT(res == 0); + ethr_write_unlock(&rwst_rwspinlock); print_line("Main thread write unlocked rwspinlock"); res = ethr_thr_join(tid, NULL); @@ -1875,23 +1209,19 @@ void * rwmt_thread(void *unused) { int data; - int res; print_line("Aux thread tries to read lock rwmutex"); - res = ethr_rwmutex_rlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rlock(&rwmt_rwmutex); print_line("Aux thread read locked rwmutex"); ASSERT(rwmt_data == 4711); print_line("Aux thread tries to read unlock rwmutex"); - res = ethr_rwmutex_runlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_runlock(&rwmt_rwmutex); print_line("Aux thread read unlocked rwmutex"); print_line("Aux thread tries to write lock rwmutex"); - res = ethr_rwmutex_rwlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwlock(&rwmt_rwmutex); print_line("Aux thread write locked rwmutex"); data = ++rwmt_data; @@ -1905,8 +1235,7 @@ rwmt_thread(void *unused) ++rwmt_data; print_line("Aux thread tries to write unlock rwmutex"); - res = ethr_rwmutex_rwunlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwunlock(&rwmt_rwmutex); print_line("Aux thread write unlocked rwmutex"); return NULL; @@ -1928,8 +1257,7 @@ rwmutex_test(void) rwmt_data = 4711; print_line("Main thread tries to read lock rwmutex"); - res = ethr_rwmutex_rlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rlock(&rwmt_rwmutex); print_line("Main thread read locked rwmutex"); ASSERT(rwmt_data == 4711); @@ -1946,13 +1274,11 @@ rwmutex_test(void) ASSERT(rwmt_data == 4711); print_line("Main thread tries to read unlock rwmutex"); - res = ethr_rwmutex_runlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_runlock(&rwmt_rwmutex); print_line("Main thread read unlocked rwmutex"); print_line("Main thread tries to write lock rwmutex"); - res = ethr_rwmutex_rwlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwlock(&rwmt_rwmutex); print_line("Main thread write locked rwmutex"); data = ++rwmt_data; @@ -1965,8 +1291,7 @@ rwmutex_test(void) ++rwmt_data; print_line("Main thread tries to write unlock rwmutex"); - res = ethr_rwmutex_rwunlock(&rwmt_rwmutex); - ASSERT(res == 0); + ethr_rwmutex_rwunlock(&rwmt_rwmutex); print_line("Main thread write unlocked rwmutex"); res = ethr_thr_join(tid, NULL); @@ -1998,65 +1323,53 @@ static ethr_atomic_t at_data; void * at_thread(void *unused) { - int res, i; + int i; long val, go; - res = ethr_atomic_inctest(&at_ready, &val); - ASSERT(res == 0); + val = ethr_atomic_inc_read(&at_ready); ASSERT(val > 0); ASSERT(val <= AT_THREADS); do { - res = ethr_atomic_read(&at_go, &go); - ASSERT(res == 0); + go = ethr_atomic_read(&at_go); } while (!go); for (i = 0; i < AT_ITER; i++) { - res = ethr_atomic_or_old(&at_data, at_set_val, &val); - ASSERT(res == 0); + val = ethr_atomic_read_bor(&at_data, at_set_val); ASSERT(val >= (i == 0 ? 0 : at_set_val) + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_and_old(&at_data, ~at_rm_val, &val); - ASSERT(res == 0); + val = ethr_atomic_read_band(&at_data, ~at_rm_val); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_read(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_data); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_inctest(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_inc_read(&at_data); ASSERT(val > at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_dectest(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_dec_read(&at_data); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); - res = ethr_atomic_inc(&at_data); - ASSERT(res == 0); + ethr_atomic_inc(&at_data); - res = ethr_atomic_dec(&at_data); - ASSERT(res == 0); + ethr_atomic_dec(&at_data); - res = ethr_atomic_addtest(&at_data, (long) 4711, &val); - ASSERT(res == 0); + val = ethr_atomic_add_read(&at_data, (long) 4711); ASSERT(val >= at_set_val + (long) 2*4711); ASSERT(val <= at_max_val); - res = ethr_atomic_add(&at_data, (long) -4711); - ASSERT(res == 0); + ethr_atomic_add(&at_data, (long) -4711); ASSERT(val >= at_set_val + (long) 4711); ASSERT(val <= at_max_val); } - res = ethr_atomic_inc(&at_done); - ASSERT(res == 0); + ethr_atomic_inc(&at_done); return NULL; } @@ -2069,14 +1382,13 @@ atomic_test(void) ethr_tid tid[AT_THREADS]; ethr_thr_opts thr_opts = ETHR_THR_OPTS_DEFAULT_INITER; - if (sizeof(long) > 4) { +#if ETHR_SIZEOF_PTR > 4 at_rm_val = ((long) 1) << 57; at_set_val = ((long) 1) << 60; - } - else { +#else at_rm_val = ((long) 1) << 27; at_set_val = ((long) 1) << 30; - } +#endif at_max_val = at_set_val + at_rm_val + ((long) AT_THREADS + 1) * 4711; data_init = at_rm_val + (long) 4711; @@ -2085,22 +1397,15 @@ atomic_test(void) thr_opts.detached = 1; print_line("Initializing"); - res = ethr_atomic_init(&at_ready, 0); - ASSERT(res == 0); - res = ethr_atomic_init(&at_go, 0); - ASSERT(res == 0); - res = ethr_atomic_init(&at_done, data_init); - ASSERT(res == 0); - res = ethr_atomic_init(&at_data, data_init); - ASSERT(res == 0); + ethr_atomic_init(&at_ready, 0); + ethr_atomic_init(&at_go, 0); + ethr_atomic_init(&at_done, data_init); + ethr_atomic_init(&at_data, data_init); - res = ethr_atomic_read(&at_data, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_data); ASSERT(val == data_init); - res = ethr_atomic_set(&at_done, 0); - ASSERT(res == 0); - res = ethr_atomic_read(&at_done, &val); - ASSERT(res == 0); + ethr_atomic_set(&at_done, 0); + val = ethr_atomic_read(&at_done); ASSERT(val == 0); print_line("Creating threads"); @@ -2111,31 +1416,27 @@ atomic_test(void) print_line("Waiting for threads to ready up"); do { - res = ethr_atomic_read(&at_ready, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_ready); ASSERT(val >= 0); ASSERT(val <= AT_THREADS); } while (val != AT_THREADS); print_line("Letting threads loose"); - res = ethr_atomic_xchg(&at_go, 17, &val); - ASSERT(res == 0); + val = ethr_atomic_xchg(&at_go, 17); ASSERT(val == 0); - res = ethr_atomic_read(&at_go, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_go); ASSERT(val == 17); print_line("Waiting for threads to finish"); do { - res = ethr_atomic_read(&at_done, &val); - ASSERT(res == 0); + val = ethr_atomic_read(&at_done); ASSERT(val >= 0); ASSERT(val <= AT_THREADS); } while (val != AT_THREADS); print_line("Checking result"); - res = ethr_atomic_read(&at_data, &val); + val = ethr_atomic_read(&at_data); ASSERT(res == 0); ASSERT(val == data_final); print_line("Result ok"); @@ -2143,190 +1444,6 @@ atomic_test(void) } -/* - * The gate test case. - * - * Tests gates. - */ - -#define GT_THREADS 10 - -static ethr_atomic_t gt_wait1; -static ethr_atomic_t gt_wait2; -static ethr_atomic_t gt_done; - -static ethr_gate gt_gate1; -static ethr_gate gt_gate2; - -void * -gt_thread(void *thr_no) -{ - int no = (int)(long) thr_no; - int swait = no % 2 == 0; - int res; - long done; - - - do { - - res = ethr_atomic_inc(>_wait1); - ASSERT(res == 0); - - if (swait) - res = ethr_gate_swait(>_gate1, INT_MAX); - else - res = ethr_gate_wait(>_gate1); - ASSERT(res == 0); - - res = ethr_atomic_dec(>_wait1); - ASSERT(res == 0); - - res = ethr_atomic_inc(>_wait2); - ASSERT(res == 0); - - if (swait) - res = ethr_gate_swait(>_gate2, INT_MAX); - else - res = ethr_gate_wait(>_gate2); - ASSERT(res == 0); - - res = ethr_atomic_dec(>_wait2); - ASSERT(res == 0); - - res = ethr_atomic_read(>_done, &done); - ASSERT(res == 0); - } while (!done); - return NULL; -} - - -static void -gate_test(void) -{ - long val; - int res, i; - ethr_tid tid[GT_THREADS]; - - print_line("Initializing"); - res = ethr_atomic_init(>_wait1, 0); - ASSERT_EQ(res, 0, "%d"); - res = ethr_atomic_init(>_wait2, 0); - ASSERT_EQ(res, 0, "%d"); - res = ethr_atomic_init(>_done, 0); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_init(>_gate1); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_init(>_gate2); - ASSERT_EQ(res, 0, "%d"); - - print_line("Creating threads"); - for (i = 0; i < GT_THREADS; i++) { - res = ethr_thr_create(&tid[i], gt_thread, (void *) i, NULL); - ASSERT_EQ(res, 0, "%d"); - } - - print_line("Waiting for threads to ready up"); - do { - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT(0 <= val && val <= GT_THREADS); - } while (val != GT_THREADS); - - print_line("Testing"); - - res = ethr_gate_let_through(>_gate1, 8); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == 8)), - 60); - - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS - 8, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 8, "%ld"); - - res = ethr_gate_let_through(>_gate2, 4); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == 4)), - 60); - - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS - 4, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 4, "%ld"); - - res = ethr_gate_let_through(>_gate1, GT_THREADS); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == GT_THREADS)), - 60); - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 0, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS, "%ld"); - - res = ethr_gate_let_through(>_gate2, GT_THREADS); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM((res = ethr_atomic_read(>_wait2, &val), - (res != 0 || val == 4)), - 60); - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, GT_THREADS - 4, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 4, "%ld"); - - res = ethr_atomic_set(>_done, 1); - ASSERT_EQ(res, 0, "%d"); - - res = ethr_gate_let_through(>_gate2, GT_THREADS); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_let_through(>_gate1, GT_THREADS - 4); - ASSERT_EQ(res, 0, "%d"); - - WAIT_UNTIL_LIM(((res = ethr_atomic_read(>_wait1, &val)) != 0 - || (val == 0 - && ((res = ethr_atomic_read(>_wait2, &val)) != 0 - || val == 0))), - 60); - - res = ethr_atomic_read(>_wait1, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 0, "%ld"); - - res = ethr_atomic_read(>_wait2, &val); - ASSERT_EQ(res, 0, "%d"); - ASSERT_EQ(val, 0, "%ld"); - - print_line("Joining threads"); - for (i = 0; i < GT_THREADS; i++) { - res = ethr_thr_join(tid[i], NULL); - ASSERT_EQ(res, 0, "%d"); - } - - res = ethr_gate_destroy(>_gate1); - ASSERT_EQ(res, 0, "%d"); - res = ethr_gate_destroy(>_gate2); - ASSERT_EQ(res, 0, "%d"); - -} - #endif /* #ifndef ETHR_NO_THREAD_LIB */ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * @@ -2342,14 +1459,12 @@ main(int argc, char *argv[]) #ifndef ETHR_NO_THREAD_LIB { char *testcase; - int res; send_my_pid(); testcase = argv[1]; - res = ethr_init(NULL); - if (res != 0) + if (ethr_init(NULL) != 0 || ethr_late_init(NULL) != 0) fail("Failed to initialize the ethread library"); if (strcmp(testcase, "create_join_thread") == 0) @@ -2360,24 +1475,14 @@ main(int argc, char *argv[]) mutex_test(); else if (strcmp(testcase, "try_lock_mutex") == 0) try_lock_mutex_test(); - else if (strcmp(testcase, "recursive_mutex") == 0) - recursive_mutex_test(); - else if (strcmp(testcase, "time_now") == 0) - time_now_test(); else if (strcmp(testcase, "cond_wait") == 0) - cond_wait_test(0); - else if (strcmp(testcase, "cond_timedwait") == 0) - cond_timedwait_test(); + cond_wait_test(); else if (strcmp(testcase, "broadcast") == 0) broadcast_test(); else if (strcmp(testcase, "detached_thread") == 0) detached_thread_test(); else if (strcmp(testcase, "max_threads") == 0) max_threads_test(); - else if (strcmp(testcase, "forksafety") == 0) - forksafety_test(); - else if (strcmp(testcase, "vfork") == 0) - vfork_test(argc, argv); else if (strcmp(testcase, "tsd") == 0) tsd_test(); else if (strcmp(testcase, "spinlock") == 0) @@ -2388,8 +1493,6 @@ main(int argc, char *argv[]) rwmutex_test(); else if (strcmp(testcase, "atomic") == 0) atomic_test(); - else if (strcmp(testcase, "gate") == 0) - gate_test(); else skip("Test case \"%s\" not implemented yet", testcase); diff --git a/erts/test/z_SUITE.erl b/erts/test/z_SUITE.erl index 8faddeb0d3..9f13a7083d 100644 --- a/erts/test/z_SUITE.erl +++ b/erts/test/z_SUITE.erl @@ -253,6 +253,8 @@ core_file_search(#core_search_conf{search_dir = Base, core_cand(Conf, Core, Cores); "core." ++ _ -> core_cand(Conf, Core, Cores); + Bin when is_binary(Bin) -> %Icky filename; ignore + Cores; BName -> case lists:suffix(".core", BName) of true -> core_cand(Conf, Core, Cores); diff --git a/erts/vsn.mk b/erts/vsn.mk index 6e1338cff5..8a1590e74c 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -17,8 +17,8 @@ # %CopyrightEnd% # -VSN = 5.8 -SYSTEM_VSN = R14A +VSN = 5.8.3 +SYSTEM_VSN = R14B02 # Port number 4365 in 4.2 # Port number 4366 in 4.3 |