diff options
Diffstat (limited to 'erts')
302 files changed, 25296 insertions, 10743 deletions
diff --git a/erts/Makefile.in b/erts/Makefile.in index dc8edcd928..2e63fc469e 100644 --- a/erts/Makefile.in +++ b/erts/Makefile.in @@ -87,17 +87,20 @@ endif # in the same directory... local_setup: @cd start_scripts && $(MAKE) + @echo `ls $(ERL_TOP)/bin/` @rm -f $(ERL_TOP)/bin/erl $(ERL_TOP)/bin/erlc $(ERL_TOP)/bin/cerl \ $(ERL_TOP)/bin/erl.exe $(ERL_TOP)/bin/erlc.exe \ $(ERL_TOP)/bin/escript $(ERL_TOP)/bin/escript.exe \ $(ERL_TOP)/bin/dialyzer $(ERL_TOP)/bin/dialyzer.exe \ $(ERL_TOP)/bin/typer $(ERL_TOP)/bin/typer.exe \ $(ERL_TOP)/bin/run_test $(ERL_TOP)/bin/run_test.exe \ + $(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/ct_run.exe \ $(ERL_TOP)/bin/start*.boot $(ERL_TOP)/bin/start*.script @if [ "X$(TARGET)" = "Xwin32" ]; then \ cp $(ERL_TOP)/bin/$(TARGET)/dialyzer.exe $(ERL_TOP)/bin/dialyzer.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/typer.exe $(ERL_TOP)/bin/typer.exe; \ - cp $(ERL_TOP)/bin/$(TARGET)/run_test.exe $(ERL_TOP)/bin/run_test.exe; \ + cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/ct_run.exe; \ + cp $(ERL_TOP)/bin/$(TARGET)/ct_run.exe $(ERL_TOP)/bin/run_test.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erlc.exe $(ERL_TOP)/bin/erlc.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/erl.exe $(ERL_TOP)/bin/erl.exe; \ cp $(ERL_TOP)/bin/$(TARGET)/werl.exe $(ERL_TOP)/bin/werl.exe; \ @@ -117,7 +120,8 @@ local_setup: $(ERL_TOP)/erts/etc/unix/cerl.src > $(ERL_TOP)/bin/cerl; \ cp $(ERL_TOP)/bin/$(TARGET)/dialyzer $(ERL_TOP)/bin/dialyzer; \ cp $(ERL_TOP)/bin/$(TARGET)/typer $(ERL_TOP)/bin/typer; \ - cp $(ERL_TOP)/bin/$(TARGET)/run_test $(ERL_TOP)/bin/run_test; \ + cp $(ERL_TOP)/bin/$(TARGET)/ct_run $(ERL_TOP)/bin/ct_run; \ + ln -s $(ERL_TOP)/bin/ct_run $(ERL_TOP)/bin/run_test; \ cp $(ERL_TOP)/bin/$(TARGET)/erlc $(ERL_TOP)/bin/erlc; \ cp $(ERL_TOP)/bin/$(TARGET)/escript $(ERL_TOP)/bin/escript; \ chmod 755 $(ERL_TOP)/bin/erl $(ERL_TOP)/bin/erlc \ diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 3b1edd7605..a1211bbf0c 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -386,14 +386,24 @@ AC_DEFUN(LM_SYS_IPV6, AC_CACHE_VAL(ac_cv_sys_ipv6_support, [ok_so_far=yes AC_TRY_COMPILE([#include <sys/types.h> -#include <netinet/in.h>], +#ifdef __WIN32__ +#include <winsock2.h> +#include <ws2tcpip.h> +#else +#include <netinet/in.h> +#endif], [struct in6_addr a6; struct sockaddr_in6 s6;], ok_so_far=yes, ok_so_far=no) if test $ok_so_far = yes; then ac_cv_sys_ipv6_support=yes else AC_TRY_COMPILE([#include <sys/types.h> -#include <netinet/in.h>], +#ifdef __WIN32__ +#include <winsock2.h> +#include <ws2tcpip.h> +#else +#include <netinet/in.h> +#endif], [struct in_addr6 a6; struct sockaddr_in6 s6;], ac_cv_sys_ipv6_support=in_addr6, ac_cv_sys_ipv6_support=no) fi @@ -737,9 +747,124 @@ case "$THR_LIB_NAME" in if test $found_win32_winnt = no; then AC_MSG_ERROR([-D_WIN32_WINNT missing in CPPFLAGS]) fi - ethr_have_native_atomics=yes - ethr_have_native_spinlock=yes + AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedCompareExchange64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedCompareExchange64(var, (__int64) 1, (__int64) 0); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()]) + + AC_CHECK_SIZEOF(void *) + case "$ac_cv_sizeof_void_p-$have_ilckd" in + 8-no) + ethr_have_native_atomics=no + ethr_have_native_spinlock=no;; + *) + ethr_have_native_atomics=yes + ethr_have_native_spinlock=yes;; + esac + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedDecrement64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedDecrement64(var); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedIncrement64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedIncrement64(var); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedExchangeAdd64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedExchangeAdd64(var, (__int64) 1); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedExchange64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedExchange64(var, (__int64) 1); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedAnd64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedAnd64(var, (__int64) 1); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()]) + + have_ilckd=no + AC_MSG_CHECKING([for _InterlockedOr64()]) + AC_TRY_LINK([ + #define WIN32_LEAN_AND_MEAN + #include <windows.h> + ], + [ + volatile __int64 *var; + _InterlockedOr64(var, (__int64) 1); + return 0; + ], + have_ilckd=yes) + AC_MSG_RESULT([$have_ilckd]) + test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()]) + ;; pthread) @@ -994,8 +1119,8 @@ case "$THR_LIB_NAME" in case "$host_cpu" in sun4u | sparc64 | sun4v) - ethr_have_native_atomics=yes;; - i86pc | i386 | i486 | i586 | i686 | x86_64 | amd64) + ethr_have_native_atomics=yes;; + i86pc | i*86 | x86_64 | amd64) ethr_have_native_atomics=yes;; macppc | ppc | "Power Macintosh") ethr_have_native_atomics=yes;; @@ -1077,6 +1202,28 @@ fi AC_CHECK_SIZEOF(void *) AC_DEFINE_UNQUOTED(ETHR_SIZEOF_PTR, $ac_cv_sizeof_void_p, [Define to the size of pointers]) +AC_CHECK_SIZEOF(int) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF_INT, $ac_cv_sizeof_int, [Define to the size of int]) +AC_CHECK_SIZEOF(long) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG, $ac_cv_sizeof_long, [Define to the size of long]) +AC_CHECK_SIZEOF(long long) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG_LONG, $ac_cv_sizeof_long_long, [Define to the size of long long]) +AC_CHECK_SIZEOF(__int64) +AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT64, $ac_cv_sizeof___int64, [Define to the size of __int64]) + + +case X$erl_xcomp_bigendian in + X) ;; + Xyes|Xno) ac_cv_c_bigendian=$erl_xcomp_bigendian;; + *) AC_MSG_ERROR([Bad erl_xcomp_bigendian value: $erl_xcomp_bigendian]);; +esac + +AC_C_BIGENDIAN + +if test "$ac_cv_c_bigendian" = "yes"; then + AC_DEFINE(ETHR_BIGENDIAN, 1, [Define if bigendian]) +fi + AC_ARG_ENABLE(native-ethr-impls, AS_HELP_STRING([--disable-native-ethr-impls], [disable native ethread implementations]), @@ -1090,7 +1237,7 @@ test "X$disable_native_ethr_impls" = "Xyes" && AC_ARG_ENABLE(prefer-gcc-native-ethr-impls, AS_HELP_STRING([--enable-prefer-gcc-native-ethr-impls], - [enable prefer gcc native ethread implementations]), + [prefer gcc native ethread implementations]), [ case "$enableval" in yes) enable_prefer_gcc_native_ethr_impls=yes ;; *) enable_prefer_gcc_native_ethr_impls=no ;; @@ -1099,21 +1246,60 @@ AC_ARG_ENABLE(prefer-gcc-native-ethr-impls, test $enable_prefer_gcc_native_ethr_impls = yes && AC_DEFINE(ETHR_PREFER_GCC_NATIVE_IMPLS, 1, [Define if you prefer gcc native ethread implementations]) +AC_ARG_WITH(libatomic_ops, + AS_HELP_STRING([--with-libatomic_ops=PATH], + [specify and prefer usage of libatomic_ops in the ethread library])) + AC_ARG_ENABLE(ethread-pre-pentium4-compatibility, AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility], [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]), -[ case "$enableval" in - yes) enable_ethread_pre_pentium4_compatibility=yes ;; - *) enable_ethread_pre_pentium4_compatibilit=no ;; - esac ], enable_ethread_pre_pentium4_compatibilit=no) +[ + case "$enable_ethread_pre_pentium4_compatibility" in + yes|no) ;; + *) enable_ethread_pre_pentium4_compatibility=check;; + esac +], +[enable_ethread_pre_pentium4_compatibility=check]) + +test "$cross_compiling" != "yes" || enable_ethread_pre_pentium4_compatibility=no + +case "$enable_ethread_pre_pentium4_compatibility-$host_cpu" in + check-i86pc | check-i*86) + AC_MSG_CHECKING([whether pre pentium 4 compatibility should forced]) + AC_RUN_IFELSE([ +#if defined(__GNUC__) +# if defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS) +# define CHECK_LIBATOMIC_OPS__ +# else +# define CHECK_GCC_ASM__ +# endif +#elif defined(ETHR_HAVE_LIBATOMIC_OPS) +# define CHECK_LIBATOMIC_OPS__ +#endif +#if defined(CHECK_LIBATOMIC_OPS__) +#include "atomic_ops.h" +#endif +int main(void) +{ +#if defined(CHECK_GCC_ASM__) + __asm__ __volatile__("mfence" : : : "memory"); +#elif defined(CHECK_LIBATOMIC_OPS__) + AO_nop_full(); +#endif + return 0; +} + ], + [enable_ethread_pre_pentium4_compatibility=no], + [enable_ethread_pre_pentium4_compatibility=yes], + [enable_ethread_pre_pentium4_compatibility=no]) + AC_MSG_RESULT([$enable_ethread_pre_pentium4_compatibility]);; + *) + ;; +esac -test $enable_ethread_pre_pentium4_compatibilit = yes && +test $enable_ethread_pre_pentium4_compatibility = yes && AC_DEFINE(ETHR_PRE_PENTIUM4_COMPAT, 1, [Define if you want compatibilty with x86 processors before pentium4.]) -AC_ARG_WITH(libatomic_ops, - AS_HELP_STRING([--with-libatomic_ops=PATH], - [use libatomic_ops with the ethread library])) - AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \ [Define if you have all ethread defines]) diff --git a/erts/autoconf/configure.vxworks b/erts/autoconf/configure.vxworks index 14fbf766dc..23a93faa31 100755 --- a/erts/autoconf/configure.vxworks +++ b/erts/autoconf/configure.vxworks @@ -2,7 +2,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2009. All Rights Reserved. +# Copyright Ericsson AB 1997-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/autoconf/vxworks/sed.general b/erts/autoconf/vxworks/sed.general index 551458daf5..88697b788d 100644 --- a/erts/autoconf/vxworks/sed.general +++ b/erts/autoconf/vxworks/sed.general @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2009. All Rights Reserved. +# Copyright Ericsson AB 1997-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/autoconf/win32.config.cache.static b/erts/autoconf/win32.config.cache.static index 31dfe510cd..d25b1df9d9 100755 --- a/erts/autoconf/win32.config.cache.static +++ b/erts/autoconf/win32.config.cache.static @@ -61,7 +61,6 @@ ac_cv_func_fork=${ac_cv_func_fork=no} ac_cv_func_fork_works=${ac_cv_func_fork_works=no} ac_cv_func_fpsetmask=${ac_cv_func_fpsetmask=no} ac_cv_func_fstat=${ac_cv_func_fstat=yes} -ac_cv_func_getaddrinfo=${ac_cv_func_getaddrinfo=no} ac_cv_func_gethostbyaddr=${ac_cv_func_gethostbyaddr=no} ac_cv_func_gethostbyaddr_r=${ac_cv_func_gethostbyaddr_r=no} ac_cv_func_gethostbyname=${ac_cv_func_gethostbyname=no} @@ -71,7 +70,6 @@ ac_cv_func_gethostname=${ac_cv_func_gethostname=no} ac_cv_func_gethrtime=${ac_cv_func_gethrtime=no} ac_cv_func_getipnodebyaddr=${ac_cv_func_getipnodebyaddr=no} ac_cv_func_getipnodebyname=${ac_cv_func_getipnodebyname=no} -ac_cv_func_getnameinfo=${ac_cv_func_getnameinfo=no} ac_cv_func_getpagesize=${ac_cv_func_getpagesize=no} ac_cv_func_gettimeofday=${ac_cv_func_gettimeofday=no} ac_cv_func_gmtime_r=${ac_cv_func_gmtime_r=no} @@ -212,7 +210,6 @@ ac_cv_sizeof_void_p=${ac_cv_sizeof_void_p=4} ac_cv_struct_exception=${ac_cv_struct_exception=no} ac_cv_struct_sockaddr_sa_len=${ac_cv_struct_sockaddr_sa_len=no} ac_cv_struct_tm=${ac_cv_struct_tm=time.h} -ac_cv_sys_ipv6_support=${ac_cv_sys_ipv6_support=no} ac_cv_sys_multicast_support=${ac_cv_sys_multicast_support=no} ac_cv_type_char=${ac_cv_type_char=yes} ac_cv_type_int=${ac_cv_type_int=yes} diff --git a/erts/configure.in b/erts/configure.in index 8c6f2ac076..a4c3ffb5c4 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script. -*-m4-*- dnl %CopyrightBegin% dnl -dnl Copyright Ericsson AB 1997-2010. All Rights Reserved. +dnl Copyright Ericsson AB 1997-2011. All Rights Reserved. dnl dnl The contents of this file are subject to the Erlang Public License, dnl Version 1.1, (the "License"); you may not use this file except in @@ -110,7 +110,8 @@ ENABLE_ALLOC_TYPE_VARS= AC_SUBST(ENABLE_ALLOC_TYPE_VARS) AC_ARG_ENABLE(bootstrap-only, -[ --enable-bootstrap-only enable bootstrap only configuration], +AS_HELP_STRING([--enable-bootstrap-only], + [enable bootstrap only configuration]), [ if test "X$enableval" = "Xyes"; then # Disable stuff not necessary in a bootstrap only system in order # to speed up things by reducing the amount of stuff needing to be @@ -126,46 +127,46 @@ AC_ARG_ENABLE(bootstrap-only, ]) AC_ARG_ENABLE(threads, -[ --enable-threads enable async thread support - --disable-threads disable async thread support], +AS_HELP_STRING([--enable-threads], [enable async thread support]) +AS_HELP_STRING([--disable-threads], [disable async thread support]), [ case "$enableval" in no) enable_threads=no ;; *) enable_threads=yes ;; esac ], enable_threads=unknown) AC_ARG_ENABLE(halfword-emulator, -[ --enable-halfword-emulator enable halfword emulator (only for 64bit builds) - --disable-halfword-emulator disable halfword emulator (only for 64bit builds)], +AS_HELP_STRING([--enable-halfword-emulator], + [enable halfword emulator (only for 64bit builds)]), [ case "$enableval" in no) enable_halfword_emualtor=no ;; *) enable_halfword_emulator=yes ;; esac ], enable_halfword_emulator=unknown) AC_ARG_ENABLE(smp-support, -[ --enable-smp-support enable smp support - --disable-smp-support disable smp support], +AS_HELP_STRING([--enable-smp-support], [enable smp support]) +AS_HELP_STRING([--disable-smp-support], [disable smp support]), [ case "$enableval" in no) enable_smp_support=no ;; *) enable_smp_support=yes ;; esac ], enable_smp_support=unknown) AC_ARG_WITH(termcap, -[ --with-termcap use termcap (default) - --without-termcap do not use any termcap libraries (ncurses,curses,termcap,termlib)], +AS_HELP_STRING([--with-termcap], [use termcap (default)]) +AS_HELP_STRING([--without-termcap], + [do not use any termcap libraries (ncurses,curses,termcap,termlib)]), [], [with_termcap=yes]) AC_ARG_ENABLE(hybrid-heap, -[ --enable-hybrid-heap enable hybrid heap - --disable-hybrid-heap disable hybrid heap], +AS_HELP_STRING([--enable-hybrid-heap], [enable hybrid heap]), [ case "$enableval" in no) enable_hybrid_heap=no ;; *) enable_hybrid_heap=yes ;; esac ], enable_hybrid_heap=unknown) AC_ARG_ENABLE(lock-checking, -[ --enable-lock-checking enable lock checking], +AS_HELP_STRING([--enable-lock-checking], [enable lock checking]), [ case "$enableval" in no) enable_lock_check=no ;; *) enable_lock_check=yes ;; @@ -174,16 +175,15 @@ AC_ARG_ENABLE(lock-checking, enable_lock_check=no) AC_ARG_ENABLE(lock-counter, -[ --enable-lock-counter enable lock counters - --disable-lock-counter disable lock counters], +AS_HELP_STRING([--enable-lock-counter], [enable lock counters]), [ case "$enableval" in no) enable_lock_count=no ;; *) enable_lock_count=yes ;; esac ], enable_lock_count=no) AC_ARG_ENABLE(kernel-poll, -[ --enable-kernel-poll enable kernel poll support - --disable-kernel-poll disable kernel poll support], +AS_HELP_STRING([--enable-kernel-poll], [enable kernel poll support]) +AS_HELP_STRING([--disable-kernel-poll], [disable kernel poll support]), [ case "$enableval" in no) enable_kernel_poll=no ;; *) enable_kernel_poll=yes ;; @@ -191,25 +191,27 @@ AC_ARG_ENABLE(kernel-poll, AC_ARG_ENABLE(sctp, -[ --enable-sctp enable sctp support - --disable-sctp disable sctp support], +AS_HELP_STRING([--enable-sctp], [enable sctp support]) +AS_HELP_STRING([--disable-sctp], [disable sctp support]), [ case "$enableval" in no) enable_sctp=no ;; *) enable_sctp=yes ;; esac ], enable_sctp=unknown) AC_ARG_ENABLE(hipe, -[ --enable-hipe enable hipe support - --disable-hipe disable hipe support]) +AS_HELP_STRING([--enable-hipe], [enable hipe support]) +AS_HELP_STRING([--disable-hipe], [disable hipe support])) AC_ARG_ENABLE(native-libs, -[ --enable-native-libs compile Erlang libraries to native code]) +AS_HELP_STRING([--enable-native-libs], + [compile Erlang libraries to native code])) AC_ARG_ENABLE(tsp, -[ --enable-tsp compile tsp app]) +AS_HELP_STRING([--enable-tsp], [compile tsp app])) AC_ARG_ENABLE(fp-exceptions, -[ --enable-fp-exceptions Use hardware floating point exceptions (default if hipe enabled)], +AS_HELP_STRING([--enable-fp-exceptions], + [use hardware floating point exceptions (default if hipe enabled)]), [ case "$enableval" in no) enable_fp_exceptions=no ;; *) enable_fp_exceptions=yes ;; @@ -217,7 +219,8 @@ AC_ARG_ENABLE(fp-exceptions, ],enable_fp_exceptions=auto) AC_ARG_ENABLE(darwin-universal, -[ --enable-darwin-universal build universal binaries on darwin i386], +AS_HELP_STRING([--enable-darwin-universal], + [build universal binaries on darwin i386]), [ case "$enableval" in no) enable_darwin_universal=no ;; *) enable_darwin_univeral=yes ;; @@ -226,7 +229,7 @@ AC_ARG_ENABLE(darwin-universal, AC_ARG_ENABLE(darwin-64bit, -[ --enable-darwin-64bit build 64bit binaries on darwin], +AS_HELP_STRING([--enable-darwin-64bit], [build 64bit binaries on darwin]), [ case "$enableval" in no) enable_darwin_64bit=no ;; *) enable_darwin_64bit=yes ;; @@ -234,7 +237,8 @@ AC_ARG_ENABLE(darwin-64bit, ],enable_darwin_64bit=no) AC_ARG_ENABLE(m64-build, -[ --enable-m64-build build 64bit binaries using the -m64 flag to (g)cc], +AS_HELP_STRING([--enable-m64-build], + [build 64bit binaries using the -m64 flag to (g)cc]), [ case "$enableval" in no) enable_m64_build=no ;; *) enable_m64_build=yes ;; @@ -242,7 +246,8 @@ AC_ARG_ENABLE(m64-build, ],enable_m64_build=no) AC_ARG_ENABLE(m32-build, -[ --enable-m32-build build 32bit binaries using the -m32 flag to (g)cc], +AS_HELP_STRING([--enable-m32-build], + [build 32bit binaries using the -m32 flag to (g)cc]), [ case "$enableval" in no) enable_m32_build=no ;; *) @@ -255,7 +260,7 @@ AC_ARG_ENABLE(m32-build, ],enable_m32_build=no) AC_ARG_ENABLE(fixalloc, -[ --disable-fixalloc disable the use of fix_alloc]) +AS_HELP_STRING([--disable-fixalloc], [disable the use of fix_alloc])) if test x${enable_fixalloc} = xno ; then AC_DEFINE(NO_FIX_ALLOC,[], [Define if you don't want the fix allocator in Erlang]) @@ -263,8 +268,9 @@ fi AC_SUBST(PERFCTR_PATH) AC_ARG_WITH(perfctr, -[ --with-perfctr=PATH specify location of perfctr include and lib - --without-perfctr don't use perfctr (default)]) +AS_HELP_STRING([--with-perfctr=PATH], + [specify location of perfctr include and lib]) +AS_HELP_STRING([--without-perfctr], [don't use perfctr (default)])) if test "x$with_perfctr" = "xno" -o "x$with_perfctr" = "x" ; then PERFCTR_PATH= @@ -278,7 +284,8 @@ else fi AC_ARG_ENABLE(clock-gettime, -[ --enable-clock-gettime Use clock-gettime for time correction], +AS_HELP_STRING([--enable-clock-gettime], + [use clock-gettime for time correction]), [ case "$enableval" in no) clock_gettime_correction=no ;; *) clock_gettime_correction=yes ;; @@ -573,6 +580,11 @@ AC_SUBST(WFLAGS) AC_SUBST(CFLAG_RUNTIME_LIBRARY_PATH) AC_CHECK_SIZEOF(void *) # Needed for ARCH and smp checks below +if test "x$ac_cv_sizeof_void_p" = x8; then + AC_SUBST(EXTERNAL_WORD_SIZE, 64) +else + AC_SUBST(EXTERNAL_WORD_SIZE, 32) +fi dnl dnl Figure out operating system and cpu architecture @@ -1293,8 +1305,7 @@ dnl zlib dnl ------------- AC_ARG_ENABLE(shared-zlib, -[ --enable-shared-zlib enable using shared zlib library - --disable-shared-zlib disable shared zlib, compile own zlib source (default)], +AS_HELP_STRING([--enable-shared-zlib], [enable using shared zlib library]), [ case "$enableval" in no) enable_shared_zlib=no ;; *) enable_shared_zlib=yes ;; @@ -1473,7 +1484,7 @@ AC_CHECK_HEADERS(fcntl.h limits.h unistd.h syslog.h dlfcn.h ieeefp.h \ sys/ioctl.h sys/time.h sys/uio.h \ sys/socket.h sys/sockio.h sys/socketio.h \ net/errno.h malloc.h mach-o/dyld.h arpa/nameser.h \ - pty.h util.h utmp.h langinfo.h poll.h) + pty.h util.h utmp.h langinfo.h poll.h sdkddkver.h) AC_CHECK_HEADER(sys/resource.h, [AC_DEFINE(HAVE_SYS_RESOURCE_H, 1, @@ -1673,18 +1684,62 @@ LIBS="$LIBS $EMU_THR_X_LIBS" dnl Check if we have these, in which case we'll try to build dnl inet_gethost with ipv6 support. -AC_CHECK_FUNC(getaddrinfo, have_getaddrinfo=yes, have_getaddrinfo=no) +AC_CHECK_HEADERS(windows.h) +AC_CHECK_HEADERS(winsock2.h) +AC_CHECK_HEADERS(ws2tcpip.h,[],[],[ +#ifdef HAVE_WINSOCK2_H +#include <winsock2.h> +#endif +#ifdef HAVE_WINDOWS_H +#include <windows.h> +#endif +]) +dnl AC_CHECK_FUNC(getaddrinfo, have_getaddrinfo=yes, have_getaddrinfo=no) +AC_MSG_CHECKING(for getaddrinfo) +AC_TRY_LINK([ +#include <stdlib.h> +#include <string.h> +#ifdef HAVE_WINSOCK2_H +#include <winsock2.h> +#endif +#ifdef HAVE_WINDOWS_H +#include <windows.h> +#endif +#ifdef HAVE_WS2TCPIP_H +#include <ws2tcpip.h> +#endif +#ifndef __WIN32__ +#include <sys/socket.h> +#include <netdb.h> +#endif +], +[ +getaddrinfo("","",NULL,NULL); +],have_getaddrinfo=yes, have_getaddrinfo=no) if test $have_getaddrinfo = yes; then + AC_MSG_RESULT([yes]) AC_MSG_CHECKING([whether getaddrinfo accepts enough flags]) - AC_TRY_RUN([ + AC_TRY_COMPILE([ #include <stdlib.h> #include <string.h> +#ifdef HAVE_WINSOCK2_H +#include <winsock2.h> +#endif +#ifdef HAVE_WINDOWS_H +#include <windows.h> +#endif +#ifdef HAVE_WS2TCPIP_H +#include <ws2tcpip.h> +#endif +#ifndef __WIN32__ #include <sys/socket.h> #include <netdb.h> -int main(int argc, char **argv) { +#endif +], +[ struct addrinfo hints, *ai; memset(&hints, 0, sizeof(hints)); - hints.ai_flags = (AI_CANONNAME|AI_V4MAPPED|AI_ADDRCONFIG); + hints.ai_flags = AI_CANONNAME; hints.ai_socktype = SOCK_STREAM; hints.ai_family = AF_INET6; if (getaddrinfo("::", NULL, &hints, &ai) == 0) { @@ -1693,26 +1748,48 @@ int main(int argc, char **argv) { } else { exit(1); } -} - ],, have_getaddrinfo=no, - [ - case X$erl_xcomp_getaddrinfo in - X) have_getaddrinfo=cross;; - Xyes|Xno) have_getaddrinfo=$erl_xcomp_getaddrinfo;; - *) AC_MSG_ERROR([Bad erl_xcomp_getaddrinfo value: $erl_xcomp_getaddrinfo]);; - esac - ]) +],, have_getaddrinfo=no) AC_MSG_RESULT($have_getaddrinfo) case $have_getaddrinfo in yes) AC_DEFINE(HAVE_GETADDRINFO, [1], [Define to 1 if you have a good `getaddrinfo' function.]);; - cross) - AC_MSG_WARN([result no guessed because of cross compilation]);; *) ;; esac +else + AC_MSG_RESULT([no]) fi -AC_CHECK_FUNCS([getnameinfo getipnodebyname getipnodebyaddr gethostbyname2]) +AC_MSG_CHECKING(for getnameinfo) +AC_TRY_LINK([ +#include <stdlib.h> +#include <string.h> +#ifdef HAVE_WINSOCK2_H +#include <winsock2.h> +#endif +#ifdef HAVE_WINDOWS_H +#include <windows.h> +#endif +#ifdef HAVE_WS2TCPIP_H +#include <ws2tcpip.h> +#endif +#ifndef __WIN32__ +#include <sys/socket.h> +#include <netdb.h> +#endif +], +[ +getnameinfo(NULL,0,NULL,0,NULL,0,0); +],have_getnameinfo=yes, have_getnameinfo=no) +if test $have_getnameinfo = yes; then + AC_MSG_RESULT([yes]) + AC_DEFINE(HAVE_GETNAMEINFO, [1], + [Define to 1 if you have a good `getnameinfo' function.]) +else + AC_MSG_RESULT([no]) +fi + + +AC_CHECK_FUNCS([getipnodebyname getipnodebyaddr gethostbyname2]) AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlopen \ pread pwrite writev memmove strerror strerror_r strncasecmp \ @@ -1788,7 +1865,7 @@ AC_CHECK_FUNCS([fdatasync]) dnl Find which C libraries are required to use fdatasync AC_SEARCH_LIBS(fdatasync, [rt]) -AC_CHECK_HEADERS(net/if_dl.h ifaddrs.h) +AC_CHECK_HEADERS(net/if_dl.h ifaddrs.h netpacket/packet.h) AC_CHECK_FUNCS([getifaddrs]) dnl ---------------------------------------------------------------------- @@ -1852,6 +1929,27 @@ if test $processor_bind_functionality = yes; then AC_DEFINE(HAVE_PROCESSOR_BIND, 1, [Define if you have processor_bind functionality]) fi +AC_MSG_CHECKING([for cpuset_getaffinity/cpuset_setaffinity]) +AC_TRY_COMPILE([ +#include <sys/param.h> +#include <sys/cpuset.h> +], +[ + int res; + cpuset_t cpuset; + CPU_ZERO(&cpuset); + CPU_SET(1, &cpuset); + res = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cpuset_t), &cpuset); + res = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, sizeof(cpuset_t), &cpuset); + res = CPU_ISSET(1, &cpuset); + CPU_CLR(1, &cpuset); +], + cpuset_xetaffinity=yes, + cpuset_xetaffinity=no) +AC_MSG_RESULT([$cpuset_xetaffinity]) +if test $cpuset_xetaffinity = yes; then + AC_DEFINE(HAVE_CPUSET_xETAFFINITY, 1, [Define if you have cpuset_getaffinity/cpuset_setaffinity]) +fi AC_CACHE_CHECK([for 'end' symbol], erts_cv_have_end_symbol, @@ -3431,9 +3529,12 @@ AC_SUBST(STATIC_ZLIB_LIBS) std_ssl_locations="/usr/local /usr/sfw /opt/local /usr /usr/pkg /usr/local/openssl /usr/lib/openssl /usr/openssl /usr/local/ssl /usr/lib/ssl /usr/ssl" AC_ARG_WITH(ssl-zlib, -[ --with-ssl-zlib=PATH specify location of ZLib to be used by OpenSSL - --with-ssl-zlib link SSL with Zlib (default if found) - --without-ssl-zlib don't link SSL with ZLib]) +AS_HELP_STRING([--with-ssl-zlib=PATH], + [specify location of ZLib to be used by OpenSSL]) +AS_HELP_STRING([--with-ssl-zlib], + [link SSL with Zlib (default if found)]) +AS_HELP_STRING([--without-ssl-zlib], + [don't link SSL with ZLib])) if test "x$with_ssl_zlib" = "xno"; then @@ -3502,13 +3603,13 @@ fi AC_ARG_WITH(ssl, -[ --with-ssl=PATH specify location of OpenSSL include and lib - --with-ssl use SSL (default) - --without-ssl don't use SSL]) +AS_HELP_STRING([--with-ssl=PATH], [specify location of OpenSSL include and lib]) +AS_HELP_STRING([--with-ssl], [use SSL (default)]) +AS_HELP_STRING([--without-ssl], [don't use SSL])) AC_ARG_ENABLE(dynamic-ssl-lib, -[ --enable-dynamic-ssl-lib enable using dynamic openssl libraries - --disable-dynamic-ssl-lib disable using dynamic openssl libraries], +AS_HELP_STRING([--disable-dynamic-ssl-lib], + [disable using dynamic openssl libraries]), [ case "$enableval" in no) enable_dynamic_ssl=no ;; *) enable_dynamic_ssl=yes ;; @@ -3559,6 +3660,12 @@ case "$erl_xcomp_without_sysroot-$with_ssl" in urp="uninstall/openssl_is1/inno setup: app path" rp="$wrp$urp" if regtool -q get "$rp" > /dev/null; then + true + else + urp="uninstall/openssl (32-bit)_is1/inno setup: app path" + rp="$wrp$urp" + fi + if regtool -q get "$rp" > /dev/null; then ssl_install_dir=`regtool -q get "$rp"` # Try hard to get rid of spaces... if cygpath -d "$ssl_install_dir" > /dev/null 2>&1; then @@ -3971,9 +4078,9 @@ esac AC_ARG_WITH(javac, -[ --with-javac=JAVAC specify Java compiler to use - --with-javac use a Java compiler if found (default) - --without-javac don't use any Java compiler]) +AS_HELP_STRING([--with-javac=JAVAC], [specify Java compiler to use]) +AS_HELP_STRING([--with-javac], [use a Java compiler if found (default)]) +AS_HELP_STRING([--without-javac], [don't use any Java compiler])) dnl dnl Then there are a number of apps which needs a java compiler... diff --git a/erts/doc/src/driver.xml b/erts/doc/src/driver.xml index 006a6160de..2b1ed398ee 100644 --- a/erts/doc/src/driver.xml +++ b/erts/doc/src/driver.xml @@ -63,7 +63,8 @@ <p>This is a simple driver for accessing a postgres database using the libpq C client library. Postgres is used because it's free and open source. For information - on postgres, refer to the website www.postgres.org.</p> + on postgres, refer to the website + <url href="http://www.postgres.org">www.postgres.org</url>.</p> <p>The driver is synchronous, it uses the synchronous calls of the client library. This is only for simplicity, and is generally not good, since it will @@ -196,18 +197,21 @@ static ErlDrvData start(ErlDrvPort port, char *command) <p>We call disconnect to log out from the database. (This should have been done from Erlang, but just in case.)</p> <code type="none"><![CDATA[ - static int do_disconnect(our_data_t* data, ei_x_buff* x); +static int do_disconnect(our_data_t* data, ei_x_buff* x); static void stop(ErlDrvData drv_data) { - do_disconnect((our_data_t*)drv_data, NULL); + our_data_t* data = (our_data_t*)drv_data; + + do_disconnect(data, NULL); + driver_free(data); } ]]></code> <p>We use the binary format only to return data to the emulator; input data is a string paramater for <c><![CDATA[connect]]></c> and <c><![CDATA[select]]></c>. The returned data consists of Erlang terms.</p> <p>The functions <c><![CDATA[get_s]]></c> and <c><![CDATA[ei_x_to_new_binary]]></c> are - utilities that is used to make the code shorter. <c><![CDATA[get_s]]></c> + utilities that are used to make the code shorter. <c><![CDATA[get_s]]></c> duplicates the string and zero-terminates it, since the postgres client library wants that. <c><![CDATA[ei_x_to_new_binary]]></c> takes an <c><![CDATA[ei_x_buff]]></c> buffer and allocates a binary and @@ -241,7 +245,7 @@ static int control(ErlDrvData drv_data, unsigned int command, char *buf, return r; } ]]></code> - <p>In <c><![CDATA[do_connect]]></c> is where we log in to the database. If the connection + <p><c><![CDATA[do_connect]]></c> is where we log in to the database. If the connection was successful we store the connection handle in our driver data, and return ok. Otherwise, we return the error message from postgres, and store <c><![CDATA[NULL]]></c> in the driver data.</p> @@ -261,7 +265,7 @@ static int do_connect(const char *s, our_data_t* data, ei_x_buff* x) } ]]></code> <p>If we are connected (if the connection handle is not <c><![CDATA[NULL]]></c>), - we log out from the database. We need to check if a we should + we log out from the database. We need to check if we should encode an ok, since we might get here from the <c><![CDATA[stop]]></c> function, which doesn't return data to the emulator.</p> <code type="none"><![CDATA[ @@ -276,7 +280,7 @@ static int do_disconnect(our_data_t* data, ei_x_buff* x) return 0; } ]]></code> - <p>We execute a query and encodes the result. Encoding is done + <p>We execute a query and encode the result. Encoding is done in another C module, <c><![CDATA[pg_encode.c]]></c> which is also provided as sample code.</p> <code type="none"><![CDATA[ @@ -288,7 +292,7 @@ static int do_select(const char* s, our_data_t* data, ei_x_buff* x) return 0; } ]]></code> - <p>Here we simply checks the result from postgres, and + <p>Here we simply check the result from postgres, and if it's data we encode it as lists of lists with column data. Everything from postgres is C strings, so we just use <c><![CDATA[ei_x_encode_string]]></c> to send @@ -389,7 +393,7 @@ disconnect(Port) -> select(Port, Query) -> binary_to_term(port_control(Port, ?DRV_SELECT, Query)). ]]></code> - <p>The api is simple: <c><![CDATA[connect/1]]></c> loads the driver, opens it + <p>The API is simple: <c><![CDATA[connect/1]]></c> loads the driver, opens it and logs on to the database, returning the Erlang port if successful, <c><![CDATA[select/2]]></c> sends a query to the driver, and returns the result, <c><![CDATA[disconnect/1]]></c> closes the @@ -414,7 +418,7 @@ select(Port, Query) -> <p>Sometimes database queries can take long time to complete, in our <c><![CDATA[pg_sync]]></c> driver, the emulator halts while the driver is doing its job. This is - often not acceptable, since no other Erlang processes + often not acceptable, since no other Erlang process gets a chance to do anything. To improve on our postgres driver, we reimplement it using the asynchronous calls in LibPQ.</p> @@ -469,7 +473,7 @@ typedef struct our_data_t { whether the driver is waiting for a connection or waiting for the result of a query. (This is needed since the entry <c><![CDATA[ready_io]]></c> will be called both when connecting and - when there is query result.)</p> + when there is a query result.)</p> <code type="none"><![CDATA[ static int do_connect(const char *s, our_data_t* data) { @@ -568,7 +572,7 @@ static void ready_io(ErlDrvData drv_data, ErlDrvEvent event) connection is successful, or error if it's not. If the connection is not yet established, we simply return; <c><![CDATA[ready_io]]></c> will be called again.</p> - <p>If we have result from a connect, indicated that we have data in + <p>If we have a result from a connect, indicated by having data in the <c><![CDATA[x]]></c> buffer, we no longer need to select on output (<c><![CDATA[ready_output]]></c>), so we remove this by calling <c><![CDATA[driver_select]]></c>.</p> @@ -627,9 +631,9 @@ return_port_data(Port) -> message queue. The function <c><![CDATA[return_port_data]]></c> above receives data from the port. Since the data is in binary format, we use <c><![CDATA[binary_to_term/1]]></c> to convert - it to Erlang term. Note that the driver is opened in - binary mode, <c><![CDATA[open_port/2]]></c> is called with the option - <c><![CDATA[[binary]]]></c>. This means that data sent from the driver + it to an Erlang term. Note that the driver is opened in + binary mode (<c><![CDATA[open_port/2]]></c> is called with the option + <c><![CDATA[[binary]]]></c>). This means that data sent from the driver to the emulator is sent as binaries. Without the <c><![CDATA[binary]]></c> option, they would have been lists of integers.</p> </section> @@ -643,15 +647,15 @@ return_port_data(Port) -> of a list of integers. For large lists (more than 100000 elements), this will take some time, so we will perform this as an asynchronous task.</p> - <p>The asynchronous api for drivers are quite complicated. First + <p>The asynchronous API for drivers is quite complicated. First of all, the work must be prepared. In our example we do this in <c><![CDATA[output]]></c>. We could have used <c><![CDATA[control]]></c> just as well, but we want some variation in our examples. In our driver, we allocate - a structure that contains all needed for the asynchronous task + a structure that contains anything that's needed for the asynchronous task to do the work. This is done in the main emulator thread. Then the asynchronous function is called from a driver thread, - separate from the main emulator thread. Note that the driver- - functions are not reentrant, so they shouldn't be used. + separate from the main emulator thread. Note that the driver-functions + are not reentrant, so they shouldn't be used. Finally, after the function is completed, the driver callback <c><![CDATA[ready_async]]></c> is called from the main emulator thread, this is where we return the result to Erlang. (We can't @@ -689,7 +693,7 @@ static ErlDrvEntry next_perm_driver_entry = { be sent later from the <c><![CDATA[ready_async]]></c> call-back.</p> <p>The <c><![CDATA[async_data]]></c> will be passed to the <c><![CDATA[do_perm]]></c> function. We do not use a <c><![CDATA[async_free]]></c> function (the last argument to - <c><![CDATA[driver_async]]></c>, it's only used if the task is cancelled + <c><![CDATA[driver_async]]></c>), it's only used if the task is cancelled programmatically.</p> <code type="none"><![CDATA[ struct our_async_data { @@ -740,7 +744,7 @@ static void ready_async(ErlDrvData drv_data, ErlDrvThreadData async_data) ErlDrvPort port = reinterpret_cast<ErlDrvPort>(drv_data); our_async_data* d = reinterpret_cast<our_async_data*>(async_data); int n = d->data.size(), result_n = n*2 + 3; - ErlDrvTermData* result = new ErlDrvTermData[result_n], * rp = result; + ErlDrvTermData *result = new ErlDrvTermData[result_n], *rp = result; for (vector<int>::iterator i = d->data.begin(); i != d->data.end(); ++i) { *rp++ = ERL_DRV_INT; diff --git a/erts/doc/src/driver_entry.xml b/erts/doc/src/driver_entry.xml index e71b48bd92..dfddbb18ea 100644 --- a/erts/doc/src/driver_entry.xml +++ b/erts/doc/src/driver_entry.xml @@ -4,7 +4,7 @@ <cref> <header> <copyright> - <year>2001</year><year>2010</year> + <year>2001</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -36,7 +36,7 @@ <description> <p>As of erts version 5.5.3 the driver interface has been extended (see <seealso marker="driver_entry#extended_marker">extended marker</seealso>). - The extended interface introduce + The extended interface introduces <seealso marker="erl_driver#version_management">version management</seealso>, the possibility to pass capability flags (see <seealso marker="driver_entry#driver_flags">driver flags</seealso>) @@ -45,21 +45,21 @@ <note> <p>Old drivers (compiled with an <c>erl_driver.h</c> from an earlier erts version than 5.5.3) have to be recompiled - (but does not have to use the extended interface).</p> + (but do not have to use the extended interface).</p> </note> <p>The <c>driver_entry</c> structure is a C struct that all erlang - drivers defines. It contains entry points for the erlang driver + drivers define. It contains entry points for the erlang driver that are called by the erlang emulator when erlang code accesses the driver.</p> <p> <marker id="emulator"></marker> The <seealso marker="driver_entry">erl_driver</seealso> driver - API functions needs a port handle + API functions need a port handle that identifies the driver instance (and the port in the emulator). This is only passed to the <c>start</c> function, but not to the other functions. The <c>start</c> function returns a driver-defined handle that is passed to the other functions. A - common practice is to have the <c>start</c> function allocating + common practice is to have the <c>start</c> function allocate some application-defined structure and stash the <c>port</c> handle in it, to use it later with the driver API functions.</p> <p>The driver call-back functions are called synchronously from the @@ -172,7 +172,7 @@ typedef struct erl_drv_entry { added to the driver list.) The driver should return 0, or if the driver can't initialize, -1.</p> </item> - <tag><marker id="start"/>int (*start)(ErlDrvPort port, char* command)</tag> + <tag><marker id="start"/>ErlDrvData (*start)(ErlDrvPort port, char* command)</tag> <item> <p>This is called when the driver is instantiated, when <c>open_port/2</c> is called. The driver should return a @@ -188,7 +188,9 @@ typedef struct erl_drv_entry { <p>This is called when the port is closed, with <c>port_close/1</c> or <c>Port ! {self(), close}</c>. Note that terminating the port owner process also closes the - port.</p> + port. If <c>drv_data</c> is a pointer to memory allocated in + <c>start</c>, then <c>stop</c> is the place to deallocate that + memory.</p> </item> <tag><marker id="output"/>void (*output)(ErlDrvData drv_data, char *buf, int len)</tag> <item> @@ -217,6 +219,10 @@ typedef struct erl_drv_entry { completes, write to the pipe (use <c>SetEvent</c> on Windows), this will make the emulator call <c>ready_input</c> or <c>ready_output</c>.</p> + <p>Spurious events may happen. That is, calls to <c>ready_input</c> + or <c>ready_output</c> even though no real events are signaled. In + reality it should be rare (and OS dependant), but a robust driver + must nevertheless be able to handle such cases.</p> </item> <tag><marker id="driver_name"/>char *driver_name</tag> <item> @@ -233,7 +239,7 @@ typedef struct erl_drv_entry { </item> <tag>void *handle</tag> <item> - <p>This field is reserved for the emulators internal use. The + <p>This field is reserved for the emulator's internal use. The emulator will modify this field; therefore, it is important that the <c>driver_entry</c> isn't declared <c>const</c>.</p> </item> @@ -397,7 +403,7 @@ typedef struct erl_drv_entry { <tag>void *handle2</tag> <item> <p> - This field is reserved for the emulators internal use. The + This field is reserved for the emulator's internal use. The emulator will modify this field; therefore, it is important that the <c>driver_entry</c> isn't declared <c>const</c>. </p> diff --git a/erts/doc/src/epmd.xml b/erts/doc/src/epmd.xml index f01cf90a36..474230cb38 100644 --- a/erts/doc/src/epmd.xml +++ b/erts/doc/src/epmd.xml @@ -119,7 +119,7 @@ <tag><c><![CDATA[-port No]]></c></tag> <item> <p>Let this instance of epmd listen to another TCP port than - default 4369. This can be also be set using the + default 4369. This can also be set using the <c><![CDATA[ERL_EPMD_PORT]]></c> environment variable, see the section <seealso marker="#environment_variables">Environment variables</seealso> below</p> @@ -186,7 +186,7 @@ <tag><c><![CDATA[-port No]]></c></tag> <item> <p>Contacts the <c>epmd</c> listening on the given TCP port number - (default 4369). This can be also be set using the + (default 4369). This can also be set using the <c><![CDATA[ERL_EPMD_PORT]]></c> environment variable, see the section <seealso marker="#environment_variables">Environment variables</seealso> below</p> diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 09c9cf6812..a66d273438 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -4,7 +4,7 @@ <comref> <header> <copyright> - <year>1996</year><year>2010</year> + <year>1996</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -231,7 +231,8 @@ <tag><c><![CDATA[-detached]]></c></tag> <item> <p>Starts the Erlang runtime system detached from the system - console. Useful for running daemons and backgrounds processes.</p> + console. Useful for running daemons and backgrounds processes. Implies + <c><![CDATA[-noinput]]></c>.</p> </item> <tag><c><![CDATA[-emu_args]]></c></tag> <item> @@ -541,6 +542,28 @@ <p>Calling <c>erlang:halt/1</c> with a string argument will still produce a crash dump.</p> </item> + <tag><c><![CDATA[+e Number]]></c></tag> + <item> + <p>Set max number of ETS tables.</p> + </item> + <tag><c><![CDATA[+ec]]></c></tag> + <item> + <p>Force the <c>compressed</c> option on all ETS tables. + Only intended for test and evaluation.</p> + </item> + <tag><c><![CDATA[+fnl]]></c></tag> + <item> + <p>The VM works with file names as if they are encoded using the ISO-latin-1 encoding, disallowing Unicode characters with codepoints beyond 255. This is default on operating systems that have transparent file naming, i.e. all Unixes except MacOSX.</p> + </item> + <tag><c><![CDATA[+fnu]]></c></tag> + <item> + <p>The VM works with file names as if they are encoded using UTF-8 (or some other system specific Unicode encoding). This is the default on operating systems that enforce Unicode encoding, i.e. Windows and MacOSX.</p> + <p>By enabling Unicode file name translation on systems where this is not default, you open up to the possibility that some file names can not be interpreted by the VM and therefore will be returned to the program as raw binaries. The option is therefore considered experimental.</p> + </item> + <tag><c><![CDATA[+fna]]></c></tag> + <item> + <p>Selection between <c>+fnl</c> and <c>+fnu</c> is done based on the current locale settings in the OS, meaning that if you have set your terminal for UTF-8 encoding, the filesystem is expected to use the same encoding for filenames (use with care).</p> + </item> <tag><c><![CDATA[+hms Size]]></c></tag> <item> <p>Sets the default heap size of processes to the size @@ -686,7 +709,7 @@ </p></item> </taglist> <p>Binding of schedulers is currently only supported on newer - Linux, Solaris, and Windows systems.</p> + Linux, Solaris, FreeBSD, and Windows systems.</p> <p>If no CPU topology is available when the <c>+sbt</c> flag is processed and <c>BindType</c> is any other type than <c>u</c>, the runtime system will fail to start. CPU @@ -914,7 +937,7 @@ <item> <p>Set the distribution buffer busy limit (<seealso marker="erlang#system_info_dist_buf_busy_limit">dist_buf_busy_limit</seealso>) - in kilobytes. Valid range is 1-2097151. Default is 128.</p> + in kilobytes. Valid range is 1-2097151. Default is 1024.</p> <p>A larger buffer limit will allow processes to buffer more outgoing messages over the distribution. When the buffer limit has been reached, sending processes will be diff --git a/erts/doc/src/erl_dist_protocol.xml b/erts/doc/src/erl_dist_protocol.xml index 1fe7ac7ecd..6c725fc82d 100644 --- a/erts/doc/src/erl_dist_protocol.xml +++ b/erts/doc/src/erl_dist_protocol.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2007</year> - <year>2007</year> + <year>2011</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml index 497a2fa01d..066a2a4b92 100644 --- a/erts/doc/src/erl_driver.xml +++ b/erts/doc/src/erl_driver.xml @@ -56,16 +56,16 @@ instance is connected to an Erlang port. Every port has a port owner process. Communication with the port is normally done through the port owner process.</p> - <p>Most of the functions takes the <c>port</c> handle as an + <p>Most of the functions take the <c>port</c> handle as an argument. This identifies the driver instance. Note that this port handle must be stored by the driver, it is not given when the driver is called from the emulator (see <seealso marker="driver_entry#emulator">driver_entry</seealso>).</p> - <p>Some of the functions takes a parameter of type + <p>Some of the functions take a parameter of type <c>ErlDrvBinary</c>, a driver binary. It should be both - allocated and freed by the caller. Using a binary directly avoid + allocated and freed by the caller. Using a binary directly avoids one extra copying of data.</p> - <p>Many of the output functions has a "header buffer", with + <p>Many of the output functions have a "header buffer", with <c>hbuf</c> and <c>hlen</c> parameters. This buffer is sent as a list before the binary (or list, depending on port mode) that is sent. This is convenient when matching on messages received from @@ -92,7 +92,7 @@ with SMP support without being rewritten if driver level locking is used.</p> <note> - <p>It is assumed that drivers does not access other drivers. If + <p>It is assumed that drivers do not access other drivers. If drivers should access each other they have to provide their own mechanism for thread safe synchronization. Such "inter driver communication" is strongly discouraged.</p> @@ -113,12 +113,12 @@ call-backs may be made from different threads.</p> </note> <p>Most functions in this API are <em>not</em> thread-safe, i.e., - they may <em>not</em> be called from an arbitrary thread. Function + they may <em>not</em> be called from an arbitrary thread. Functions that are not documented as thread-safe may only be called from driver call-backs or function calls descending from a driver call-back call. Note that driver call-backs may be called from different threads. This, however, is not a problem for any - functions in this API, since the emulator have control over + function in this API, since the emulator has control over these threads.</p> <note> <p>Functions not explicitly documented as thread-safe are @@ -155,10 +155,10 @@ more information.</p> </item> <tag>Output functions</tag> - <item>With the output functions, the driver sends data back + <item>With the output functions, the driver sends data back to the emulator. They will be received as messages by the port owner process, see <c>open_port/2</c>. The vector function and the - function taking a driver binary is faster, because that avoid + function taking a driver binary are faster, because they avoid copying the data buffer. There is also a fast way of sending terms from the driver, without going through the binary term format.</item> @@ -193,14 +193,14 @@ use functionality from the POSIX thread API or the Windows native thread API. </p> - <p>The Erlang driver thread API only return error codes when it is + <p>The Erlang driver thread API only returns error codes when it is reasonable to recover from an error condition. If it isn't reasonable to recover from an error condition, the whole runtime system is terminated. For example, if a create mutex operation fails, an error code is returned, but if a lock operation on a mutex fails, the whole runtime system is terminated. </p> - <p>Note that there exist no "condition variable wait with timeout" in + <p>Note that there exists no "condition variable wait with timeout" in the Erlang driver thread API. This is due to issues with <c>pthread_cond_timedwait()</c>. When the system clock suddenly is changed, it isn't always guaranteed that you will wake up from @@ -241,7 +241,7 @@ to give you better error reports. </p> </item> - <tag>Adding / remove drivers</tag> + <tag>Adding / removing drivers</tag> <item>A driver can add and later remove drivers.</item> <tag>Monitoring processes</tag> <item>A driver can monitor a process that does not own a port.</item> @@ -262,7 +262,7 @@ could, under rare circumstances, mean that drivers have to be slightly modified. If so, this will of course be documented. <c>ERL_DRV_EXTENDED_MINOR_VERSION</c> will be incremented when - new features are added. The runtime system use the minor version + new features are added. The runtime system uses the minor version of the driver to determine what features to use. The runtime system will refuse to load a driver if the major versions differ, or if the major versions are equal and the @@ -273,7 +273,7 @@ It can, however, not make sure that it isn't incompatible. Therefore, when loading a driver that doesn't use the extended driver interface, there is a risk that it will be loaded also when - the driver is incompatible. When the driver use the extended driver + the driver is incompatible. When the driver uses the extended driver interface, the emulator can verify that it isn't of an incompatible driver version. You are therefore advised to use the extended driver interface.</p> @@ -309,7 +309,7 @@ typedef struct ErlDrvSysInfo { <seealso marker="#driver_system_info">driver_system_info()</seealso> will write the system information when passed a reference to a <c>ErlDrvSysInfo</c> structure. A description of the - fields in the structure follow: + fields in the structure follows: </p> <taglist> <tag><c>driver_major_version</c></tag> @@ -347,14 +347,6 @@ typedef struct ErlDrvSysInfo { <item>A value <c>!= 0</c> if the runtime system has SMP support; otherwise, <c>0</c>. </item> - <tag><c>thread_support</c></tag> - <item>A value <c>!= 0</c> if the runtime system has thread support; - otherwise, <c>0</c>. - </item> - <tag><c>smp_support</c></tag> - <item>A value <c>!= 0</c> if the runtime system has SMP support; - otherwise, <c>0</c>. - </item> <tag><c>async_threads</c></tag> <item>The number of async threads in the async thread pool used by <seealso marker="#driver_async">driver_async()</seealso> @@ -401,8 +393,8 @@ typedef struct ErlDrvBinary { <seealso marker="#driver_binary_dec_refc">driver_binary_dec_refc()</seealso>.</p> </note> <p>Some driver calls, such as <c>driver_enq_binary</c>, - increments the driver reference count, and others, such as - <c>driver_deq</c> decrements it.</p> + increment the driver reference count, and others, such as + <c>driver_deq</c> decrement it.</p> <p>Using a driver binary instead of a normal buffer, is often faster, since the emulator doesn't need to copy the data, only the pointer is used.</p> @@ -415,7 +407,7 @@ typedef struct ErlDrvBinary { <c>driver_outputv</c> calls, and in the queue. Also the driver call-back <seealso marker="driver_entry#outputv">outputv</seealso> uses driver binaries.</p> - <p>If the driver of some reason or another, wants to keep a + <p>If the driver for some reason or another, wants to keep a driver binary around, in a static variable for instance, the reference count should be incremented, and the binary can later be freed in the <seealso marker="driver_entry#stop">stop</seealso> call-back, with @@ -423,7 +415,7 @@ typedef struct ErlDrvBinary { <p>Note that since a driver binary is shared by the driver and the emulator, a binary received from the emulator or sent to the emulator, must not be changed by the driver.</p> - <p>From erts version 5.5 (OTP release R11B), orig_bytes is + <p>Since erts version 5.5 (OTP release R11B), orig_bytes is guaranteed to be properly aligned for storage of an array of doubles (usually 8-byte aligned).</p> </item> @@ -447,7 +439,7 @@ typedef struct ErlIOVec { int vsize; int size; SysIOVec* iov; - >ErlDrvBinary** binv; + ErlDrvBinary** binv; } ErlIOVec; </code> <p>The I/O vector used by the emulator and drivers, is a list @@ -495,17 +487,17 @@ typedef struct ErlIOVec { Currently, the only port specific data that the emulator associates with the port data lock is the driver queue.</p> <p>Normally a driver instance does not have a port data lock. If - the driver instance want to use a port data lock, it has to + the driver instance wants to use a port data lock, it has to create the port data lock by calling <seealso marker="#driver_pdl_create">driver_pdl_create()</seealso>. <em>NOTE</em>: Once the port data lock has been created, every - access to data associated with the port data lock have to be done + access to data associated with the port data lock has to be done while having the port data lock locked. The port data lock is locked, and unlocked, respectively, by use of <seealso marker="#driver_pdl_lock">driver_pdl_lock()</seealso>, and <seealso marker="#driver_pdl_unlock">driver_pdl_unlock()</seealso>.</p> <p>A port data lock is reference counted, and when the reference - count reach zero, it will be destroyed. The emulator will at + count reaches zero, it will be destroyed. The emulator will at least increment the reference count once when the lock is created and decrement it once when the port associated with the lock terminates. The emulator will also increment the @@ -545,7 +537,7 @@ typedef struct ErlIOVec { </p> <taglist> <tag>suggested_stack_size</tag> - <item>A suggestion, in kilo-words, on how large stack to use. A value less + <item>A suggestion, in kilo-words, on how large a stack to use. A value less than zero means default size. </item> </taglist> @@ -648,7 +640,7 @@ typedef struct ErlIOVec { opened.</p> <p>The data is queued in the port owner process' message queue. Note that this does not yield to the emulator. (Since - the driver and the emulator runs in the same thread.)</p> + the driver and the emulator run in the same thread.)</p> <p>The parameter <c>buf</c> points to the data to send, and <c>len</c> is the number of bytes.</p> <p>The return value for all output functions is 0. (Unless the @@ -749,7 +741,7 @@ typedef struct ErlIOVec { function <seealso marker="driver_entry#emulator">timeout</seealso> is called.</p> <p>Note that there is only one timer on each driver instance; setting a new timer will replace an older one.</p> - <p>Return value i 0 (-1 only when the <c>timeout</c> driver + <p>Return value is 0 (-1 only when the <c>timeout</c> driver function is <c>NULL</c>).</p> </desc> </func> @@ -799,20 +791,20 @@ typedef struct ErlIOVec { event object must be a socket or pipe (or other object that <c>select</c>/<c>poll</c> can use). On windows, the Win32 API function <c>WaitForMultipleObjects</c> - is used. This places other restriction on the event object. + is used. This places other restrictions on the event object. Refer to the Win32 SDK documentation.</p> <p>The <c>on</c> parameter should be <c>1</c> for setting events and <c>0</c> for clearing them.</p> - <p>The <c>mode</c> argument is bitwise-or combination of + <p>The <c>mode</c> argument is a bitwise-or combination of <c>ERL_DRV_READ</c>, <c>ERL_DRV_WRITE</c> and <c>ERL_DRV_USE</c>. - The first two specifies whether to wait for read events and/or write + The first two specify whether to wait for read events and/or write events. A fired read event will call <seealso marker="driver_entry#ready_input">ready_input</seealso> while a fired write event will call <seealso marker="driver_entry#ready_output">ready_output</seealso>. </p> <note> - <p>Some OS (Windows) does not differ between read and write events. + <p>Some OS (Windows) do not differentiate between read and write events. The call-back for a fired event then only depends on the value of <c>mode</c>.</p> </note> <p><c>ERL_DRV_USE</c> specifies if we are using the event object or if we want to close it. @@ -834,9 +826,9 @@ typedef struct ErlIOVec { as before. But it is recommended to update them to use <c>ERL_DRV_USE</c> and <c>stop_select</c> to make sure that event objects are closed in a safe way.</p> </note> - <p>The return value is 0 (Failure, -1, only if the + <p>The return value is 0 (failure, -1, only if the <c>ready_input</c>/<c>ready_output</c> is - <c>NULL</c>.</p> + <c>NULL</c>).</p> </desc> </func> <func> @@ -1076,7 +1068,7 @@ typedef struct ErlIOVec { array of <c>SysIOVec</c>s. It also returns the number of elements in <c>vlen</c>. This is the only way to get data out of the queue.</p> - <p>Nothing is remove from the queue by this function, that must be done + <p>Nothing is removed from the queue by this function, that must be done with <c>driver_deq</c>.</p> <p>The returned array is suitable to use with the Unix system call <c>writev</c>.</p> @@ -1209,7 +1201,7 @@ typedef struct ErlIOVec { <fsummary>Stop monitoring a process from a driver</fsummary> <desc> <marker id="driver_demonitor_process"></marker> - <p>This function cancels an monitor created earlier. </p> + <p>This function cancels a monitor created earlier. </p> <p>The function returns 0 if a monitor was removed and > 0 if the monitor did no longer exist.</p> </desc> @@ -1326,7 +1318,7 @@ typedef struct ErlIOVec { <p>This function signals to erlang that the driver has encountered an EOF and should be closed, unless the port was opened with the <c>eof</c> option, in that case eof is sent - to the port. Otherwise, the port is close and an + to the port. Otherwise, the port is closed and an <c>'EXIT'</c> message is sent to the port owner process.</p> <p>The return value is 0.</p> </desc> @@ -1349,8 +1341,8 @@ typedef struct ErlIOVec { (<c>driver_failure</c>).</p> <p>The driver should fail only when in severe error situations, when the driver cannot possibly keep open, for instance - buffer allocation gets out of memory. Normal errors is more - appropriate to handle with sending error codes with + buffer allocation gets out of memory. For normal errors + it is more appropriate to send error codes with <c>driver_output</c>.</p> <p>The return value is 0.</p> </desc> @@ -1371,7 +1363,7 @@ typedef struct ErlIOVec { <p>This function returns the process id of the process that made the current call to the driver. The process id can be used with <c>driver_send_term</c> to send back data to the - caller. <c>driver_caller()</c> only return valid data + caller. <c>driver_caller()</c> only returns valid data when currently executing in one of the following driver callbacks:</p> <taglist> @@ -1409,7 +1401,7 @@ typedef struct ErlIOVec { tuple, the elements are given first, and then the tuple term, with a count. Likewise for lists.</p> <p>A tuple must be specified with the number of elements. (The - elements precedes the <c>ERL_DRV_TUPLE</c> term.)</p> + elements precede the <c>ERL_DRV_TUPLE</c> term.)</p> <p>A list must be specified with the number of elements, including the tail, which is the last term preceding <c>ERL_DRV_LIST</c>.</p> @@ -1518,7 +1510,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len }; driver_output_term(port, spec, sizeof(spec) / sizeof(spec[0])); ]]></code> - <p>If you want to pass a binary and doesn't already have the content + <p>If you want to pass a binary and don't already have the content of the binary in an <c>ErlDrvBinary</c>, you can benefit from using <c>ERL_DRV_BUF2BINARY</c> instead of creating an <c>ErlDrvBinary</c> via <c>driver_alloc_binary()</c> and then pass the binary via @@ -1565,7 +1557,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len <em>other</em> processes than the port owner process. The <c>receiver</c> parameter specifies the process to receive the data.</p> - <p>The parameters <c>term</c> and <c>n</c> does the same thing + <p>The parameters <c>term</c> and <c>n</c> do the same thing as in <seealso marker="#driver_output_term">driver_output_term</seealso>.</p> <p>This function is only thread-safe when the emulator with SMP support is used.</p> @@ -1660,7 +1652,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len <desc> <marker id="driver_lock_driver"></marker> <p>This function locks the driver used by the port <c>port</c> - in memory for the rest of the emulator process + in memory for the rest of the emulator process' lifetime. After this call, the driver behaves as one of Erlang's statically linked in drivers.</p> </desc> @@ -1904,7 +1896,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len corresponding to one of the involved thread identifiers has terminated since the thread identifier was saved, the result of <c>erl_drv_equal_tids()</c> might not give - expected result. + the expected result. </p></note> <p>This function is thread-safe.</p> </desc> diff --git a/erts/doc/src/erl_ext_dist.xml b/erts/doc/src/erl_ext_dist.xml index c2d58d1ef1..fd2da2cfe3 100644 --- a/erts/doc/src/erl_ext_dist.xml +++ b/erts/doc/src/erl_ext_dist.xml @@ -5,7 +5,7 @@ <header> <copyright> <year>2007</year> - <year>2007</year> + <year>2011</year> <holder>Ericsson AB, All Rights Reserved</holder> </copyright> <legalnotice> diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index 27887cbdf6..4bbd4e2a54 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -4,7 +4,7 @@ <cref> <header> <copyright> - <year>2001</year><year>2010</year> + <year>2001</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -193,9 +193,9 @@ ok A handle ("safe pointer") to this memory block can then be returned to Erlang by the use of <seealso marker="#enif_make_resource">enif_make_resource</seealso>. The term returned by <c>enif_make_resource</c> - is totally opaque in nature. It can be stored and passed between processses - on the same node, but the only real end usage is to pass it back as argument to a NIF. - The NIF can then do <seealso marker="#enif_get_resource">enif_get_resource</seealso> + is totally opaque in nature. It can be stored and passed between processes + on the same node, but the only real end usage is to pass it back as an argument to a NIF. + The NIF can then call <seealso marker="#enif_get_resource">enif_get_resource</seealso> and get back a pointer to the memory block that is guaranteed to still be valid. A resource object will not be deallocated until the last handle term has been garbage collected by the VM and the resource has been @@ -212,17 +212,7 @@ ok the garbage collector or <c>enif_release_resource</c>). Resource types are uniquely identified by a supplied name string and the name of the implementing module.</p> - <p>Resource types support upgrade in runtime by allowing a loaded NIF - library to takeover an already existing resource type and thereby - "inherit" all existing objects of that type. The destructor of the new - library will thereafter be called for the inherited objects and the - library with the old destructor function can be safely unloaded. Existing - resource objects, of a module that is upgraded, must either be deleted - or taken over by the new NIF library. The unloading of a library will be - postponed as long as there exist resource objects with a destructor - function in the library. - </p> - <p>Here is a template example of how to create and return a resource object.</p> + <marker id="enif_resource_example"/><p>Here is a template example of how to create and return a resource object.</p> <p/> <code type="none"> ERL_NIF_TERM term; @@ -240,8 +230,13 @@ ok /* resource now only owned by "Erlang" */ } return term; -} -</code> + </code> + <p>Note that once <c>enif_make_resource</c> creates the term to + return to Erlang, the code can choose to either keep its own + native pointer to the allocated struct and release it later, or + release it immediately and rely solely on the garbage collector + to eventually deallocate the resource object when it collects + the term.</p> <p>Another usage of resource objects is to create binary terms with user defined memory management. <seealso marker="#enif_make_resource_binary">enif_make_resource_binary</seealso> @@ -251,6 +246,16 @@ ok this can be a binary term consisting of data from a <c>mmap</c>'ed file. The destructor can then do <c>munmap</c> to release the memory region.</p> + <p>Resource types support upgrade in runtime by allowing a loaded NIF + library to takeover an already existing resource type and thereby + "inherit" all existing objects of that type. The destructor of the new + library will thereafter be called for the inherited objects and the + library with the old destructor function can be safely unloaded. Existing + resource objects, of a module that is upgraded, must either be deleted + or taken over by the new NIF library. The unloading of a library will be + postponed as long as there exist resource objects with a destructor + function in the library. + </p> </item> <tag>Threads and concurrency</tag> <item><p>A NIF is thread-safe without any explicit synchronization as @@ -368,7 +373,7 @@ ok environments between NIF calls. </p> <p>A <em>process independent environment</em> is created by calling <seealso marker="#enif_alloc_env">enif_alloc_env</seealso>. It can be - used to store terms beteen NIF calls and to send terms with + used to store terms between NIF calls and to send terms with <seealso marker="#enif_send">enif_send</seealso>. A process independent environment with all its terms is valid until you explicitly invalidates it with <seealso marker="#enif_free_env">enif_free_env</seealso> @@ -464,7 +469,7 @@ typedef enum { </section> <funcs> - <func><name><ret>void*</ret><nametext>enif_alloc(ErlNifEnv* env, size_t size)</nametext></name> + <func><name><ret>void*</ret><nametext>enif_alloc(size_t size)</nametext></name> <fsummary>Allocate dynamic memory.</fsummary> <desc><p>Allocate memory of <c>size</c> bytes. Return NULL if allocation failed.</p></desc> </func> @@ -539,7 +544,7 @@ typedef enum { <desc><p>Same as <seealso marker="erl_driver#erl_drv_equal_tids">erl_drv_equal_tids</seealso>. </p></desc> </func> - <func><name><ret>void</ret><nametext>enif_free(ErlNifEnv* env, void* ptr)</nametext></name> + <func><name><ret>void</ret><nametext>enif_free(void* ptr)</nametext></name> <fsummary>Free dynamic memory</fsummary> <desc><p>Free memory allocated by <c>enif_alloc</c>.</p></desc> </func> @@ -832,8 +837,14 @@ typedef enum { <fsummary>Create an opaque handle to a resource object</fsummary> <desc><p>Create an opaque handle to a memory managed resource object obtained by <seealso marker="#enif_alloc_resource">enif_alloc_resource</seealso>. - No ownership transfer is done, the resource object still needs to be released by - <seealso marker="#enif_release_resource">enif_release_resource</seealso>.</p> + No ownership transfer is done, as the resource object still needs to be released by + <seealso marker="#enif_release_resource">enif_release_resource</seealso>, + but note that the call to <c>enif_release_resource</c> can occur + immediately after obtaining the term from <c>enif_make_resource</c>, + in which case the resource object will be deallocated when the + term is garbage collected. See the + <seealso marker="#enif_resource_example">example of creating and + returning a resource object</seealso> for more details.</p> <p>Note that the only defined behaviour of using a resource term in an Erlang program is to store it and send it between processes on the same node. Other operations such as matching or <c>term_to_binary</c> @@ -857,11 +868,6 @@ typedef enum { <seealso marker="#enif_release_resource">enif_release_resource</seealso>.</p> </desc> </func> - <func><name><ret>ErlNifPid*</ret><nametext>enif_self(ErlNifEnv* caller_env, ErlNifPid* pid)</nametext></name> - <fsummary>Get the pid of the calling process.</fsummary> - <desc><p>Initialize the pid variable <c>*pid</c> to represent the - calling process. Return <c>pid</c>.</p></desc> - </func> <func><name><ret>ERL_NIF_TERM</ret><nametext>enif_make_string(ErlNifEnv* env, const char* string, ErlNifCharEncoding encoding)</nametext></name> <fsummary>Create a string.</fsummary> <desc><p>Create a list containing the characters of the @@ -980,11 +986,12 @@ typedef enum { <c>reload</c> or <c>upgrade</c>.</p> <p>Was previously named <c>enif_get_data</c>.</p></desc> </func> - <func><name><ret>void</ret><nametext>enif_realloc_binary(ErlNifBinary* bin, size_t size)</nametext></name> + <func><name><ret>int</ret><nametext>enif_realloc_binary(ErlNifBinary* bin, size_t size)</nametext></name> <fsummary>Change the size of a binary.</fsummary> <desc><p>Change the size of a binary <c>bin</c>. The source binary may be read-only, in which case it will be left untouched and - a mutable copy is allocated and assigned to <c>*bin</c>.</p></desc> + a mutable copy is allocated and assigned to <c>*bin</c>. Return true on success, + false if memory allocation failed.</p></desc> </func> <func><name><ret>void</ret><nametext>enif_release_binary(ErlNifBinary* bin)</nametext></name> <fsummary>Release a binary.</fsummary> @@ -1041,7 +1048,12 @@ typedef enum { <desc><p>Same as <seealso marker="erl_driver#erl_drv_rwlock_tryrwlock">erl_drv_rwlock_tryrwlock</seealso>. </p></desc> </func> - <func><name><ret>unsigned</ret><nametext>enif_send(ErlNifEnv* env, ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)</nametext></name> + <func><name><ret>ErlNifPid*</ret><nametext>enif_self(ErlNifEnv* caller_env, ErlNifPid* pid)</nametext></name> + <fsummary>Get the pid of the calling process.</fsummary> + <desc><p>Initialize the pid variable <c>*pid</c> to represent the + calling process. Return <c>pid</c>.</p></desc> + </func> + <func><name><ret>int</ret><nametext>enif_send(ErlNifEnv* env, ErlNifPid* to_pid, ErlNifEnv* msg_env, ERL_NIF_TERM msg)</nametext></name> <fsummary>Send a message to a process.</fsummary> <desc><p>Send a message to a process.</p> <taglist> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 25c92bdbb7..19f501391f 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -4,7 +4,7 @@ <erlref> <header> <copyright> - <year>1996</year><year>2010</year> + <year>1996</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -2781,14 +2781,17 @@ os_prompt%</pre> <name>open_port(PortName, PortSettings) -> port()</name> <fsummary>Open a port</fsummary> <type> - <v>PortName = {spawn, Command} | {spawn_driver, Command} | {spawn_executable, Command} | {fd, In, Out}</v> + <v>PortName = {spawn, Command} | {spawn_driver, Command} | {spawn_executable, FileName} | {fd, In, Out}</v> <v> Command = string()</v> + <v> FileName = [ FileNameChar ] | binary()</v> + <v> FileNameChar = int() (1..255 or any Unicode codepoint, see description)</v> <v> In = Out = int()</v> <v>PortSettings = [Opt]</v> - <v> Opt = {packet, N} | stream | {line, L} | {cd, Dir} | {env, Env} | {args, [ string() ]} | {arg0, string()} | exit_status | use_stdio | nouse_stdio | stderr_to_stdout | in | out | binary | eof</v> + <v> Opt = {packet, N} | stream | {line, L} | {cd, Dir} | {env, Env} | {args, [ ArgString ]} | {arg0, ArgString} | exit_status | use_stdio | nouse_stdio | stderr_to_stdout | in | out | binary | eof</v> <v> N = 1 | 2 | 4</v> <v> L = int()</v> <v> Dir = string()</v> + <v> ArgString = [ FileNameChar ] | binary()</v> <v> Env = [{Name, Val}]</v> <v> Name = string()</v> <v> Val = string() | false</v> @@ -2851,7 +2854,26 @@ os_prompt%</pre> executed, the appropriate command interpreter will implicitly be invoked, but there will still be no command argument expansion or implicit PATH search.</p> - + + <p>The name of the executable as well as the arguments + given in <c>args</c> and <c>arg0</c> is subject to + Unicode file name translation if the system is running + in Unicode file name mode. To avoid + translation or force i.e. UTF-8, supply the executable + and/or arguments as a binary in the correct + encoding. See the <seealso + marker="kernel:file">file</seealso> module, the + <seealso marker="kernel:file#native_name_encoding/0"> + file:native_name_encoding/0</seealso> function and the + <seealso marker="stdlib:unicode_usage">stdlib users guide + </seealso> for details.</p> + + <note>The characters in the name (if given as a list) + can only be > 255 if the Erlang VM is started in + Unicode file name translation mode, otherwise the name + of the executable is limited to the ISO-latin-1 + character set.</note> + <p>If the <c>Command</c> cannot be run, an error exception, with the posix error code as the reason, is raised. The error reason may differ between operating @@ -2954,6 +2976,21 @@ os_prompt%</pre> should not be given in this list. The proper executable name will automatically be used as argv[0] where applicable.</p> + <p>When the Erlang VM is running in Unicode file name + mode, the arguments can contain any Unicode characters and + will be translated into whatever is appropriate on the + underlying OS, which means UTF-8 for all platforms except + Windows, which has other (more transparent) ways of + dealing with Unicode arguments to programs. To avoid + Unicode translation of arguments, they can be supplied as + binaries in whatever encoding is deemed appropriate.</p> + + <note>The characters in the arguments (if given as a + list of characters) can only be > 255 if the Erlang + VM is started in Unicode file name mode, + otherwise the arguments are limited to the + ISO-latin-1 character set.</note> + <p>If one, for any reason, wants to explicitly set the program name in the argument vector, the <c>arg0</c> option can be used.</p> @@ -2969,6 +3006,9 @@ os_prompt%</pre> responds to this is highly system dependent and no specific effect is guaranteed.</p> + <p>The unicode file name translation rules of the + <c>args</c> option apply to this option as well.</p> + </item> <tag><c>exit_status</c></tag> @@ -3940,7 +3980,8 @@ os_prompt%</pre> <tag><c>{status, Status}</c></tag> <item> <p><c>Status</c> is the status of the process. <c>Status</c> - is <c>waiting</c> (waiting for a message), <c>running</c>, + is <c>exiting</c>, <c>garbage_collecting</c>, + <c>waiting</c> (for a message), <c>running</c>, <c>runnable</c> (ready to run, but another process is running), or <c>suspended</c> (suspended on a "busy" port or by the <c>erlang:suspend_process/[1,2]</c> BIF).</p> @@ -5178,7 +5219,7 @@ true</pre> <seealso marker="#system_info_scheduler_bindings">erlang:system_info(scheduler_bindings)</seealso>. </p> <p>Schedulers can currently only be bound on newer Linux, - Solaris, and Windows systems, but more systems will be + Solaris, FreeBSD, and Windows systems, but more systems will be supported in the future. </p> <p>In order for the runtime system to be able to bind schedulers, @@ -5559,7 +5600,7 @@ true</pre> <item> <p>Returns the automatically detected <c>CpuTopology</c>. The emulator currently only detects the CPU topology on some newer - Linux, Solaris, and Windows systems. On Windows system with + Linux, Solaris, FreeBSD, and Windows systems. On Windows system with more than 32 logical processors the CPU topology is not detected. </p> <p>For more information see the documentation of the diff --git a/erts/doc/src/erlc.xml b/erts/doc/src/erlc.xml index 1e8960c22c..ebf76a2afe 100644 --- a/erts/doc/src/erlc.xml +++ b/erts/doc/src/erlc.xml @@ -4,7 +4,7 @@ <comref> <header> <copyright> - <year>1997</year><year>2010</year> + <year>1997</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -141,6 +141,50 @@ for compiling native code, which needs to be compiled with the same run-time system that it should be run on.</p> </item> + <tag>-M</tag> + <item> + <p>Produces a Makefile rule to track headers dependencies. The + rule is sent to stdout. No object file is produced. + </p> + </item> + <tag>-MF <em>Makefile</em></tag> + <item> + <p>Like the <c><![CDATA[-M]]></c> option above, except that the + Makefile is written to <em>Makefile</em>. No object + file is produced. + </p> + </item> + <tag>-MD</tag> + <item> + <p>Same as <c><![CDATA[-M -MF <File>.Pbeam]]></c>. + </p> + </item> + <tag>-MT <em>Target</em></tag> + <item> + <p>In conjunction with <c><![CDATA[-M]]></c> or + <c><![CDATA[-MF]]></c>, change the name of the rule emitted + to <em>Target</em>. + </p> + </item> + <tag>-MQ <em>Target</em></tag> + <item> + <p>Like the <c><![CDATA[-MT]]></c> option above, except that + characters special to make(1) are quoted. + </p> + </item> + <tag>-MP</tag> + <item> + <p>In conjunction with <c><![CDATA[-M]]></c> or + <c><![CDATA[-MF]]></c>, add a phony target for each dependency. + </p> + </item> + <tag>-MG</tag> + <item> + <p>In conjunction with <c><![CDATA[-M]]></c> or + <c><![CDATA[-MF]]></c>, consider missing headers as generated + files and add them to the dependencies. + </p> + </item> <tag>--</tag> <item> <p>Signals that no more options will follow. diff --git a/erts/doc/src/escript.xml b/erts/doc/src/escript.xml index 44c9a5ac68..66e904f64f 100644 --- a/erts/doc/src/escript.xml +++ b/erts/doc/src/escript.xml @@ -4,7 +4,7 @@ <comref> <header> <copyright> - <year>2007</year><year>2010</year> + <year>2007</year><year>2011</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -153,7 +153,10 @@ halt(1).</pre> <p>Execution of interpreted code is slower than compiled code. If much of the execution takes place in interpreted code it may be worthwhile to compile it, even though the compilation - itself will take a little while.</p> + itself will take a little while. It is also possible to supply + <c>native</c> instead of compile, this will compile the script + using the native flag, again depending on the characteristics + of the escript this could or could not be worth while.</p> <p>As mentioned earlier, it is possible to have a script which contains precompiled <c>beam</c> code. In a precompiled @@ -397,6 +400,9 @@ ok Warnings and errors (if any) are written to the standard output, but the script will not be run. The exit status will be 0 if there were no errors, and 127 otherwise.</item> + + <tag>-n</tag> + <item>Compile the escript using the +native flag.</item> </taglist> </section> </comref> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 1703ce0942..102fa43c1f 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -30,6 +30,513 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 5.8.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + The scroll wheel now scrolls the werl window on Windows.</p> + <p> + Own Id: OTP-8985</p> + </item> + <item> + <p> + Some malformed distribution messages could cause VM to + crash, this is now corrected.</p> + <p> + Own Id: OTP-8993</p> + </item> + <item> + <p> + The OS function getifaddrs() can return NULL in some + address fields for e.g PPP and tunnel devices which + caused the emulator to segfault. This bug has now been + corrected.</p> + <p> + Own Id: OTP-8996</p> + </item> + <item> + <p> + The expression <<A:0>> would always produce + an empty binary, even if <c>A</c> was not an integer. + Corrected to cause a <c>badarg</c> exception if the type + of <c>A</c> is invalid. (Thanks to Zvi.)</p> + <p> + Own Id: OTP-8997</p> + </item> + <item> + <p> + A bug that potentially could cause an emulator crash when + deleting an ETS-table has been fixed. A resource leak + when hitting the maximum amount of ETS-tables allowed has + also been fixed.</p> + <p> + Own Id: OTP-8999</p> + </item> + <item> + <p> + A bug in the <c>exit/2</c> BIF could potentially cause an + emulator crash.</p> + <p> + Own Id: OTP-9005</p> + </item> + <item> + <p> + Due to a bug in glibc the runtime system could abort + while trying to destroy a mutex. The runtime system will + now issue a warning instead of aborting.</p> + <p> + Own Id: OTP-9009</p> + </item> + <item> + <p> + A bug in epmd could create strange behaviour when + listen() calls failed. This is now corrected thanks to + Steve Vinoski.</p> + <p> + Own Id: OTP-9024</p> + </item> + <item> + <p>When setting file_info the win32_driver will now + correctly set access and modified time. Previously these + entities were swapped.</p> + <p> + Own Id: OTP-9046</p> + </item> + <item> + <p> + Setting scheduler bind type to <c>unbound</c> failed if + binding of schedulers wasn't supported, or if CPU + topology wasn't present. This even though the + documentation stated that it is possible to set the bind + type to <c>unbound</c>.</p> + <p> + Own Id: OTP-9056 Aux Id: Seq11779 </p> + </item> + <item> + <p>Two problems were fixed in crash dump: The time left + for timers are now shown as unsigned integers and the + contents of ordered_set ETS tables is no longer + included.</p> + <p> + Own Id: OTP-9057</p> + </item> + <item> + <p> + The VM could fail to set IP_TOS and SO_PRIORITY in + certain situations, either because sockets were supplied + as open file descriptors, or because SO_PRIORITY by + default was set higher than the user can explicitly set + it to. Those situations are now handled.</p> + <p> + Own Id: OTP-9069</p> + </item> + <item> + <p> + Wx on MacOS X generated complains on stderr about certain + cocoa functions not beeing called from the "Main thread". + This is now corrected.</p> + <p> + Own Id: OTP-9081</p> + </item> + <item> + <p> + Fix a couple typos in driver_entry(3) (thanks to Tuncer + Ayaz).</p> + <p> + Own Id: OTP-9085</p> + </item> + <item> + <p> + Mention that "-detached" implies "-noinput"</p> + <p> + Clarify that specifying "-noinput" is unnecessary if the + "-detached" flag is given. (thanks to Holger Wei�)</p> + <p> + Own Id: OTP-9086</p> + </item> + <item> + <p> + A potential problem (found by code inspection) when + calling a fun whose code was not loaded has been fixed.</p> + <p> + Own Id: OTP-9095</p> + </item> + <item> + <p> + The emulator could get into a state where it didn't check + for I/O.</p> + <p> + Own Id: OTP-9105 Aux Id: Seq11798 </p> + </item> + <item> + <p> + Attempting to create binaries exceeding 2Gb (using for + example <c>term_to_binary/1</c>) would crash the emulator + with an attempt to allocate huge amounts of memory. + (Thanks to Jon Meredith.)</p> + <p> + Own Id: OTP-9117</p> + </item> + <item> + <p> + Fix erlang:hibernate/3 on HiPE enabled emulator (Thanks + to Paul Guyot)</p> + <p> + Own Id: OTP-9125</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p>From this release, the previously experimental + halfword emulator is now official. It can be enabled by + giving the <c>--enable-halfword-emulator</c> option to + the <c>configure</c> script.</p> + <p>The halfword emulator is a 64-bit application, but + uses halfwords (32-bit words) for all data in Erlang + processes, therefore using less memory and being faster + than the standard 64-bit emulator. The total size of all + BEAM code and all process data for all processes is + limited to 4Gb, but ETS tables and off-heap binaries are + only limited by the amount of available memory.</p> + <p> + Own Id: OTP-8941</p> + </item> + <item> + <p> + 32-bit atomic memory operations have been introduced + internally in the run time system, and are now used where + appropriate. There were previously only atomic memory + operations of word size available. The 32-bit atomic + memory operations slightly reduce memory consumption, and + slightly improve performance on 64-bit runtime systems.</p> + <p> + Own Id: OTP-8974</p> + </item> + <item> + <p> + Performance enhancements for looking up timer-entries and + removing timers from the wheel.</p> + <p> + Own Id: OTP-8990</p> + </item> + <item> + <p> + Write accesses to ETS tables have been optimized by + reducing the amount of atomic memory operations needed + during a write access.</p> + <p> + Own Id: OTP-9000</p> + </item> + <item> + <p> + Strange C coding in the VM made the -D_FORTIFY_SOURCE + option to gcc-4.5 react badly. The code is now cleaned up + so that it's accepted by gcc-4.5.</p> + <p> + Own Id: OTP-9025</p> + </item> + <item> + <p> + The memory footprint for loaded code has been somewhat + reduced (especially in the 64-bit BEAM machine).</p> + <p> + Own Id: OTP-9030</p> + </item> + <item> + <p> + The maximum number of allowed arguments for an Erlang + function has been lowered from 256 to 255, so that the + number of arguments can now fit in a byte.</p> + <p> + Own Id: OTP-9049</p> + </item> + <item> + <p> + Dependency generation for Makefiles has been added to the + compiler and erlc. See the manual pages for + <c>compile</c> and <c>erlc</c>. (Thanks to Jean-Sebastien + Pedron.)</p> + <p> + Own Id: OTP-9065</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 5.8.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> Fix format_man_pages so it handles all man sections + and remove warnings/errors in various man pages. </p> + <p> + Own Id: OTP-8600</p> + </item> + <item> + <p> + The <c>configure</c> command line argument <seealso + marker="doc/installation_guide:INSTALL#How-to-Build-and-Install-ErlangOTP_A-Closer-Look-at-the-individual-Steps_Configuring">--enable-ethread-pre-pentium4-compatibility</seealso> + had no effect. This option is now also automatically + enabled if required on the build machine.</p> + <p> + Own Id: OTP-8847</p> + </item> + <item> + <p> + Windows 2003 and Windows XP pre SP3 would sometimes not + start the Erlang R14B VM at all due to a bug in the cpu + topology detection. The bug affects Windows only, no + other platform is even remotely affected. The bug is now + corrected.</p> + <p> + Own Id: OTP-8876</p> + </item> + <item> + <p> + The HiPE run-time in the 64-bit emulator could do a + 64-bit write to a 32-bit struct field. It happened to be + harmless on Intel/AMD processors. Corrected. (Thanks to + Mikael Pettersson.)</p> + <p> + Own Id: OTP-8877</p> + </item> + <item> + <p> + A bug in <seealso + marker="erl_driver#erl_drv_tsd_get">erl_drv_tsd_get()</seealso> + and <seealso + marker="erl_nif#enif_tsd_get">enif_tsd_get()</seealso> + could cause an emulator crash. These functions are + currently not used in OTP. That is, the crash only occur + on systems with user implemented NIF libraries, or + drivers that use one of these functions.</p> + <p> + Own Id: OTP-8889</p> + </item> + <item> + <p> + Calling <c>erlang:system_info({cpu_topology, + CpuTopologyType})</c> with another <c>CpuTopologyType</c> + element than one of the documented atoms <c>defined</c>, + <c>detected</c>, or <c>used</c> caused an emulator crash. + (Thanks to Paul Guyot)</p> + <p> + Own Id: OTP-8914</p> + </item> + <item> + <p> + The ERTS internal rwlock implementation could get into an + inconsistent state. This bug was very seldom triggered, + but could be during heavy contention. The bug was + introduced in R14B (erts-5.8.1).</p> + <p> + The bug was most likely to be triggered when using the + <c>read_concurrency</c> option on an ETS table that was + frequently accessed from multiple processes doing lots of + writes and reads. That is, in a situation where you + typically don't want to use the <c>read_concurrency</c> + option in the first place.</p> + <p> + Own Id: OTP-8925 Aux Id: OTP-8544 </p> + </item> + <item> + <p> + Tracing to port could cause an emulator crash when + unloading the trace driver.</p> + <p> + Own Id: OTP-8932</p> + </item> + <item> + <p> + Removed use of CancelIoEx on Windows that had been shown + to cause problems with some drivers.</p> + <p> + Own Id: OTP-8937</p> + </item> + <item> + <p> + The fallback implementation used when no native atomic + implementation was found did not compile. (Thanks to + Patrick Baggett, and Tuncer Ayaz)</p> + <p> + Own Id: OTP-8944</p> + </item> + <item> + <p> + Some integer values used during load balancing could + under rare circumstances wrap causing a load unbalance + between schedulers.</p> + <p> + Own Id: OTP-8950</p> + </item> + <item> + <p> + The windows VM now correctly handles appending to large + files (> 4GB).</p> + <p> + Own Id: OTP-8958</p> + </item> + <item> + <p> + Name resolving of IPv6 addresses has been implemented for + Windows versions that support it. The use of ancient + resolver flags (AI_V4MAPPED | AI_ADDRCONFIG) to the + getaddrinfo() function has been removed since e.g FreeBSD + regard mapped IPv4 addresses to be a security problem and + the semantics of the address configured flag is + uncertain.</p> + <p> + Own Id: OTP-8969</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + The help texts produced by the <c>configure</c> scripts + in the top directory and in the erts directory have been + aligned and cleaned up.</p> + <p> + Own Id: OTP-8859</p> + </item> + <item> + <p> + When the runtime system had fewer schedulers than logical + processors, the system could get an unnecessarily large + amount reader groups.</p> + <p> + Own Id: OTP-8861</p> + </item> + <item> + <p> + <c>run_rel</c> has been updated to support Solaris's + /dev/ptmx device and to load the necessary STREAMS + modules so that <c>to_erl</c> can provide terminal echo + of keyboard input. (Thanks to Ryan Tilder.)</p> + <p> + Own Id: OTP-8878</p> + </item> + <item> + <p> + The Erlang VM now supports Unicode filenames. The feature + is turned on by default on systems where Unicode + filenames are mandatory (Windows and MacOSX), but can be + enabled on other systems with the '+fnu' emulator option. + Enabling the Unicode filename feature on systems where it + is not default is however considered experimental and not + to be used for production. Together with the Unicode file + name support, the concept of "raw filenames" is + introduced, which means filenames provided without + implicit unicode encoding translation. Raw filenames are + provided as binaries, not lists. For further information, + see stdlib users guide and the chapter about using + Unicode in Erlang. Also see the file module manual page.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-8887</p> + </item> + <item> + <p>Buffer overflows have been prevented in <c>erlc</c>, + <c>dialyzer</c>, <c>typer</c>, <c>run_test</c>, + <c>heart</c>, <c>escript</c>, and <c>erlexec</c>.</p> + (Thanks to Michael Santos.) + <p> + Own Id: OTP-8892</p> + </item> + <item> + <p> + The runtime system is now less eager to suspend processes + sending messages over the distribution. The default value + of the distribution buffer busy limit has also been + increased from 128 KB to 1 MB. This in order to improve + throughput.</p> + <p> + Own Id: OTP-8901</p> + </item> + <item> + <p> + The distribution buffer busy limit can now be configured + at system startup. For more information see the + documentation of the <c>erl</c> <seealso + marker="erl#+zdbbl">+zdbbl</seealso> command line flag. + (Thanks to Scott Lystig Fritchie)</p> + <p> + Own Id: OTP-8912</p> + </item> + <item> + <p> + The inet driver internal buffer stack implementation has + been rewritten in order to reduce lock contention.</p> + <p> + Own Id: OTP-8916</p> + </item> + <item> + <p> + New ETS option <c>compressed</c>, to enable a more + compact storage format at the expence of heavier table + operations. For test and evaluation, <c>erl +ec</c> can + be used to force compression on all ETS tables.</p> + <p> + Own Id: OTP-8922 Aux Id: seq11658 </p> + </item> + <item> + <p> + There is now a new function inet:getifaddrs/0 modeled + after C library function getifaddrs() on BSD and LInux + that reports existing interfaces and their addresses on + the host. This replaces the undocumented and unsupported + inet:getiflist/0 and inet:ifget/2.</p> + <p> + Own Id: OTP-8926</p> + </item> + <item> + <p> + Support for detection of CPU topology and binding of + schedulers on FreeBSD 8 have been added. (Thanks to Paul + Guyot)</p> + <p> + Own Id: OTP-8939</p> + </item> + <item> + <p> + Several bugs related to hibernate/3 and HiPE have been + corrected. (Thanks to Paul Guyot.)</p> + <p> + Own Id: OTP-8952</p> + </item> + <item> + <p> + Support for soft and hard links on Windows versions and + filesystems that support them is added.</p> + <p> + Own Id: OTP-8955</p> + </item> + <item> + <p> + The win32 virtual machine is now linked large address + aware. his allows the Erlang VM to use up to 3 gigs of + address space on Windows instead of the default of 2 + gigs.</p> + <p> + Own Id: OTP-8956</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 5.8.1.2</title> <section><title>Fixed Bugs and Malfunctions</title> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 76d782b159..f04df354a8 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -505,8 +505,10 @@ ifdef HIPE_ENABLED OPCODE_TABLES += hipe/hipe_ops.tab endif -$(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/beam_opcodes.c: $(OPCODE_TABLES) - LANG=C $(PERL) utils/beam_makeops -outdir $(TTF_DIR) \ +$(TTF_DIR)/beam_opcodes.h $(TTF_DIR)/beam_opcodes.c: $(OPCODE_TABLES) utils/beam_makeops + LANG=C $(PERL) utils/beam_makeops \ + -wordsize @EXTERNAL_WORD_SIZE@ \ + -outdir $(TTF_DIR) \ -emulator $(OPCODE_TABLES) # bif and atom table @@ -734,7 +736,7 @@ RUN_OBJS = \ $(OBJDIR)/erl_fun.o $(OBJDIR)/erl_bif_port.o \ $(OBJDIR)/erl_term.o $(OBJDIR)/erl_node_tables.o \ $(OBJDIR)/erl_monitors.o $(OBJDIR)/erl_process_dump.o \ - $(OBJDIR)/erl_bif_timer.o \ + $(OBJDIR)/erl_bif_timer.o $(OBJDIR)/erl_cpu_topology.o \ $(OBJDIR)/erl_drv_thread.o $(OBJDIR)/erl_bif_chksum.o \ $(OBJDIR)/erl_bif_re.o $(OBJDIR)/erl_unicode.o \ $(OBJDIR)/packet_parser.o $(OBJDIR)/safe_hash.o \ @@ -796,7 +798,8 @@ endif OS_OBJS += $(OBJDIR)/erl_mseg.o \ $(OBJDIR)/erl_$(ERLANG_OSTYPE)_sys_ddll.o \ - $(OBJDIR)/erl_mtrace_sys_wrap.o + $(OBJDIR)/erl_mtrace_sys_wrap.o \ + $(OBJDIR)/erl_sys_common_misc.o HIPE_x86_OS_OBJS=$(HIPE_x86_$(OPSYS)_OBJS) HIPE_x86_OBJS=$(OBJDIR)/hipe_x86.o $(OBJDIR)/hipe_x86_glue.o $(OBJDIR)/hipe_x86_bifs.o $(OBJDIR)/hipe_x86_signal.o $(OBJDIR)/hipe_x86_stack.o $(HIPE_x86_OS_OBJS) diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 682f31b83f..31910888d1 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -950,8 +950,8 @@ static int set_function_break(Module *modp, BeamInstr *pc, int bif, MatchSetUnref(old_match_spec); } else { BpDataCount *bdc = (BpDataCount *) bd; - long count = 0; - long res = 0; + erts_aint_t count = 0; + erts_aint_t res = 0; ASSERT(! match_spec); ASSERT(is_nil(tracer_pid)); diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index ebc171078d..bd8a7249a7 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -157,7 +157,7 @@ do { \ BpData **bds = (BpData **) (pc)[-4]; \ BpDataCount *bdc = NULL; \ Uint ix = bp_sched2ix_proc( (p) ); \ - long count = 0; \ + erts_aint_t count = 0; \ \ ASSERT((pc)[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); \ ASSERT(bds); \ diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index b0bf14b94f..8a48049921 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -48,7 +48,6 @@ void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); -static void print_big(int to, void *to_arg, Eterm* addr); static int print_op(int to, void *to_arg, int op, int size, BeamInstr* addr); Eterm erts_debug_same_2(Process* p, Eterm term1, Eterm term2) @@ -157,6 +156,25 @@ void debug_dump_code(BeamInstr *I, int num) } #endif +BIF_RETTYPE +erts_debug_instructions_0(BIF_ALIST_0) +{ + int i = 0; + Uint needed = num_instructions * 2; + Eterm* hp; + Eterm res = NIL; + + for (i = 0; i < num_instructions; i++) { + needed += 2*strlen(opc[i].name); + } + hp = HAlloc(BIF_P, needed); + for (i = num_instructions-1; i >= 0; i--) { + Eterm s = erts_bld_string_n(&hp, 0, opc[i].name, strlen(opc[i].name)); + res = erts_bld_cons(&hp, 0, s, res); + } + return res; +} + Eterm erts_debug_disassemble_1(Process* p, Eterm addr) { @@ -249,7 +267,7 @@ erts_debug_disassemble_1(Process* p, Eterm addr) "unknown " HEXF "\n", instr); code_ptr++; } - bin = new_binary(p, (byte *) dsbufp->str, (int) dsbufp->str_len); + bin = new_binary(p, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); hsz = 4+4; (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); @@ -312,6 +330,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) BeamInstr packed = 0; /* Accumulator for packed operations. */ BeamInstr args[8]; /* Arguments for this instruction. */ BeamInstr* ap; /* Pointer to arguments. */ + BeamInstr* unpacked; /* Unpacked arguments */ start_prog = opc[op].pack; @@ -360,6 +379,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) *ap++ = packed & BEAM_LOOSE_MASK; packed >>= BEAM_LOOSE_SHIFT; break; +#ifdef ARCH_64 + case 'w': /* Shift 32 steps */ + *ap++ = packed & BEAM_WIDE_MASK; + packed >>= BEAM_WIDE_SHIFT; + break; +#endif case 'p': *sp++ = *--ap; break; @@ -386,7 +411,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'x': /* x(N) */ if (reg_index(ap[0]) == 0) { - erts_print(to, to_arg, "X[0]"); + erts_print(to, to_arg, "x[0]"); } else { erts_print(to, to_arg, "x(%d)", reg_index(ap[0])); } @@ -506,6 +531,7 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) ap++; break; case 'P': /* Byte offset into tuple (see beam_load.c) */ + case 'Q': /* Like 'P', but packable */ erts_print(to, to_arg, "%d", (*ap / sizeof(Eterm)) - 1); ap++; break; @@ -526,9 +552,12 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) * Print more information about certain instructions. */ + unpacked = ap; ap = addr + size; switch (op) { - case op_i_select_val_sfI: + case op_i_select_val_rfI: + case op_i_select_val_xfI: + case op_i_select_val_yfI: { int n = ap[-1]; @@ -540,7 +569,24 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_jump_on_val_sfII: + case op_i_select_tuple_arity_rfI: + case op_i_select_tuple_arity_xfI: + case op_i_select_tuple_arity_yfI: + { + int n = ap[-1]; + + while (n > 0) { + Uint arity = arityval(ap[0]); + erts_print(to, to_arg, " {%d} f(" HEXF ")", arity, ap[1]); + ap += 2; + size += 2; + n--; + } + } + break; + case op_i_jump_on_val_rfII: + case op_i_jump_on_val_xfII: + case op_i_jump_on_val_yfII: { int n; for (n = ap[-2]; n > 0; n--) { @@ -550,39 +596,46 @@ print_op(int to, void *to_arg, int op, int size, BeamInstr* addr) } } break; - case op_i_select_big_sf: - while (ap[0]) { - Eterm *bigp = (Eterm *) ap; - int arity = thing_arityval(*bigp); - print_big(to, to_arg, bigp); - size += TermWords(arity+1); - ap += TermWords(arity+1); - erts_print(to, to_arg, " f(" HEXF ") ", ap[0]); - ap++; - size++; + case op_i_jump_on_val_zero_rfI: + case op_i_jump_on_val_zero_xfI: + case op_i_jump_on_val_zero_yfI: + { + int n; + for (n = ap[-1]; n > 0; n--) { + erts_print(to, to_arg, "f(" HEXF ") ", ap[0]); + ap++; + size++; + } + } + break; + case op_i_put_tuple_rI: + case op_i_put_tuple_xI: + case op_i_put_tuple_yI: + { + int n = unpacked[-1]; + + while (n > 0) { + if (!is_header(ap[0])) { + erts_print(to, to_arg, " %T", (Eterm) ap[0]); + } else { + switch ((ap[0] >> 2) & 0x03) { + case R_REG_DEF: + erts_print(to, to_arg, " x(0)"); + break; + case X_REG_DEF: + erts_print(to, to_arg, " x(%d)", ap[0] >> 4); + break; + case Y_REG_DEF: + erts_print(to, to_arg, " y(%d)", ap[0] >> 4); + break; + } + } + ap++, size++, n--; + } } - ap++; - size++; break; } erts_print(to, to_arg, "\n"); return size; } - -static void -print_big(int to, void *to_arg, Eterm* addr) -{ - int i; - int k; - - i = BIG_SIZE(addr); - if (BIG_SIGN(addr)) - erts_print(to, to_arg, "-#integer(%d) = {", i); - else - erts_print(to, to_arg, "#integer(%d) = {", i); - erts_print(to, to_arg, "0x%x", BIG_DIGIT(addr, 0)); - for (k = 1; k < i; k++) - erts_print(to, to_arg, ",0x%x", BIG_DIGIT(addr, k)); - erts_print(to, to_arg, "}"); -} diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 8a0e12dd4f..8991f7b198 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -321,6 +321,7 @@ extern int count_instructions; # define POST_BIF_GC_SWAPIN_0(_p, _res) \ ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \ PROCESS_MAIN_CHK_LOCKS((_p)); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \ if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ _res = erts_gc_after_bif_call((_p), (_res), NULL, 0); \ E = (_p)->stop; \ @@ -328,6 +329,7 @@ extern int count_instructions; HTOP = HEAP_TOP((_p)) # define POST_BIF_GC_SWAPIN(_p, _res, _regs, _arity) \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC((_p)); \ ERTS_SMP_REQ_PROC_MAIN_LOCK((_p)); \ PROCESS_MAIN_CHK_LOCKS((_p)); \ if (((_p)->mbuf) || (MSO(_p).overhead >= BIN_VHEAP_SZ(_p)) ) { \ @@ -344,6 +346,8 @@ extern int count_instructions; #define xb(N) (*(Eterm *) (((unsigned char *)reg) + (N))) #define yb(N) (*(Eterm *) (((unsigned char *)E) + (N))) #define fb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N))) +#define Qb(N) (N) +#define Ib(N) (N) #define x(N) reg[N] #define y(N) E[N] #define r(N) x##N @@ -365,6 +369,7 @@ extern int count_instructions; reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, needed + (HeapNeed), reg, (M)); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ r(0) = reg[0]; \ SWAPIN; \ @@ -418,6 +423,7 @@ extern int count_instructions; reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ r(0) = reg[0]; \ SWAPIN; \ @@ -440,6 +446,7 @@ extern int count_instructions; reg[0] = r(0); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ r(0) = reg[0]; \ SWAPIN; \ @@ -462,6 +469,7 @@ extern int count_instructions; reg[Live] = Extra; \ PROCESS_MAIN_CHK_LOCKS(c_p); \ FCALLS -= erts_garbage_collect(c_p, need, reg, (Live)+1); \ + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); \ PROCESS_MAIN_CHK_LOCKS(c_p); \ if (Live > 0) { \ r(0) = reg[0]; \ @@ -472,6 +480,13 @@ extern int count_instructions; HEAP_SPACE_VERIFIED(need); \ } while (0) +#define TestHeapPutList(Need, Reg) \ + do { \ + TestHeap((Need), 1); \ + PutList(Reg, r(0), r(0), StoreSimpleDest); \ + CHECK_TERM(r(0)); \ + } while (0) + #ifdef HYBRID #ifdef INCREMENTAL #define TestGlobalHeap(Nh, Live, hp) \ @@ -516,6 +531,11 @@ extern int count_instructions; SWAPIN; \ } while (0) +#define PutTuple(Dst, Arity) \ + do { \ + Dst = make_tuple(HTOP); \ + pt_arity = (Arity); \ + } while (0) /* * Check that we haven't used the reductions and jump to function pointed to by @@ -674,6 +694,11 @@ extern int count_instructions; SET_I((BeamInstr *) CallDest); \ Dispatch(); +#define MoveJump(Src) \ + r(0) = (Src); \ + SET_I((BeamInstr *) Arg(0)); \ + Goto(*I); + #define GetList(Src, H, T) do { \ Eterm* tmp_ptr = list_val(Src); \ H = CAR(tmp_ptr); \ @@ -723,16 +748,8 @@ extern int count_instructions; (Dest) = (* (Eterm *) EXPAND_POINTER(tmp_arg1)); \ } while (0) -#define PutTuple(Arity, Src, Dest) \ - ASSERT(is_arity_value(Arity)); \ - Dest = make_tuple(HTOP); \ - HTOP[0] = (Arity); \ - HTOP[1] = (Src); \ - HTOP += 2 - -#define Put(Word) *HTOP++ = (Word) - #define EqualImmed(X, Y, Action) if (X != Y) { Action; } +#define NotEqualImmed(X, Y, Action) if (X == Y) { Action; } #define IsFloat(Src, Fail) if (is_not_float(Src)) { Fail; } @@ -984,8 +1001,39 @@ extern int count_instructions; #define IsPid(Src, Fail) if (is_not_pid(Src)) { Fail; } #define IsRef(Src, Fail) if (is_not_ref(Src)) { Fail; } -static BifFunction translate_gc_bif(void* gcf); -static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf); +/* + * process_main() is already huge, so we want to avoid inlining + * into it. Especially functions that are seldom used. + */ +#ifdef __GNUC__ +# define NOINLINE __attribute__((__noinline__)) +#else +# define NOINLINE +#endif + +/* + * The following functions are called directly by process_main(). + * Don't inline them. + */ +static BifFunction translate_gc_bif(void* gcf) NOINLINE; +static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, + Eterm* reg, BifFunction bf) NOINLINE; +static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, + Eterm* reg, Eterm func) NOINLINE; +static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; +static BeamInstr* apply(Process* p, Eterm module, Eterm function, + Eterm args, Eterm* reg) NOINLINE; +static BeamInstr* call_fun(Process* p, int arity, + Eterm* reg, Eterm args) NOINLINE; +static BeamInstr* apply_fun(Process* p, Eterm fun, + Eterm args, Eterm* reg) NOINLINE; +static Eterm new_fun(Process* p, Eterm* reg, + ErlFunEntry* fe, int num_free) NOINLINE; + + +/* + * Functions not directly called by process_main(). OK to inline. + */ static BeamInstr* next_catch(Process* c_p, Eterm *reg); static void terminate_proc(Process* c_p, Eterm Value); static Eterm add_stacktrace(Process* c_p, Eterm Value, Eterm exc); @@ -993,16 +1041,6 @@ static void save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, Eterm args); static struct StackTrace * get_trace_from_exc(Eterm exc); static Eterm make_arglist(Process* c_p, Eterm* reg, int a); -static Eterm call_error_handler(Process* p, BeamInstr* ip, Eterm* reg); -static Eterm call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg); -static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity); -static BeamInstr* apply(Process* p, Eterm module, Eterm function, - Eterm args, Eterm* reg); -static int hibernate(Process* c_p, Eterm module, Eterm function, - Eterm args, Eterm* reg); -static BeamInstr* call_fun(Process* p, int arity, Eterm* reg, Eterm args); -static BeamInstr* apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg); -static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free); #if defined(VXWORKS) static int init_done; @@ -1146,6 +1184,8 @@ void process_main(void) Uint temp_bits; /* Temporary used by BsSkipBits2 & BsGetInteger2 */ + Eterm pt_arity; /* Used by do_put_tuple */ + ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */ @@ -1178,7 +1218,12 @@ void process_main(void) do_schedule1: PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); +#if HALFWORD_HEAP + ASSERT(erts_get_scheduler_data()->num_tmp_heap_used == 0); +#endif + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = schedule(c_p, reds_used); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); #ifdef DEBUG pid = c_p->id; #endif @@ -1246,6 +1291,52 @@ void process_main(void) #define STORE_ARITH_RESULT(res) StoreBifResult(2, (res)); #define ARITH_FUNC(name) erts_gc_##name + { + Eterm increment_reg_val; + Eterm increment_val; + Uint live; + Eterm result; + + OpCase(i_increment_yIId): + increment_reg_val = yb(Arg(0)); + goto do_increment; + + OpCase(i_increment_xIId): + increment_reg_val = xb(Arg(0)); + goto do_increment; + + OpCase(i_increment_rIId): + increment_reg_val = r(0); + I--; + + do_increment: + increment_val = Arg(1); + if (is_small(increment_reg_val)) { + Sint i = signed_val(increment_reg_val) + increment_val; + ASSERT(MY_IS_SSMALL(i) == IS_SSMALL(i)); + if (MY_IS_SSMALL(i)) { + result = make_small(i); + store_result: + StoreBifResult(3, result); + } + } + + live = Arg(2); + SWAPOUT; + reg[0] = r(0); + reg[live] = increment_reg_val; + reg[live+1] = make_small(increment_val); + result = erts_gc_mixed_plus(c_p, reg, live); + r(0) = reg[0]; + SWAPIN; + ERTS_HOLE_CHECK(c_p); + if (is_value(result)) { + goto store_result; + } + ASSERT(c_p->freason != BADMATCH || is_value(c_p->fvalue)); + goto find_func_info; + } + OpCase(i_plus_jId): { Eterm result; @@ -1309,6 +1400,52 @@ void process_main(void) } Next(1); + { + Eterm is_eq_exact_lit_val; + + OpCase(i_is_eq_exact_literal_xfc): + is_eq_exact_lit_val = xb(Arg(0)); + I++; + goto do_is_eq_exact_literal; + + OpCase(i_is_eq_exact_literal_yfc): + is_eq_exact_lit_val = yb(Arg(0)); + I++; + goto do_is_eq_exact_literal; + + OpCase(i_is_eq_exact_literal_rfc): + is_eq_exact_lit_val = r(0); + + do_is_eq_exact_literal: + if (!eq(Arg(1), is_eq_exact_lit_val)) { + ClauseFail(); + } + Next(2); + } + + { + Eterm is_ne_exact_lit_val; + + OpCase(i_is_ne_exact_literal_xfc): + is_ne_exact_lit_val = xb(Arg(0)); + I++; + goto do_is_ne_exact_literal; + + OpCase(i_is_ne_exact_literal_yfc): + is_ne_exact_lit_val = yb(Arg(0)); + I++; + goto do_is_ne_exact_literal; + + OpCase(i_is_ne_exact_literal_rfc): + is_ne_exact_lit_val = r(0); + + do_is_ne_exact_literal: + if (eq(Arg(1), is_ne_exact_lit_val)) { + ClauseFail(); + } + Next(2); + } + OpCase(i_move_call_only_fcr): { r(0) = Arg(1); } @@ -1392,6 +1529,17 @@ void process_main(void) NextPF(1, next); } + OpCase(move_x1_c): { + x(1) = Arg(0); + Next(1); + } + + OpCase(move_x2_c): { + x(2) = Arg(0); + Next(1); + } + + OpCase(return): { SET_I(c_p->cp); /* @@ -1405,16 +1553,6 @@ void process_main(void) Goto(*I); } - OpCase(test_heap_1_put_list_Iy): { - BeamInstr *next; - - PreFetch(2, next); - TestHeap(Arg(0), 1); - PutList(yb(Arg(1)), r(0), r(0), StoreSimpleDest); - CHECK_TERM(r(0)); - NextPF(2, next); - } - /* * Send is almost a standard call-BIF with two arguments, except for: * 1) It cannot be traced. @@ -1447,24 +1585,36 @@ void process_main(void) goto find_func_info; } - OpCase(i_element_jssd): { - Eterm index; - Eterm tuple; - - /* - * Inlined version of element/2 for speed. - */ - GetArg2(1, index, tuple); - if (is_small(index) && is_tuple(tuple)) { - Eterm* tp = tuple_val(tuple); - - if ((signed_val(index) >= 1) && - (signed_val(index) <= arityval(*tp))) { - Eterm result = tp[signed_val(index)]; - StoreBifResult(3, result); - } - } - } + { + Eterm element_index; + Eterm element_tuple; + + OpCase(i_element_xjsd): + element_tuple = xb(Arg(0)); + I++; + goto do_element; + + OpCase(i_element_yjsd): + element_tuple = yb(Arg(0)); + I++; + goto do_element; + + OpCase(i_element_rjsd): + element_tuple = r(0); + /* Fall through */ + + do_element: + GetArg1(1, element_index); + if (is_small(element_index) && is_tuple(element_tuple)) { + Eterm* tp = tuple_val(element_tuple); + + if ((signed_val(element_index) >= 1) && + (signed_val(element_index) <= arityval(*tp))) { + Eterm result = tp[signed_val(element_index)]; + StoreBifResult(2, result); + } + } + } /* Fall through */ OpCase(badarg_j): @@ -1472,24 +1622,32 @@ void process_main(void) c_p->freason = BADARG; goto lb_Cl_error; - OpCase(i_fast_element_jIsd): { - Eterm tuple; - - /* - * Inlined version of element/2 for even more speed. - * The first argument is an untagged integer >= 1. - * The second argument is guaranteed to be a register operand. - */ - GetArg1(2, tuple); - if (is_tuple(tuple)) { - Eterm* tp = tuple_val(tuple); - tmp_arg2 = Arg(1); - if (tmp_arg2 <= arityval(*tp)) { - Eterm result = tp[tmp_arg2]; - StoreBifResult(3, result); - } - } + { + Eterm fast_element_tuple; + + OpCase(i_fast_element_rjId): + fast_element_tuple = r(0); + + do_fast_element: + if (is_tuple(fast_element_tuple)) { + Eterm* tp = tuple_val(fast_element_tuple); + Eterm pos = Arg(1); /* Untagged integer >= 1 */ + if (pos <= arityval(*tp)) { + Eterm result = tp[pos]; + StoreBifResult(2, result); + } + } goto badarg; + + OpCase(i_fast_element_xjId): + fast_element_tuple = xb(Arg(0)); + I++; + goto do_fast_element; + + OpCase(i_fast_element_yjId): + fast_element_tuple = yb(Arg(0)); + I++; + goto do_fast_element; } OpCase(catch_yf): @@ -1515,6 +1673,7 @@ void process_main(void) SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); FCALLS -= erts_garbage_collect(c_p, 3, reg+2, 1); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; } @@ -1633,6 +1792,7 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); }, { + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); r(0) = reg[0]; SWAPIN; @@ -1691,6 +1851,7 @@ void process_main(void) CANCEL_TIMER(c_p); free_message(msgp); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); NextPF(0, next); @@ -1842,8 +2003,87 @@ void process_main(void) NextPF(0, next); } - OpCase(i_select_val_sfI): - GetArg1(0, tmp_arg1); + + { + Eterm select_val2; + + OpCase(i_select_tuple_arity2_yfAfAf): + select_val2 = yb(Arg(0)); + goto do_select_tuple_arity2; + + OpCase(i_select_tuple_arity2_xfAfAf): + select_val2 = xb(Arg(0)); + goto do_select_tuple_arity2; + + OpCase(i_select_tuple_arity2_rfAfAf): + select_val2 = r(0); + I--; + + do_select_tuple_arity2: + if (is_not_tuple(select_val2)) { + goto select_val2_fail; + } + select_val2 = *tuple_val(select_val2); + goto do_select_val2; + + OpCase(i_select_val2_yfcfcf): + select_val2 = yb(Arg(0)); + goto do_select_val2; + + OpCase(i_select_val2_xfcfcf): + select_val2 = xb(Arg(0)); + goto do_select_val2; + + OpCase(i_select_val2_rfcfcf): + select_val2 = r(0); + I--; + + do_select_val2: + if (select_val2 == Arg(2)) { + I += 2; + } else if (select_val2 == Arg(4)) { + I += 4; + } + + select_val2_fail: + SET_I((BeamInstr *) Arg(1)); + Goto(*I); + } + + { + Eterm select_val; + + OpCase(i_select_tuple_arity_xfI): + select_val = xb(Arg(0)); + goto do_select_tuple_arity; + + OpCase(i_select_tuple_arity_yfI): + select_val = yb(Arg(0)); + goto do_select_tuple_arity; + + OpCase(i_select_tuple_arity_rfI): + select_val = r(0); + I--; + + do_select_tuple_arity: + if (is_tuple(select_val)) { + select_val = *tuple_val(select_val); + goto do_binary_search; + } + SET_I((BeamInstr *) Arg(1)); + Goto(*I); + + OpCase(i_select_val_xfI): + select_val = xb(Arg(0)); + goto do_binary_search; + + OpCase(i_select_val_yfI): + select_val = yb(Arg(0)); + goto do_binary_search; + + OpCase(i_select_val_rfI): + select_val = r(0); + I--; do_binary_search: { @@ -1880,9 +2120,9 @@ void process_main(void) unsigned int boffset = ((unsigned int)bdiff >> 1) & ~(sizeof(struct Pairs)-1); mid = (struct Pairs*)((char*)low + boffset); - if (tmp_arg1 < mid->val) { + if (select_val < mid->val) { high = mid; - } else if (tmp_arg1 > mid->val) { + } else if (select_val > mid->val) { low = mid + 1; } else { SET_I(mid->addr); @@ -1892,16 +2132,28 @@ void process_main(void) SET_I((BeamInstr *) Arg(1)); Goto(*I); } + } - OpCase(i_jump_on_val_zero_sfI): { - Eterm index; - - GetArg1(0, index); - if (is_small(index)) { - index = signed_val(index); - if (index < Arg(2)) { - SET_I((BeamInstr *) (&Arg(3))[index]); + Eterm jump_on_val_zero_index; + + OpCase(i_jump_on_val_zero_yfI): + jump_on_val_zero_index = yb(Arg(0)); + goto do_jump_on_val_zero_index; + + OpCase(i_jump_on_val_zero_xfI): + jump_on_val_zero_index = xb(Arg(0)); + goto do_jump_on_val_zero_index; + + OpCase(i_jump_on_val_zero_rfI): + jump_on_val_zero_index = r(0); + I--; + + do_jump_on_val_zero_index: + if (is_small(jump_on_val_zero_index)) { + jump_on_val_zero_index = signed_val(jump_on_val_zero_index); + if (jump_on_val_zero_index < Arg(2)) { + SET_I((BeamInstr *) (&Arg(3))[jump_on_val_zero_index]); Goto(*I); } } @@ -1909,15 +2161,27 @@ void process_main(void) Goto(*I); } - OpCase(i_jump_on_val_sfII): { - Eterm index; + Eterm jump_on_val_index; - GetArg1(0, index); - if (is_small(index)) { - index = (Uint) (signed_val(index) - Arg(3)); - if (index < Arg(2)) { - SET_I((BeamInstr *) (&Arg(4))[index]); + + OpCase(i_jump_on_val_yfII): + jump_on_val_index = yb(Arg(0)); + goto do_jump_on_val_index; + + OpCase(i_jump_on_val_xfII): + jump_on_val_index = xb(Arg(0)); + goto do_jump_on_val_index; + + OpCase(i_jump_on_val_rfII): + jump_on_val_index = r(0); + I--; + + do_jump_on_val_index: + if (is_small(jump_on_val_index)) { + jump_on_val_index = (Uint) (signed_val(jump_on_val_index) - Arg(3)); + if (jump_on_val_index < Arg(2)) { + SET_I((BeamInstr *) (&Arg(4))[jump_on_val_index]); Goto(*I); } } @@ -1925,6 +2189,32 @@ void process_main(void) Goto(*I); } + do_put_tuple: { + Eterm* hp = HTOP; + + *hp++ = make_arityval(pt_arity); + + do { + Eterm term = *I++; + switch (term & _TAG_IMMED1_MASK) { + case (R_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: + *hp++ = r(0); + break; + case (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: + *hp++ = x(term >> _TAG_IMMED1_SIZE); + break; + case (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER: + *hp++ = y(term >> _TAG_IMMED1_SIZE); + break; + default: + *hp++ = term; + break; + } + } while (--pt_arity != 0); + HTOP = hp; + Goto(*I); + } + /* * All guards with zero arguments have special instructions: * self/0 @@ -1952,6 +2242,7 @@ void process_main(void) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); result = (*bf)(c_p, arg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; @@ -1980,6 +2271,7 @@ void process_main(void) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); result = (*bf)(c_p, arg); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; @@ -2009,6 +2301,7 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); result = (*bf)(c_p, reg, live); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -2044,6 +2337,7 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); result = (*bf)(c_p, reg, live); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -2082,6 +2376,7 @@ void process_main(void) PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); result = (*bf)(c_p, reg, live); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); SWAPIN; @@ -2116,6 +2411,7 @@ void process_main(void) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); result = (*bf)(c_p, tmp_arg1, tmp_arg2); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); FCALLS = c_p->fcalls; @@ -2139,6 +2435,7 @@ void process_main(void) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); result = (*bf)(c_p, tmp_arg1, tmp_arg2); ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_HOLE_CHECK(c_p); if (is_value(result)) { @@ -2562,23 +2859,25 @@ void process_main(void) OpCase(i_int_bnot_jsId): { - GetArg1(1, tmp_arg1); - if (is_small(tmp_arg1)) { - tmp_arg1 = make_small(~signed_val(tmp_arg1)); + Eterm bnot_val; + + GetArg1(1, bnot_val); + if (is_small(bnot_val)) { + bnot_val = make_small(~signed_val(bnot_val)); } else { Uint live = Arg(2); SWAPOUT; reg[0] = r(0); - reg[live] = tmp_arg1; - tmp_arg1 = erts_gc_bnot(c_p, reg, live); + reg[live] = bnot_val; + bnot_val = erts_gc_bnot(c_p, reg, live); r(0) = reg[0]; SWAPIN; ERTS_HOLE_CHECK(c_p); - if (is_nil(tmp_arg1)) { + if (is_nil(bnot_val)) { goto lb_Cl_error; } } - StoreBifResult(3, tmp_arg1); + StoreBifResult(3, bnot_val); } badarith: @@ -2833,121 +3132,6 @@ void process_main(void) goto do_schedule1; } - OpCase(i_select_tuple_arity_sfI): - { - GetArg1(0, tmp_arg1); - - if (is_tuple(tmp_arg1)) { - tmp_arg1 = *tuple_val(tmp_arg1); - goto do_binary_search; - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - - OpCase(i_select_big_sf): - { - Eterm* bigp; - Uint arity; - Eterm* given; - Uint given_arity; - Uint given_size; - - GetArg1(0, tmp_arg1); - if (is_big(tmp_arg1)) { - - /* - * The loader has sorted the bignumbers in descending order - * on the arity word. Therefore, we know that the search - * has failed as soon as we encounter an arity word less than - * the arity word of the given number. There is a zero word - * (less than any valid arity word) stored after the last bignumber. - */ - - given = big_val(tmp_arg1); - given_arity = given[0]; - given_size = thing_arityval(given_arity); - bigp = (Eterm *) &Arg(2); - while ((arity = bigp[0]) > given_arity) { - bigp += (TermWords(thing_arityval(arity) + 1) + 1) * (sizeof(BeamInstr)/sizeof(Eterm)); - } - while (bigp[0] == given_arity) { - if (memcmp(bigp+1, given+1, sizeof(Eterm)*given_size) == 0) { - BeamInstr *tmp = - ((BeamInstr *) (UWord) bigp) + TermWords(given_size + 1); - SET_I((BeamInstr *) *tmp); - Goto(*I); - } - bigp += (TermWords(thing_arityval(arity) + 1) + 1) * (sizeof(BeamInstr)/sizeof(Eterm)); - } - } - - /* - * Failed. - */ - - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } - -#if defined(ARCH_64) && !HALFWORD_HEAP - OpCase(i_select_float_sfI): - { - Uint f; - int n; - struct ValLabel { - Uint f; - BeamInstr* addr; - }; - struct ValLabel* ptr; - - GetArg1(0, tmp_arg1); - ASSERT(is_float(tmp_arg1)); - f = float_val(tmp_arg1)[1]; - n = Arg(2); - ptr = (struct ValLabel *) &Arg(3); - while (n-- > 0) { - if (ptr->f == f) { - SET_I(ptr->addr); - Goto(*I); - } - ptr++; - } - SET_I((Eterm *) Arg(1)); - Goto(*I); - } -#else - OpCase(i_select_float_sfI): - { - Uint fpart1; - Uint fpart2; - int n; - struct ValLabel { - Uint fpart1; - Uint fpart2; - BeamInstr* addr; - }; - struct ValLabel* ptr; - - GetArg1(0, tmp_arg1); - ASSERT(is_float(tmp_arg1)); - fpart1 = float_val(tmp_arg1)[1]; - fpart2 = float_val(tmp_arg1)[2]; - - n = Arg(2); - ptr = (struct ValLabel *) &Arg(3); - while (n-- > 0) { - if (ptr->fpart1 == fpart1 && ptr->fpart2 == fpart2) { - SET_I(ptr->addr); - Goto(*I); - } - ptr++; - } - SET_I((BeamInstr *) Arg(1)); - Goto(*I); - } -#endif - OpCase(set_tuple_element_sdP): { Eterm element; Eterm tuple; @@ -2993,15 +3177,17 @@ void process_main(void) the first argument. We also handle atom tags in the first argument for backwards compatibility. */ - GetArg2(0, tmp_arg1, tmp_arg2); - c_p->fvalue = tmp_arg2; + Eterm raise_val1; + Eterm raise_val2; + GetArg2(0, raise_val1, raise_val2); + c_p->fvalue = raise_val2; if (c_p->freason == EXC_NULL) { /* a safety check for the R10-0 case; should not happen */ c_p->ftrace = NIL; c_p->freason = EXC_ERROR; } /* for R10-0 code, keep existing c_p->ftrace and hope it's correct */ - switch (tmp_arg1) { + switch (raise_val1) { case am_throw: c_p->freason = EXC_THROWN & ~EXF_SAVETRACE; break; @@ -3017,8 +3203,8 @@ void process_main(void) passed from a user! Currently only expecting generated calls. */ struct StackTrace *s; - c_p->ftrace = tmp_arg1; - s = get_trace_from_exc(tmp_arg1); + c_p->ftrace = raise_val1; + s = get_trace_from_exc(raise_val1); if (s == NULL) { c_p->freason = EXC_ERROR; } else { @@ -3029,11 +3215,24 @@ void process_main(void) goto find_func_info; } - OpCase(badmatch_s): { - GetArg1(0, tmp_arg1); - c_p->fvalue = tmp_arg1; - c_p->freason = BADMATCH; - } + { + Eterm badmatch_val; + + OpCase(badmatch_y): + badmatch_val = yb(Arg(0)); + goto do_badmatch; + + OpCase(badmatch_x): + badmatch_val = xb(Arg(0)); + goto do_badmatch; + + OpCase(badmatch_r): + badmatch_val = r(0); + + do_badmatch: + c_p->fvalue = badmatch_val; + c_p->freason = BADMATCH; + } /* Fall through here */ find_func_info: { @@ -3056,12 +3255,11 @@ void process_main(void) */ SWAPOUT; reg[0] = r(0); - tmp_arg1 = call_error_handler(c_p, I-3, reg); + I = call_error_handler(c_p, I-3, reg, am_undefined_function); r(0) = reg[0]; SWAPIN; - if (tmp_arg1) { - SET_I(c_p->i); - Dispatch(); + if (I) { + Goto(*I); } /* Fall through */ @@ -3084,128 +3282,153 @@ void process_main(void) } } - OpCase(call_nif): - { - /* - * call_nif is always first instruction in function: - * - * I[-3]: Module - * I[-2]: Function - * I[-1]: Arity - * I[0]: &&call_nif - * I[1]: Function pointer to NIF function - * I[2]: Pointer to erl_module_nif - */ - BifFunction vbf; - - c_p->current = I-3; /* current and vbf set to please handle_error */ - SWAPOUT; - c_p->fcalls = FCALLS - 1; - PROCESS_MAIN_CHK_LOCKS(c_p); - tmp_arg2 = I[-1]; - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + { + Eterm nif_bif_result; + Eterm bif_nif_arity; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - { - typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); - NifF* fp = vbf = (NifF*) I[1]; - struct enif_environment_t env; - erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); - reg[0] = r(0); - tmp_arg1 = (*fp)(&env, tmp_arg2, reg); - erts_post_nif(&env); - } - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); - goto apply_bif_or_nif_epilogue; - - OpCase(apply_bif): - /* - * At this point, I points to the code[3] in the export entry for - * the BIF: - * - * code[0]: Module - * code[1]: Function - * code[2]: Arity - * code[3]: &&apply_bif - * code[4]: Function pointer to BIF function - */ + OpCase(call_nif): + { + /* + * call_nif is always first instruction in function: + * + * I[-3]: Module + * I[-2]: Function + * I[-1]: Arity + * I[0]: &&call_nif + * I[1]: Function pointer to NIF function + * I[2]: Pointer to erl_module_nif + */ + BifFunction vbf; - c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ - c_p->i = I; /* In case we apply check_process_code/2. */ - c_p->arity = 0; /* To allow garbage collection on ourselves - * (check_process_code/2). - */ - SWAPOUT; - c_p->fcalls = FCALLS - 1; - vbf = (BifFunction) Arg(0); - PROCESS_MAIN_CHK_LOCKS(c_p); - tmp_arg2 = I[-1]; - ASSERT(tmp_arg2 <= 3); - ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - switch (tmp_arg2) { - case 3: + c_p->current = I-3; /* current and vbf set to please handle_error */ + SWAPOUT; + c_p->fcalls = FCALLS - 1; + PROCESS_MAIN_CHK_LOCKS(c_p); + bif_nif_arity = I[-1]; + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); { - Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, r(0), x(1), x(2), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); + typedef Eterm NifF(struct enif_environment_t*, int argc, Eterm argv[]); + NifF* fp = vbf = (NifF*) I[1]; + struct enif_environment_t env; + erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2]); + reg[0] = r(0); + nif_bif_result = (*fp)(&env, bif_nif_arity, reg); + erts_post_nif(&env); } - break; - case 2: - { - Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, r(0), x(1), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(nif_bif_result)); + PROCESS_MAIN_CHK_LOCKS(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + goto apply_bif_or_nif_epilogue; + + OpCase(apply_bif): + /* + * At this point, I points to the code[3] in the export entry for + * the BIF: + * + * code[0]: Module + * code[1]: Function + * code[2]: Arity + * code[3]: &&apply_bif + * code[4]: Function pointer to BIF function + */ + + c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ + c_p->i = I; /* In case we apply check_process_code/2. */ + c_p->arity = 0; /* To allow garbage collection on ourselves + * (check_process_code/2). + */ + SWAPOUT; + c_p->fcalls = FCALLS - 1; + vbf = (BifFunction) Arg(0); + PROCESS_MAIN_CHK_LOCKS(c_p); + bif_nif_arity = I[-1]; + ASSERT(bif_nif_arity <= 3); + ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + switch (bif_nif_arity) { + case 3: + { + Eterm (*bf)(Process*, Eterm, Eterm, Eterm, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, r(0), x(1), x(2), I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + break; + case 2: + { + Eterm (*bf)(Process*, Eterm, Eterm, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, r(0), x(1), I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + break; + case 1: + { + Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, r(0), I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + } + break; + case 0: + { + Eterm (*bf)(Process*, BeamInstr*) = vbf; + ASSERT(!ERTS_PROC_IS_EXITING(c_p)); + nif_bif_result = (*bf)(c_p, I); + ASSERT(!ERTS_PROC_IS_EXITING(c_p) || + is_non_value(nif_bif_result)); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + PROCESS_MAIN_CHK_LOCKS(c_p); + break; + } + default: + erl_exit(1, "apply_bif: invalid arity: %u\n", + bif_nif_arity); } - break; - case 1: - { - Eterm (*bf)(Process*, Eterm, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, r(0), I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); + + apply_bif_or_nif_epilogue: + ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); + ERTS_HOLE_CHECK(c_p); + if (c_p->mbuf) { + reg[0] = r(0); + nif_bif_result = erts_gc_after_bif_call(c_p, nif_bif_result, + reg, bif_nif_arity); + r(0) = reg[0]; } - break; - case 0: - { - Eterm (*bf)(Process*, BeamInstr*) = vbf; - ASSERT(!ERTS_PROC_IS_EXITING(c_p)); - tmp_arg1 = (*bf)(c_p, I); - ASSERT(!ERTS_PROC_IS_EXITING(c_p) || is_non_value(tmp_arg1)); - PROCESS_MAIN_CHK_LOCKS(c_p); - break; + SWAPIN; /* There might have been a garbage collection. */ + FCALLS = c_p->fcalls; + if (is_value(nif_bif_result)) { + r(0) = nif_bif_result; + CHECK_TERM(r(0)); + SET_I(c_p->cp); + c_p->cp = 0; + Goto(*I); + } else if (c_p->freason == TRAP) { + SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); + r(0) = c_p->def_arg_reg[0]; + x(1) = c_p->def_arg_reg[1]; + x(2) = c_p->def_arg_reg[2]; + if (c_p->status == P_WAITING) { + goto do_schedule; + } + Dispatch(); } - } -apply_bif_or_nif_epilogue: - ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); - ERTS_HOLE_CHECK(c_p); - if (c_p->mbuf) { reg[0] = r(0); - tmp_arg1 = erts_gc_after_bif_call(c_p, tmp_arg1, reg, tmp_arg2); - r(0) = reg[0]; + I = handle_error(c_p, c_p->cp, reg, vbf); + goto post_error_handling; } - SWAPIN; /* There might have been a garbage collection. */ - FCALLS = c_p->fcalls; - if (is_value(tmp_arg1)) { - r(0) = tmp_arg1; - CHECK_TERM(r(0)); - SET_I(c_p->cp); - Goto(*I); - } else if (c_p->freason == TRAP) { - SET_I(*((BeamInstr **) (UWord) ((c_p)->def_arg_reg + 3))); - r(0) = c_p->def_arg_reg[0]; - x(1) = c_p->def_arg_reg[1]; - x(2) = c_p->def_arg_reg[2]; - Dispatch(); - } - reg[0] = r(0); - I = handle_error(c_p, c_p->cp, reg, vbf); - goto post_error_handling; } OpCase(i_get_sd): @@ -3218,11 +3441,25 @@ apply_bif_or_nif_epilogue: StoreBifResult(1, result); } - OpCase(case_end_s): - GetArg1(0, tmp_arg1); - c_p->fvalue = tmp_arg1; - c_p->freason = EXC_CASE_CLAUSE; - goto find_func_info; + { + Eterm case_end_val; + + OpCase(case_end_x): + case_end_val = xb(Arg(0)); + goto do_case_end; + + OpCase(case_end_y): + case_end_val = yb(Arg(0)); + goto do_case_end; + + OpCase(case_end_r): + case_end_val = r(0); + + do_case_end: + c_p->fvalue = case_end_val; + c_p->freason = EXC_CASE_CLAUSE; + goto find_func_info; + } OpCase(if_end): c_p->freason = EXC_IF_CLAUSE; @@ -3235,10 +3472,13 @@ apply_bif_or_nif_epilogue: } OpCase(try_case_end_s): - GetArg1(0, tmp_arg1); - c_p->fvalue = tmp_arg1; - c_p->freason = EXC_TRY_CLAUSE; - goto find_func_info; + { + Eterm try_case_end_val; + GetArg1(0, try_case_end_val); + c_p->fvalue = try_case_end_val; + c_p->freason = EXC_TRY_CLAUSE; + goto find_func_info; + } /* * Construction of binaries using new instructions. @@ -3786,19 +4026,20 @@ apply_bif_or_nif_epilogue: Eterm header; BeamInstr *next; Uint slots; + Eterm context; OpCase(i_bs_start_match2_rfIId): { - tmp_arg1 = r(0); + context = r(0); do_start_match: slots = Arg(2); - if (!is_boxed(tmp_arg1)) { + if (!is_boxed(context)) { ClauseFail(); } PreFetch(4, next); - header = *boxed_val(tmp_arg1); + header = *boxed_val(context); if (header_is_bin_matchstate(header)) { - ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1); + ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(context); Uint actual_slots = HEADER_NUM_SLOTS(header); ms->save_offset[0] = ms->mb.offset; if (actual_slots < slots) { @@ -3806,8 +4047,8 @@ apply_bif_or_nif_epilogue: Uint live = Arg(1); Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - TestHeapPreserve(wordsneeded, live, tmp_arg1); - ms = (ErlBinMatchState *) boxed_val(tmp_arg1); + TestHeapPreserve(wordsneeded, live, context); + ms = (ErlBinMatchState *) boxed_val(context); dst = (ErlBinMatchState *) HTOP; *dst = *ms; *HTOP = HEADER_BIN_MATCHSTATE(slots); @@ -3819,12 +4060,12 @@ apply_bif_or_nif_epilogue: Eterm result; Uint live = Arg(1); Uint wordsneeded = ERL_BIN_MATCHSTATE_SIZE(slots); - TestHeapPreserve(wordsneeded, live, tmp_arg1); + TestHeapPreserve(wordsneeded, live, context); HEAP_TOP(c_p) = HTOP; #ifdef DEBUG c_p->stop = E; /* Needed for checking in HeapOnlyAlloc(). */ #endif - result = erts_bs_start_match_2(c_p, tmp_arg1, slots); + result = erts_bs_start_match_2(c_p, context, slots); HTOP = HEAP_TOP(c_p); HEAP_SPACE_VERIFIED(0); if (is_non_value(result)) { @@ -3838,12 +4079,12 @@ apply_bif_or_nif_epilogue: NextPF(4, next); } OpCase(i_bs_start_match2_xfIId): { - tmp_arg1 = xb(Arg(0)); + context = xb(Arg(0)); I++; goto do_start_match; } OpCase(i_bs_start_match2_yfIId): { - tmp_arg1 = yb(Arg(0)); + context = yb(Arg(0)); I++; goto do_start_match; } @@ -3936,93 +4177,105 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } + { + Eterm bs_get_integer8_context; + OpCase(i_bs_get_integer_8_rfd): { - tmp_arg1 = r(0); - goto do_bs_get_integer_8; - } + bs_get_integer8_context = r(0); + goto do_bs_get_integer_8; + } OpCase(i_bs_get_integer_8_xfd): { - tmp_arg1 = xb(Arg(0)); - I++; - } + bs_get_integer8_context = xb(Arg(0)); + I++; + } do_bs_get_integer_8: { - ErlBinMatchBuffer *_mb; - Eterm _result; - _mb = ms_matchbuffer(tmp_arg1); - if (_mb->size - _mb->offset < 8) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 8, 0, _mb); - } else { - _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]); - _mb->offset += 8; + ErlBinMatchBuffer *_mb; + Eterm _result; + _mb = ms_matchbuffer(bs_get_integer8_context); + if (_mb->size - _mb->offset < 8) { + ClauseFail(); + } + if (BIT_OFFSET(_mb->offset) != 0) { + _result = erts_bs_get_integer_2(c_p, 8, 0, _mb); + } else { + _result = make_small(_mb->base[BYTE_OFFSET(_mb->offset)]); + _mb->offset += 8; + } + StoreBifResult(1, _result); } - StoreBifResult(1, _result); } - OpCase(i_bs_get_integer_16_rfd): { - tmp_arg1 = r(0); + { + Eterm bs_get_integer_16_context; + + OpCase(i_bs_get_integer_16_rfd): + bs_get_integer_16_context = r(0); goto do_bs_get_integer_16; - } - OpCase(i_bs_get_integer_16_xfd): { - tmp_arg1 = xb(Arg(0)); + OpCase(i_bs_get_integer_16_xfd): + bs_get_integer_16_context = xb(Arg(0)); I++; - } - do_bs_get_integer_16: { - ErlBinMatchBuffer *_mb; - Eterm _result; - _mb = ms_matchbuffer(tmp_arg1); - if (_mb->size - _mb->offset < 16) { - ClauseFail(); - } - if (BIT_OFFSET(_mb->offset) != 0) { - _result = erts_bs_get_integer_2(c_p, 16, 0, _mb); - } else { - _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset))); - _mb->offset += 16; + do_bs_get_integer_16: + { + ErlBinMatchBuffer *_mb; + Eterm _result; + _mb = ms_matchbuffer(bs_get_integer_16_context); + if (_mb->size - _mb->offset < 16) { + ClauseFail(); + } + if (BIT_OFFSET(_mb->offset) != 0) { + _result = erts_bs_get_integer_2(c_p, 16, 0, _mb); + } else { + _result = make_small(get_int16(_mb->base+BYTE_OFFSET(_mb->offset))); + _mb->offset += 16; + } + StoreBifResult(1, _result); } - StoreBifResult(1, _result); } - OpCase(i_bs_get_integer_32_rfId): { - tmp_arg1 = r(0); + { + Eterm bs_get_integer_32_context; + + OpCase(i_bs_get_integer_32_rfId): + bs_get_integer_32_context = r(0); goto do_bs_get_integer_32; - } + - OpCase(i_bs_get_integer_32_xfId): { - tmp_arg1 = xb(Arg(0)); + OpCase(i_bs_get_integer_32_xfId): + bs_get_integer_32_context = xb(Arg(0)); I++; - } - do_bs_get_integer_32: { - ErlBinMatchBuffer *_mb; - Uint32 _integer; - Eterm _result; - _mb = ms_matchbuffer(tmp_arg1); - if (_mb->size - _mb->offset < 32) { ClauseFail(); } - if (BIT_OFFSET(_mb->offset) != 0) { - _integer = erts_bs_get_unaligned_uint32(_mb); - } else { - _integer = get_int32(_mb->base + _mb->offset/8); - } - _mb->offset += 32; + + do_bs_get_integer_32: + { + ErlBinMatchBuffer *_mb; + Uint32 _integer; + Eterm _result; + _mb = ms_matchbuffer(bs_get_integer_32_context); + if (_mb->size - _mb->offset < 32) { ClauseFail(); } + if (BIT_OFFSET(_mb->offset) != 0) { + _integer = erts_bs_get_unaligned_uint32(_mb); + } else { + _integer = get_int32(_mb->base + _mb->offset/8); + } + _mb->offset += 32; #if !defined(ARCH_64) || HALFWORD_HEAP - if (IS_USMALL(0, _integer)) { + if (IS_USMALL(0, _integer)) { #endif - _result = make_small(_integer); + _result = make_small(_integer); #if !defined(ARCH_64) || HALFWORD_HEAP - } else { - TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); - _result = uint_to_big((Uint) _integer, HTOP); - HTOP += BIG_UINT_HEAP_SIZE; - HEAP_SPACE_VERIFIED(0); - } + } else { + TestHeap(BIG_UINT_HEAP_SIZE, Arg(1)); + _result = uint_to_big((Uint) _integer, HTOP); + HTOP += BIG_UINT_HEAP_SIZE; + HEAP_SPACE_VERIFIED(0); + } #endif - StoreBifResult(2, _result); + StoreBifResult(2, _result); + } } /* Operands: Size Live Fail Flags Dst */ @@ -4120,54 +4373,64 @@ apply_bif_or_nif_epilogue: StoreBifResult(3, result); } - /* Operands: MatchContext Fail Dst */ + { + Eterm get_utf8_context; + + /* Operands: MatchContext Fail Dst */ OpCase(i_bs_get_utf8_rfd): { - tmp_arg1 = r(0); - goto do_bs_get_utf8; - } + get_utf8_context = r(0); + goto do_bs_get_utf8; + } OpCase(i_bs_get_utf8_xfd): { - tmp_arg1 = xb(Arg(0)); - I++; - } + get_utf8_context = xb(Arg(0)); + I++; + } - /* - * tmp_arg1 = match_context - * Operands: Fail Dst - */ + /* + * get_utf8_context = match_context + * Operands: Fail Dst + */ - do_bs_get_utf8: { - Eterm result = erts_bs_get_utf8(ms_matchbuffer(tmp_arg1)); - if (is_non_value(result)) { - ClauseFail(); + do_bs_get_utf8: { + Eterm result = erts_bs_get_utf8(ms_matchbuffer(get_utf8_context)); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(1, result); } - StoreBifResult(1, result); } - /* Operands: MatchContext Fail Flags Dst */ + { + Eterm get_utf16_context; + + /* Operands: MatchContext Fail Flags Dst */ OpCase(i_bs_get_utf16_rfId): { - tmp_arg1 = r(0); - goto do_bs_get_utf16; - } + get_utf16_context = r(0); + goto do_bs_get_utf16; + } OpCase(i_bs_get_utf16_xfId): { - tmp_arg1 = xb(Arg(0)); - I++; - } + get_utf16_context = xb(Arg(0)); + I++; + } - /* - * tmp_arg1 = match_context - * Operands: Fail Flags Dst - */ - do_bs_get_utf16: { - Eterm result = erts_bs_get_utf16(ms_matchbuffer(tmp_arg1), Arg(1)); - if (is_non_value(result)) { - ClauseFail(); + /* + * get_utf16_context = match_context + * Operands: Fail Flags Dst + */ + do_bs_get_utf16: { + Eterm result = erts_bs_get_utf16(ms_matchbuffer(get_utf16_context), + Arg(1)); + if (is_non_value(result)) { + ClauseFail(); + } + StoreBifResult(2, result); } - StoreBifResult(2, result); } { + Eterm context_to_binary_context; ErlBinMatchBuffer* mb; ErlSubBin* sb; Uint size; @@ -4176,27 +4439,29 @@ apply_bif_or_nif_epilogue: Uint hole_size; OpCase(bs_context_to_binary_r): { - tmp_arg1 = x0; + context_to_binary_context = x0; I -= 2; goto do_context_to_binary; } /* Unfortunately, inlining can generate this instruction. */ OpCase(bs_context_to_binary_y): { - tmp_arg1 = yb(Arg(0)); + context_to_binary_context = yb(Arg(0)); goto do_context_to_binary0; } OpCase(bs_context_to_binary_x): { - tmp_arg1 = xb(Arg(0)); + context_to_binary_context = xb(Arg(0)); do_context_to_binary0: I--; } do_context_to_binary: - if (is_boxed(tmp_arg1) && header_is_bin_matchstate(*boxed_val(tmp_arg1))) { - ErlBinMatchState* ms = (ErlBinMatchState *) boxed_val(tmp_arg1); + if (is_boxed(context_to_binary_context) && + header_is_bin_matchstate(*boxed_val(context_to_binary_context))) { + ErlBinMatchState* ms; + ms = (ErlBinMatchState *) boxed_val(context_to_binary_context); mb = &ms->mb; offs = ms->save_offset[0]; size = mb->size - offs; @@ -4205,17 +4470,17 @@ apply_bif_or_nif_epilogue: Next(2); OpCase(i_bs_get_binary_all_reuse_rfI): { - tmp_arg1 = x0; + context_to_binary_context = x0; goto do_bs_get_binary_all_reuse; } OpCase(i_bs_get_binary_all_reuse_xfI): { - tmp_arg1 = xb(Arg(0)); + context_to_binary_context = xb(Arg(0)); I++; } do_bs_get_binary_all_reuse: - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(context_to_binary_context); size = mb->size - mb->offset; if (size % Arg(1) != 0) { ClauseFail(); @@ -4224,7 +4489,7 @@ apply_bif_or_nif_epilogue: do_bs_get_binary_all_reuse_common: orig = mb->orig; - sb = (ErlSubBin *) boxed_val(tmp_arg1); + sb = (ErlSubBin *) boxed_val(context_to_binary_context); hole_size = 1 + header_arity(sb->thing_word) - ERL_SUB_BIN_SIZE; sb->thing_word = HEADER_SUB_BIN; sb->size = BYTE_OFFSET(size); @@ -4240,12 +4505,14 @@ apply_bif_or_nif_epilogue: } { + Eterm match_string_context; + OpCase(i_bs_match_string_rfII): { - tmp_arg1 = r(0); + match_string_context = r(0); goto do_bs_match_string; } OpCase(i_bs_match_string_xfII): { - tmp_arg1 = xb(Arg(0)); + match_string_context = xb(Arg(0)); I++; } @@ -4260,7 +4527,7 @@ apply_bif_or_nif_epilogue: PreFetch(3, next); bits = Arg(1); bytes = (byte *) Arg(2); - mb = ms_matchbuffer(tmp_arg1); + mb = ms_matchbuffer(match_string_context); if (mb->size - mb->offset < bits) { ClauseFail(); } @@ -4353,6 +4620,7 @@ apply_bif_or_nif_epilogue: ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); flags = erts_call_trace(c_p, ep->code, ep->match_prog_set, reg, 0, &c_p->tracer_proc); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -4364,6 +4632,7 @@ apply_bif_or_nif_epilogue: /* SWAPOUT, SWAPIN was done and r(0) was saved above */ PROCESS_MAIN_CHK_LOCKS(c_p); FCALLS -= erts_garbage_collect(c_p, 3, reg, ep->code[2]); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); r(0) = reg[0]; SWAPIN; @@ -4453,6 +4722,7 @@ apply_bif_or_nif_epilogue: reg[0] = r(0); PROCESS_MAIN_CHK_LOCKS(c_p); FCALLS -= erts_garbage_collect(c_p, 2, reg, I[-1]); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); r(0) = reg[0]; } @@ -4556,6 +4826,7 @@ apply_bif_or_nif_epilogue: /* SWAPOUT was done and r(0) was saved above */ PROCESS_MAIN_CHK_LOCKS(c_p); FCALLS -= erts_garbage_collect(c_p, need, reg, I[-1]); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); r(0) = reg[0]; SWAPIN; @@ -4723,7 +4994,7 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } - OpCase(fmove_new_ld): { + OpCase(fmove_ld): { Eterm fr = Arg(0); Eterm dest = make_float(HTOP); @@ -4753,11 +5024,6 @@ apply_bif_or_nif_epilogue: NextPF(2, next); } - /* - * Old allocating fmove. - */ - - #ifdef NO_FPE_SIGNALS OpCase(fclearerror): OpCase(i_fcheckerror): @@ -4958,7 +5224,7 @@ apply_bif_or_nif_epilogue: OpCase(i_hibernate): { SWAPOUT; - if (hibernate(c_p, r(0), x(1), x(2), reg)) { + if (erts_hibernate(c_p, r(0), x(1), x(2), reg)) { goto do_schedule; } else { I = handle_error(c_p, I, reg, hibernate_3); @@ -4969,12 +5235,11 @@ apply_bif_or_nif_epilogue: OpCase(i_debug_breakpoint): { SWAPOUT; reg[0] = r(0); - tmp_arg1 = call_breakpoint_handler(c_p, I-3, reg); + I = call_error_handler(c_p, I-3, reg, am_breakpoint); r(0) = reg[0]; SWAPIN; - if (tmp_arg1) { - SET_I(c_p->i); - Dispatch(); + if (I) { + Goto(*I); } goto no_error_handler; } @@ -5633,9 +5898,6 @@ build_stacktrace(Process* c_p, Eterm exc) { Eterm args; int depth; BeamInstr* current; -#if HALFWORD_HEAP - BeamInstr current_buff[3]; -#endif Eterm Where = NIL; Eterm *next_p = &Where; @@ -5665,14 +5927,7 @@ build_stacktrace(Process* c_p, Eterm exc) { * (e.g. spawn_link(erlang, abs, [1])). */ if (current == NULL) { -#if HALFWORD_HEAP - current = current_buff; - current[0] = (BeamInstr) c_p->initial[0]; - current[1] = (BeamInstr) c_p->initial[1]; - current[2] = (BeamInstr) c_p->initial[2]; -#else current = c_p->initial; -#endif args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -5724,8 +5979,8 @@ build_stacktrace(Process* c_p, Eterm exc) { } -static Eterm -call_error_handler(Process* p, BeamInstr* fi, Eterm* reg) +static BeamInstr* +call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) { Eterm* hp; Export* ep; @@ -5737,62 +5992,12 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg) /* * Search for the error_handler module. */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_undefined_function, 3); - if (ep == NULL) { /* No error handler */ - p->current = fi; - p->freason = EXC_UNDEF; - return 0; - } - p->i = ep->address; - - /* - * Create a list with all arguments in the x registers. - */ - - arity = fi[2]; - sz = 2 * arity; - if (HeapWordsLeft(p) < sz) { - erts_garbage_collect(p, sz, reg, arity); - } - hp = HEAP_TOP(p); - HEAP_TOP(p) += sz; - args = NIL; - for (i = arity-1; i >= 0; i--) { - args = CONS(hp, reg[i], args); - hp += 2; - } - - /* - * Set up registers for call to error_handler:undefined_function/3. - */ - reg[0] = fi[0]; - reg[1] = fi[1]; - reg[2] = args; - return 1; -} - -static Eterm -call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg) -{ - Eterm* hp; - Export* ep; - int arity; - Eterm args; - Uint sz; - int i; - - /* - * Search for error handler module. - */ - ep = erts_find_function(erts_proc_get_error_handler(p), - am_breakpoint, 3); + ep = erts_find_function(erts_proc_get_error_handler(p), func, 3); if (ep == NULL) { /* No error handler */ p->current = fi; p->freason = EXC_UNDEF; return 0; } - p->i = ep->address; /* * Create a list with all arguments in the x registers. @@ -5812,15 +6017,14 @@ call_breakpoint_handler(Process* p, BeamInstr* fi, Eterm* reg) } /* - * Set up registers for call to error_handler:breakpoint/3. + * Set up registers for call to error_handler:<func>/3. */ reg[0] = fi[0]; reg[1] = fi[1]; reg[2] = args; - return 1; + return ep->address; } - static Export* apply_setup_error_handler(Process* p, Eterm module, Eterm function, Uint arity, Eterm* reg) @@ -5997,8 +6201,8 @@ fixed_apply(Process* p, Eterm* reg, Uint arity) return ep->address; } -static int -hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) +int +erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) { int arity; Eterm tmp; @@ -6069,6 +6273,7 @@ hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg) c_p->fvalue = NIL; PROCESS_MAIN_CHK_LOCKS(c_p); erts_garbage_collect_hibernate(c_p); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); erts_smp_proc_lock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -6195,6 +6400,7 @@ call_fun(Process* p, /* Current process. */ reg[0] = module; reg[1] = fun; reg[2] = args; + reg[3] = NIL; return ep->address; } } @@ -6314,6 +6520,7 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) if (HEAP_LIMIT(p) - HEAP_TOP(p) <= needed) { PROCESS_MAIN_CHK_LOCKS(p); erts_garbage_collect(p, needed, reg, num_free); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(p); PROCESS_MAIN_CHK_LOCKS(p); } hp = p->htop; diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index df5602b040..c697b1ef31 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -89,13 +89,12 @@ typedef struct { } Label; /* - * Type for a operand for a generic instruction. + * Type for an operand for a generic instruction. */ typedef struct { unsigned type; /* Type of operand. */ - BeamInstr val; /* Value of operand. */ - Uint bigarity; /* Arity for bignumbers (only). */ + BeamInstr val; /* Value of operand. */ } GenOpArg; /* @@ -326,11 +325,6 @@ typedef struct { Literal* literals; /* Array of literals. */ LiteralPatch* literal_patches; /* Operands that need to be patched. */ Uint total_literal_size; /* Total heap size for all literals. */ - - /* - * Floating point. - */ - int new_float_instructions; /* New allocation scheme for floating point. */ } LoaderState; typedef struct { @@ -476,16 +470,16 @@ static int read_code_header(LoaderState* stp); static int load_code(LoaderState* stp); static GenOp* gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOpArg Tuple, GenOpArg Dst); -static GenOp* gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, +static GenOp* gen_split_values(LoaderState* stp, GenOpArg S, + GenOpArg TypeFail, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); static GenOp* gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); -static GenOp* gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail, - GenOpArg Size, GenOpArg* Rest); +static GenOp* gen_select_literals(LoaderState* stp, GenOpArg S, + GenOpArg Fail, GenOpArg Size, + GenOpArg* Rest); static GenOp* const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest); -static GenOp* gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg Func, - GenOpArg arity, GenOpArg label); static int freeze_code(LoaderState* stp); @@ -818,7 +812,6 @@ init_state(LoaderState* stp) stp->total_literal_size = 0; stp->literal_patches = 0; stp->string_patches = 0; - stp->new_float_instructions = 0; stp->may_load_nif = 0; stp->on_load = 0; } @@ -1618,7 +1611,6 @@ load_code(LoaderState* stp) BeamInstr val; BeamInstr words = 0; - stp->new_float_instructions = 1; GetTagAndValue(stp, tag, n); VerifyTag(stp, tag, TAG_u); while (n-- > 0) { @@ -1772,7 +1764,7 @@ load_code(LoaderState* stp) } stp->specific_op = specific; - CodeNeed(opc[stp->specific_op].sz+2); /* Extra margin for packing */ + CodeNeed(opc[stp->specific_op].sz+16); /* Extra margin for packing */ code[ci++] = BeamOpCode(stp->specific_op); } @@ -1936,7 +1928,8 @@ load_code(LoaderState* stp) } code[ci++] = (BeamInstr) stp->import[i].bf; break; - case 'P': /* Byte offset into tuple */ + case 'P': /* Byte offset into tuple or stack */ + case 'Q': /* Like 'P', but packable */ VerifyTag(stp, tag, TAG_u); tmp = tmp_op->a[arg].val; code[ci++] = (BeamInstr) ((tmp_op->a[arg].val+1) * sizeof(Eterm)); @@ -1957,84 +1950,6 @@ load_code(LoaderState* stp) } /* - * Load any list arguments using the primitive tags. - */ - - for ( ; arg < tmp_op->arity; arg++) { - switch (tmp_op->a[arg].type) { - case TAG_i: - CodeNeed(1); - code[ci++] = make_small(tmp_op->a[arg].val); - break; - case TAG_u: - case TAG_a: - case TAG_v: - CodeNeed(1); - code[ci++] = tmp_op->a[arg].val; - break; - case TAG_f: - CodeNeed(1); - code[ci] = stp->labels[tmp_op->a[arg].val].patches; - stp->labels[tmp_op->a[arg].val].patches = ci; - ci++; - break; - case TAG_q: - { - Eterm lit; - - lit = stp->literals[tmp_op->a[arg].val].term; - if (is_big(lit)) { - Eterm* bigp; - Eterm *tmp; - Uint size; - Uint term_size; - - bigp = big_val(lit); - term_size = bignum_header_arity(*bigp); - size = TermWords(term_size + 1); - CodeNeed(size); - tmp = (Eterm *) (code + ci); - *tmp++ = *bigp++; - while (term_size-- > 0) { - *tmp++ = *bigp++; - } - ci +=size; - } else if (is_float(lit)) { -#if defined(ARCH_64) && !HALFWORD_HEAP - CodeNeed(1); - code[ci++] = float_val(stp->literals[tmp_op->a[arg].val].term)[1]; -#elif HALFWORD_HEAP - Eterm* fptr; - Uint size; - Eterm *tmp; - - fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1; - size = TermWords(2); - CodeNeed(size); - tmp = (Eterm *) (code + ci); - *tmp++ = *fptr++; - *tmp = *fptr; - ci += size; -#else - Eterm* fptr; - - fptr = float_val(stp->literals[tmp_op->a[arg].val].term)+1; - CodeNeed(2); - code[ci++] = *fptr++; - code[ci++] = *fptr; -#endif - } else { - LoadError0(stp, "literal is neither float nor big"); - } - } - break; - default: - LoadError1(stp, "unsupported primitive type '%c'", - tag_to_letter[tmp_op->a[arg].type]); - } - } - - /* * The packing engine. */ if (opc[stp->specific_op].pack[0]) { @@ -2057,6 +1972,11 @@ load_code(LoaderState* stp) case '6': /* Shift 16 steps */ packed = (packed << BEAM_LOOSE_SHIFT) | code[--ci]; break; +#ifdef ARCH_64 + case 'w': /* Shift 32 steps */ + packed = (packed << BEAM_WIDE_SHIFT) | code[--ci]; + break; +#endif case 'p': /* Put instruction (from stack). */ code[ci++] = *--sp; break; @@ -2072,6 +1992,58 @@ load_code(LoaderState* stp) } /* + * Load any list arguments using the primitive tags. + */ + + for ( ; arg < tmp_op->arity; arg++) { + switch (tmp_op->a[arg].type) { + case TAG_i: + CodeNeed(1); + code[ci++] = make_small(tmp_op->a[arg].val); + break; + case TAG_u: + case TAG_a: + case TAG_v: + CodeNeed(1); + code[ci++] = tmp_op->a[arg].val; + break; + case TAG_f: + CodeNeed(1); + code[ci] = stp->labels[tmp_op->a[arg].val].patches; + stp->labels[tmp_op->a[arg].val].patches = ci; + ci++; + break; + case TAG_r: + CodeNeed(1); + code[ci++] = (R_REG_DEF << _TAG_PRIMARY_SIZE) | + TAG_PRIMARY_HEADER; + break; + case TAG_x: + CodeNeed(1); + code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | + (X_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + break; + case TAG_y: + CodeNeed(1); + code[ci++] = (tmp_op->a[arg].val << _TAG_IMMED1_SIZE) | + (Y_REG_DEF << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; + break; + case TAG_n: + CodeNeed(1); + code[ci++] = NIL; + break; + case TAG_q: + CodeNeed(1); + new_literal_patch(stp, ci); + code[ci++] = tmp_op->a[arg].val; + break; + default: + LoadError1(stp, "unsupported primitive type '%c'", + tag_to_letter[tmp_op->a[arg].type]); + } + } + + /* * Handle a few special cases. */ switch (stp->specific_op) { @@ -2239,11 +2211,12 @@ use_jump_tab(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) } /* - * Predicate to test whether all values in a table are big numbers. + * Predicate to test whether all values in a table are either + * floats or bignums. */ static int -all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) +floats_or_bignums(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) { int i; @@ -2255,9 +2228,6 @@ all_values_are_big(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) if (Rest[i].type != TAG_q) { return 0; } - if (is_not_big(stp->literals[Rest[i].val].term)) { - return 0; - } if (Rest[i+1].type != TAG_f) { return 0; } @@ -2317,6 +2287,14 @@ mixed_types(LoaderState* stp, GenOpArg Size, GenOpArg* Rest) return 0; } +static int +same_label(LoaderState* stp, GenOpArg Target, GenOpArg Label) +{ + return Target.type = TAG_f && Label.type == TAG_u && + Target.val == Label.val; +} + + /* * Generate an instruction for element/2. */ @@ -2328,23 +2306,23 @@ gen_element(LoaderState* stp, GenOpArg Fail, GenOpArg Index, GenOp* op; NEW_GENOP(stp, op); - op->op = genop_i_element_4; op->arity = 4; - op->a[0] = Fail; - op->a[1] = Index; - op->a[2] = Tuple; - op->a[3] = Dst; op->next = NULL; - /* - * If safe, generate a faster instruction. - */ - if (Index.type == TAG_i && Index.val > 0 && (Tuple.type == TAG_r || Tuple.type == TAG_x || Tuple.type == TAG_y)) { op->op = genop_i_fast_element_4; - op->a[1].type = TAG_u; - op->a[1].val = Index.val; + op->a[0] = Tuple; + op->a[1] = Fail; + op->a[2].type = TAG_u; + op->a[2].val = Index.val; + op->a[3] = Dst; + } else { + op->op = genop_i_element_4; + op->a[0] = Tuple; + op->a[1] = Fail; + op->a[2] = Index; + op->a[3] = Dst; } return op; @@ -2595,8 +2573,6 @@ binary_too_big_bits(LoaderState* stp, GenOpArg Size) return Size.type == TAG_u && (((Size.val+7)/8) >> (8*sizeof(Uint)-3) != 0); } -#define new_float_allocation(Stp) ((Stp)->new_float_instructions) - static GenOp* gen_put_binary(LoaderState* stp, GenOpArg Fail,GenOpArg Size, GenOpArg Unit, GenOpArg Flags, GenOpArg Src) @@ -2809,6 +2785,52 @@ gen_skip_bits2(LoaderState* stp, GenOpArg Fail, GenOpArg Ms, return op; } +static GenOp* +gen_increment(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, + GenOpArg Live, GenOpArg Dst) +{ + GenOp* op; + + NEW_GENOP(stp, op); + op->op = genop_i_increment_4; + op->arity = 4; + op->next = NULL; + op->a[0] = Reg; + op->a[1].type = TAG_u; + op->a[1].val = Integer.val; + op->a[2] = Live; + op->a[3] = Dst; + return op; +} + +static GenOp* +gen_increment_from_minus(LoaderState* stp, GenOpArg Reg, GenOpArg Integer, + GenOpArg Live, GenOpArg Dst) +{ + GenOp* op; + + NEW_GENOP(stp, op); + op->op = genop_i_increment_4; + op->arity = 4; + op->next = NULL; + op->a[0] = Reg; + op->a[1].type = TAG_u; + op->a[1].val = -Integer.val; + op->a[2] = Live; + op->a[3] = Dst; + return op; +} + +/* + * Test whether the negation of the given number is small. + */ +static int +negation_is_small(LoaderState* stp, GenOpArg Int) +{ + return Int.type == TAG_i && IS_SSMALL(-Int.val); +} + + static int smp(LoaderState* stp) { @@ -3000,6 +3022,21 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, ASSERT(op->a[i].val < op->a[i+2].val); } #endif + + /* + * Use a special-cased instruction if there are only two values. + */ + if (size == 2) { + op->op = genop_i_select_tuple_arity2_6; + op->arity--; + op->a[2].type = TAG_u; + op->a[2].val = arityval(op->a[3].val); + op->a[3] = op->a[4]; + op->a[4].type = TAG_u; + op->a[4].val = arityval(op->a[5].val); + op->a[5] = op->a[6]; + } + return op; } @@ -3009,18 +3046,24 @@ gen_select_tuple_arity(LoaderState* stp, GenOpArg S, GenOpArg Fail, */ static GenOp* -gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, - GenOpArg Size, GenOpArg* Rest) +gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg TypeFail, + GenOpArg Fail, GenOpArg Size, GenOpArg* Rest) { GenOp* op1; GenOp* op2; GenOp* label; - Uint type; + GenOp* is_integer; int i; ASSERT(Size.val >= 2 && Size.val % 2 == 0); + NEW_GENOP(stp, is_integer); + is_integer->op = genop_is_integer_2; + is_integer->arity = 2; + is_integer->a[0] = TypeFail; + is_integer->a[1] = S; + NEW_GENOP(stp, label); label->op = genop_label_1; label->arity = 1; @@ -3046,15 +3089,13 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, op2->a[2].type = TAG_u; op2->a[2].val = 0; - op1->next = label; - label->next = op2; - op2->next = NULL; - - type = Rest[0].type; + /* + * Split the list. + */ ASSERT(Size.type == TAG_u); for (i = 0; i < Size.val; i += 2) { - GenOp* op = (Rest[i].type == type) ? op1 : op2; + GenOp* op = (Rest[i].type == TAG_q) ? op2 : op1; int dst = 3 + op->a[2].val; ASSERT(Rest[i+1].type == TAG_f); @@ -3063,13 +3104,36 @@ gen_split_values(LoaderState* stp, GenOpArg S, GenOpArg Fail, op->arity += 2; op->a[2].val += 2; } + ASSERT(op1->a[2].val > 0); + ASSERT(op2->a[2].val > 0); /* - * None of the instructions should have zero elements in the list. + * Order the instruction sequence appropriately. */ - ASSERT(op1->a[2].val > 0); - ASSERT(op2->a[2].val > 0); + if (TypeFail.val == Fail.val) { + /* + * select_val L1 S ... (small numbers) + * label L1 + * is_integer Fail S + * select_val Fail S ... (bignums) + */ + op1->next = label; + label->next = is_integer; + is_integer->next = op2; + } else { + /* + * is_integer TypeFail S + * select_val L1 S ... (small numbers) + * label L1 + * select_val Fail S ... (bignums) + */ + is_integer->next = op1; + op1->next = label; + label->next = op2; + op1 = is_integer; + } + op2->next = NULL; return op1; } @@ -3091,6 +3155,29 @@ gen_jump_tab(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpAr ASSERT(Size.val >= 2 && Size.val % 2 == 0); /* + * If there is only one choice, don't generate a jump table. + */ + if (Size.val == 2) { + GenOp* jump; + + NEW_GENOP(stp, op); + op->arity = 3; + op->op = genop_is_ne_exact_3; + op->a[0] = Rest[1]; + op->a[1] = S; + op->a[2] = Rest[0]; + + NEW_GENOP(stp, jump); + jump->next = NULL; + jump->arity = 1; + jump->op = genop_jump_1; + jump->a[0] = Fail; + + op->next = jump; + return op; + } + + /* * Calculate the minimum and maximum values and size of jump table. */ @@ -3162,8 +3249,9 @@ genopargcompare(GenOpArg* a, GenOpArg* b) } /* - * Generate a select_val instruction. We know that a jump table is not suitable, - * and that all values are of the same type (integer, atoms, floats; never bignums). + * Generate a select_val instruction. We know that a jump table + * is not suitable, and that all values are of the same type + * (integer or atoms). */ static GenOp* @@ -3177,12 +3265,7 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, NEW_GENOP(stp, op); op->next = NULL; - if (Rest[0].type != TAG_q) { - op->op = genop_i_select_val_3; - } else { - ASSERT(is_float(stp->literals[Rest[0].val].term)); - op->op = genop_i_select_float_3; - } + op->op = genop_i_select_val_3; GENOP_ARITY(op, arity); op->a[0] = S; op->a[1] = Fail; @@ -3204,19 +3287,19 @@ gen_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, } #endif - return op; -} - -/* - * Compare function for qsort(). - */ + /* + * Use a special-cased instruction if there are only two values. + */ + if (size == 2) { + op->op = genop_i_select_val2_6; + op->arity--; + op->a[2] = op->a[3]; + op->a[3] = op->a[4]; + op->a[4] = op->a[5]; + op->a[5] = op->a[6]; + } -static int -genbigcompare(GenOpArg* a, GenOpArg* b) -{ - int val = (int)(b->bigarity - a->bigarity); - - return val != 0 ? val : ((int) (a->val - b->val)); + return op; } /* @@ -3224,37 +3307,35 @@ genbigcompare(GenOpArg* a, GenOpArg* b) */ static GenOp* -gen_select_big(LoaderState* stp, GenOpArg S, GenOpArg Fail, +gen_select_literals(LoaderState* stp, GenOpArg S, GenOpArg Fail, GenOpArg Size, GenOpArg* Rest) { GenOp* op; - int arity = Size.val + 2 + 1; - int size = Size.val / 2; + GenOp* jump; + GenOp** prev_next = &op; + int i; - NEW_GENOP(stp, op); - op->next = NULL; - op->op = genop_i_select_big_2; - GENOP_ARITY(op, arity); - op->a[0] = S; - op->a[1] = Fail; for (i = 0; i < Size.val; i += 2) { + GenOp* op; ASSERT(Rest[i].type == TAG_q); - op->a[i+2] = Rest[i]; - op->a[i+2].bigarity = *big_val(stp->literals[op->a[i+2].val].term); - op->a[i+3] = Rest[i+1]; - } - ASSERT(i+2 == arity-1); - op->a[arity-1].type = TAG_u; - op->a[arity-1].val = 0; - - /* - * Sort the values in descending arity order. - */ - - qsort(op->a+2, size, 2*sizeof(GenOpArg), - (int (*)(const void *, const void *)) genbigcompare); + NEW_GENOP(stp, op); + op->op = genop_is_ne_exact_3; + op->arity = 3; + op->a[0] = Rest[i+1]; + op->a[1] = S; + op->a[2] = Rest[i]; + *prev_next = op; + prev_next = &op->next; + } + + NEW_GENOP(stp, jump); + jump->next = NULL; + jump->op = genop_jump_1; + jump->arity = 1; + jump->a[0] = Fail; + *prev_next = jump; return op; } @@ -3272,7 +3353,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, int i; ASSERT(Size.type == TAG_u); - ASSERT(S.type == TAG_q); NEW_GENOP(stp, op); op->next = NULL; @@ -3283,18 +3363,32 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, * Search for a literal matching the controlling expression. */ - if (S.type == TAG_q) { - Eterm expr = stp->literals[S.val].term; - for (i = 0; i < Size.val; i += 2) { - if (Rest[i].type == TAG_q) { - Eterm term = stp->literals[Rest[i].val].term; - if (eq(term, expr)) { - ASSERT(Rest[i+1].type == TAG_f); - op->a[0] = Rest[i+1]; - return op; + switch (S.type) { + case TAG_q: + { + Eterm expr = stp->literals[S.val].term; + for (i = 0; i < Size.val; i += 2) { + if (Rest[i].type == TAG_q) { + Eterm term = stp->literals[Rest[i].val].term; + if (eq(term, expr)) { + ASSERT(Rest[i+1].type == TAG_f); + op->a[0] = Rest[i+1]; + return op; + } } } } + break; + case TAG_i: + case TAG_a: + for (i = 0; i < Size.val; i += 2) { + if (Rest[i].val == S.val && Rest[i].type == S.type) { + ASSERT(Rest[i+1].type == TAG_f); + op->a[0] = Rest[i+1]; + return op; + } + } + break; } /* @@ -3305,36 +3399,6 @@ const_select_val(LoaderState* stp, GenOpArg S, GenOpArg Fail, return op; } - -static GenOp* -gen_func_info(LoaderState* stp, GenOpArg mod, GenOpArg func, - GenOpArg arity, GenOpArg label) -{ - GenOp* fi; - GenOp* op; - - NEW_GENOP(stp, fi); - fi->op = genop_i_func_info_4; - fi->arity = 4; - fi->a[0].type = TAG_u; /* untagged Zero */ - fi->a[0].val = 0; - fi->a[1] = mod; - fi->a[2] = func; - fi->a[3] = arity; - - NEW_GENOP(stp, op); - op->op = genop_label_1; - op->arity = 1; - op->a[0] = label; - - fi->next = op; - op->next = NULL; - - return fi; -} - - - static GenOp* gen_make_fun2(LoaderState* stp, GenOpArg idx) { @@ -3477,6 +3541,56 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, return op; } +static GenOp* +tuple_append_put5(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, + GenOpArg* Puts, GenOpArg S1, GenOpArg S2, GenOpArg S3, + GenOpArg S4, GenOpArg S5) +{ + GenOp* op; + int arity = Arity.val; /* Arity of tuple, not the instruction */ + int i; + + NEW_GENOP(stp, op); + op->next = NULL; + GENOP_ARITY(op, arity+2+5); + op->op = genop_i_put_tuple_2; + op->a[0] = Dst; + op->a[1].type = TAG_u; + op->a[1].val = arity + 5; + for (i = 0; i < arity; i++) { + op->a[i+2] = Puts[i]; + } + op->a[arity+2] = S1; + op->a[arity+3] = S2; + op->a[arity+4] = S3; + op->a[arity+5] = S4; + op->a[arity+6] = S5; + return op; +} + +static GenOp* +tuple_append_put(LoaderState* stp, GenOpArg Arity, GenOpArg Dst, + GenOpArg* Puts, GenOpArg S) +{ + GenOp* op; + int arity = Arity.val; /* Arity of tuple, not the instruction */ + int i; + + NEW_GENOP(stp, op); + op->next = NULL; + GENOP_ARITY(op, arity+2+1); + op->op = genop_i_put_tuple_2; + op->a[0] = Dst; + op->a[1].type = TAG_u; + op->a[1].val = arity + 1; + for (i = 0; i < arity; i++) { + op->a[i+2] = Puts[i]; + } + op->a[arity+2] = S; + return op; +} + + /* * Freeze the code in memory, move the string table into place, @@ -3624,25 +3738,32 @@ freeze_code(LoaderState* stp) CHKBLK(ERTS_ALC_T_CODE,code); if (compile_size) { byte* compile_info = str_table + strtab_size + attr_size; - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code); sys_memcpy(compile_info, stp->chunks[COMPILE_CHUNK].start, stp->chunks[COMPILE_CHUNK].size); - CHKBLK(ERTS_ALC_T_CODE,code); + + CHKBLK(ERTS_ALC_T_CODE,code); code[MI_COMPILE_PTR] = (BeamInstr) compile_info; - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code); code[MI_COMPILE_SIZE] = (BeamInstr) stp->chunks[COMPILE_CHUNK].size; - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code); decoded_size = erts_decode_ext_size(compile_info, compile_size, 0); - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code); if (decoded_size < 0) { LoadError0(stp, "bad external term representation of compilation information"); } - CHKBLK(ERTS_ALC_T_CODE,code); + CHKBLK(ERTS_ALC_T_CODE,code); code[MI_COMPILE_SIZE_ON_HEAP] = decoded_size; } CHKBLK(ERTS_ALC_T_CODE,code); /* + * Make sure that we have not overflowed the allocated code space. + */ + ASSERT(str_table + strtab_size + attr_size + compile_size == + ((byte *) code) + size); + + /* * Go through all i_new_bs_put_strings instructions, restore the pointer to * the instruction and convert string offsets to pointers (to the * FIRST character). @@ -3876,11 +3997,23 @@ transform_engine(LoaderState* st) if (i == 0) goto restart; break; +#if defined(TOP_is_eq) case TOP_is_eq: ASSERT(ap < instr->arity); if (*pc++ != instr->a[ap].val) goto restart; break; +#endif + case TOP_is_type_eq: + mask = *pc++; + + ASSERT(ap < instr->arity); + ASSERT(instr->a[ap].type < BEAM_NUM_TAGS); + if (((1 << instr->a[ap].type) & mask) == 0) + goto restart; + if (*pc++ != instr->a[ap].val) + goto restart; + break; case TOP_is_same_var: ASSERT(ap < instr->arity); i = *pc++; @@ -4001,14 +4134,17 @@ transform_engine(LoaderState* st) case TOP_rest_args: { int n = *pc++; + int formal_arity = gen_opc[instr->op].arity; + int num_vars = n + (instr->arity - formal_arity); + int j = formal_arity; + var = erts_alloc(ERTS_ALC_T_LOADER_TMP, - instr->arity * sizeof(GenOpArg)); + num_vars * sizeof(GenOpArg)); for (i = 0; i < n; i++) { var[i] = def_vars[i]; } - while (i < instr->arity) { - var[i] = instr->a[i]; - i++; + while (i < num_vars) { + var[i++] = instr->a[j++]; } } break; @@ -5315,6 +5451,9 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (state.lambdas != state.def_lambdas) { erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas); } + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels); + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom); + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export); if (bin != NULL) { driver_free_binary(bin); } @@ -5326,9 +5465,18 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) if (code != NULL) { erts_free(ERTS_ALC_T_CODE, code); } + if (state.labels != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.labels); + } if (state.lambdas != state.def_lambdas) { erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.lambdas); } + if (state.atom != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.atom); + } + if (state.export != NULL) { + erts_free(ERTS_ALC_T_LOADER_TMP, (void *) state.export); + } if (bin != NULL) { driver_free_binary(bin); } diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 6e9755ad48..f01580eb2b 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -813,7 +813,7 @@ BIF_RETTYPE spawn_opt_1(BIF_ALIST_1) so.min_heap_size = H_MIN_SIZE; so.min_vheap_size = BIN_VH_MIN_SIZE; so.priority = PRIORITY_NORMAL; - so.max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); + so.max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); so.scheduler = 0; /* @@ -1091,10 +1091,20 @@ BIF_RETTYPE unlink_1(BIF_ALIST_1) BIF_RETTYPE hibernate_3(BIF_ALIST_3) { /* - * hibernate/3 is implemented as an instruction; therefore - * this function will never be called. + * hibernate/3 is usually translated to an instruction; therefore + * this function is only called from HiPE or when the call could not + * be translated. */ - BIF_ERROR(BIF_P, BADARG); + Eterm reg[3]; + + if (erts_hibernate(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3, reg)) { + /* + * If hibernate succeeded, TRAP. The process will be suspended + * if status is P_WAITING or continue (if any message was in the queue). + */ + BIF_TRAP_CODE_PTR_(BIF_P, BIF_P->i); + } + return THE_NON_VALUE; } /**********************************************************************/ @@ -1351,9 +1361,10 @@ BIF_RETTYPE exit_2(BIF_ALIST_2) #ifdef ERTS_SMP if (rp == BIF_P) rp_locks &= ~ERTS_PROC_LOCK_MAIN; - else + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + if (rp != BIF_P) erts_smp_proc_dec_refc(rp); - erts_smp_proc_unlock(rp, rp_locks); #endif /* * We may have exited ourselves and may have to take action. @@ -3269,12 +3280,13 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) erts_smp_mtx_lock(&ports_snapshot_mtx); /* One snapshot at a time */ - erts_smp_atomic_set(&erts_dead_ports_ptr, (long) (port_buf + erts_max_ports)); + erts_smp_atomic_set(&erts_dead_ports_ptr, + (erts_aint_t) (port_buf + erts_max_ports)); next_ss = erts_smp_atomic_inctest(&erts_ports_snapshot); if (erts_smp_atomic_read(&erts_ports_alive) > 0) { - long i; + erts_aint_t i; for (i = erts_max_ports-1; i >= 0; i--) { Port* prt = &erts_port[i]; erts_smp_port_state_lock(prt); @@ -3289,7 +3301,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) } dead_ports = (Eterm*)erts_smp_atomic_xchg(&erts_dead_ports_ptr, - (long)NULL); + (erts_aint_t) NULL); erts_smp_mtx_unlock(&ports_snapshot_mtx); ASSERT(pp <= dead_ports); @@ -3300,7 +3312,7 @@ BIF_RETTYPE ports_0(BIF_ALIST_0) ASSERT((alive+dead) <= erts_max_ports); if (alive+dead > 0) { - long i; + erts_aint_t i; Eterm *hp = HAlloc(BIF_P, (alive+dead)*2); for (i = 0; i < alive; i++) { @@ -3796,7 +3808,8 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) goto error; } nval = (n > (Sint) ((Uint16) -1)) ? ((Uint16) -1) : ((Uint16) n); - oval = (Uint) erts_smp_atomic_xchg(&erts_max_gen_gcs, (long) nval); + oval = (Uint) erts_smp_atomic32_xchg(&erts_max_gen_gcs, + (erts_aint32_t) nval); BIF_RET(make_small(oval)); } else if (BIF_ARG_1 == am_min_heap_size) { int oval = H_MIN_SIZE; @@ -4139,7 +4152,7 @@ void erts_init_bif(void) erts_smp_spinlock_init(&make_ref_lock, "make_ref"); erts_smp_mtx_init(&ports_snapshot_mtx, "ports_snapshot"); - erts_smp_atomic_init(&erts_dead_ports_ptr, (long)NULL); + erts_smp_atomic_init(&erts_dead_ports_ptr, (erts_aint_t) NULL); /* * bif_return_trap/1 is a hidden BIF that bifs that need to diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index a84ee7bb23..8faa09feb8 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -201,6 +201,12 @@ do { \ return THE_NON_VALUE; \ } while(0) +#define BIF_TRAP_CODE_PTR_(p, Code_) do { \ + *((UWord *) (UWord) ((p)->def_arg_reg + 3)) = (UWord) (Code_); \ + (p)->freason = TRAP; \ + return THE_NON_VALUE; \ + } while(0) + extern Export bif_return_trap_export; #ifdef DEBUG #define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL) \ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 0674aae77f..d9dd80fa8b 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -660,6 +660,7 @@ bif erts_debug:display/1 bif 'erl.system.debug':display/1 ebif_erts_debug_display_1 bif erts_debug:dist_ext_to_term/2 bif 'erl.system.debug':dist_ext_to_term/2 ebif_erts_debug_dist_ext_to_term_2 +bif erts_debug:instructions/0 # # Monitor testing bif's... @@ -795,6 +796,13 @@ bif erlang:nif_error/1 bif erlang:nif_error/2 # +# Helpers for unicode filenames +# +bif prim_file:internal_name2native/1 +bif prim_file:internal_native2name/1 +bif prim_file:internal_normalize_utf8/1 +bif file:native_name_encoding/0 +# # Obsolete # diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index ff15d834ab..f47f5a9c0c 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1558,7 +1558,7 @@ Eterm erts_sint64_to_big(Sint64 x, Eterm **hpp) ** Convert a bignum to a double float */ int -big_to_double(Eterm x, double* resp) +big_to_double(Wterm x, double* resp) { double d = 0.0; Eterm* xp = big_val(x); @@ -1725,7 +1725,7 @@ static Eterm big_norm(Eterm *x, dsize_t xl, short sign) /* ** Compare bignums */ -int big_comp(Eterm x, Eterm y) +int big_comp(Wterm x, Wterm y) { Eterm* xp = big_val(x); Eterm* yp = big_val(y); @@ -2060,7 +2060,7 @@ static Eterm B_plus_minus(ErtsDigit *x, dsize_t xl, short xsgn, /* ** Add bignums */ -Eterm big_plus(Eterm x, Eterm y, Eterm *r) +Eterm big_plus(Wterm x, Wterm y, Eterm *r) { Eterm* xp = big_val(x); Eterm* yp = big_val(y); diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 25466cd3c2..f28a390aea 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -120,7 +120,7 @@ char *erts_big_to_string(Eterm x, char *buf, Uint buf_sz); Eterm small_times(Sint, Sint, Eterm*); -Eterm big_plus(Eterm, Eterm, Eterm*); +Eterm big_plus(Wterm, Wterm, Eterm*); Eterm big_minus(Eterm, Eterm, Eterm*); Eterm big_times(Eterm, Eterm, Eterm*); Eterm big_div(Eterm, Eterm, Eterm*); @@ -137,9 +137,9 @@ Eterm big_bxor(Eterm, Eterm, Eterm*); Eterm big_bnot(Eterm, Eterm*); Eterm big_lshift(Eterm, Sint, Eterm*); -int big_comp (Eterm, Eterm); +int big_comp (Wterm, Wterm); int big_ucomp (Eterm, Eterm); -int big_to_double(Eterm x, double* resp); +int big_to_double(Wterm x, double* resp); Eterm small_to_big(Sint, Eterm*); Eterm uint_to_big(Uint, Eterm*); Eterm uword_to_big(UWord, Eterm*); diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index 8ee8fbcb29..9486602633 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -56,7 +56,7 @@ erts_init_binary(void) */ Eterm -new_binary(Process *p, byte *buf, int len) +new_binary(Process *p, byte *buf, Uint len) { ProcBin* pb; Binary* bptr; @@ -217,8 +217,8 @@ erts_get_aligned_binary_bytes_extra(Eterm bin, byte** base_ptr, ErtsAlcType_t al return bytes; } -static Eterm -bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs) +Eterm +erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs) { if (bitoffs == 0) { while (size) { @@ -263,7 +263,7 @@ BIF_RETTYPE binary_to_list_1(BIF_ALIST_1) Eterm* hp = HAlloc(BIF_P, 2 * size); byte* bytes = binary_bytes(real_bin)+offset; - BIF_RET(bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); + BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); } error: @@ -295,7 +295,7 @@ BIF_RETTYPE binary_to_list_3(BIF_ALIST_3) } i = stop-start+1; hp = HAlloc(BIF_P, 2*i); - BIF_RET(bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs)); + BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes+start-1, i, bitoffs)); error: BIF_ERROR(BIF_P, BADARG); @@ -339,7 +339,7 @@ BIF_RETTYPE bitstring_to_list_1(BIF_ALIST_1) previous = CONS(hp, make_binary(last), previous); hp += 2; } - BIF_RET(bin_bytes_to_list(previous, hp, bytes, size, bitoffs)); + BIF_RET(erts_bin_bytes_to_list(previous, hp, bytes, size, bitoffs)); } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index f339e19761..d255cf3558 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -98,7 +98,7 @@ process_killer(void) switch(j) { case 'k': if (rp->status == P_WAITING) { - Uint32 rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; + ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; erts_smp_proc_inc_refc(rp); erts_smp_proc_lock(rp, rp_locks); (void) erts_send_exit_signal(NULL, @@ -558,7 +558,7 @@ do_break(void) #endif #ifdef DEBUG case 't': - p_slpq(); + erts_p_slpq(); return; case 'b': bin_check(); @@ -624,9 +624,9 @@ bin_check(void) erts_printf("Process %T holding binary data \n", rp->id); printed = 1; } - erts_printf("0x%08lx orig_size: %ld, norefs = %ld\n", - (unsigned long)bp->val, - (long)bp->val->orig_size, + erts_printf("%p orig_size: %bpd, norefs = %bpd\n", + bp->val, + bp->val->orig_size, erts_smp_atomic_read(&bp->val->refc)); } } diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 8bee47232e..243e8973cf 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -72,8 +72,11 @@ copy_object(Eterm obj, Process* to) * Return the "flat" size of the object. */ -Uint -size_object(Eterm obj) +#if HALFWORD_HEAP +Uint size_object_rel(Eterm obj, Eterm* base) +#else +Uint size_object(Eterm obj) +#endif { Uint sum = 0; Eterm* ptr; @@ -84,7 +87,7 @@ size_object(Eterm obj) switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: sum += 2; - ptr = list_val(obj); + ptr = list_val_rel(obj,base); obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); @@ -93,11 +96,11 @@ size_object(Eterm obj) break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(obj); + Eterm hdr = *boxed_val_rel(obj,base); ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: - ptr = tuple_val(obj); + ptr = tuple_val_rel(obj,base); arity = header_arity(hdr); sum += arity + 1; if (arity == 0) { /* Empty tuple -- unusual. */ @@ -113,7 +116,7 @@ size_object(Eterm obj) break; case FUN_SUBTAG: { - Eterm* bptr = fun_val(obj); + Eterm* bptr = fun_val_rel(obj,base); ErlFunThing* funp = (ErlFunThing *) bptr; unsigned eterms = 1 /* creator */ + funp->num_free; unsigned sz = thing_arityval(hdr); @@ -136,7 +139,7 @@ size_object(Eterm obj) Uint bitoffs; Uint extra_bytes; Eterm hdr; - ERTS_GET_REAL_BIN(obj, real_bin, offset, bitoffs, bitsize); + ERTS_GET_REAL_BIN_REL(obj, real_bin, offset, bitoffs, bitsize, base); if ((bitsize + bitoffs) > 8) { sum += ERL_SUB_BIN_SIZE; extra_bytes = 2; @@ -146,11 +149,11 @@ size_object(Eterm obj) } else { extra_bytes = 0; } - hdr = *binary_val(real_bin); + hdr = *binary_val_rel(real_bin,base); if (thing_subtag(hdr) == REFC_BINARY_SUBTAG) { sum += PROC_BIN_SIZE; } else { - sum += heap_bin_size(binary_size(obj)+extra_bytes); + sum += heap_bin_size(binary_size_rel(obj,base)+extra_bytes); } goto pop_next; } @@ -181,8 +184,12 @@ size_object(Eterm obj) /* * Copy a structure to a heap. */ -Eterm -copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +#if HALFWORD_HEAP +Eterm copy_struct_rel(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, + Eterm* src_base, Eterm* dst_base) +#else +Eterm copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +#endif { char* hstart; Uint hsize; @@ -214,7 +221,10 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) /* Copy the object onto the heap */ switch (primary_tag(obj)) { - case TAG_PRIMARY_LIST: argp = &res; goto L_copy_list; + case TAG_PRIMARY_LIST: + argp = &res; + objp = list_val_rel(obj,src_base); + goto L_copy_list; case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed; default: erl_exit(ERTS_ABORT_EXIT, @@ -231,32 +241,46 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) hp++; break; case TAG_PRIMARY_LIST: - objp = list_val(obj); + objp = list_val_rel(obj,src_base); + #if !HALFWORD_HEAP || defined(DEBUG) if (in_area(objp,hstart,hsize)) { + ASSERT(!HALFWORD_HEAP); hp++; break; } + #endif argp = hp++; /* Fall through */ L_copy_list: tailp = argp; - while (is_list(obj)) { - objp = list_val(obj); + for (;;) { tp = tailp; - elem = *objp; + elem = CAR(objp); if (IS_CONST(elem)) { - *(hbot-2) = elem; - tailp = hbot-1; hbot -= 2; + CAR(hbot) = elem; + tailp = &CDR(hbot); } else { - *htop = elem; - tailp = htop+1; + CAR(htop) = elem; + #if HALFWORD_HEAP + CDR(htop) = CDR(objp); + *tailp = make_list_rel(htop,dst_base); + htop += 2; + goto L_copy; + #else + tailp = &CDR(htop); htop += 2; + #endif + } + ASSERT(!HALFWORD_HEAP || tp < hp || tp >= hbot); + *tp = make_list_rel(tailp - 1, dst_base); + obj = CDR(objp); + if (!is_list(obj)) { + break; } - *tp = make_list(tailp - 1); - obj = *(objp+1); + objp = list_val_rel(obj,src_base); } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; @@ -268,21 +292,24 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } case TAG_PRIMARY_BOXED: - if (in_area(boxed_val(obj),hstart,hsize)) { + #if !HALFWORD_HEAP || defined(DEBUG) + if (in_area(boxed_val_rel(obj,src_base),hstart,hsize)) { + ASSERT(!HALFWORD_HEAP); hp++; break; } + #endif argp = hp++; L_copy_boxed: - objp = boxed_val(obj); + objp = boxed_val_rel(obj, src_base); hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { int const_flag = 1; /* assume constant tuple */ i = arityval(hdr); - *argp = make_tuple(htop); + *argp = make_tuple_rel(htop, dst_base); tp = htop; /* tp is pointer to new arity value */ *htop++ = *objp++; /* copy arity value */ while (i--) { @@ -311,7 +338,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) while (i--) { *tp++ = *objp++; } - *argp = make_binary(hbot); + *argp = make_binary_rel(hbot, dst_base); pb = (ProcBin*) hbot; erts_refc_inc(&pb->val->refc, 2); pb->next = off_heap->first; @@ -338,7 +365,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) extra_bytes = 0; } real_size = size+extra_bytes; - objp = binary_val(real_bin); + objp = binary_val_rel(real_bin,src_base); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { ErlHeapBin* from = (ErlHeapBin *) objp; ErlHeapBin* to; @@ -368,7 +395,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*) to; OH_OVERHEAD(off_heap, to->size / sizeof(Eterm)); } - *argp = make_binary(hbot); + *argp = make_binary_rel(hbot, dst_base); if (extra_bytes != 0) { ErlSubBin* res; hbot -= ERL_SUB_BIN_SIZE; @@ -380,7 +407,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) res->offs = 0; res->is_writable = 0; res->orig = *argp; - *argp = make_binary(hbot); + *argp = make_binary_rel(hbot, dst_base); } break; } @@ -400,7 +427,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*) funp; erts_refc_inc(&funp->fe->refc, 2); #endif - *argp = make_fun(tp); + *argp = make_fun_rel(tp, dst_base); } break; case EXTERNAL_PID_SUBTAG: @@ -420,7 +447,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) off_heap->first = (struct erl_off_heap_header*)etp; erts_refc_inc(&etp->node->refc, 2); - *argp = make_external(tp); + *argp = make_external_rel(tp, dst_base); } break; case BIN_MATCHSTATE_SUBTAG: @@ -430,7 +457,7 @@ copy_struct(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) i = thing_arityval(hdr)+1; hbot -= i; tp = hbot; - *argp = make_boxed(hbot); + *argp = make_boxed_rel(hbot, dst_base); while (i--) { *tp++ = *objp++; } @@ -885,12 +912,21 @@ Eterm copy_struct_lazy(Process *from, Eterm orig, Uint offs) * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -Eterm -copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +#if HALFWORD_HEAP +Eterm copy_shallow_rel(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, + Eterm* src_base) +#else +Eterm copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) +#endif { Eterm* tp = ptr; Eterm* hp = *hpp; - Sint offs = hp - tp; + const Eterm res = make_tuple(hp); +#if HALFWORD_HEAP + const Sint offs = COMPRESS_POINTER(hp - (tp - src_base)); +#else + const Sint offs = (hp - tp) * sizeof(Eterm); +#endif while (sz--) { Eterm val = *tp++; @@ -901,7 +937,7 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) break; case TAG_PRIMARY_LIST: case TAG_PRIMARY_BOXED: - *hp++ = offset_ptr(val, offs); + *hp++ = byte_offset_ptr(val, offs); break; case TAG_PRIMARY_HEADER: *hp++ = val; @@ -958,7 +994,8 @@ copy_shallow(Eterm* ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap) } } *hpp = hp; - return make_tuple(ptr + offs); + + return res; } /* Move all terms in heap fragments into heap. The terms must be guaranteed to diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 4497e17d79..044fd045a6 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -162,7 +162,7 @@ Uint erts_dist_cache_size(void) static ErtsProcList * get_suspended_on_de(DistEntry *dep, Uint32 unset_qflgs) { - ERTS_SMP_LC_ASSERT(erts_smp_lc_spinlock_is_locked(&dep->qlock)); + ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&dep->qlock)); dep->qflgs &= ~unset_qflgs; if (dep->qflgs & ERTS_DE_QFLG_EXIT) { /* No resume when exit has been scheduled */ @@ -455,17 +455,17 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) if (dep->status & ERTS_DE_SFLG_EXITING) { #ifdef DEBUG - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qflgs & ERTS_DE_QFLG_EXIT); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); #endif } else { dep->status |= ERTS_DE_SFLG_EXITING; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); dep->qflgs |= ERTS_DE_QFLG_EXIT; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } erts_smp_de_links_lock(dep); @@ -579,7 +579,7 @@ static void clear_dist_entry(DistEntry *dep) erts_smp_de_links_unlock(dep); #endif - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); if (!dep->out_queue.last) obuf = dep->finalized_out_queue.first; @@ -595,7 +595,7 @@ static void clear_dist_entry(DistEntry *dep) dep->status = 0; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLGS_ALL); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); erts_smp_atomic_set(&dep->dist_cmd_scheduled, 0); dep->send = NULL; erts_smp_de_rwunlock(dep); @@ -613,10 +613,10 @@ static void clear_dist_entry(DistEntry *dep) } if (obufsize) { - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } } @@ -864,11 +864,15 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) #include <valgrind/valgrind.h> #include <valgrind/memcheck.h> +#ifndef HAVE_VALGRIND_PRINTF_XML +#define VALGRIND_PRINTF_XML VALGRIND_PRINTF +#endif + # define PURIFY_MSG(msg) \ do { \ char buf__[1]; size_t bufsz__ = sizeof(buf__); \ if (erts_sys_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \ - VALGRIND_PRINTF("<erlang_error_log>" \ + VALGRIND_PRINTF_XML("<erlang_error_log>" \ "%s, line %d: %s</erlang_error_log>\n", \ __FILE__, __LINE__, msg); \ } else { \ @@ -917,6 +921,7 @@ int erts_net_message(Port *prt, Eterm token_size; ErtsMonitor *mon; ErtsLink *lnk; + Uint tuple_arity; int res; #ifdef ERTS_DIST_MSG_DBG int orig_len = len; @@ -1003,29 +1008,23 @@ int erts_net_message(Port *prt, #endif if (is_not_tuple(arg) || - (tuple = tuple_val(arg), arityval(*tuple) < 1) || + (tuple = tuple_val(arg), (tuple_arity = arityval(*tuple)) < 1) || is_not_small(tuple[1])) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg); - erts_send_error_to_logger_nogl(dsbufp); - goto data_error; + goto invalid_message; } token_size = 0; switch (type = unsigned_val(tuple[1])) { case DOP_LINK: + if (tuple_arity != 3) { + goto invalid_message; + } from = tuple[2]; to = tuple[3]; /* local proc to link to */ if (is_not_pid(from) || is_not_pid(to)) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - PURIFY_MSG("data error"); - erts_dsprintf(dsbufp, - "Invalid DOP_LINK distribution message: %.200T", - arg); - erts_send_error_to_logger_nogl(dsbufp); - goto data_error; + goto invalid_message; } rp = erts_pid2proc_opt(NULL, 0, @@ -1064,8 +1063,14 @@ int erts_net_message(Port *prt, case DOP_UNLINK: { ErtsDistLinkData dld; + if (tuple_arity != 3) { + goto invalid_message; + } from = tuple[2]; to = tuple[3]; + if (is_not_pid(from) || is_not_pid(to)) { + goto invalid_message; + } rp = erts_pid2proc_opt(NULL, 0, to, ERTS_PROC_LOCK_LINK, @@ -1092,11 +1097,19 @@ int erts_net_message(Port *prt, /* A remote process wants to monitor us, we get: {DOP_MONITOR_P, Remote pid, local pid or name, ref} */ Eterm name; + + if (tuple_arity != 4) { + goto invalid_message; + } watcher = tuple[2]; watched = tuple[3]; /* local proc to monitor */ ref = tuple[4]; + if (is_not_ref(ref)) { + goto invalid_message; + } + if (is_atom(watched)) { name = watched; rp = erts_whereis_process(NULL, 0, @@ -1138,10 +1151,17 @@ int erts_net_message(Port *prt, We get {DOP_DEMONITOR_P, Remote pid, Local pid or name, ref}, We need only the ref of course */ + if (tuple_arity != 4) { + goto invalid_message; + } /* watcher = tuple[2]; */ /* watched = tuple[3]; May be an atom in case of monitor name */ ref = tuple[4]; + if(is_not_ref(ref)) { + goto invalid_message; + } + erts_smp_de_links_lock(dep); mon = erts_remove_monitor(&(dep->monitors),ref); erts_smp_de_links_unlock(dep); @@ -1166,10 +1186,11 @@ int erts_net_message(Port *prt, erts_destroy_monitor(mon); break; - case DOP_NODE_LINK: /* XXX never sent ?? */ - break; - case DOP_REG_SEND_TT: + if (tuple_arity != 5) { + goto invalid_message; + } + token_size = size_object(tuple[5]); /* Fall through ... */ case DOP_REG_SEND: @@ -1180,12 +1201,19 @@ int erts_net_message(Port *prt, * There is intentionally no testing of the cookie (it is always '') * from R9B and onwards. */ + if (type != DOP_REG_SEND_TT && tuple_arity != 4) { + goto invalid_message; + } + #ifdef ERTS_DIST_MSG_DBG dist_msg_dbg(&ede, "MSG", buf, orig_len); #endif from = tuple[2]; to = tuple[4]; + if (is_not_pid(from) || is_not_atom(to)){ + goto invalid_message; + } rp = erts_whereis_process(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); if (rp) { Uint xsize = (type == DOP_REG_SEND @@ -1217,6 +1245,10 @@ int erts_net_message(Port *prt, break; case DOP_SEND_TT: + if (tuple_arity != 4) { + goto invalid_message; + } + token_size = size_object(tuple[4]); /* Fall through ... */ case DOP_SEND: @@ -1227,8 +1259,13 @@ int erts_net_message(Port *prt, #ifdef ERTS_DIST_MSG_DBG dist_msg_dbg(&ede, "MSG", buf, orig_len); #endif - + if (type != DOP_SEND_TT && tuple_arity != 3) { + goto invalid_message; + } to = tuple[3]; + if (is_not_pid(to)) { + goto invalid_message; + } rp = erts_pid2proc_opt(NULL, 0, to, 0, ERTS_P2P_FLG_SMP_INC_REFC); if (rp) { Uint xsize = type == DOP_SEND ? 0 : ERTS_HEAP_FRAG_SIZE(token_size); @@ -1266,11 +1303,19 @@ int erts_net_message(Port *prt, Eterm sysname; ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_MSG_SEND|ERTS_PROC_LOCK_LINK; + if (tuple_arity != 5) { + goto invalid_message; + } + /* watched = tuple[2]; */ /* remote proc which died */ /* watcher = tuple[3]; */ ref = tuple[4]; reason = tuple[5]; + if(is_not_ref(ref)) { + goto invalid_message; + } + erts_smp_de_links_lock(dep); sysname = dep->sysname; mon = erts_remove_monitor(&(dep->monitors), ref); @@ -1317,24 +1362,25 @@ int erts_net_message(Port *prt, ErtsProcLocks rp_locks = ERTS_PROC_LOCK_LINK|ERTS_PROC_LOCKS_XSIG_SEND; /* 'from', which 'to' is linked to, died */ if (type == DOP_EXIT) { - from = tuple[2]; - to = tuple[3]; - reason = tuple[4]; - token = NIL; + if (tuple_arity != 4) { + goto invalid_message; + } + + from = tuple[2]; + to = tuple[3]; + reason = tuple[4]; + token = NIL; } else { - from = tuple[2]; - to = tuple[3]; - token = tuple[4]; - reason = tuple[5]; + if (tuple_arity != 5) { + goto invalid_message; + } + from = tuple[2]; + to = tuple[3]; + token = tuple[4]; + reason = tuple[5]; } - if (is_not_internal_pid(to)) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - PURIFY_MSG("data error"); - erts_dsprintf(dsbufp, - "Invalid DOP_EXIT distribution message: %.200T", - arg); - erts_send_error_to_logger_nogl(dsbufp); - goto data_error; + if (is_not_pid(from) || is_not_internal_pid(to)) { + goto invalid_message; } rp = erts_pid2proc(NULL, 0, to, rp_locks); @@ -1381,15 +1427,24 @@ int erts_net_message(Port *prt, ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; /* 'from' is send an exit signal to 'to' */ if (type == DOP_EXIT2) { - from = tuple[2]; - to = tuple[3]; - reason = tuple[4]; - token = NIL; + if (tuple_arity != 4) { + goto invalid_message; + } + from = tuple[2]; + to = tuple[3]; + reason = tuple[4]; + token = NIL; } else { - from = tuple[2]; - to = tuple[3]; - token = tuple[4]; - reason = tuple[5]; + if (tuple_arity != 5) { + goto invalid_message; + } + from = tuple[2]; + to = tuple[3]; + token = tuple[4]; + reason = tuple[5]; + } + if (is_not_pid(from) || is_not_internal_pid(to)) { + goto invalid_message; } rp = erts_pid2proc_opt(NULL, 0, to, rp_locks, ERTS_P2P_FLG_SMP_INC_REFC); @@ -1408,10 +1463,14 @@ int erts_net_message(Port *prt, break; } case DOP_GROUP_LEADER: + if (tuple_arity != 3) { + goto invalid_message; + } from = tuple[2]; /* Group leader */ to = tuple[3]; /* new member */ - if (is_not_pid(from)) - break; + if (is_not_pid(from) || is_not_pid(to)) { + goto invalid_message; + } rp = erts_pid2proc(NULL, 0, to, ERTS_PROC_LOCK_MAIN); if (!rp) @@ -1420,16 +1479,8 @@ int erts_net_message(Port *prt, erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); break; - default: { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Illegal value in distribution dispatch switch: " - "%.200T", - arg); - erts_send_error_to_logger_nogl(dsbufp); - PURIFY_MSG("data error"); - goto data_error; - } + default: + goto invalid_message; } erts_cleanup_offheap(&off_heap); @@ -1441,8 +1492,14 @@ int erts_net_message(Port *prt, UnUseTmpHeapNoproc(DIST_CTL_DEFAULT_SIZE); ERTS_SMP_CHK_NO_PROC_LOCKS; return 0; - + invalid_message: + { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Invalid distribution message: %.200T", arg); + erts_send_error_to_logger_nogl(dsbufp); + } data_error: + PURIFY_MSG("data error"); erts_cleanup_offheap(&off_heap); #ifndef HYBRID /* FIND ME! */ if (ctl != ctl_default) { @@ -1538,18 +1595,18 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) } else { ErtsProcList *plp = NULL; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); dep->qsize += size_obuf(obuf); if (dep->qsize >= erts_dist_buf_busy_limit) dep->qflgs |= ERTS_DE_QFLG_BUSY; if (!force_busy && (dep->qflgs & ERTS_DE_QFLG_BUSY)) { - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); plp = erts_proclist_create(c_p); plp->next = NULL; erts_suspend(c_p, ERTS_PROC_LOCK_MAIN, NULL); suspended = 1; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); } /* Enqueue obuf on dist entry */ @@ -1575,7 +1632,7 @@ dsig_send(ErtsDSigData *dsdp, Eterm ctl, Eterm msg, int force_busy) } } - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); erts_schedule_dist_command(NULL, dep); erts_smp_de_runlock(dep); @@ -1708,10 +1765,8 @@ erts_dist_command(Port *prt, int reds_limit) { Sint reds = ERTS_PORT_REDS_DIST_CMD_START; int prt_busy; - int de_busy; Uint32 status; Uint32 flags; - Uint32 qflgs; Sint obufsize = 0; ErtsDistOutputQueue oq, foq; DistEntry *dep = prt->dist_entry; @@ -1746,13 +1801,12 @@ erts_dist_command(Port *prt, int reds_limit) * a mess. */ - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); oq.first = dep->out_queue.first; oq.last = dep->out_queue.last; dep->out_queue.first = NULL; dep->out_queue.last = NULL; - qflgs = dep->qflgs; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); foq.first = dep->finalized_out_queue.first; foq.last = dep->finalized_out_queue.last; @@ -1763,17 +1817,8 @@ erts_dist_command(Port *prt, int reds_limit) goto preempted; prt_busy = (int) (prt->status & ERTS_PORT_SFLG_PORT_BUSY); - de_busy = (int) (qflgs & ERTS_DE_QFLG_BUSY); - if (prt_busy) { - if (!de_busy) { - erts_smp_spin_lock(&dep->qlock); - dep->qflgs |= ERTS_DE_QFLG_BUSY; - erts_smp_spin_unlock(&dep->qlock); - de_busy = 1; - } - } - else if (foq.first) { + if (!prt_busy && foq.first) { int preempt = 0; do { Uint size; @@ -1791,10 +1836,7 @@ erts_dist_command(Port *prt, int reds_limit) free_dist_obuf(fob); preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - erts_smp_spin_lock(&dep->qlock); - dep->qflgs |= ERTS_DE_QFLG_BUSY; - erts_smp_spin_unlock(&dep->qlock); - de_busy = prt_busy = 1; + prt_busy = 1; break; } } while (foq.first && !preempt); @@ -1877,10 +1919,7 @@ erts_dist_command(Port *prt, int reds_limit) free_dist_obuf(fob); preempt = reds > reds_limit || (prt->status & ERTS_PORT_SFLGS_DEAD); if (prt->status & ERTS_PORT_SFLG_PORT_BUSY) { - erts_smp_spin_lock(&dep->qlock); - dep->qflgs |= ERTS_DE_QFLG_BUSY; - erts_smp_spin_unlock(&dep->qlock); - de_busy = prt_busy = 1; + prt_busy = 1; if (oq.first && !preempt) goto finalize_only; } @@ -1907,22 +1946,23 @@ erts_dist_command(Port *prt, int reds_limit) * dist entry in a non-busy state and resume suspended * processes. */ - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; obufsize = 0; - if (de_busy && !prt_busy && dep->qsize < erts_dist_buf_busy_limit) { + if (!prt_busy + && (dep->qflgs & ERTS_DE_QFLG_BUSY) + && dep->qsize < erts_dist_buf_busy_limit) { ErtsProcList *suspendees; int resumed; suspendees = get_suspended_on_de(dep, ERTS_DE_QFLG_BUSY); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); resumed = erts_resume_processes(suspendees); reds += resumed*ERTS_PORT_REDS_DIST_CMD_RESUMED; - de_busy = 0; } else - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } ASSERT(!oq.first && !oq.last); @@ -1931,10 +1971,10 @@ erts_dist_command(Port *prt, int reds_limit) if (obufsize != 0) { ASSERT(obufsize > 0); - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize >= obufsize); dep->qsize -= obufsize; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } ASSERT(foq.first || !foq.last); @@ -1984,9 +2024,9 @@ erts_dist_command(Port *prt, int reds_limit) foq.last = NULL; #ifdef DEBUG - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize == obufsize); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); #endif } else { @@ -1995,14 +2035,14 @@ erts_dist_command(Port *prt, int reds_limit) * Unhandle buffers need to be put back first * in out_queue. */ - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); dep->qsize -= obufsize; obufsize = 0; oq.last->next = dep->out_queue.first; dep->out_queue.first = oq.first; if (!dep->out_queue.last) dep->out_queue.last = oq.last; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); } erts_schedule_dist_command(prt, NULL); @@ -2026,10 +2066,10 @@ erts_kill_dist_connection(DistEntry *dep, Uint32 connection_id) dep->status |= ERTS_DE_SFLG_EXITING; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(!(dep->qflgs & ERTS_DE_QFLG_EXIT)); dep->qflgs |= ERTS_DE_QFLG_EXIT; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); erts_schedule_dist_command(NULL, dep); } @@ -2400,13 +2440,13 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) ErtsProcList *plp = erts_proclist_create(BIF_P); plp->next = NULL; erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); if (dep->suspended.last) dep->suspended.last->next = plp; else dep->suspended.first = plp; dep->suspended.last = plp; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); goto yield; } @@ -2434,9 +2474,9 @@ BIF_RETTYPE setnode_3(BIF_ALIST_3) ASSERT(dep->send); #ifdef DEBUG - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); ASSERT(dep->qsize == 0); - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); #endif erts_set_dist_entry_connected(dep, BIF_ARG_2, flags); diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 28cdd05c3c..695a4fc3fe 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -38,6 +38,7 @@ #define DFLAG_UNICODE_IO 0x1000 #define DFLAG_DIST_HDR_ATOM_CACHE 0x2000 #define DFLAG_SMALL_ATOM_TAGS 0x4000 +#define DFLAGS_INTERNAL_TAGS 0x8000 /* All flags that should be enabled when term_to_binary/1 is used. */ #define TERM_TO_BINARY_DFLAGS (DFLAG_EXTENDED_REFERENCES \ @@ -51,7 +52,7 @@ #define DOP_SEND 2 #define DOP_EXIT 3 #define DOP_UNLINK 4 -#define DOP_NODE_LINK 5 +/* Ancient DOP_NODE_LINK (5) was here, can be reused */ #define DOP_REG_SEND 6 #define DOP_GROUP_LEADER 7 #define DOP_EXIT2 8 @@ -68,7 +69,6 @@ /* distribution trap functions */ extern Export* dsend2_trap; extern Export* dsend3_trap; -/*extern Export* dsend_nosuspend_trap;*/ extern Export* dlink_trap; extern Export* dunlink_trap; extern Export* dmonitor_node_trap; @@ -99,7 +99,7 @@ typedef struct { #define ERTS_DE_IS_CONNECTED(DEP) \ (!ERTS_DE_IS_NOT_CONNECTED((DEP))) -#define ERTS_DE_BUSY_LIMIT (128*1024) +#define ERTS_DE_BUSY_LIMIT (1024*1024) extern int erts_dist_buf_busy_limit; extern int erts_is_alive; @@ -154,10 +154,10 @@ erts_dsig_prepare(ErtsDSigData *dsdp, } if (no_suspend) { failure = ERTS_DSIG_PREP_CONNECTED; - erts_smp_spin_lock(&dep->qlock); + erts_smp_mtx_lock(&dep->qlock); if (dep->qflgs & ERTS_DE_QFLG_BUSY) failure = ERTS_DSIG_PREP_WOULD_SUSPEND; - erts_smp_spin_unlock(&dep->qlock); + erts_smp_mtx_unlock(&dep->qlock); if (failure == ERTS_DSIG_PREP_WOULD_SUSPEND) goto fail; } diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 07b4167b27..775f4435a9 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1348,6 +1348,13 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) argv[j++] = argv[i]; } *argc = j; +#if HALFWORD_HEAP + /* If halfword heap, silently ignore any disabling of internal + allocators */ + for (i = 0; i < aui_sz; ++i) + aui[i]->enable = 1; +#endif + } @@ -1404,6 +1411,33 @@ void erts_alloc_reg_scheduler_id(Uint id) erts_tsd_set(thr_ix_key, (void *)(long) ix); } +static void +no_verify(Allctr_t *allctr) +{ + +} + +erts_alloc_verify_func_t +erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr) +{ + if (erts_allctrs_info[ERTS_ALC_A_TEMPORARY].alloc_util + && erts_allctrs_info[ERTS_ALC_A_TEMPORARY].thr_spec) { + ErtsAllocatorThrSpec_t *tspec; + tspec = &erts_allctr_thr_spec[ERTS_ALC_A_TEMPORARY]; + if (!tspec->all_thr_safe) { + int ix = erts_alc_get_thr_ix(); + + if (ix < tspec->size) { + *allctr = tspec->allctr[ix]; + return erts_alcu_verify_unused; + } + } + } + + *allctr = NULL; + return no_verify; +} + __decl_noreturn void erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) { @@ -1561,7 +1595,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) Eterm atoms[sizeof(size)/sizeof(Uint)]; Uint *uintps[sizeof(size)/sizeof(Uint)]; Eterm euints[sizeof(size)/sizeof(Uint)]; - int need_atom; int want_tot_or_sys; int length; Eterm res = THE_NON_VALUE; @@ -1749,7 +1782,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) /* Calculate values needed... */ want_tot_or_sys = want.total || want.system; - need_atom = ERTS_MEM_NEED_ALL_ALCU || want.atom; if (ERTS_MEM_NEED_ALL_ALCU) { size.total = 0; diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 3e96c76dbf..ce792d4d17 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -172,9 +172,17 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size); void erts_free(ErtsAlcType_t type, void *ptr); void *erts_alloc_fnf(ErtsAlcType_t type, Uint size); void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size); +void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size); + #endif /* #if !ERTS_ALC_DO_INLINE */ +#ifndef ERTS_CACHE_LINE_SIZE +/* Assume a cache line size of 64 bytes */ +# define ERTS_CACHE_LINE_SIZE ((UWord) 64) +# define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1) +#endif + #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) ERTS_ALC_INLINE @@ -234,13 +242,25 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) size); } +ERTS_ALC_INLINE +void *erts_alloc_permanent_cache_aligned(ErtsAlcType_t type, Uint size) +{ + UWord v = (UWord) erts_alloc(type, size + (ERTS_CACHE_LINE_SIZE-1)); + + if (v & ERTS_CACHE_LINE_MASK) { + v = (v & ~ERTS_CACHE_LINE_MASK) + ERTS_CACHE_LINE_SIZE; + } + ASSERT((v & ERTS_CACHE_LINE_MASK) == 0); + return (void*)v; +} + #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ -#ifndef ERTS_CACHE_LINE_SIZE -/* Assume a cache line size of 64 bytes */ -# define ERTS_CACHE_LINE_SIZE ((UWord) 64) -# define ERTS_CACHE_LINE_MASK (ERTS_CACHE_LINE_SIZE - 1) -#endif +typedef void (*erts_alloc_verify_func_t)(Allctr_t *); + +erts_alloc_verify_func_t +erts_alloc_get_verify_unused_temp_alloc(Allctr_t **allctr); + #define ERTS_ALC_CACHE_LINE_ALIGN_SIZE(SZ) \ (((((SZ) - 1) / ERTS_CACHE_LINE_SIZE) + 1) * ERTS_CACHE_LINE_SIZE) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 7df9f19af0..ca71798917 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2010. All Rights Reserved. +# Copyright Ericsson AB 2003-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -193,6 +193,7 @@ type DB_FIXATION SHORT_LIVED ETS db_fixation type DB_FIX_DEL SHORT_LIVED ETS fixed_del type DB_TABLES LONG_LIVED ETS db_tabs type DB_NTAB_ENT STANDARD ETS db_named_table_entry +type DB_HEIR_DATA STANDARD ETS db_heir_data type DB_TMP TEMPORARY ETS db_tmp type DB_MC_STK TEMPORARY ETS db_mc_stack type DB_MS_PSDO_PROC LONG_LIVED ETS db_match_pseudo_proc @@ -247,7 +248,7 @@ type CPUDATA LONG_LIVED SYSTEM cpu_data type TMP_CPU_IDS SHORT_LIVED SYSTEM tmp_cpu_ids type EXT_TERM_DATA SHORT_LIVED PROCESSES external_term_data type ZLIB STANDARD SYSTEM zlib -type RDR_GRPS_MAP LONG_LIVED SYSTEM reader_groups_map +type CPU_GRPS_MAP LONG_LIVED SYSTEM cpu_groups_map +if smp type ASYNC SHORT_LIVED SYSTEM async @@ -263,6 +264,8 @@ type XPORTS_LIST SHORT_LIVED SYSTEM extra_port_list type PROC_LCK_WTR LONG_LIVED SYSTEM proc_lock_waiter type PROC_LCK_QS LONG_LIVED SYSTEM proc_lock_queues type RUNQ_BLNS LONG_LIVED SYSTEM run_queue_balancing +type MISC_AUX_WORK_Q LONG_LIVED SYSTEM misc_aux_work_q +type MISC_AUX_WORK SHORT_LIVED SYSTEM misc_aux_work +endif # diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 8b184899c9..1394b7e829 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -3368,6 +3368,38 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) * Debug functions * \* */ +void +erts_alcu_verify_unused(Allctr_t *allctr) +{ + UWord no; + + no = allctr->sbcs.curr_mseg.no; + no += allctr->sbcs.curr_sys_alloc.no; + no += allctr->mbcs.blocks.curr.no; + + if (no) { + UWord sz = allctr->sbcs.blocks.curr.size; + sz += allctr->mbcs.blocks.curr.size; + erl_exit(ERTS_ABORT_EXIT, + "%salloc() used when expected to be unused!\n" + "Total amount of blocks allocated: %bpu\n" + "Total amount of bytes allocated: %bpu\n", + allctr->name_prefix, no, sz); + } +} + +void +erts_alcu_verify_unused_ts(Allctr_t *allctr) +{ +#ifdef USE_THREADS + erts_mtx_lock(&allctr->mutex); +#endif + erts_alcu_verify_unused(allctr); +#ifdef USE_THREADS + erts_mtx_unlock(&allctr->mutex); +#endif +} + #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f2b951bca6..d296081714 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -333,6 +333,9 @@ struct Allctr_t_ { int erts_alcu_start(Allctr_t *, AllctrInit_t *); void erts_alcu_stop(Allctr_t *); +void erts_alcu_verify_unused(Allctr_t *); +void erts_alcu_verify_unused_ts(Allctr_t *allctr); + unsigned long erts_alcu_test(unsigned long, unsigned long, unsigned long); diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index b6a445c55c..684fa5d12f 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -1477,7 +1477,7 @@ BIF_RETTYPE binary_matches_3(BIF_ALIST_3) goto badarg; } if (hsend == 0) { - BIF_RET(am_nomatch); + BIF_RET(NIL); } if (is_tuple(BIF_ARG_2)) { tp = tuple_val(BIF_ARG_2); diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 2c2e283f65..c9cdcb87a6 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1193,7 +1193,7 @@ int erts_ddll_driver_ok(DE_Handle *dh) static void ddll_no_more_references(void *vdh) { DE_Handle *dh = (DE_Handle *) vdh; - int x; + erts_aint_t x; lock_drv_list(); @@ -1604,7 +1604,7 @@ static int do_load_driver_entry(DE_Handle *dh, char *path, char *name) erts_sys_ddll_close(dh->handle); return ERL_DE_LOAD_ERROR_BAD_NAME; } - erts_smp_atomic_init(&(dh->refc), (long) 0); + erts_smp_atomic_init(&(dh->refc), (erts_aint_t) 0); dh->port_count = 0; dh->full_path = erts_alloc(ERTS_ALC_T_DDLL_HANDLE, sys_strlen(path) + 1); sys_strcpy(dh->full_path, path); @@ -1672,7 +1672,7 @@ static int load_driver_entry(DE_Handle **dhp, char *path, char *name) dh->handle = NULL; dh->procs = NULL; dh->port_count = 0; - erts_refc_init(&(dh->refc), (long) 0); + erts_refc_init(&(dh->refc), (erts_aint_t) 0); dh->status = -1; dh->reload_full_path = NULL; dh->reload_driver_name = NULL; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 801263ec26..71206c48b2 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -38,6 +38,7 @@ #include "erl_instrument.h" #include "dist.h" #include "erl_gc.h" +#include "erl_cpu_topology.h" #ifdef HIPE #include "hipe_arch.h" #endif @@ -1544,7 +1545,7 @@ process_info_aux(Process *BIF_P, case am_backtrace: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); erts_stack_dump(ERTS_PRINT_DSBUF, (void *) dsbufp, rp); - res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len); + res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); hp = HAlloc(BIF_P, 3); break; @@ -1687,6 +1688,8 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return erts_get_cpu_topology_term(BIF_P, *tp); } else if (ERTS_IS_ATOM_STR("cpu_topology", sel) && arity == 2) { Eterm res = erts_get_cpu_topology_term(BIF_P, *tp); + if (res == THE_NON_VALUE) + goto badarg; ERTS_BIF_PREP_TRAP1(ret, erts_format_cpu_topology_trap, BIF_P, res); return ret; #if defined(PURIFY) || defined(VALGRIND) @@ -1720,8 +1723,14 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ } else if (is_list(*tp)) { #if defined(PURIFY) #define ERTS_ERROR_CHECKER_PRINTF purify_printf +#define ERTS_ERROR_CHECKER_PRINTF_XML purify_printf #elif defined(VALGRIND) #define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF +# ifndef HAVE_VALGRIND_PRINTF_XML +# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF +# else +# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML +# endif #endif int buf_size = 8*1024; /* Try with 8KB first */ char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); @@ -1738,8 +1747,8 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ } buf[buf_size - 1 - r] = '\0'; if (check_if_xml()) { - ERTS_ERROR_CHECKER_PRINTF("<erlang_info_log>" - "%s</erlang_info_log>\n", buf); + ERTS_ERROR_CHECKER_PRINTF_XML("<erlang_info_log>" + "%s</erlang_info_log>\n", buf); } else { ERTS_ERROR_CHECKER_PRINTF("%s\n", buf); } @@ -1999,6 +2008,8 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(db_get_trace_control_word_0(BIF_P)); } else if (ERTS_IS_ATOM_STR("ets_realloc_moves", BIF_ARG_1)) { BIF_RET((erts_ets_realloc_always_moves) ? am_true : am_false); + } else if (ERTS_IS_ATOM_STR("ets_always_compress", BIF_ARG_1)) { + BIF_RET((erts_ets_always_compress) ? am_true : am_false); } else if (ERTS_IS_ATOM_STR("snifs", BIF_ARG_1)) { Uint size = 0; Uint *szp; @@ -2015,7 +2026,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = TUPLE2(hp, am_sequential_tracer, val); BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection){ - Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); Eterm tup; hp = HAlloc(BIF_P, 3+2 + 3+2 + 3+2); @@ -2030,7 +2041,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(res); } else if (BIF_ARG_1 == am_fullsweep_after){ - Uint val = (Uint) erts_smp_atomic_read(&erts_max_gen_gcs); + Uint val = (Uint) erts_smp_atomic32_read(&erts_max_gen_gcs); hp = HAlloc(BIF_P, 3); res = TUPLE2(hp, am_fullsweep_after, make_small(val)); BIF_RET(res); @@ -2069,7 +2080,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); ASSERT(dsbufp && dsbufp->str); - res = new_binary(BIF_P, (byte *) dsbufp->str, (int) dsbufp->str_len); + res = new_binary(BIF_P, (byte *) dsbufp->str, dsbufp->str_len); erts_destroy_info_dsbuf(dsbufp); BIF_RET(res); } else if (ERTS_IS_ATOM_STR("dist_ctrl", BIF_ARG_1)) { @@ -2345,9 +2356,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) /* Arguments that are unusual follow ... */ else if (ERTS_IS_ATOM_STR("logical_processors", BIF_ARG_1)) { int no; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - no = erts_get_cpu_configured(erts_cpuinfo); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + erts_get_logical_processors(&no, NULL, NULL); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2357,9 +2366,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("logical_processors_online", BIF_ARG_1)) { int no; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - no = erts_get_cpu_online(erts_cpuinfo); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + erts_get_logical_processors(NULL, &no, NULL); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -2369,9 +2376,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) } else if (ERTS_IS_ATOM_STR("logical_processors_available", BIF_ARG_1)) { int no; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - no = erts_get_cpu_available(erts_cpuinfo); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + erts_get_logical_processors(NULL, NULL, &no); if (no > 0) BIF_RET(make_small((Uint) no)); else { @@ -3431,8 +3436,8 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) */ if (ERTS_IS_ATOM_STR("available_internal_state", BIF_ARG_1) && (BIF_ARG_2 == am_true || BIF_ARG_2 == am_false)) { - long on = (long) (BIF_ARG_2 == am_true); - long prev_on = erts_smp_atomic_xchg(&available_internal_state, on); + erts_aint_t on = (erts_aint_t) (BIF_ARG_2 == am_true); + erts_aint_t prev_on = erts_smp_atomic_xchg(&available_internal_state, on); if (on) { erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); erts_dsprintf(dsbufp, "Process %T ", BIF_P->id); @@ -3629,7 +3634,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } else if (ERTS_IS_ATOM_STR("hipe_test_reschedule_suspend", BIF_ARG_1)) { /* Used by hipe test suites */ - long flag = erts_smp_atomic_read(&hipe_test_reschedule_flag); + erts_aint_t flag = erts_smp_atomic_read(&hipe_test_reschedule_flag); if (!flag && BIF_ARG_2 != am_false) { erts_smp_atomic_set(&hipe_test_reschedule_flag, 1); erts_suspend(BIF_P, ERTS_PROC_LOCK_MAIN, NULL); @@ -3704,7 +3709,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) #ifdef ERTS_ENABLE_LOCK_COUNT static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_stats_t *stats, Eterm res) { - unsigned long tries = 0, colls = 0; + Uint tries = 0, colls = 0; unsigned long timer_s = 0, timer_ns = 0, timer_n = 0; unsigned int line = 0; @@ -3717,8 +3722,8 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}}] */ - tries = (unsigned long) ethr_atomic_read(&stats->tries); - colls = (unsigned long) ethr_atomic_read(&stats->colls); + tries = (Uint) ethr_atomic_read(&stats->tries); + colls = (Uint) ethr_atomic_read(&stats->colls); line = stats->line; timer_s = stats->timer.s; diff --git a/erts/emulator/beam/erl_bif_lists.c b/erts/emulator/beam/erl_bif_lists.c index ce13469801..47c48e74d6 100644 --- a/erts/emulator/beam/erl_bif_lists.c +++ b/erts/emulator/beam/erl_bif_lists.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -378,7 +378,7 @@ keyfind(int Bif, Process* p, Eterm Key, Eterm Pos, Eterm List) Eterm *tuple_ptr = tuple_val(term); if (pos <= arityval(*tuple_ptr)) { Eterm element = tuple_ptr[pos]; - if (cmp(Key, element) == 0) { + if (CMP(Key, element) == 0) { return term; } } diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 378c5e73fd..fbc92b9730 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -610,6 +610,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) int binary_io; int soft_eof; Sint linebuf; + Eterm edir = NIL; byte dir[MAXPATHLEN]; /* These are the defaults */ @@ -686,19 +687,10 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) } else if (option == am_arg0) { char *a0; - int n; - if (is_nil(*tp)) { - n = 0; - } else if( (n = is_string(*tp)) == 0) { + + if ((a0 = erts_convert_filename_to_native(*tp, ERTS_ALC_T_TMP, 1)) == NULL) { goto badarg; } - a0 = (char *) erts_alloc(ERTS_ALC_T_TMP, - (n + 1) * sizeof(byte)); - if (intlist_to_buf(*tp, a0, n) != n) { - erl_exit(1, "%s:%d: Internal error\n", - __FILE__, __LINE__); - } - a0[n] = '\0'; if (opts.argv == NULL) { opts.argv = erts_alloc(ERTS_ALC_T_TMP, 2 * sizeof(char **)); @@ -711,22 +703,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) opts.argv[0] = a0; } } else if (option == am_cd) { - Eterm iolist; - DeclareTmpHeap(heap,4,p); - int r; - - UseTmpHeap(4,p); - heap[0] = *tp; - heap[1] = make_list(heap+2); - heap[2] = make_small(0); - heap[3] = NIL; - iolist = make_list(heap); - r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN); - UnUseTmpHeap(4,p); - if (r < 0) { - goto badarg; - } - opts.wd = (char *) dir; + edir = *tp; } else { goto badarg; } @@ -838,19 +815,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) goto badarg; } name = tp[1]; - if (is_atom(name)) { - name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, - atom_tab(atom_val(name))->len+1); - sys_memcpy((void *) name_buf, - (void *) atom_tab(atom_val(name))->name, - atom_tab(atom_val(name))->len); - name_buf[atom_tab(atom_val(name))->len] = '\0'; - } else if ((i = is_string(name))) { - name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); - if (intlist_to_buf(name, name_buf, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - name_buf[i] = '\0'; - } else { + if ((name_buf = erts_convert_filename_to_native(name,ERTS_ALC_T_TMP,0)) == NULL) { goto badarg; } opts.spawn_type = ERTS_SPAWN_EXECUTABLE; @@ -892,7 +857,33 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) /* Argument vector only if explicit spawn_executable */ goto badarg; } - + + if (edir != NIL) { + /* A working directory is expressed differently if spawn_executable, i.e. Unicode is handles + for spawn_executable... */ + if (opts.spawn_type != ERTS_SPAWN_EXECUTABLE) { + Eterm iolist; + DeclareTmpHeap(heap,4,p); + int r; + + UseTmpHeap(4,p); + heap[0] = edir; + heap[1] = make_list(heap+2); + heap[2] = make_small(0); + heap[3] = NIL; + iolist = make_list(heap); + r = io_list_to_buf(iolist, (char*) dir, MAXPATHLEN); + UnUseTmpHeap(4,p); + if (r < 0) { + goto badarg; + } + opts.wd = (char *) dir; + } else { + if ((opts.wd = erts_convert_filename_to_native(edir,ERTS_ALC_T_TMP,0)) == NULL) { + goto badarg; + } + } + } if (driver != &spawn_driver && opts.exit_status) { goto badarg; @@ -941,6 +932,9 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) if (opts.argv) { free_args(opts.argv); } + if (opts.wd && opts.wd != ((char *)dir)) { + erts_free(ERTS_ALC_T_TMP, (void *) opts.wd); + } return port_num; badarg: @@ -950,6 +944,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_nump) #undef OPEN_PORT_ERROR } +/* Arguments can be given i unicode and as raw binaries, convert filename is used to convert */ static char **convert_args(Eterm l) { char **pp; @@ -966,22 +961,14 @@ static char **convert_args(Eterm l) pp[i++] = erts_default_arg0; while (is_list(l)) { str = CAR(list_val(l)); - - if (is_nil(str)) { - n = 0; - } else if( (n = is_string(str)) == 0) { - /* Not a string... */ + if ((b = erts_convert_filename_to_native(str,ERTS_ALC_T_TMP,1)) == NULL) { int j; for (j = 1; j < i; ++j) erts_free(ERTS_ALC_T_TMP, pp[j]); erts_free(ERTS_ALC_T_TMP, pp); return NULL; - } - b = (char *) erts_alloc(ERTS_ALC_T_TMP, (n + 1) * sizeof(byte)); - pp[i++] = (char *) b; - if (intlist_to_buf(str, b, n) != n) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); - b[n] = '\0'; + } + pp[i++] = b; l = CDR(list_val(l)); } pp[i] = NULL; diff --git a/erts/emulator/beam/erl_bif_timer.c b/erts/emulator/beam/erl_bif_timer.c index 4ae2f6ebf4..db771bd216 100644 --- a/erts/emulator/beam/erl_bif_timer.c +++ b/erts/emulator/beam/erl_bif_timer.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -478,7 +478,7 @@ setup_bif_timer(Uint32 xflags, tab_insert(btm); ASSERT(btm == tab_find(ref)); btm->tm.active = 0; /* MUST be initalized */ - erl_set_timer(&btm->tm, + erts_set_timer(&btm->tm, (ErlTimeoutProc) bif_timer_timeout, (ErlCancelProc) bif_timer_cleanup, (void *) btm, @@ -550,7 +550,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) res = am_false; } else { - Uint left = time_left(&btm->tm); + Uint left = erts_time_left(&btm->tm); if (!(btm->flags & BTM_FLG_BYNAME)) { erts_smp_proc_lock(btm->receiver.proc.ess, ERTS_PROC_LOCK_MSGQ); unlink_proc(btm); @@ -558,7 +558,7 @@ BIF_RETTYPE cancel_timer_1(BIF_ALIST_1) } tab_remove(btm); ASSERT(!tab_find(BIF_ARG_1)); - erl_cancel_timer(&btm->tm); + erts_cancel_timer(&btm->tm); erts_smp_btm_rwunlock(); res = erts_make_integer(left, BIF_P); } @@ -587,7 +587,7 @@ BIF_RETTYPE read_timer_1(BIF_ALIST_1) res = am_false; } else { - Uint left = time_left(&btm->tm); + Uint left = erts_time_left(&btm->tm); res = erts_make_integer(left, BIF_P); } @@ -613,7 +613,8 @@ erts_print_bif_timer_info(int to, void *to_arg) : btm->receiver.proc.ess->id); erts_print(to, to_arg, "=timer:%T\n", receiver); erts_print(to, to_arg, "Message: %T\n", btm->message); - erts_print(to, to_arg, "Time left: %d ms\n", time_left(&btm->tm)); + erts_print(to, to_arg, "Time left: %u ms\n", + erts_time_left(&btm->tm)); } } @@ -640,7 +641,7 @@ erts_cancel_bif_timers(Process *p, ErtsProcLocks plocks) tab_remove(btm); tmp_btm = btm; btm = btm->receiver.proc.next; - erl_cancel_timer(&tmp_btm->tm); + erts_cancel_timer(&tmp_btm->tm); } p->bif_timers = NULL; diff --git a/erts/emulator/beam/erl_binary.h b/erts/emulator/beam/erl_binary.h index a569fe2e85..506c4813fa 100644 --- a/erts/emulator/beam/erl_binary.h +++ b/erts/emulator/beam/erl_binary.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -71,6 +71,7 @@ typedef struct erl_heap_bin { */ #define binary_size(Bin) (binary_val(Bin)[1]) +#define binary_size_rel(Bin,BasePtr) (binary_val_rel(Bin,BasePtr)[1]) #define binary_bitsize(Bin) \ ((*binary_val(Bin) == HEADER_SUB_BIN) ? \ @@ -93,9 +94,12 @@ typedef struct erl_heap_bin { * Bitsize: output variable (Uint) */ -#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ +#define ERTS_GET_BINARY_BYTES(Bin,Bytep,Bitoffs,Bitsize) \ + ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,NULL) + +#define ERTS_GET_BINARY_BYTES_REL(Bin,Bytep,Bitoffs,Bitsize,BasePtr) \ do { \ - Eterm* _real_bin = binary_val(Bin); \ + Eterm* _real_bin = binary_val_rel(Bin,BasePtr); \ Uint _offs = 0; \ Bitoffs = Bitsize = 0; \ if (*_real_bin == HEADER_SUB_BIN) { \ @@ -103,7 +107,7 @@ do { \ _offs = _sb->offs; \ Bitoffs = _sb->bitoffs; \ Bitsize = _sb->bitsize; \ - _real_bin = binary_val(_sb->orig); \ + _real_bin = binary_val_rel(_sb->orig,BasePtr); \ } \ if (*_real_bin == HEADER_PROC_BIN) { \ Bytep = ((ProcBin *) _real_bin)->bytes + _offs; \ @@ -125,9 +129,12 @@ do { \ * BitSize: Extra bit size (Uint) */ -#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \ +#define ERTS_GET_REAL_BIN(Bin, RealBin, ByteOffset, BitOffset, BitSize) \ + ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, NULL) + +#define ERTS_GET_REAL_BIN_REL(Bin, RealBin, ByteOffset, BitOffset, BitSize, BasePtr) \ do { \ - ErlSubBin* _sb = (ErlSubBin *) binary_val(Bin); \ + ErlSubBin* _sb = (ErlSubBin *) binary_val_rel(Bin,BasePtr); \ if (_sb->thing_word == HEADER_SUB_BIN) { \ RealBin = _sb->orig; \ ByteOffset = _sb->offs; \ @@ -152,6 +159,8 @@ do { \ void erts_init_binary(void); byte* erts_get_aligned_binary_bytes_extra(Eterm, byte**, ErtsAlcType_t, unsigned extra); +/* Used by unicode module */ +Eterm erts_bin_bytes_to_list(Eterm previous, Eterm* hp, byte* bytes, Uint size, Uint bitoffs); /* * Common implementation for erlang:list_to_binary/1 and binary:list_to_bin/1 diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 88d2c06246..6f8a7436d5 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -555,10 +555,11 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags) { unsigned long offs; - ASSERT(size != 0); offs = BIT_OFFSET(size); if (is_small(val)) { Sint v = signed_val(val); + + ASSERT(size != 0); /* Tested by caller */ if (flags & BSF_LITTLE) { /* Little endian */ sz--; COPY_VAL(buf,1,v,sz); @@ -578,6 +579,9 @@ fmt_int(byte *buf, Uint sz, Eterm val, Uint size, Uint flags) ErtsDigit* dp = big_v(val); int n = MIN(sz,ds); + if (size == 0) { + return 0; + } if (flags & BSF_LITTLE) { sz -= n; /* pad with this amount */ if (sign) { @@ -729,15 +733,13 @@ erts_new_bs_put_integer(ERL_BITS_PROTO_3(Eterm arg, Uint num_bits, unsigned flag Uint b; byte *iptr; - if (num_bits == 0) { - return 1; - } - bit_offset = BIT_OFFSET(bin_offset); if (is_small(arg)) { Uint rbits = 8 - bit_offset; - if (bit_offset + num_bits <= 8) { + if (num_bits == 0) { + return 1; + } else if (bit_offset + num_bits <= 8) { /* * All bits are in the same byte. */ diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c new file mode 100644 index 0000000000..bcf8bcf270 --- /dev/null +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -0,0 +1,2361 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010-2011. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: CPU topology and related functionality + * + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include <ctype.h> + +#include "global.h" +#include "error.h" +#include "bif.h" +#include "erl_cpu_topology.h" + +#define ERTS_MAX_READER_GROUPS 8 + +/* + * Cpu topology hierarchy. + */ +#define ERTS_TOPOLOGY_NODE 0 +#define ERTS_TOPOLOGY_PROCESSOR 1 +#define ERTS_TOPOLOGY_PROCESSOR_NODE 2 +#define ERTS_TOPOLOGY_CORE 3 +#define ERTS_TOPOLOGY_THREAD 4 +#define ERTS_TOPOLOGY_LOGICAL 5 + +#define ERTS_TOPOLOGY_MAX_DEPTH 6 + +typedef struct { + int bind_id; + int bound_id; +} ErtsCpuBindData; + +static erts_cpu_info_t *cpuinfo; + +static int max_main_threads; +static int reader_groups; + +static ErtsCpuBindData *scheduler2cpu_map; +static erts_smp_rwmtx_t cpuinfo_rwmtx; + +typedef enum { + ERTS_CPU_BIND_UNDEFINED, + ERTS_CPU_BIND_SPREAD, + ERTS_CPU_BIND_PROCESSOR_SPREAD, + ERTS_CPU_BIND_THREAD_SPREAD, + ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD, + ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD, + ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD, + ERTS_CPU_BIND_NO_SPREAD, + ERTS_CPU_BIND_NONE +} ErtsCpuBindOrder; + +#define ERTS_CPU_BIND_DEFAULT_BIND \ + ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD + +static int no_cpu_groups_callbacks; +static ErtsCpuBindOrder cpu_bind_order; + +static erts_cpu_topology_t *user_cpudata; +static int user_cpudata_size; +static erts_cpu_topology_t *system_cpudata; +static int system_cpudata_size; + +typedef struct { + int level[ERTS_TOPOLOGY_MAX_DEPTH+1]; +} erts_avail_cput; + +typedef struct { + int id; + int sub_levels; + int cpu_groups; +} erts_cpu_groups_count_t; + +typedef struct { + int logical; + int cpu_group; +} erts_cpu_groups_map_array_t; + +typedef struct erts_cpu_groups_callback_list_t_ erts_cpu_groups_callback_list_t; +struct erts_cpu_groups_callback_list_t_ { + erts_cpu_groups_callback_list_t *next; + erts_cpu_groups_callback_t callback; + void *arg; +}; + +typedef struct erts_cpu_groups_map_t_ erts_cpu_groups_map_t; +struct erts_cpu_groups_map_t_ { + erts_cpu_groups_map_t *next; + int groups; + erts_cpu_groups_map_array_t *array; + int size; + int logical_processors; + erts_cpu_groups_callback_list_t *callback_list; +}; + +typedef struct { + erts_cpu_groups_callback_t callback; + int ix; + void *arg; +} erts_cpu_groups_callback_call_t; + +static erts_cpu_groups_map_t *cpu_groups_maps; + +static erts_cpu_groups_map_t *reader_groups_map; + +#define ERTS_TOPOLOGY_CG ERTS_TOPOLOGY_MAX_DEPTH + +#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff) + +#ifdef ERTS_SMP +static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata, + int size, + ErtsCpuBindOrder bind_order, + int mk_seq); +static void write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size); +#endif + +static void reader_groups_callback(int, ErtsSchedulerData *, int, void *); +static erts_cpu_groups_map_t *add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg); +static void update_cpu_groups_maps(void); +static void make_cpu_groups_map(erts_cpu_groups_map_t *map, int test); +static int cpu_groups_lookup(erts_cpu_groups_map_t *map, + ErtsSchedulerData *esdp); + +static void create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, + int *cpudata_size); +static void destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata); + +static int +int_cmp(const void *vx, const void *vy) +{ + return *((int *) vx) - *((int *) vy); +} + +static int +cpu_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->core != y->core) + return x->core - y->core; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->node != y->node) + return x->node - y->node; + return 0; +} + +static int +cpu_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_thread_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + return 0; +} + +static int +cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->node != y->node) + return x->node - y->node; + if (x->core != y->core) + return x->core - y->core; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->core != y->core) + return x->core - y->core; + if (x->processor != y->processor) + return x->processor - y->processor; + return 0; +} + +static int +cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->thread != y->thread) + return x->thread - y->thread; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->core != y->core) + return x->core - y->core; + return 0; +} + +static int +cpu_no_spread_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->node != y->node) + return x->node - y->node; + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->thread != y->thread) + return x->thread - y->thread; + return 0; +} + +static ERTS_INLINE void +make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node) +{ + int ix; + int node = -1; + int processor = -1; + int processor_node = -1; + int processor_node_node = -1; + int core = -1; + int thread = -1; + int old_node = -1; + int old_processor = -1; + int old_processor_node = -1; + int old_core = -1; + int old_thread = -1; + + for (ix = 0; ix < size; ix++) { + if (!no_node || cpudata[ix].node >= 0) { + if (old_node == cpudata[ix].node) + cpudata[ix].node = node; + else { + old_node = cpudata[ix].node; + old_processor = processor = -1; + if (!no_node) + old_processor_node = processor_node = -1; + old_core = core = -1; + old_thread = thread = -1; + if (no_node || cpudata[ix].node >= 0) + cpudata[ix].node = ++node; + } + } + if (old_processor == cpudata[ix].processor) + cpudata[ix].processor = processor; + else { + old_processor = cpudata[ix].processor; + if (!no_node) + processor_node_node = old_processor_node = processor_node = -1; + old_core = core = -1; + old_thread = thread = -1; + cpudata[ix].processor = ++processor; + } + if (no_node && cpudata[ix].processor_node < 0) + old_processor_node = -1; + else { + if (old_processor_node == cpudata[ix].processor_node) { + if (no_node) + cpudata[ix].node = cpudata[ix].processor_node = node; + else { + if (processor_node_node >= 0) + cpudata[ix].node = processor_node_node; + cpudata[ix].processor_node = processor_node; + } + } + else { + old_processor_node = cpudata[ix].processor_node; + old_core = core = -1; + old_thread = thread = -1; + if (no_node) + cpudata[ix].node = cpudata[ix].processor_node = ++node; + else { + cpudata[ix].node = processor_node_node = ++node; + cpudata[ix].processor_node = ++processor_node; + } + } + } + if (!no_node && cpudata[ix].processor_node < 0) + cpudata[ix].processor_node = 0; + if (old_core == cpudata[ix].core) + cpudata[ix].core = core; + else { + old_core = cpudata[ix].core; + old_thread = thread = -1; + cpudata[ix].core = ++core; + } + if (old_thread == cpudata[ix].thread) + cpudata[ix].thread = thread; + else + old_thread = cpudata[ix].thread = ++thread; + } +} + +static void +cpu_bind_order_sort(erts_cpu_topology_t *cpudata, + int size, + ErtsCpuBindOrder bind_order, + int mk_seq) +{ + if (size > 1) { + int no_node = 0; + int (*cmp_func)(const void *, const void *); + switch (bind_order) { + case ERTS_CPU_BIND_SPREAD: + cmp_func = cpu_spread_order_cmp; + break; + case ERTS_CPU_BIND_PROCESSOR_SPREAD: + cmp_func = cpu_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_THREAD_SPREAD: + cmp_func = cpu_thread_spread_order_cmp; + break; + case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: + no_node = 1; + cmp_func = cpu_thread_no_node_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: + no_node = 1; + cmp_func = cpu_no_node_processor_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: + no_node = 1; + cmp_func = cpu_no_node_thread_spread_order_cmp; + break; + case ERTS_CPU_BIND_NO_SPREAD: + cmp_func = cpu_no_spread_order_cmp; + break; + default: + cmp_func = NULL; + erl_exit(ERTS_ABORT_EXIT, + "Bad cpu bind type: %d\n", + (int) cpu_bind_order); + break; + } + + if (mk_seq) + make_cpudata_id_seq(cpudata, size, no_node); + + qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func); + } +} + +static int +processor_order_cmp(const void *vx, const void *vy) +{ + erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; + erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; + + if (x->processor != y->processor) + return x->processor - y->processor; + if (x->node != y->node) + return x->node - y->node; + if (x->processor_node != y->processor_node) + return x->processor_node - y->processor_node; + if (x->core != y->core) + return x->core - y->core; + if (x->thread != y->thread) + return x->thread - y->thread; + return 0; +} + +#ifdef ERTS_SMP +void +erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp) +{ + erts_cpu_groups_map_t *cgm; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_callback_call_t *cgcc; + int cgcc_ix; + + /* Unbind from cpu */ + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + if (scheduler2cpu_map[esdp->no].bound_id >= 0 + && erts_unbind_from_cpu(cpuinfo) == 0) { + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + } + + cgcc = erts_alloc(ERTS_ALC_T_TMP, + (no_cpu_groups_callbacks + * sizeof(erts_cpu_groups_callback_call_t))); + cgcc_ix = 0; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + cgcc[cgcc_ix].callback = cgcl->callback; + cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp); + cgcc[cgcc_ix].arg = cgcl->arg; + cgcc_ix++; + } + } + ASSERT(no_cpu_groups_callbacks == cgcc_ix); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + + for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) + cgcc[cgcc_ix].callback(1, + esdp, + cgcc[cgcc_ix].ix, + cgcc[cgcc_ix].arg); + + erts_free(ERTS_ALC_T_TMP, cgcc); + + if (esdp->no <= max_main_threads) + erts_thr_set_main_status(0, 0); + +} + +void +erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp) +{ + ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(esdp->run_queue)); + + if (esdp->no <= max_main_threads) + erts_thr_set_main_status(1, (int) esdp->no); + + /* Make sure we check if we should bind to a cpu or not... */ + if (esdp->run_queue->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) + erts_smp_atomic32_set(&esdp->chk_cpu_bind, 1); + else + esdp->run_queue->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; +} + +#endif + +void +erts_sched_check_cpu_bind(ErtsSchedulerData *esdp) +{ + int res, cpu_id, cgcc_ix; + erts_cpu_groups_map_t *cgm; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_callback_call_t *cgcc; +#ifdef ERTS_SMP + if (erts_common_run_queue) + erts_smp_atomic32_set(&esdp->chk_cpu_bind, 0); + else { + esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; + } +#endif + erts_smp_runq_unlock(esdp->run_queue); + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + cpu_id = scheduler2cpu_map[esdp->no].bind_id; + if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) { + res = erts_bind_to_cpu(cpuinfo, cpu_id); + if (res == 0) + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id; + else { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n", + (int) esdp->no, cpu_id, erl_errno_id(-res)); + erts_send_error_to_logger_nogl(dsbufp); + if (scheduler2cpu_map[esdp->no].bound_id >= 0) + goto unbind; + } + } + else if (cpu_id < 0) { + unbind: + /* Get rid of old binding */ + res = erts_unbind_from_cpu(cpuinfo); + if (res == 0) + esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; + else if (res != -ENOTSUP) { + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n", + (int) esdp->no, cpu_id, erl_errno_id(-res)); + erts_send_error_to_logger_nogl(dsbufp); + } + } + + cgcc = erts_alloc(ERTS_ALC_T_TMP, + (no_cpu_groups_callbacks + * sizeof(erts_cpu_groups_callback_call_t))); + cgcc_ix = 0; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + cgcc[cgcc_ix].callback = cgcl->callback; + cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp); + cgcc[cgcc_ix].arg = cgcl->arg; + cgcc_ix++; + } + } + + ASSERT(no_cpu_groups_callbacks == cgcc_ix); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + + for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) + cgcc[cgcc_ix].callback(0, + esdp, + cgcc[cgcc_ix].ix, + cgcc[cgcc_ix].arg); + + erts_free(ERTS_ALC_T_TMP, cgcc); + + erts_smp_runq_lock(esdp->run_queue); +} + +#ifdef ERTS_SMP +void +erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp) +{ + int cgcc_ix; + erts_cpu_groups_map_t *cgm; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_callback_call_t *cgcc; + + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + + cgcc = erts_alloc(ERTS_ALC_T_TMP, + (no_cpu_groups_callbacks + * sizeof(erts_cpu_groups_callback_call_t))); + cgcc_ix = 0; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + cgcc[cgcc_ix].callback = cgcl->callback; + cgcc[cgcc_ix].ix = cpu_groups_lookup(cgm, esdp); + cgcc[cgcc_ix].arg = cgcl->arg; + cgcc_ix++; + } + } + + ASSERT(no_cpu_groups_callbacks == cgcc_ix); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + + for (cgcc_ix = 0; cgcc_ix < no_cpu_groups_callbacks; cgcc_ix++) + cgcc[cgcc_ix].callback(0, + esdp, + cgcc[cgcc_ix].ix, + cgcc[cgcc_ix].arg); + + erts_free(ERTS_ALC_T_TMP, cgcc); + + if (esdp->no <= max_main_threads) + erts_thr_set_main_status(1, (int) esdp->no); +} +#endif + +static void +write_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) +{ + int s_ix = 1; + int cpu_ix; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + if (cpu_bind_order != ERTS_CPU_BIND_NONE && size) { + + cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1); + + for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++) + if (erts_is_cpu_available(cpuinfo, cpudata[cpu_ix].logical)) + scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical; + } + + if (s_ix <= erts_no_schedulers) + for (; s_ix <= erts_no_schedulers; s_ix++) + scheduler2cpu_map[s_ix].bind_id = -1; +} + +int +erts_init_scheduler_bind_type_string(char *how) +{ + if (sys_strcmp(how, "u") == 0) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) + return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED; + else if (!system_cpudata && !user_cpudata) + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; + else if (sys_strcmp(how, "db") == 0) + cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (sys_strcmp(how, "s") == 0) + cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (sys_strcmp(how, "ps") == 0) + cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "ts") == 0) + cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (sys_strcmp(how, "tnnps") == 0) + cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "nnps") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (sys_strcmp(how, "nnts") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (sys_strcmp(how, "ns") == 0) + cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else + return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE; + return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; +} + +static Eterm +bound_schedulers_term(ErtsCpuBindOrder order) +{ + switch (order) { + case ERTS_CPU_BIND_SPREAD: { + ERTS_DECL_AM(spread); + return AM_spread; + } + case ERTS_CPU_BIND_PROCESSOR_SPREAD: { + ERTS_DECL_AM(processor_spread); + return AM_processor_spread; + } + case ERTS_CPU_BIND_THREAD_SPREAD: { + ERTS_DECL_AM(thread_spread); + return AM_thread_spread; + } + case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: { + ERTS_DECL_AM(thread_no_node_processor_spread); + return AM_thread_no_node_processor_spread; + } + case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: { + ERTS_DECL_AM(no_node_processor_spread); + return AM_no_node_processor_spread; + } + case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: { + ERTS_DECL_AM(no_node_thread_spread); + return AM_no_node_thread_spread; + } + case ERTS_CPU_BIND_NO_SPREAD: { + ERTS_DECL_AM(no_spread); + return AM_no_spread; + } + case ERTS_CPU_BIND_NONE: { + ERTS_DECL_AM(unbound); + return AM_unbound; + } + default: + ASSERT(0); + return THE_NON_VALUE; + } +} + +Eterm +erts_bound_schedulers_term(Process *c_p) +{ + ErtsCpuBindOrder order; + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + order = cpu_bind_order; + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return bound_schedulers_term(order); +} + +Eterm +erts_bind_schedulers(Process *c_p, Eterm how) +{ + int notify = 0; + Eterm res; + erts_cpu_topology_t *cpudata; + int cpudata_size; + ErtsCpuBindOrder old_cpu_bind_order; + + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + + if (erts_bind_to_cpu(cpuinfo, -1) == -ENOTSUP) { + if (cpu_bind_order == ERTS_CPU_BIND_NONE + && ERTS_IS_ATOM_STR("unbound", how)) { + res = bound_schedulers_term(ERTS_CPU_BIND_NONE); + goto done; + } + ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP); + } + else { + + old_cpu_bind_order = cpu_bind_order; + + if (ERTS_IS_ATOM_STR("default_bind", how)) + cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (ERTS_IS_ATOM_STR("spread", how)) + cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (ERTS_IS_ATOM_STR("processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("no_spread", how)) + cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (ERTS_IS_ATOM_STR("unbound", how)) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else { + cpu_bind_order = old_cpu_bind_order; + ERTS_BIF_PREP_ERROR(res, c_p, BADARG); + goto done; + } + + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + + if (!cpudata) { + cpu_bind_order = old_cpu_bind_order; + ERTS_BIF_PREP_ERROR(res, c_p, BADARG); + goto done; + } + + write_schedulers_bind_change(cpudata, cpudata_size); + notify = 1; + + destroy_tmp_cpu_topology_copy(cpudata); + + res = bound_schedulers_term(old_cpu_bind_order); + } + + done: + + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + + if (notify) + erts_sched_notify_check_cpu_bind(); + + return res; +} + +int +erts_sched_bind_atthrcreate_prepare(void) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + return esdp != NULL && erts_is_scheduler_bound(esdp); +} + +int +erts_sched_bind_atthrcreate_child(int unbind) +{ + int res = 0; + if (unbind) { + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + res = erts_unbind_from_cpu(cpuinfo); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + } + return res; +} + +void +erts_sched_bind_atthrcreate_parent(int unbind) +{ + +} + +int +erts_sched_bind_atfork_prepare(void) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + int unbind = esdp != NULL && erts_is_scheduler_bound(esdp); + if (unbind) + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + return unbind; +} + +int +erts_sched_bind_atfork_child(int unbind) +{ + if (unbind) { + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + return erts_unbind_from_cpu(cpuinfo); + } + return 0; +} + +char * +erts_sched_bind_atvfork_child(int unbind) +{ + if (unbind) { + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + return erts_get_unbind_from_cpu_str(cpuinfo); + } + return "false"; +} + +void +erts_sched_bind_atfork_parent(int unbind) +{ + if (unbind) + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); +} + +Eterm +erts_fake_scheduler_bindings(Process *p, Eterm how) +{ + ErtsCpuBindOrder fake_cpu_bind_order; + erts_cpu_topology_t *cpudata; + int cpudata_size; + Eterm res; + + if (ERTS_IS_ATOM_STR("default_bind", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; + else if (ERTS_IS_ATOM_STR("spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD; + else if (ERTS_IS_ATOM_STR("processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; + else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; + else if (ERTS_IS_ATOM_STR("no_spread", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; + else if (ERTS_IS_ATOM_STR("unbound", how)) + fake_cpu_bind_order = ERTS_CPU_BIND_NONE; + else { + ERTS_BIF_PREP_ERROR(res, p, BADARG); + return res; + } + + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + + if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE) + ERTS_BIF_PREP_RET(res, am_false); + else { + int i; + Eterm *hp; + + cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1); + +#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA + + erts_fprintf(stderr, "node: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].node); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "processor: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].processor); + erts_fprintf(stderr, "\n"); + if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD + && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD + && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) { + erts_fprintf(stderr, "processor_node:"); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].processor_node); + erts_fprintf(stderr, "\n"); + } + erts_fprintf(stderr, "core: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].core); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "thread: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].thread); + erts_fprintf(stderr, "\n"); + erts_fprintf(stderr, "logical: "); + for (i = 0; i < cpudata_size; i++) + erts_fprintf(stderr, " %2d", cpudata[i].logical); + erts_fprintf(stderr, "\n"); +#endif + + hp = HAlloc(p, cpudata_size+1); + ERTS_BIF_PREP_RET(res, make_tuple(hp)); + *hp++ = make_arityval((Uint) cpudata_size); + for (i = 0; i < cpudata_size; i++) + *hp++ = make_small((Uint) cpudata[i].logical); + } + + destroy_tmp_cpu_topology_copy(cpudata); + + return res; +} + +Eterm +erts_get_schedulers_binds(Process *c_p) +{ + int ix; + ERTS_DECL_AM(unbound); + Eterm *hp = HAlloc(c_p, erts_no_schedulers+1); + Eterm res = make_tuple(hp); + + *(hp++) = make_arityval(erts_no_schedulers); + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + for (ix = 1; ix <= erts_no_schedulers; ix++) + *(hp++) = (scheduler2cpu_map[ix].bound_id >= 0 + ? make_small(scheduler2cpu_map[ix].bound_id) + : AM_unbound); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return res; +} + +/* + * CPU topology + */ + +typedef struct { + int *id; + int used; + int size; +} ErtsCpuTopIdSeq; + +typedef struct { + ErtsCpuTopIdSeq logical; + ErtsCpuTopIdSeq thread; + ErtsCpuTopIdSeq core; + ErtsCpuTopIdSeq processor_node; + ErtsCpuTopIdSeq processor; + ErtsCpuTopIdSeq node; +} ErtsCpuTopEntry; + +static void +init_cpu_top_entry(ErtsCpuTopEntry *cte) +{ + int size = 10; + cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->logical.size = size; + cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->thread.size = size; + cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->core.size = size; + cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->processor_node.size = size; + cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->processor.size = size; + cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, + sizeof(int)*size); + cte->node.size = size; +} + +static void +destroy_cpu_top_entry(ErtsCpuTopEntry *cte) +{ + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id); + erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id); +} + +static int +get_cput_value_or_range(int *v, int *vr, char **str) +{ + long l; + char *c = *str; + errno = 0; + if (!isdigit((unsigned char)*c)) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; + l = strtol(c, &c, 10); + if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; + *v = (int) l; + if (*c == '-') { + c++; + if (!isdigit((unsigned char)*c)) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + l = strtol(c, &c, 10); + if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + *vr = (int) l; + } + *str = c; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str) +{ + int ix = 0; + int need_size = 0; + char *c = *str; + + while (1) { + int res; + int val; + int nids; + int val_range = -1; + res = get_cput_value_or_range(&val, &val_range, &c); + if (res != ERTS_INIT_CPU_TOPOLOGY_OK) + return res; + if (val_range < 0 || val_range == val) + nids = 1; + else { + if (val_range > val) + nids = val_range - val + 1; + else + nids = val - val_range + 1; + } + need_size += nids; + if (need_size > idseq->size) { + idseq->size = need_size + 10; + idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS, + idseq->id, + sizeof(int)*idseq->size); + } + if (nids == 1) + idseq->id[ix++] = val; + else if (val_range > val) { + for (; val <= val_range; val++) + idseq->id[ix++] = val; + } + else { + for (; val >= val_range; val--) + idseq->id[ix++] = val; + } + if (*c != ',') + break; + c++; + } + *str = c; + idseq->used = ix; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +get_cput_entry(ErtsCpuTopEntry *cput, char **str) +{ + int h; + char *c = *str; + + cput->logical.used = 0; + cput->thread.id[0] = 0; + cput->thread.used = 1; + cput->core.id[0] = 0; + cput->core.used = 1; + cput->processor_node.id[0] = -1; + cput->processor_node.used = 1; + cput->processor.id[0] = 0; + cput->processor.used = 1; + cput->node.id[0] = -1; + cput->node.used = 1; + + h = ERTS_TOPOLOGY_MAX_DEPTH; + while (*c != ':' && *c != '\0') { + int res; + ErtsCpuTopIdSeq *idseqp; + switch (*c++) { + case 'L': + if (h <= ERTS_TOPOLOGY_LOGICAL) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->logical; + h = ERTS_TOPOLOGY_LOGICAL; + break; + case 't': + case 'T': + if (h <= ERTS_TOPOLOGY_THREAD) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->thread; + h = ERTS_TOPOLOGY_THREAD; + break; + case 'c': + case 'C': + if (h <= ERTS_TOPOLOGY_CORE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->core; + h = ERTS_TOPOLOGY_CORE; + break; + case 'p': + case 'P': + if (h <= ERTS_TOPOLOGY_PROCESSOR) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->processor; + h = ERTS_TOPOLOGY_PROCESSOR; + break; + case 'n': + case 'N': + if (h <= ERTS_TOPOLOGY_PROCESSOR) { + do_node: + if (h <= ERTS_TOPOLOGY_NODE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->node; + h = ERTS_TOPOLOGY_NODE; + } + else { + int p_node = 0; + char *p_chk = c; + while (*p_chk != '\0' && *p_chk != ':') { + if (*p_chk == 'p' || *p_chk == 'P') { + p_node = 1; + break; + } + p_chk++; + } + if (!p_node) + goto do_node; + if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; + idseqp = &cput->processor_node; + h = ERTS_TOPOLOGY_PROCESSOR_NODE; + } + break; + default: + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE; + } + res = get_cput_id_seq(idseqp, &c); + if (res != ERTS_INIT_CPU_TOPOLOGY_OK) + return res; + } + + if (cput->logical.used < 1) + return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID; + + if (*c == ':') { + c++; + } + + if (cput->thread.used != 1 + && cput->thread.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->core.used != 1 + && cput->core.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->processor_node.used != 1 + && cput->processor_node.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->processor.used != 1 + && cput->processor.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + if (cput->node.used != 1 + && cput->node.used != cput->logical.used) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; + + *str = c; + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +static int +verify_topology(erts_cpu_topology_t *cpudata, int size) +{ + if (size > 0) { + int *logical; + int node, processor, no_nodes, i; + + /* Verify logical ids */ + logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size); + + for (i = 0; i < size; i++) + logical[i] = cpudata[i].logical; + + qsort(logical, size, sizeof(int), int_cmp); + for (i = 0; i < size-1; i++) { + if (logical[i] == logical[i+1]) { + erts_free(ERTS_ALC_T_TMP, logical); + return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS; + } + } + + erts_free(ERTS_ALC_T_TMP, logical); + + qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp); + + /* Verify unique entities */ + + for (i = 1; i < size; i++) { + if (cpudata[i-1].processor == cpudata[i].processor + && cpudata[i-1].node == cpudata[i].node + && (cpudata[i-1].processor_node + == cpudata[i].processor_node) + && cpudata[i-1].core == cpudata[i].core + && cpudata[i-1].thread == cpudata[i].thread) { + return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES; + } + } + + /* Verify numa nodes */ + node = cpudata[0].node; + processor = cpudata[0].processor; + no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0; + for (i = 1; i < size; i++) { + if (no_nodes) { + if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + } + else { + if (cpudata[i].processor == processor && cpudata[i].node != node) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + node = cpudata[i].node; + processor = cpudata[i].processor; + if (node >= 0 && cpudata[i].processor_node >= 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + if (node < 0 && cpudata[i].processor_node < 0) + return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; + } + } + } + + return ERTS_INIT_CPU_TOPOLOGY_OK; +} + +int +erts_init_cpu_topology_string(char *topology_str) +{ + ErtsCpuTopEntry cput; + int need_size; + char *c; + int ix; + int error = ERTS_INIT_CPU_TOPOLOGY_OK; + + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata_size = 10; + + user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + + init_cpu_top_entry(&cput); + + ix = 0; + need_size = 0; + + c = topology_str; + if (*c == '\0') { + error = ERTS_INIT_CPU_TOPOLOGY_MISSING; + goto fail; + } + do { + int r; + error = get_cput_entry(&cput, &c); + if (error != ERTS_INIT_CPU_TOPOLOGY_OK) + goto fail; + need_size += cput.logical.used; + if (user_cpudata_size < need_size) { + user_cpudata_size = need_size + 10; + user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, + user_cpudata, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + } + + ASSERT(cput.thread.used == 1 + || cput.thread.used == cput.logical.used); + ASSERT(cput.core.used == 1 + || cput.core.used == cput.logical.used); + ASSERT(cput.processor_node.used == 1 + || cput.processor_node.used == cput.logical.used); + ASSERT(cput.processor.used == 1 + || cput.processor.used == cput.logical.used); + ASSERT(cput.node.used == 1 + || cput.node.used == cput.logical.used); + + for (r = 0; r < cput.logical.used; r++) { + user_cpudata[ix].logical = cput.logical.id[r]; + user_cpudata[ix].thread = + cput.thread.id[cput.thread.used == 1 ? 0 : r]; + user_cpudata[ix].core = + cput.core.id[cput.core.used == 1 ? 0 : r]; + user_cpudata[ix].processor_node = + cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r]; + user_cpudata[ix].processor = + cput.processor.id[cput.processor.used == 1 ? 0 : r]; + user_cpudata[ix].node = + cput.node.id[cput.node.used == 1 ? 0 : r]; + ix++; + } + } while (*c != '\0'); + + if (user_cpudata_size != ix) { + user_cpudata_size = ix; + user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, + user_cpudata, + (sizeof(erts_cpu_topology_t) + * user_cpudata_size)); + } + + error = verify_topology(user_cpudata, user_cpudata_size); + if (error == ERTS_INIT_CPU_TOPOLOGY_OK) { + destroy_cpu_top_entry(&cput); + return ERTS_INIT_CPU_TOPOLOGY_OK; + } + + fail: + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata_size = 0; + destroy_cpu_top_entry(&cput); + return error; +} + +#define ERTS_GET_CPU_TOPOLOGY_ERROR -1 +#define ERTS_GET_USED_CPU_TOPOLOGY 0 +#define ERTS_GET_DETECTED_CPU_TOPOLOGY 1 +#define ERTS_GET_DEFINED_CPU_TOPOLOGY 2 + +static Eterm get_cpu_topology_term(Process *c_p, int type); + +Eterm +erts_set_cpu_topology(Process *c_p, Eterm term) +{ + erts_cpu_topology_t *cpudata = NULL; + int cpudata_size = 0; + Eterm res; + + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY); + if (term == am_undefined) { + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata = NULL; + user_cpudata_size = 0; + + if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) { + cpudata_size = system_cpudata_size; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + + sys_memcpy((void *) cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*cpudata_size); + } + } + else if (is_not_list(term)) { + error: + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + res = THE_NON_VALUE; + goto done; + } + else { + Eterm list = term; + int ix = 0; + + cpudata_size = 100; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + + while (is_list(list)) { + Eterm *lp = list_val(list); + Eterm cpu = CAR(lp); + Eterm* tp; + Sint id; + + if (is_not_tuple(cpu)) + goto error; + + tp = tuple_val(cpu); + + if (arityval(tp[0]) != 7 || tp[1] != am_cpu) + goto error; + + if (ix >= cpudata_size) { + cpudata_size += 100; + cpudata = erts_realloc(ERTS_ALC_T_TMP, + cpudata, + (sizeof(erts_cpu_topology_t) + * cpudata_size)); + } + + id = signed_val(tp[2]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].node = (int) id; + + id = signed_val(tp[3]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].processor = (int) id; + + id = signed_val(tp[4]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].processor_node = (int) id; + + id = signed_val(tp[5]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].core = (int) id; + + id = signed_val(tp[6]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].thread = (int) id; + + id = signed_val(tp[7]); + if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) + goto error; + cpudata[ix].logical = (int) id; + + list = CDR(lp); + ix++; + } + + if (is_not_nil(list)) + goto error; + + cpudata_size = ix; + + if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size)) + goto error; + + if (user_cpudata_size != cpudata_size) { + if (user_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); + user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + sizeof(erts_cpu_topology_t)*cpudata_size); + user_cpudata_size = cpudata_size; + } + + sys_memcpy((void *) user_cpudata, + (void *) cpudata, + sizeof(erts_cpu_topology_t)*cpudata_size); + } + + update_cpu_groups_maps(); + + write_schedulers_bind_change(cpudata, cpudata_size); + + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_sched_notify_check_cpu_bind(); + + done: + + if (cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); + + return res; +} + +static void +create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size) +{ + if (user_cpudata) { + *cpudata_size = user_cpudata_size; + *cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * (*cpudata_size))); + sys_memcpy((void *) *cpudata, + (void *) user_cpudata, + sizeof(erts_cpu_topology_t)*(*cpudata_size)); + } + else if (system_cpudata) { + *cpudata_size = system_cpudata_size; + *cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * (*cpudata_size))); + sys_memcpy((void *) *cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*(*cpudata_size)); + } + else { + *cpudata = NULL; + *cpudata_size = 0; + } +} + +static void +destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata) +{ + if (cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); +} + + +static Eterm +bld_topology_term(Eterm **hpp, + Uint *hszp, + erts_cpu_topology_t *cpudata, + int size) +{ + Eterm res = NIL; + int i; + + if (size == 0) + return am_undefined; + + for (i = size-1; i >= 0; i--) { + res = erts_bld_cons(hpp, + hszp, + erts_bld_tuple(hpp, + hszp, + 7, + am_cpu, + make_small(cpudata[i].node), + make_small(cpudata[i].processor), + make_small(cpudata[i].processor_node), + make_small(cpudata[i].core), + make_small(cpudata[i].thread), + make_small(cpudata[i].logical)), + res); + } + return res; +} + +static Eterm +get_cpu_topology_term(Process *c_p, int type) +{ +#ifdef DEBUG + Eterm *hp_end; +#endif + Eterm *hp; + Uint hsz; + Eterm res = THE_NON_VALUE; + erts_cpu_topology_t *cpudata = NULL; + int size = 0; + + switch (type) { + case ERTS_GET_USED_CPU_TOPOLOGY: + if (user_cpudata) + goto defined; + else + goto detected; + case ERTS_GET_DETECTED_CPU_TOPOLOGY: + detected: + if (!system_cpudata) + res = am_undefined; + else { + size = system_cpudata_size; + cpudata = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(erts_cpu_topology_t) + * size)); + sys_memcpy((void *) cpudata, + (void *) system_cpudata, + sizeof(erts_cpu_topology_t)*size); + } + break; + case ERTS_GET_DEFINED_CPU_TOPOLOGY: + defined: + if (!user_cpudata) + res = am_undefined; + else { + size = user_cpudata_size; + cpudata = user_cpudata; + } + break; + default: + erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); + break; + } + + if (res == am_undefined) { + ASSERT(!cpudata); + return res; + } + + hsz = 0; + + bld_topology_term(NULL, &hsz, + cpudata, size); + + hp = HAlloc(c_p, hsz); + +#ifdef DEBUG + hp_end = hp + hsz; +#endif + + res = bld_topology_term(&hp, NULL, + cpudata, size); + + ASSERT(hp_end == hp); + + if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata) + erts_free(ERTS_ALC_T_TMP, cpudata); + + return res; +} + +Eterm +erts_get_cpu_topology_term(Process *c_p, Eterm which) +{ + Eterm res; + int type; + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + if (ERTS_IS_ATOM_STR("used", which)) + type = ERTS_GET_USED_CPU_TOPOLOGY; + else if (ERTS_IS_ATOM_STR("detected", which)) + type = ERTS_GET_DETECTED_CPU_TOPOLOGY; + else if (ERTS_IS_ATOM_STR("defined", which)) + type = ERTS_GET_DEFINED_CPU_TOPOLOGY; + else + type = ERTS_GET_CPU_TOPOLOGY_ERROR; + if (type == ERTS_GET_CPU_TOPOLOGY_ERROR) + res = THE_NON_VALUE; + else + res = get_cpu_topology_term(c_p, type); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return res; +} + +static void +get_logical_processors(int *conf, int *onln, int *avail) +{ + if (conf) + *conf = erts_get_cpu_configured(cpuinfo); + if (onln) + *onln = erts_get_cpu_online(cpuinfo); + if (avail) + *avail = erts_get_cpu_available(cpuinfo); +} + +void +erts_get_logical_processors(int *conf, int *onln, int *avail) +{ + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + get_logical_processors(conf, onln, avail); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); +} + +void +erts_pre_early_init_cpu_topology(int *max_rg_p, + int *conf_p, + int *onln_p, + int *avail_p) +{ + cpu_groups_maps = NULL; + no_cpu_groups_callbacks = 0; + *max_rg_p = ERTS_MAX_READER_GROUPS; + cpuinfo = erts_cpu_info_create(); + get_logical_processors(conf_p, onln_p, avail_p); +} + +void +erts_early_init_cpu_topology(int no_schedulers, + int *max_main_threads_p, + int max_reader_groups, + int *reader_groups_p) +{ + user_cpudata = NULL; + user_cpudata_size = 0; + + system_cpudata_size = erts_get_cpu_topology_size(cpuinfo); + system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * system_cpudata_size)); + + cpu_bind_order = ERTS_CPU_BIND_UNDEFINED; + + if (!erts_get_cpu_topology(cpuinfo, system_cpudata) + || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, + system_cpudata_size)) { + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + system_cpudata = NULL; + system_cpudata_size = 0; + } + + max_main_threads = erts_get_cpu_configured(cpuinfo); + if (max_main_threads > no_schedulers) + max_main_threads = no_schedulers; + *max_main_threads_p = max_main_threads; + + reader_groups = max_main_threads; + if (reader_groups <= 1 || max_reader_groups <= 1) + reader_groups = 0; + if (reader_groups > max_reader_groups) + reader_groups = max_reader_groups; + *reader_groups_p = reader_groups; +} + +void +erts_init_cpu_topology(void) +{ + int ix; + + erts_smp_rwmtx_init(&cpuinfo_rwmtx, "cpu_info"); + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + + scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(ErtsCpuBindData) + * (erts_no_schedulers+1))); + for (ix = 1; ix <= erts_no_schedulers; ix++) { + scheduler2cpu_map[ix].bind_id = -1; + scheduler2cpu_map[ix].bound_id = -1; + } + + if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED) { + int ncpus = erts_get_cpu_configured(cpuinfo); + if (ncpus < 1 || erts_no_schedulers < ncpus) + cpu_bind_order = ERTS_CPU_BIND_NONE; + else + cpu_bind_order = ((system_cpudata || user_cpudata) + && (erts_bind_to_cpu(cpuinfo, -1) != -ENOTSUP) + ? ERTS_CPU_BIND_DEFAULT_BIND + : ERTS_CPU_BIND_NONE); + } + + reader_groups_map = add_cpu_groups(reader_groups, + reader_groups_callback, + NULL); + + if (cpu_bind_order == ERTS_CPU_BIND_NONE) + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + else { + erts_cpu_topology_t *cpudata; + int cpudata_size; + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + write_schedulers_bind_change(cpudata, cpudata_size); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + erts_sched_notify_check_cpu_bind(); + destroy_tmp_cpu_topology_copy(cpudata); + } +} + +int +erts_update_cpu_info(void) +{ + int changed; + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + changed = erts_cpu_info_update(cpuinfo); + if (changed) { + erts_cpu_topology_t *cpudata; + int cpudata_size; + + if (system_cpudata) + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + + system_cpudata_size = erts_get_cpu_topology_size(cpuinfo); + if (!system_cpudata_size) + system_cpudata = NULL; + else { + system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, + (sizeof(erts_cpu_topology_t) + * system_cpudata_size)); + + if (!erts_get_cpu_topology(cpuinfo, system_cpudata) + || (ERTS_INIT_CPU_TOPOLOGY_OK + != verify_topology(system_cpudata, + system_cpudata_size))) { + erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); + system_cpudata = NULL; + system_cpudata_size = 0; + } + } + + update_cpu_groups_maps(); + + create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); + write_schedulers_bind_change(cpudata, cpudata_size); + destroy_tmp_cpu_topology_copy(cpudata); + } + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); + if (changed) + erts_sched_notify_check_cpu_bind(); + return changed; +} + +/* + * reader groups map + */ + +void +reader_groups_callback(int suspending, + ErtsSchedulerData *esdp, + int group, + void *unused) +{ + if (reader_groups && esdp->no <= max_main_threads) + erts_smp_rwmtx_set_reader_group(suspending ? 0 : group+1); +} + +static Eterm get_cpu_groups_map(Process *c_p, + erts_cpu_groups_map_t *map, + int offset); +Eterm +erts_debug_reader_groups_map(Process *c_p, int groups) +{ + Eterm res; + erts_cpu_groups_map_t test; + + test.array = NULL; + test.groups = groups; + make_cpu_groups_map(&test, 1); + if (!test.array) + res = NIL; + else { + res = get_cpu_groups_map(c_p, &test, 1); + erts_free(ERTS_ALC_T_TMP, test.array); + } + return res; +} + + +Eterm +erts_get_reader_groups_map(Process *c_p) +{ + Eterm res; + erts_smp_rwmtx_rlock(&cpuinfo_rwmtx); + res = get_cpu_groups_map(c_p, reader_groups_map, 1); + erts_smp_rwmtx_runlock(&cpuinfo_rwmtx); + return res; +} + +/* + * CPU groups + */ + +static Eterm +get_cpu_groups_map(Process *c_p, + erts_cpu_groups_map_t *map, + int offset) +{ +#ifdef DEBUG + Eterm *endp; +#endif + Eterm res = NIL, tuple; + Eterm *hp; + int i; + + hp = HAlloc(c_p, map->logical_processors*(2+3)); +#ifdef DEBUG + endp = hp + map->logical_processors*(2+3); +#endif + for (i = map->size - 1; i >= 0; i--) { + if (map->array[i].logical >= 0) { + tuple = TUPLE2(hp, + make_small(map->array[i].logical), + make_small(map->array[i].cpu_group + offset)); + hp += 3; + res = CONS(hp, tuple, res); + hp += 2; + } + } + ASSERT(hp == endp); + return res; +} + +static void +make_available_cpu_topology(erts_avail_cput *no, + erts_avail_cput *avail, + erts_cpu_topology_t *cpudata, + int *size, + int test) +{ + int len = *size; + erts_cpu_topology_t last; + int a, i, j; + + no->level[ERTS_TOPOLOGY_NODE] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR] = -1; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE] = -1; + no->level[ERTS_TOPOLOGY_CORE] = -1; + no->level[ERTS_TOPOLOGY_THREAD] = -1; + no->level[ERTS_TOPOLOGY_LOGICAL] = -1; + + last.node = INT_MIN; + last.processor = INT_MIN; + last.processor_node = INT_MIN; + last.core = INT_MIN; + last.thread = INT_MIN; + last.logical = INT_MIN; + + a = 0; + + for (i = 0; i < len; i++) { + + if (!test && !erts_is_cpu_available(cpuinfo, cpudata[i].logical)) + continue; + + if (last.node != cpudata[i].node) + goto node; + if (last.processor != cpudata[i].processor) + goto processor; + if (last.processor_node != cpudata[i].processor_node) + goto processor_node; + if (last.core != cpudata[i].core) + goto core; + ASSERT(last.thread != cpudata[i].thread); + goto thread; + + node: + no->level[ERTS_TOPOLOGY_NODE]++; + processor: + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + processor_node: + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + core: + no->level[ERTS_TOPOLOGY_CORE]++; + thread: + no->level[ERTS_TOPOLOGY_THREAD]++; + + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + for (j = 0; j < ERTS_TOPOLOGY_LOGICAL; j++) + avail[a].level[j] = no->level[j]; + + avail[a].level[ERTS_TOPOLOGY_LOGICAL] = cpudata[i].logical; + avail[a].level[ERTS_TOPOLOGY_CG] = 0; + + ASSERT(last.logical != cpudata[i].logical); + + last = cpudata[i]; + a++; + } + + no->level[ERTS_TOPOLOGY_NODE]++; + no->level[ERTS_TOPOLOGY_PROCESSOR]++; + no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; + no->level[ERTS_TOPOLOGY_CORE]++; + no->level[ERTS_TOPOLOGY_THREAD]++; + no->level[ERTS_TOPOLOGY_LOGICAL]++; + + *size = a; +} + +static void +cpu_group_insert(erts_cpu_groups_map_t *map, + int logical, int cpu_group) +{ + int start = logical % map->size; + int ix = start; + + do { + if (map->array[ix].logical < 0) { + map->array[ix].logical = logical; + map->array[ix].cpu_group = cpu_group; + return; + } + ix++; + if (ix == map->size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); +} + + +static int +sub_levels(erts_cpu_groups_count_t *cgc, int level, int aix, + int avail_sz, erts_avail_cput *avail) +{ + int sub_level = level+1; + int last = -1; + cgc->sub_levels = 0; + + do { + if (last != avail[aix].level[sub_level]) { + cgc->sub_levels++; + last = avail[aix].level[sub_level]; + } + aix++; + } + while (aix < avail_sz && cgc->id == avail[aix].level[level]); + cgc->cpu_groups = 0; + return aix; +} + +static int +write_cpu_groups(int *cgp, erts_cpu_groups_count_t *cgcp, + int level, int a, + int avail_sz, erts_avail_cput *avail) +{ + int cg = *cgp; + int sub_level = level+1; + int sl_per_gr = cgcp->sub_levels / cgcp->cpu_groups; + int xsl = cgcp->sub_levels % cgcp->cpu_groups; + int sls = 0; + int last = -1; + int xsl_cg_lim = (cgcp->cpu_groups - xsl) + cg + 1; + + ASSERT(level < 0 || avail[a].level[level] == cgcp->id); + + do { + if (last != avail[a].level[sub_level]) { + if (!sls) { + sls = sl_per_gr; + cg++; + if (cg >= xsl_cg_lim) + sls++; + } + last = avail[a].level[sub_level]; + sls--; + } + avail[a].level[ERTS_TOPOLOGY_CG] = cg; + a++; + } while (a < avail_sz && (level < 0 + || avail[a].level[level] == cgcp->id)); + + ASSERT(cgcp->cpu_groups == cg - *cgp); + + *cgp = cg; + + return a; +} + +static int +cg_count_sub_levels_compare(const void *vx, const void *vy) +{ + erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx; + erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy; + if (x->sub_levels != y->sub_levels) + return y->sub_levels - x->sub_levels; + return x->id - y->id; +} + +static int +cg_count_id_compare(const void *vx, const void *vy) +{ + erts_cpu_groups_count_t *x = (erts_cpu_groups_count_t *) vx; + erts_cpu_groups_count_t *y = (erts_cpu_groups_count_t *) vy; + return x->id - y->id; +} + +static void +make_cpu_groups_map(erts_cpu_groups_map_t *map, int test) +{ + int i, spread_level, avail_sz; + erts_avail_cput no, *avail; + erts_cpu_topology_t *cpudata; + ErtsAlcType_t alc_type = (test + ? ERTS_ALC_T_TMP + : ERTS_ALC_T_CPU_GRPS_MAP); + + if (map->array) + erts_free(alc_type, map->array); + + map->array = NULL; + map->logical_processors = 0; + map->size = 0; + + if (!map->groups) + return; + + create_tmp_cpu_topology_copy(&cpudata, &avail_sz); + + if (!cpudata) + return; + + cpu_bind_order_sort(cpudata, + avail_sz, + ERTS_CPU_BIND_NO_SPREAD, + 1); + + avail = erts_alloc(ERTS_ALC_T_TMP, + sizeof(erts_avail_cput)*avail_sz); + + make_available_cpu_topology(&no, avail, cpudata, + &avail_sz, test); + + destroy_tmp_cpu_topology_copy(cpudata); + + map->size = avail_sz*2+1; + + map->array = erts_alloc(alc_type, + (sizeof(erts_cpu_groups_map_array_t) + * map->size));; + map->logical_processors = avail_sz; + + for (i = 0; i < map->size; i++) { + map->array[i].logical = -1; + map->array[i].cpu_group = -1; + } + + spread_level = ERTS_TOPOLOGY_CORE; + for (i = ERTS_TOPOLOGY_NODE; i < ERTS_TOPOLOGY_THREAD; i++) { + if (no.level[i] > map->groups) { + spread_level = i; + break; + } + } + + if (no.level[spread_level] <= map->groups) { + int a, cg, last = -1; + cg = -1; + ASSERT(spread_level == ERTS_TOPOLOGY_CORE); + for (a = 0; a < avail_sz; a++) { + if (last != avail[a].level[spread_level]) { + cg++; + last = avail[a].level[spread_level]; + } + cpu_group_insert(map, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + cg); + } + } + else { /* map->groups < no.level[spread_level] */ + erts_cpu_groups_count_t *cg_count; + int a, cg, tl, toplevels; + + tl = spread_level-1; + + if (spread_level == ERTS_TOPOLOGY_NODE) + toplevels = 1; + else + toplevels = no.level[tl]; + + cg_count = erts_alloc(ERTS_ALC_T_TMP, + toplevels*sizeof(erts_cpu_groups_count_t)); + + if (toplevels == 1) { + cg_count[0].id = 0; + cg_count[0].sub_levels = no.level[spread_level]; + cg_count[0].cpu_groups = map->groups; + } + else { + int cgs_per_tl, cgs; + cgs = map->groups; + cgs_per_tl = cgs / toplevels; + + a = 0; + for (i = 0; i < toplevels; i++) { + cg_count[i].id = avail[a].level[tl]; + a = sub_levels(&cg_count[i], tl, a, avail_sz, avail); + } + + qsort(cg_count, + toplevels, + sizeof(erts_cpu_groups_count_t), + cg_count_sub_levels_compare); + + for (i = 0; i < toplevels; i++) { + if (cg_count[i].sub_levels < cgs_per_tl) { + cg_count[i].cpu_groups = cg_count[i].sub_levels; + cgs -= cg_count[i].sub_levels; + } + else { + cg_count[i].cpu_groups = cgs_per_tl; + cgs -= cgs_per_tl; + } + } + + while (cgs > 0) { + for (i = 0; i < toplevels; i++) { + if (cg_count[i].sub_levels == cg_count[i].cpu_groups) + break; + else { + cg_count[i].cpu_groups++; + if (--cgs == 0) + break; + } + } + } + + qsort(cg_count, + toplevels, + sizeof(erts_cpu_groups_count_t), + cg_count_id_compare); + } + + a = i = 0; + cg = -1; + while (a < avail_sz) { + a = write_cpu_groups(&cg, &cg_count[i], tl, + a, avail_sz, avail); + i++; + } + + ASSERT(map->groups == cg + 1); + + for (a = 0; a < avail_sz; a++) + cpu_group_insert(map, + avail[a].level[ERTS_TOPOLOGY_LOGICAL], + avail[a].level[ERTS_TOPOLOGY_CG]); + + erts_free(ERTS_ALC_T_TMP, cg_count); + } + + erts_free(ERTS_ALC_T_TMP, avail); +} + +static erts_cpu_groups_map_t * +add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg) +{ + int use_groups = groups; + erts_cpu_groups_callback_list_t *cgcl; + erts_cpu_groups_map_t *cgm; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + if (use_groups > max_main_threads) + use_groups = max_main_threads; + + if (!use_groups) + return NULL; + + no_cpu_groups_callbacks++; + cgcl = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP, + sizeof(erts_cpu_groups_callback_list_t)); + cgcl->callback = callback; + cgcl->arg = arg; + + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + if (cgm->groups == use_groups) { + cgcl->next = cgm->callback_list; + cgm->callback_list = cgcl; + return cgm; + } + } + + + cgm = erts_alloc(ERTS_ALC_T_CPU_GRPS_MAP, + sizeof(erts_cpu_groups_map_t)); + cgm->next = cpu_groups_maps; + cgm->groups = use_groups; + cgm->array = NULL; + cgm->size = 0; + cgm->logical_processors = 0; + cgm->callback_list = cgcl; + + cgcl->next = NULL; + + make_cpu_groups_map(cgm, 0); + + cpu_groups_maps = cgm; + + return cgm; +} + +static void +remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg) +{ + erts_cpu_groups_map_t *prev_cgm, *cgm; + erts_cpu_groups_callback_list_t *prev_cgcl, *cgcl; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + no_cpu_groups_callbacks--; + + prev_cgm = NULL; + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) { + prev_cgcl = NULL; + for (cgcl = cgm->callback_list; cgcl; cgcl = cgcl->next) { + if (cgcl->callback == callback && cgcl->arg == arg) { + if (prev_cgcl) + prev_cgcl->next = cgcl->next; + else + cgm->callback_list = cgcl->next; + erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgcl); + if (!cgm->callback_list) { + if (prev_cgm) + prev_cgm->next = cgm->next; + else + cpu_groups_maps = cgm->next; + if (cgm->array) + erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm->array); + erts_free(ERTS_ALC_T_CPU_GRPS_MAP, cgm); + } + return; + } + prev_cgcl = cgcl; + } + prev_cgm = cgm; + } + + erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); +} + +static int +cpu_groups_lookup(erts_cpu_groups_map_t *map, + ErtsSchedulerData *esdp) +{ + int start, logical, ix; + + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rlocked(&cpuinfo_rwmtx) + || erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + if (esdp->cpu_id < 0) + return (((int) esdp->no) - 1) % map->groups; + + logical = esdp->cpu_id; + start = logical % map->size; + ix = start; + + do { + if (map->array[ix].logical == logical) { + int group = map->array[ix].cpu_group; + ASSERT(0 <= group && group < map->groups); + return group; + } + ix++; + if (ix == map->size) + ix = 0; + } while (ix != start); + + erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); +} + +static void +update_cpu_groups_maps(void) +{ + erts_cpu_groups_map_t *cgm; + ERTS_SMP_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&cpuinfo_rwmtx)); + + for (cgm = cpu_groups_maps; cgm; cgm = cgm->next) + make_cpu_groups_map(cgm, 0); +} + +void +erts_add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg) +{ + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + add_cpu_groups(groups, callback, arg); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); +} + +void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback, + void *arg) +{ + erts_smp_rwmtx_rwlock(&cpuinfo_rwmtx); + remove_cpu_groups(callback, arg); + erts_smp_rwmtx_rwunlock(&cpuinfo_rwmtx); +} diff --git a/erts/emulator/beam/erl_cpu_topology.h b/erts/emulator/beam/erl_cpu_topology.h new file mode 100644 index 0000000000..c5a9520b61 --- /dev/null +++ b/erts/emulator/beam/erl_cpu_topology.h @@ -0,0 +1,105 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: CPU topology and related functionality + * + * Author: Rickard Green + */ + +#ifndef ERL_CPU_TOPOLOGY_H__ +#define ERL_CPU_TOPOLOGY_H__ + +void erts_pre_early_init_cpu_topology(int *max_rg_p, + int *conf_p, + int *onln_p, + int *avail_p); +void erts_early_init_cpu_topology(int no_schedulers, + int *max_main_threads_p, + int max_reader_groups, + int *reader_groups_p); +void erts_init_cpu_topology(void); + + +#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS 0 +#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED 1 +#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY 2 +#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE 3 + +int erts_init_scheduler_bind_type_string(char *how); + + +#define ERTS_INIT_CPU_TOPOLOGY_OK 0 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID 1 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE 2 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY 3 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE 4 +#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES 5 +#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID 6 +#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS 7 +#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES 8 +#define ERTS_INIT_CPU_TOPOLOGY_MISSING 9 + +int erts_init_cpu_topology_string(char *topology_str); + +void erts_sched_check_cpu_bind(ErtsSchedulerData *esdp); +#ifdef ERTS_SMP +void erts_sched_init_check_cpu_bind(ErtsSchedulerData *esdp); +void erts_sched_check_cpu_bind_prep_suspend(ErtsSchedulerData *esdp); +void erts_sched_check_cpu_bind_post_suspend(ErtsSchedulerData *esdp); +#endif + +int erts_update_cpu_info(void); + +Eterm erts_bind_schedulers(Process *c_p, Eterm how); +Eterm erts_get_schedulers_binds(Process *c_p); + +Eterm erts_get_reader_groups_map(Process *c_p); + +Eterm erts_set_cpu_topology(Process *c_p, Eterm term); +Eterm erts_get_cpu_topology_term(Process *c_p, Eterm which); + +int erts_update_cpu_info(void); +void erts_get_logical_processors(int *conf, int *onln, int *avail); + +int erts_sched_bind_atthrcreate_prepare(void); +int erts_sched_bind_atthrcreate_child(int unbind); +void erts_sched_bind_atthrcreate_parent(int unbind); + +int erts_sched_bind_atfork_prepare(void); +int erts_sched_bind_atfork_child(int unbind); +char *erts_sched_bind_atvfork_child(int unbind); +void erts_sched_bind_atfork_parent(int unbind); + +Eterm erts_fake_scheduler_bindings(Process *p, Eterm how); +Eterm erts_debug_cpu_groups_map(Process *c_p, int groups); + + +typedef void (*erts_cpu_groups_callback_t)(int, + ErtsSchedulerData *, + int, + void *); + +void erts_add_cpu_groups(int groups, + erts_cpu_groups_callback_t callback, + void *arg); +void erts_remove_cpu_groups(erts_cpu_groups_callback_t callback, + void *arg); + +#endif diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 1c2c0fe4cb..61e8a595be 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -179,6 +179,7 @@ extern DbTableMethod db_tree; int user_requested_db_max_tabs; int erts_ets_realloc_always_moves; +int erts_ets_always_compress; static int db_max_tabs; static DbTable *meta_pid_to_tab; /* Pid mapped to owned tables */ static DbTable *meta_pid_to_fixed_tab; /* Pid mapped to fixed tables */ @@ -218,47 +219,68 @@ Export ets_select_continue_exp; * Static traps */ static Export ets_delete_continue_exp; - -static ERTS_INLINE DbTable* db_ref(DbTable* tb, db_lock_kind_t kind) -{ - if (tb != NULL && kind != LCK_READ) { - erts_refc_inc(&tb->common.ref, 2); - } - return tb; -} - -static ERTS_INLINE DbTable* db_unref(DbTable* tb, db_lock_kind_t kind) + +static void +free_dbtable(DbTable* tb) { - if (kind != LCK_READ && !erts_refc_dectest(&tb->common.ref, 0)) { #ifdef HARDDEBUG if (erts_smp_atomic_read(&tb->common.memory_size) != sizeof(DbTable)) { - erts_fprintf(stderr, "ets: db_unref memory remain=%ld fix=%x\n", - erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), + erts_fprintf(stderr, "ets: free_dbtable memory remain=%ld fix=%x\n", + erts_smp_atomic_read(&tb->common.memory_size)-sizeof(DbTable), tb->common.fixations); } - erts_fprintf(stderr, "ets: db_unref(%T) deleted!!!\r\n", + erts_fprintf(stderr, "ets: free_dbtable(%T) deleted!!!\r\n", tb->common.id); - erts_fprintf(stderr, "ets: db_unref: meta_pid_to_tab common.memory_size = %ld\n", + erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_tab common.memory_size = %ld\n", erts_smp_atomic_read(&meta_pid_to_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_tab); - erts_fprintf(stderr, "ets: db_unref: meta_pid_to_fixed_tab common.memory_size = %ld\n", + erts_fprintf(stderr, "ets: free_dbtable: meta_pid_to_fixed_tab common.memory_size = %ld\n", erts_smp_atomic_read(&meta_pid_to_fixed_tab->common.memory_size)); print_table(ERTS_PRINT_STDOUT, NULL, 1, meta_pid_to_fixed_tab); - #endif #ifdef ERTS_SMP erts_smp_rwmtx_destroy(&tb->common.rwlock); erts_smp_mtx_destroy(&tb->common.fixlock); #endif ASSERT(is_immed(tb->common.heir_data)); - erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); + erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); - return NULL; - } - return tb; +} + +#ifdef ERTS_SMP +static void +chk_free_dbtable(void *vtb) +{ + DbTable * tb = (DbTable *) vtb; + ERTS_THR_MEMORY_BARRIER; + if (erts_refc_dectest(&tb->common.ref, 0) == 0) + free_dbtable(tb); +} +#endif + +static void schedule_free_dbtable(DbTable* tb) +{ + /* + * NON-SMP case: Caller is *not* allowed to access the *tb + * structure after this function has returned! + * SMP case: Caller is allowed to access the *tb structure + * until the bif has returned (we typically + * need to unlock the table lock after this + * function has returned). + */ +#ifdef ERTS_SMP + int scheds = erts_get_max_no_executing_schedulers(); + ASSERT(scheds >= 1); + ASSERT(erts_refc_read(&tb->common.ref, 0) == 0); + erts_refc_init(&tb->common.ref, scheds); + ERTS_THR_MEMORY_BARRIER; + erts_smp_schedule_misc_aux_work(0, scheds, chk_free_dbtable, tb); +#else + free_dbtable(tb); +#endif } static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, @@ -269,8 +291,6 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, if (use_frequent_read_lock) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; #endif - erts_refc_init(&tb->common.ref, 1); - erts_refc_init(&tb->common.fixref, 0); #ifdef ERTS_SMP erts_smp_rwmtx_init_opt_x(&tb->common.rwlock, &rwmtx_opt, rwname, tb->common.the_name); @@ -279,7 +299,7 @@ static ERTS_INLINE void db_init_lock(DbTable* tb, int use_frequent_read_lock, #endif } -static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind) +static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) { #ifdef ERTS_SMP ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); @@ -307,16 +327,13 @@ static ERTS_INLINE void db_lock_take_over_ref(DbTable* tb, db_lock_kind_t kind) #endif } -static ERTS_INLINE void db_lock(DbTable* tb, db_lock_kind_t kind) -{ - (void) db_ref(tb, kind); -#ifdef ERTS_SMP - db_lock_take_over_ref(tb, kind); -#endif -} - static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) { + /* + * In NON-SMP case tb may refer to an already deallocated + * DbTable structure. That is, ONLY the SMP case is allowed + * to follow the tb pointer! + */ #ifdef ERTS_SMP ASSERT(tb != meta_pid_to_tab && tb != meta_pid_to_fixed_tab); @@ -343,7 +360,6 @@ static ERTS_INLINE void db_unlock(DbTable* tb, db_lock_kind_t kind) } } #endif - (void) db_unref(tb, kind); /* May delete table... */ } @@ -370,6 +386,13 @@ DbTable* db_get_table_aux(Process *p, DbTable *tb = NULL; erts_smp_rwmtx_t *mtl = NULL; + /* + * IMPORTANT: Only scheduler threads are allowed + * to access tables. Memory management + * depend on it. + */ + ASSERT(erts_get_scheduler_data()); + if (is_small(id)) { Uint slot = unsigned_val(id) & meta_main_tab_slot_mask; if (!meta_already_locked) { @@ -383,12 +406,8 @@ DbTable* db_get_table_aux(Process *p, || erts_lc_rwmtx_is_rwlocked(test_mtl)); } #endif - if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) { - /* SMP: inc to prevent race, between unlock of meta_main_tab_lock - * and the table locking outside the meta_main_tab_lock - */ - tb = db_ref(meta_main_tab[slot].u.tb, kind); - } + if (slot < db_max_tabs && IS_SLOT_ALIVE(slot)) + tb = meta_main_tab[slot].u.tb; } else if (is_atom(id)) { struct meta_name_tab_entry* bucket = meta_name_tab_bucket(id,&mtl); @@ -402,16 +421,15 @@ DbTable* db_get_table_aux(Process *p, if (bucket->pu.tb != NULL) { if (is_atom(bucket->u.name_atom)) { /* single */ - if (bucket->u.name_atom == id) { - tb = db_ref(bucket->pu.tb, kind); - } + if (bucket->u.name_atom == id) + tb = bucket->pu.tb; } else { /* multi */ Uint cnt = unsigned_val(bucket->u.mcnt); Uint i; for (i=0; i<cnt; i++) { if (bucket->pu.mvec[i].u.name_atom == id) { - tb = db_ref(bucket->pu.mvec[i].pu.tb, kind); + tb = bucket->pu.mvec[i].pu.tb; break; } } @@ -419,7 +437,7 @@ DbTable* db_get_table_aux(Process *p, } } if (tb) { - db_lock_take_over_ref(tb, kind); + db_lock(tb, kind); if (tb->common.id != id || ((tb->common.status & what) == 0 && p->id != tb->common.owner)) { db_unlock(tb, kind); @@ -593,11 +611,11 @@ done: */ static ERTS_INLINE void local_fix_table(DbTable* tb) { - erts_refc_inc(&tb->common.fixref, 1); + erts_refc_inc(&tb->common.ref, 1); } static ERTS_INLINE void local_unfix_table(DbTable* tb) { - if (erts_refc_dectest(&tb->common.fixref, 0) == 0) { + if (erts_refc_dectest(&tb->common.ref, 0) == 0) { ASSERT(IS_HASH_TABLE(tb->common.status)); db_unfix_table_hash(&(tb->hash)); } @@ -892,7 +910,8 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm upop; Eterm* tpl; Sint position; - Eterm incr, warp, oldcnt; + Eterm incr, warp; + Wterm oldcnt; if (is_not_list(iter)) { goto finalize; @@ -931,7 +950,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) position > arityval(handle.dbterm->tpl[0])) { goto finalize; } - oldcnt = handle.dbterm->tpl[position]; + oldcnt = db_do_read_element(&handle, position); if (is_big(oldcnt)) { halloc_size += BIG_NEED_SIZE(big_arity(oldcnt)); } @@ -967,7 +986,7 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) Eterm* tpl = tuple_val(CAR(list_val(iter))); Sint position = signed_val(tpl[1]); Eterm incr = tpl[2]; - Eterm oldcnt = handle.dbterm->tpl[position]; + Wterm oldcnt = db_do_read_element(&handle,position); Eterm newcnt = db_add_counter(&htop, oldcnt, incr); if (newcnt == NIL) { @@ -980,9 +999,9 @@ BIF_RETTYPE ets_update_counter_3(BIF_ALIST_3) if (arityval(*tpl) == 4) { /* Maybe warp it */ Eterm threshold = tpl[3]; - if ((cmp(incr,make_small(0)) < 0) ? /* negative increment? */ - (cmp(newcnt,threshold) < 0) : /* if negative, check if below */ - (cmp(newcnt,threshold) > 0)) { /* else check if above threshold */ + if ((CMP(incr,make_small(0)) < 0) ? /* negative increment? */ + (CMP(newcnt,threshold) < 0) : /* if negative, check if below */ + (CMP(newcnt,threshold) > 0)) { /* else check if above threshold */ newcnt = tpl[4]; } @@ -1276,7 +1295,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) UWord heir_data; Uint32 status; Sint keypos; - int is_named, is_fine_locked, frequent_read; + int is_named, is_fine_locked, frequent_read, is_compressed; int cret; DeclareTmpHeap(meta_tuple,3,BIF_P); DbTableMethod* meth; @@ -1296,6 +1315,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) frequent_read = 0; heir = am_none; heir_data = (UWord) am_undefined; + is_compressed = erts_ets_always_compress; list = BIF_ARG_2; while(is_list(list)) { @@ -1358,6 +1378,9 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) else if (val == am_named_table) { is_named = 1; } + else if (val == am_compressed) { + is_compressed = 1; + } else if (val == am_set || val == am_protected) ; else break; @@ -1409,6 +1432,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) tb->common.type = status & ERTS_ETS_TABLE_TYPES; /* Note, 'type' is *read only* from now on... */ #endif + erts_refc_init(&tb->common.ref, 0); db_init_lock(tb, status & (DB_FINE_LOCKED|DB_FREQ_READ), "db_tab", "db_tab_fix"); tb->common.keypos = keypos; @@ -1418,6 +1442,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) erts_smp_atomic_init(&tb->common.nitems, 0); tb->common.fixations = NULL; + tb->common.compress = is_compressed; cret = meth->db_create(BIF_P, tb); ASSERT(cret == DB_ERROR_NONE); @@ -1430,8 +1455,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) "** Too many db tables **\n"); free_heir_data(tb); tb->common.meth->db_free_table(tb); - erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); - ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); + free_dbtable(tb); BIF_ERROR(BIF_P, SYSTEM_LIMIT); } @@ -1465,9 +1489,10 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) free_slot(slot); erts_smp_rwmtx_rwunlock(mmtl); - db_lock_take_over_ref(tb,LCK_WRITE); + db_lock(tb,LCK_WRITE); free_heir_data(tb); tb->common.meth->db_free_table(tb); + schedule_free_dbtable(tb); db_unlock(tb,LCK_WRITE); BIF_ERROR(BIF_P, BADARG); } @@ -2572,7 +2597,7 @@ BIF_RETTYPE ets_match_object_3(BIF_ALIST_3) BIF_RETTYPE ets_info_1(BIF_ALIST_1) { static Eterm fields[] = {am_protection, am_keypos, am_type, am_named_table, - am_node, am_size, am_name, am_heir, am_owner, am_memory}; + am_node, am_size, am_name, am_heir, am_owner, am_memory, am_compressed}; Eterm results[sizeof(fields)/sizeof(Eterm)]; DbTable* tb; Eterm res; @@ -2688,7 +2713,6 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) Binary *mp; Eterm res; Uint32 dummy; - Uint sz; if (!(is_list(BIF_ARG_1) || BIF_ARG_1 == NIL) || !is_binary(BIF_ARG_2)) { error: @@ -2713,11 +2737,10 @@ BIF_RETTYPE ets_match_spec_run_r_3(BIF_ALIST_3) BIF_TRAP3(bif_export[BIF_ets_match_spec_run_r_3], BIF_P,lst,BIF_ARG_2,ret); } - res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, 0, &dummy); + res = db_prog_match(BIF_P, mp, CAR(list_val(lst)), NULL, NULL, 0, + ERTS_PAM_COPY_RESULT, &dummy); if (is_value(res)) { - sz = size_object(res); - hp = HAlloc(BIF_P, sz + 2); - res = copy_struct(res, sz, &hp, &MSO(BIF_P)); + hp = HAlloc(BIF_P, 2); ret = CONS(hp,res,ret); /*hp += 2;*/ } @@ -2750,17 +2773,10 @@ void init_db(void) rwmtx_opt.type = ERTS_SMP_RWMTX_TYPE_FREQUENT_READ; rwmtx_opt.lived = ERTS_SMP_RWMTX_LONG_LIVED; - meta_main_tab_locks = erts_alloc(ERTS_ALC_T_DB_TABLES, - (sizeof(erts_meta_main_tab_lock_t) - * (ERTS_META_MAIN_TAB_LOCK_TAB_SIZE+1))); - - if ((((UWord) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) != 0) - meta_main_tab_locks = ((erts_meta_main_tab_lock_t *) - ((((UWord) meta_main_tab_locks) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); - - ASSERT((((UWord) meta_main_tab_locks) & ERTS_CACHE_LINE_MASK) == 0); + meta_main_tab_locks = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_DB_TABLES, + sizeof(erts_meta_main_tab_lock_t) + * ERTS_META_MAIN_TAB_LOCK_TAB_SIZE); for (i = 0; i < ERTS_META_MAIN_TAB_LOCK_TAB_SIZE; i++) { erts_smp_rwmtx_init_opt_x(&meta_main_tab_locks[i].rwmtx, &rwmtx_opt, @@ -2837,9 +2853,9 @@ void init_db(void) erts_smp_atomic_init(&meta_pid_to_tab->common.nitems, 0); meta_pid_to_tab->common.slot = -1; meta_pid_to_tab->common.meth = &db_hash; + meta_pid_to_tab->common.compress = 0; - erts_refc_init(&meta_pid_to_tab->common.ref, 1); - erts_refc_init(&meta_pid_to_tab->common.fixref, 0); + erts_refc_init(&meta_pid_to_tab->common.ref, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ @@ -2869,9 +2885,9 @@ void init_db(void) erts_smp_atomic_init(&meta_pid_to_fixed_tab->common.nitems, 0); meta_pid_to_fixed_tab->common.slot = -1; meta_pid_to_fixed_tab->common.meth = &db_hash; + meta_pid_to_fixed_tab->common.compress = 0; - erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 1); - erts_refc_init(&meta_pid_to_fixed_tab->common.fixref, 0); + erts_refc_init(&meta_pid_to_fixed_tab->common.ref, 0); /* Neither rwlock or fixlock used db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ @@ -3029,12 +3045,10 @@ retry: to_pid, to_locks, ERTS_P2P_FLG_TRY_LOCK); if (to_proc == ERTS_PROC_LOCK_BUSY) { - db_ref(tb, LCK_NONE); /* while unlocked */ db_unlock(tb,LCK_WRITE); to_proc = erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, to_pid, to_locks); db_lock(tb,LCK_WRITE); - tb = db_unref(tb, LCK_NONE); ASSERT(tb != NULL); if (tb->common.owner != p->id) { @@ -3077,7 +3091,7 @@ retry: db_unlock(tb,LCK_WRITE); heir_data = tb->common.heir_data; if (!is_immed(heir_data)) { - Eterm* tpv = DBTERM_BUF((DbTerm*)heir_data); /* tuple_val */ + Eterm* tpv = ((DbTerm*)heir_data)->tpl; /* tuple_val */ ASSERT(arityval(*tpv) == 1); heir_data = tpv[1]; } @@ -3145,13 +3159,13 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); erts_smp_rwmtx_rlock(mmtl); if (!IS_SLOT_FREE(ix)) { - tb = db_ref(GET_ANY_SLOT_TAB(ix), LCK_WRITE); + tb = GET_ANY_SLOT_TAB(ix); ASSERT(tb); } erts_smp_rwmtx_runlock(mmtl); if (tb) { int do_yield; - db_lock_take_over_ref(tb, LCK_WRITE); + db_lock(tb, LCK_WRITE); /* Ownership may have changed since we looked up the table. */ if (tb->common.owner != pid) { @@ -3233,7 +3247,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) erts_smp_rwmtx_t *mmtl = get_meta_main_tab_lock(ix); erts_smp_rwmtx_rlock(mmtl); if (IS_SLOT_ALIVE(ix)) { - tb = db_ref(meta_main_tab[ix].u.tb, LCK_WRITE_REC); + tb = meta_main_tab[ix].u.tb; ASSERT(tb); } erts_smp_rwmtx_runlock(mmtl); @@ -3241,7 +3255,7 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) int reds; DbFixation** pp; - db_lock_take_over_ref(tb, LCK_WRITE_REC); + db_lock(tb, LCK_WRITE_REC); #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif @@ -3251,7 +3265,8 @@ erts_db_process_exiting(Process *c_p, ErtsProcLocks c_p_locks) pp = &(*pp)->next) { if ((*pp)->pid == pid) { DbFixation* fix = *pp; - erts_refc_add(&tb->common.fixref,-fix->counter,0); + erts_aint_t diff = -((erts_aint_t) fix->counter); + erts_refc_add(&tb->common.ref,diff,0); *pp = fix->next; erts_db_free(ERTS_ALC_T_DB_FIXATION, tb, fix, sizeof(DbFixation)); @@ -3326,7 +3341,7 @@ static void fix_table_locked(Process* p, DbTable* tb) #ifdef ERTS_SMP erts_smp_mtx_lock(&tb->common.fixlock); #endif - erts_refc_inc(&tb->common.fixref,1); + erts_refc_inc(&tb->common.ref,1); fix = tb->common.fixations; if (fix == NULL) { get_now(&(tb->common.megasec), @@ -3380,7 +3395,7 @@ static void unfix_table_locked(Process* p, DbTable* tb, for (pp = &tb->common.fixations; *pp != NULL; pp = &(*pp)->next) { if ((*pp)->pid == p->id) { DbFixation* fix = *pp; - erts_refc_dec(&tb->common.fixref,0); + erts_refc_dec(&tb->common.ref,0); --(fix->counter); ASSERT(fix->counter >= 0); if (fix->counter > 0) { @@ -3406,11 +3421,10 @@ static void unfix_table_locked(Process* p, DbTable* tb, unlocked: if (!IS_FIXED(tb) && IS_HASH_TABLE(tb->common.status) - && erts_smp_atomic_read(&tb->hash.fixdel) != (long)NULL) { + && erts_smp_atomic_read(&tb->hash.fixdel) != (erts_aint_t)NULL) { #ifdef ERTS_SMP if (*kind_p == LCK_READ && tb->common.is_thread_safe) { /* Must have write lock while purging pseudo-deleted (OTP-8166) */ - db_ref(tb, LCK_WRITE); /* LCK_WRITE need it, but not LCK_READ */ erts_smp_rwmtx_runlock(&tb->common.rwlock); erts_smp_rwmtx_rwlock(&tb->common.rwlock); *kind_p = LCK_WRITE; @@ -3429,6 +3443,8 @@ static void free_fixations_locked(DbTable *tb) fix = tb->common.fixations; while (fix != NULL) { + erts_aint_t diff = -((erts_aint_t) fix->counter); + erts_refc_add(&tb->common.ref,diff,0); next_fix = fix->next; db_meta_lock(meta_pid_to_fixed_tab, LCK_WRITE_REC); db_erase_bag_exact2(meta_pid_to_fixed_tab, @@ -3466,11 +3482,24 @@ static void set_heir(Process* me, DbTable* tb, Eterm heir, UWord heir_data) if (!is_immed(heir_data)) { DeclareTmpHeap(tmp,2,me); + Eterm wrap_tpl; + int size; + DbTerm* dbterm; + Eterm* top; + ErlOffHeap tmp_offheap; UseTmpHeap(2,me); - /* Make a dummy 1-tuple around data to use db_get_term() */ - heir_data = (UWord) db_get_term(&tb->common, NULL, 0, - TUPLE1(tmp,heir_data)); + /* Make a dummy 1-tuple around data to use DbTerm */ + wrap_tpl = TUPLE1(tmp,heir_data); + size = size_object(wrap_tpl); + dbterm = erts_db_alloc(ERTS_ALC_T_DB_HEIR_DATA, (DbTable *)tb, + (sizeof(DbTerm) + sizeof(Eterm)*(size-1))); + dbterm->size = size; + top = dbterm->tpl; + tmp_offheap.first = NULL; + copy_struct(wrap_tpl, size, &top, &tmp_offheap); + dbterm->first_oh = tmp_offheap.first; + heir_data = (UWord)dbterm; UnUseTmpHeap(2,me); ASSERT(!is_immed(heir_data)); } @@ -3481,8 +3510,8 @@ static void free_heir_data(DbTable* tb) { if (tb->common.heir != am_none && !is_immed(tb->common.heir_data)) { DbTerm* p = (DbTerm*) tb->common.heir_data; - db_free_term_data(p); - erts_db_free(ERTS_ALC_T_DB_TERM, tb, (void *)p, + db_cleanup_offheap_comp(p); + erts_db_free(ERTS_ALC_T_DB_HEIR_DATA, tb, (void *)p, sizeof(DbTerm) + (p->size-1)*sizeof(Eterm)); } #ifdef DEBUG @@ -3552,10 +3581,6 @@ static int free_table_cont(Process *p, mmtl = get_meta_main_tab_lock(tb->common.slot); #ifdef ERTS_SMP if (erts_smp_rwmtx_tryrwlock(mmtl) == EBUSY) { - /* - * We keep our increased refc over this op in order to - * prevent the table from disapearing. - */ erts_smp_rwmtx_rwunlock(&tb->common.rwlock); erts_smp_rwmtx_rwlock(mmtl); erts_smp_rwmtx_rwlock(&tb->common.rwlock); @@ -3570,7 +3595,7 @@ static int free_table_cont(Process *p, make_small(tb->common.slot)); db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); } - db_unref(tb, LCK_NONE); + schedule_free_dbtable(tb); BUMP_REDS(p, 100); return 0; } @@ -3618,10 +3643,13 @@ static Eterm table_info(Process* p, DbTable* tb, Eterm What) ret = erts_this_dist_entry->sysname; } else if (What == am_named_table) { ret = is_atom(tb->common.id) ? am_true : am_false; + } else if (What == am_compressed) { + ret = tb->common.compress ? am_true : am_false; + } /* * For debugging purposes */ - } else if (What == am_data) { + else if (What == am_data) { print_table(ERTS_PRINT_STDOUT, NULL, 1, tb); ret = am_true; } else if (What == am_atom_put("fixed",5)) { diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index 7da28fad29..e0bdebcb01 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -61,6 +61,7 @@ void erts_db_foreach_offheap(DbTable *, extern int user_requested_db_max_tabs; /* set in erl_init */ extern int erts_ets_realloc_always_moves; /* set in erl_init */ +extern int erts_ets_always_compress; /* set in erl_init */ extern Export ets_select_delete_continue_exp; extern Export ets_select_count_continue_exp; extern Export ets_select_continue_exp; @@ -82,7 +83,8 @@ Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); #define ERTS_DB_ALC_MEM_UPDATE_(TAB, FREE_SZ, ALLOC_SZ) \ do { \ - long sz__ = ((long) (ALLOC_SZ)) - ((long) (FREE_SZ)); \ + erts_aint_t sz__ = (((erts_aint_t) (ALLOC_SZ)) \ + - ((erts_aint_t) (FREE_SZ))); \ ASSERT((TAB)); \ erts_smp_atomic_add(&(TAB)->common.memory_size, sz__); \ } while (0) diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index fa707f4eed..9ef990cc4f 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -135,8 +135,8 @@ static ERTS_INLINE Uint hash_to_ix(DbTableHash* tb, HashValue hval) */ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) { - long was_next; - long exp_next; + erts_aint_t was_next; + erts_aint_t exp_next; FixedDeletion* fixd = (FixedDeletion*) erts_db_alloc(ERTS_ALC_T_DB_FIX_DEL, (DbTable *) tb, sizeof(FixedDeletion)); @@ -146,7 +146,9 @@ static ERTS_INLINE void add_fixed_deletion(DbTableHash* tb, int ix) do { /* Lockless atomic insertion in linked list: */ exp_next = was_next; fixd->next = (FixedDeletion*) exp_next; - was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixd, exp_next); + was_next = erts_smp_atomic_cmpxchg(&tb->fixdel, + (erts_aint_t) fixd, + exp_next); }while (was_next != exp_next); } @@ -256,22 +258,16 @@ static ERTS_INLINE Sint next_slot_w(DbTableHash* tb, Uint ix, } -/* - * tplp is an untagged pointer to a tuple we know is large enough - * and dth is a pointer to a DbTableHash. - */ -#define GETKEY(dth, tplp) (*((tplp) + (dth)->common.keypos)) - /* * Some special binary flags */ #define BIN_FLAG_ALL_OBJECTS BIN_FLAG_USR1 -/* - * Size calculations - */ -#define SIZ_OVERHEAD ((sizeof(HashDbTerm)/sizeof(Eterm)) - 1) -#define SIZ_DBTERM(HDT) (SIZ_OVERHEAD + (HDT)->dbterm.size) + +static ERTS_INLINE void free_term(DbTableHash *tb, HashDbTerm* p) +{ + db_free_term((DbTable*)tb, p, offsetof(HashDbTerm, dbterm)); +} /* * Local types @@ -358,10 +354,8 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, HashValue hval, HashDbTerm *list); static void shrink(DbTableHash* tb, int nactive); static void grow(DbTableHash* tb, int nactive); -static void free_term(DbTableHash *tb, HashDbTerm* p); -static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2); -static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, - Eterm obj, HashValue hval); +static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, + DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, struct mp_info *mpi); @@ -434,6 +428,9 @@ static ERTS_INLINE void try_shrink(DbTableHash* tb) } } +#define EQ_REL(x,y,y_base) \ + (is_same(x,NULL,y,y_base) || (is_not_both_immed((x),(y)) && eq_rel((x),NULL,(y),y_base))) + /* Is this a live object (not pseodo-deleted) with the specified key? */ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, @@ -442,7 +439,8 @@ static ERTS_INLINE int has_live_key(DbTableHash* tb, HashDbTerm* b, if (b->hvalue != hval) return 0; else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); - return EQ(key,itemKey); + ASSERT(!is_header(itemKey)); + return EQ_REL(key, itemKey, b->dbterm.tpl); } } @@ -454,10 +452,38 @@ static ERTS_INLINE int has_key(DbTableHash* tb, HashDbTerm* b, if (b->hvalue != hval && b->hvalue != INVALID_HASH) return 0; else { Eterm itemKey = GETKEY(tb, b->dbterm.tpl); - return EQ(key,itemKey); + ASSERT(!is_header(itemKey)); + return EQ_REL(key, itemKey, b->dbterm.tpl); } } +static ERTS_INLINE HashDbTerm* new_dbterm(DbTableHash* tb, Eterm obj) +{ + HashDbTerm* p; + if (tb->common.compress) { + p = db_store_term_comp(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj); + } + else { + p = db_store_term(&tb->common, NULL, offsetof(HashDbTerm,dbterm), obj); + } + return p; +} + +static ERTS_INLINE HashDbTerm* replace_dbterm(DbTableHash* tb, HashDbTerm* old, + Eterm obj) +{ + HashDbTerm* ret; + ASSERT(old != NULL); + if (tb->common.compress) { + ret = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj); + } + else { + ret = db_store_term(&tb->common, &(old->dbterm), offsetof(HashDbTerm,dbterm), obj); + } + return ret; +} + + /* ** External interface @@ -514,12 +540,12 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) { /*int tries = 0;*/ DEBUG_WAIT(); - if (erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel, - (long)NULL) != (long)NULL) { + if (erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, + (erts_aint_t)NULL) != (erts_aint_t)NULL) { /* Oboy, must join lists */ FixedDeletion* last = fixdel; - long was_tail; - long exp_tail; + erts_aint_t was_tail; + erts_aint_t exp_tail; while (last->next != NULL) last = last->next; was_tail = erts_smp_atomic_read(&tb->fixdel); @@ -528,7 +554,7 @@ static void restore_fixdel(DbTableHash* tb, FixedDeletion* fixdel) last->next = (FixedDeletion*) exp_tail; /*++tries;*/ DEBUG_WAIT(); - was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (long)fixdel, + was_tail = erts_smp_atomic_cmpxchg(&tb->fixdel, (erts_aint_t)fixdel, exp_tail); }while (was_tail != exp_tail); } @@ -546,7 +572,7 @@ void db_unfix_table_hash(DbTableHash *tb) || (erts_smp_lc_rwmtx_is_rlocked(&tb->common.rwlock) && !tb->common.is_thread_safe)); restart: - fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (long)NULL); + fixdel = (FixedDeletion*) erts_smp_atomic_xchg(&tb->fixdel, (erts_aint_t)NULL); while (fixdel != NULL) { FixedDeletion *fx = fixdel; int ix = fx->slot; @@ -615,8 +641,8 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_atomic_init(&tb->szm, SEGSZ_MASK); erts_smp_atomic_init(&tb->nactive, SEGSZ); - erts_smp_atomic_init(&tb->fixdel, (long)NULL); - erts_smp_atomic_init(&tb->segtab, (long) alloc_ext_seg(tb,0,NULL)->segtab); + erts_smp_atomic_init(&tb->fixdel, (erts_aint_t)NULL); + erts_smp_atomic_init(&tb->segtab, (erts_aint_t) alloc_ext_seg(tb,0,NULL)->segtab); tb->nsegs = NSEG_1; tb->nslots = SEGSZ; @@ -667,9 +693,7 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret) } } if (list != NULL) { - Eterm key = GETKEY(tb, list->dbterm.tpl); - - COPY_OBJECT(key, p, ret); + *ret = db_copy_key(p, tbl, &list->dbterm); RUNLOCK_HASH(lck); } else { @@ -717,7 +741,7 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) *ret = am_EOT; } else { - COPY_OBJECT(GETKEY(tb, b->dbterm.tpl), p, ret); + *ret = db_copy_key(p, tbl, &b->dbterm); RUNLOCK_HASH(lck); } return DB_ERROR_NONE; @@ -764,7 +788,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) ret = DB_ERROR_BADKEY; goto Ldone; } - q = get_term(tb, b, obj, hval); + q = replace_dbterm(tb, b, obj); q->next = bnext; q->hvalue = hval; /* In case of INVALID_HASH */ *bp = q; @@ -784,7 +808,7 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) HashDbTerm** qp = bp; q = b; do { - if (eq(make_tuple(q->dbterm.tpl), obj)) { + if (db_eq(&tb->common,obj,&q->dbterm)) { if (q->hvalue == INVALID_HASH) { erts_smp_atomic_inc(&tb->common.nitems); q->hvalue = hval; @@ -803,7 +827,8 @@ int db_put_hash(DbTable *tbl, Eterm obj, int key_clash_fail) /*else DB_DUPLICATE_BAG */ Lnew: - q = get_term(tb, NULL, obj, hval); + q = new_dbterm(tb, obj); + q->hvalue = hval; q->next = b; *bp = q; nitems = erts_smp_atomic_inctest(&tb->common.nitems); @@ -844,7 +869,7 @@ int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) while(b2 != NULL && has_key(tb,b2,key,hval)) b2 = b2->next; } - copy = put_term_list(p, b1, b2); + copy = build_term_list(p, b1, b2, tb); CHECK_TABLES(); *ret = copy; goto done; @@ -967,13 +992,10 @@ static int db_get_element_hash(Process *p, DbTable *tbl, while(b1 != 0) { if (has_live_key(tb,b1,key,hval)) { - Eterm copy; - if (ndex > arityval(b1->dbterm.tpl[0])) { retval = DB_ERROR_BADITEM; goto done; } - if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { HashDbTerm* b; HashDbTerm* b2 = b1->next; @@ -987,15 +1009,12 @@ static int db_get_element_hash(Process *p, DbTable *tbl, } b2 = b2->next; } - b = b1; while(b != b2) { if (b->hvalue != INVALID_HASH) { Eterm *hp; - Uint sz = size_object(b->dbterm.tpl[ndex])+2; - - hp = HAlloc(p, sz); - copy = copy_struct(b->dbterm.tpl[ndex], sz-2, &hp, &MSO(p)); + Eterm copy = db_copy_element_from_ets(&tb->common, p, + &b->dbterm, ndex, &hp, 2); elem_list = CONS(hp, copy, elem_list); hp += 2; } @@ -1004,8 +1023,8 @@ static int db_get_element_hash(Process *p, DbTable *tbl, *ret = elem_list; } else { - COPY_OBJECT(b1->dbterm.tpl[ndex], p, ©); - *ret = copy; + Eterm* hp; + *ret = db_copy_element_from_ets(&tb->common, p, &b1->dbterm, ndex, &hp, 0); } retval = DB_ERROR_NONE; goto done; @@ -1040,6 +1059,7 @@ int db_erase_bag_exact2(DbTable *tbl, Eterm key, Eterm value) ASSERT(!IS_FIXED(tb)); ASSERT((tb->common.status & DB_BAG)); + ASSERT(!tb->common.compress); while(b != 0) { if (has_live_key(tb,b,key,hval)) { @@ -1139,7 +1159,7 @@ static int db_erase_object_hash(DbTable *tbl, Eterm object, Eterm *ret) while(b != 0) { if (has_live_key(tb,b,key,hval)) { ++nkeys; - if (eq(object, make_tuple(b->dbterm.tpl))) { + if (db_eq(&tb->common,object, &b->dbterm)) { --nitems_diff; if (nkeys==1 && IS_FIXED(tb)) { /* Pseudo remove */ add_fixed_deletion(tb,ix); @@ -1188,7 +1208,7 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret) lck = RLOCK_HASH(tb, slot); nactive = NACTIVE(tb); if (slot < nactive) { - *ret = put_term_list(p, BUCKET(tb, slot), 0); + *ret = build_term_list(p, BUCKET(tb, slot), 0, tb); retval = DB_ERROR_NONE; } else if (slot == nactive) { @@ -1232,8 +1252,6 @@ static int db_select_continue_hash(Process *p, int num_left = 1000; HashDbTerm *current = 0; Eterm match_list; - Uint32 dummy; - unsigned sz; Eterm *hp; Eterm match_res; Sint got; @@ -1285,26 +1303,14 @@ static int db_select_continue_hash(Process *p, } for(;;) { if (current->hvalue != INVALID_HASH && - (match_res = - db_prog_match(p,mp, - make_tuple(current->dbterm.tpl), - NULL,0,&dummy), + (match_res = db_match_dbterm(&tb->common, p, mp, all_objects, + ¤t->dbterm, &hp, 2), is_value(match_res))) { - if (all_objects) { - hp = HAlloc(p, current->dbterm.size + 2); - match_res = copy_shallow(DBTERM_BUF(¤t->dbterm), - current->dbterm.size, - &hp, - &MSO(p)); - } else { - sz = size_object(match_res); - - hp = HAlloc(p, sz + 2); - match_res = copy_struct(match_res, sz, &hp, &MSO(p)); - } - match_list = CONS(hp, match_res, match_list); + + match_list = CONS(hp, match_res, match_list); ++got; } + --num_left; save_slot_ix = slot_ix; if ((current = next(tb, (Uint*)&slot_ix, &lck, current)) == NULL) { @@ -1395,9 +1401,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, HashDbTerm *current = 0; unsigned current_list_pos = 0; Eterm match_list; - Uint32 dummy; Eterm match_res; - unsigned sz; Eterm *hp; int num_left = 1000; Uint got = 0; @@ -1464,22 +1468,9 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, for(;;) { if (current != NULL) { if (current->hvalue != INVALID_HASH) { - match_res = db_prog_match(p,mpi.mp, - make_tuple(current->dbterm.tpl), - NULL,0,&dummy); + match_res = db_match_dbterm(&tb->common, p, mpi.mp, 0, + ¤t->dbterm, &hp, 2); if (is_value(match_res)) { - if (mpi.all_objects) { - hp = HAlloc(p, current->dbterm.size + 2); - match_res = copy_shallow(DBTERM_BUF(¤t->dbterm), - current->dbterm.size, - &hp, - &MSO(p)); - } else { - sz = size_object(match_res); - - hp = HAlloc(p, sz + 2); - match_res = copy_struct(match_res, sz, &hp, &MSO(p)); - } match_list = CONS(hp, match_res, match_list); ++got; } @@ -1594,7 +1585,6 @@ static int db_select_count_hash(Process *p, Uint slot_ix = 0; HashDbTerm* current = NULL; unsigned current_list_pos = 0; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got = 0; @@ -1644,8 +1634,8 @@ static int db_select_count_hash(Process *p, for(;;) { if (current != NULL) { if (current->hvalue != INVALID_HASH) { - if (db_prog_match(p, mpi.mp, make_tuple(current->dbterm.tpl), - NULL,0, &dummy) == am_true) { + if (db_match_dbterm(&tb->common, p, mpi.mp, 0, + ¤t->dbterm, NULL,0) == am_true) { ++got; } --num_left; @@ -1713,7 +1703,6 @@ static int db_select_delete_hash(Process *p, Uint slot_ix = 0; HashDbTerm **current = NULL; unsigned current_list_pos = 0; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got = 0; @@ -1723,9 +1712,9 @@ static int db_select_delete_hash(Process *p, Eterm mpb; Eterm egot; #ifdef ERTS_SMP - int fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */ + erts_aint_t fixated_by_me = tb->common.is_thread_safe ? 0 : 1; /* ToDo: something nicer */ #else - int fixated_by_me = 0; + erts_aint_t fixated_by_me = 0; #endif erts_smp_rwmtx_t* lck; @@ -1794,9 +1783,8 @@ static int db_select_delete_hash(Process *p, } else { int did_erase = 0; - if ((db_prog_match(p,mpi.mp, - make_tuple((*current)->dbterm.tpl), - NULL,0,&dummy)) == am_true) { + if (db_match_dbterm(&tb->common, p, mpi.mp, 0, + &(*current)->dbterm, NULL, 0) == am_true) { if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { add_fixed_deletion(tb, slot_ix); @@ -1859,7 +1847,6 @@ static int db_select_delete_continue_hash(Process *p, Uint slot_ix; Uint last_pseudo_delete = (Uint)-1; HashDbTerm **current = NULL; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got; @@ -1907,8 +1894,8 @@ static int db_select_delete_continue_hash(Process *p, } else { int did_erase = 0; - if ((db_prog_match(p,mp,make_tuple((*current)->dbterm.tpl), - NULL,0,&dummy)) == am_true) { + if (db_match_dbterm(&tb->common, p, mp, 0, + &(*current)->dbterm, NULL, 0) == am_true) { if (NFIXED(tb) > fixated_by_me) { /* fixated by others? */ if (slot_ix != last_pseudo_delete) { add_fixed_deletion(tb, slot_ix); @@ -1970,7 +1957,6 @@ static int db_select_count_continue_hash(Process *p, DbTableHash *tb = &tbl->hash; Uint slot_ix; HashDbTerm* current; - Uint32 dummy; Eterm *hp; int num_left = 1000; Uint got; @@ -2008,8 +1994,8 @@ static int db_select_count_continue_hash(Process *p, current = current->next; continue; } - if (db_prog_match(p, mp, make_tuple(current->dbterm.tpl), - NULL,0,&dummy) == am_true) { + if (db_match_dbterm(&tb->common, p, mp, 0, ¤t->dbterm, + NULL, 0) == am_true) { ++got; } --num_left; @@ -2135,11 +2121,11 @@ static int db_free_table_continue_hash(DbTable *tbl) sizeof(FixedDeletion)); ERTS_ETS_MISC_MEM_ADD(-sizeof(FixedDeletion)); if (++done >= 2*DELETE_RECORD_LIMIT) { - erts_smp_atomic_set(&tb->fixdel, (long)fixdel); + erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)fixdel); return 0; /* Not done */ } } - erts_smp_atomic_set(&tb->fixdel, (long)NULL); + erts_smp_atomic_set(&tb->fixdel, (erts_aint_t)NULL); done /= 2; while(tb->nslots != 0) { @@ -2188,7 +2174,7 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, HashValue hval = NIL; int num_heads = 0; int i; - + mpi->lists = mpi->dlists; mpi->num_lists = 0; mpi->key_given = 1; @@ -2356,7 +2342,7 @@ static int alloc_seg(DbTableHash *tb) struct ext_segment* eseg; eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - erts_smp_atomic_set(&tb->segtab, (long) eseg->segtab); + erts_smp_atomic_set(&tb->segtab, (erts_aint_t) eseg->segtab); tb->nsegs = eseg->nsegs; } ASSERT(seg_ix < tb->nsegs); @@ -2428,7 +2414,7 @@ static int free_seg(DbTableHash *tb, int free_records) MY_ASSERT(newtop->s.is_ext_segment); if (newtop->prev_segtab != NULL) { /* Time to use a smaller segtab */ - erts_smp_atomic_set(&tb->segtab, (long)newtop->prev_segtab); + erts_smp_atomic_set(&tb->segtab, (erts_aint_t)newtop->prev_segtab); tb->nsegs = seg_ix; ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); } @@ -2445,7 +2431,7 @@ static int free_seg(DbTableHash *tb, int free_records) if (seg_ix > 0) { if (seg_ix < tb->nsegs) SEGTAB(tb)[seg_ix] = NULL; } else { - erts_smp_atomic_set(&tb->segtab, (long)NULL); + erts_smp_atomic_set(&tb->segtab, (erts_aint_t)NULL); } #endif tb->nslots -= SEGSZ; @@ -2454,31 +2440,19 @@ static int free_seg(DbTableHash *tb, int free_records) } -static HashDbTerm* get_term(DbTableHash* tb, HashDbTerm* old, - Eterm obj, HashValue hval) -{ - HashDbTerm* p = db_get_term((DbTableCommon *) tb, - (old != NULL) ? &(old->dbterm) : NULL, - ((char *) &(old->dbterm)) - ((char *) old), - obj); - p->hvalue = hval; - /*p->next = NULL;*/ /*No Need */ - return p; -} - - /* ** Copy terms from ptr1 until ptr2 ** works for ptr1 == ptr2 == 0 => [] ** or ptr2 == 0 */ -static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2) +static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, + DbTableHash* tb) { int sz = 0; HashDbTerm* ptr; Eterm list = NIL; Eterm copy; - Eterm *hp; + Eterm *hp, *hend; ptr = ptr1; while(ptr != ptr2) { @@ -2490,26 +2464,20 @@ static Eterm put_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2) } hp = HAlloc(p, sz); + hend = hp + sz; ptr = ptr1; while(ptr != ptr2) { if (ptr->hvalue != INVALID_HASH) { - copy = copy_shallow(DBTERM_BUF(&ptr->dbterm), ptr->dbterm.size, &hp, &MSO(p)); + copy = db_copy_object_from_ets(&tb->common, &ptr->dbterm, &hp, &MSO(p)); list = CONS(hp, copy, list); hp += 2; } ptr = ptr->next; } - return list; -} + HRelease(p,hend,hp); -static void free_term(DbTableHash *tb, HashDbTerm* p) -{ - db_free_term_data(&(p->dbterm)); - erts_db_free(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - (void *) p, - SIZ_DBTERM(p)*sizeof(Eterm)); + return list; } /* Grow table with one new bucket. @@ -2720,8 +2688,11 @@ static int db_lookup_dbterm_hash(DbTable *tbl, Eterm key, DbUpdateHandle* handle handle->tb = tbl; handle->bp = (void**) prevp; handle->dbterm = &b->dbterm; - handle->new_size = b->dbterm.size; handle->mustResize = 0; + handle->new_size = b->dbterm.size; + #if HALFWORD_HEAP + handle->abs_vec = NULL; + #endif handle->lck = lck; /* KEEP hval WLOCKED, db_finalize_dbterm_hash will WUNLOCK */ return 1; @@ -2742,37 +2713,14 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle) erts_smp_rwmtx_t* lck = (erts_smp_rwmtx_t*) handle->lck; ERTS_SMP_LC_ASSERT(IS_HASH_WLOCKED(&tbl->hash,lck)); /* locked by db_lookup_dbterm_hash */ - ASSERT(&oldp->dbterm == handle->dbterm); - if (handle->mustResize) { - ErlOffHeap tmp_offheap; - Eterm* top; - Eterm copy; - DbTerm* newDbTerm; - HashDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl, - sizeof(HashDbTerm)+sizeof(Eterm)*(handle->new_size-1)); - sys_memcpy(newp, oldp, sizeof(HashDbTerm)-sizeof(DbTerm)); /* copy only hashtab header */ - *(handle->bp) = newp; - newDbTerm = &newp->dbterm; - - newDbTerm->size = handle->new_size; - tmp_offheap.first = NULL; - tmp_offheap.overhead = 0; - - /* make a flat copy */ - top = DBTERM_BUF(newDbTerm); - copy = copy_struct(make_tuple(handle->dbterm->tpl), - handle->new_size, - &top, &tmp_offheap); - newDbTerm->first_oh = tmp_offheap.first; - DBTERM_SET_TPL(newDbTerm,tuple_val(copy)); + ASSERT((&oldp->dbterm == handle->dbterm) == !(tbl->common.compress && handle->mustResize)); + if (handle->mustResize) { + db_finalize_resize(handle, offsetof(HashDbTerm,dbterm)); WUNLOCK_HASH(lck); - - db_free_term_data(handle->dbterm); - erts_db_free(ERTS_ALC_T_DB_TERM, tbl, - (void *) (((char *) handle->dbterm) - (sizeof(HashDbTerm) - sizeof(DbTerm))), - sizeof(HashDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1)); + + free_term(&tbl->hash, oldp); } else { WUNLOCK_HASH(lck); @@ -2781,7 +2729,7 @@ static void db_finalize_dbterm_hash(DbUpdateHandle* handle) handle->dbterm = 0; #endif return; -} +} static int db_delete_all_objects_hash(Process* p, DbTable* tbl) { diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 5644e85f97..6cdbec3213 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -48,9 +48,6 @@ #include "erl_db_tree.h" - - -#define GETKEY(dtt, tplp) (*((tplp) + (dtt)->common.keypos)) #define GETKEY_WITH_POS(Keypos, Tplp) (*((Tplp) + Keypos)) #define NITEMS(tb) ((int)erts_smp_atomic_read(&(tb)->common.nitems)) @@ -122,12 +119,41 @@ static void release_stack(DbTableTree* tb, DbTreeStack* stack) } } -static void reset_static_stack(DbTableTree* tb) +static ERTS_INLINE void reset_static_stack(DbTableTree* tb) { tb->static_stack.pos = 0; tb->static_stack.slot = 0; } +static ERTS_INLINE void free_term(DbTableTree *tb, TreeDbTerm* p) +{ + db_free_term((DbTable*)tb, p, offsetof(TreeDbTerm, dbterm)); +} + +static ERTS_INLINE TreeDbTerm* new_dbterm(DbTableTree *tb, Eterm obj) +{ + TreeDbTerm* p; + if (tb->common.compress) { + p = db_store_term_comp(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj); + } + else { + p = db_store_term(&tb->common, NULL, offsetof(TreeDbTerm,dbterm), obj); + } + return p; +} +static ERTS_INLINE TreeDbTerm* replace_dbterm(DbTableTree *tb, TreeDbTerm* old, + Eterm obj) +{ + TreeDbTerm* p; + ASSERT(old != NULL); + if (tb->common.compress) { + p = db_store_term_comp(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj); + } + else { + p = db_store_term(&tb->common, &(old->dbterm), offsetof(TreeDbTerm,dbterm), obj); + } + return p; +} /* ** Some macros for "direction stacks" @@ -178,12 +204,6 @@ static void do_dump_tree2(int to, void *to_arg, int show, TreeDbTerm *t, #endif /* - * Size calculations - */ -#define SIZ_OVERHEAD ((sizeof(TreeDbTerm)/sizeof(Eterm)) - 1) -#define SIZ_DBTERM(TDT) (SIZ_OVERHEAD + (TDT)->dbterm.size) - -/* ** Datatypes */ @@ -259,13 +279,10 @@ struct select_delete_context { /* ** Forward declarations */ -static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key); +static TreeDbTerm *linkout_tree(DbTableTree *tb, Eterm key, Eterm* key_base); static TreeDbTerm *linkout_object_tree(DbTableTree *tb, Eterm object); static int do_free_tree_cont(DbTableTree *tb, int num_left); -static TreeDbTerm* get_term(DbTableTree *tb, - TreeDbTerm* old, - Eterm obj); static void free_term(DbTableTree *tb, TreeDbTerm* p); static int balance_left(TreeDbTerm **this); static int balance_right(TreeDbTerm **this); @@ -273,15 +290,15 @@ static int delsub(TreeDbTerm **this); static TreeDbTerm *slot_search(Process *p, DbTableTree *tb, Sint slot); static TreeDbTerm *find_node(DbTableTree *tb, Eterm key); static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key); -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key); -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key); +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack*, Eterm key, Eterm* kbase); static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack*, Eterm key); static void traverse_backwards(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, + Eterm lastkey, Eterm* lk_base, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -289,7 +306,7 @@ static void traverse_backwards(DbTableTree *tb, void *context); static void traverse_forward(DbTableTree *tb, DbTreeStack*, - Eterm lastkey, + Eterm lastkey, Eterm* lk_base, int (*doit)(DbTableTree *tb, TreeDbTerm *, void *, @@ -297,8 +314,8 @@ static void traverse_forward(DbTableTree *tb, void *context); static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, Eterm *partly_bound_key); -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key); -static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done); +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base); +static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done); static int analyze_pattern(DbTableTree *tb, Eterm pattern, struct mp_info *mpi); @@ -318,7 +335,6 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, int forward); -static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t); static int partly_bound_can_match_lesser(Eterm partly_bound_1, Eterm partly_bound_2); @@ -472,9 +488,6 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret) DbTableTree *tb = &tbl->tree; DbTreeStack* stack; TreeDbTerm *this; - Eterm e; - Eterm *hp; - Uint sz; if (( this = tb->root ) == NULL) { *ret = am_EOT; @@ -493,13 +506,7 @@ static int db_first_tree(Process *p, DbTable *tbl, Eterm *ret) stack->slot = 1; release_stack(tb,stack); } - e = GETKEY(tb, this->dbterm.tpl); - sz = size_object(e); - - hp = HAlloc(p, sz); - - *ret = copy_struct(e,sz,&hp,&MSO(p)); - + *ret = db_copy_key(p, tbl, &this->dbterm); return DB_ERROR_NONE; } @@ -508,26 +515,17 @@ static int db_next_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) DbTableTree *tb = &tbl->tree; DbTreeStack* stack; TreeDbTerm *this; - Eterm e; - Eterm *hp; - Uint sz; if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_next(tb, stack, key); + this = find_next(tb, stack, key, NULL); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; return DB_ERROR_NONE; } - e = GETKEY(tb, this->dbterm.tpl); - sz = size_object(e); - - hp = HAlloc(p, sz); - - *ret = copy_struct(e,sz,&hp,&MSO(p)); - + *ret = db_copy_key(p, tbl, &this->dbterm); return DB_ERROR_NONE; } @@ -536,9 +534,6 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret) DbTableTree *tb = &tbl->tree; TreeDbTerm *this; DbTreeStack* stack; - Eterm e; - Eterm *hp; - Uint sz; if (( this = tb->root ) == NULL) { *ret = am_EOT; @@ -557,13 +552,7 @@ static int db_last_tree(Process *p, DbTable *tbl, Eterm *ret) stack->slot = NITEMS(tb); release_stack(tb,stack); } - e = GETKEY(tb, this->dbterm.tpl); - sz = size_object(e); - - hp = HAlloc(p, sz); - - *ret = copy_struct(e,sz,&hp,&MSO(p)); - + *ret = db_copy_key(p, tbl, &this->dbterm); return DB_ERROR_NONE; } @@ -572,27 +561,33 @@ static int db_prev_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) DbTableTree *tb = &tbl->tree; TreeDbTerm *this; DbTreeStack* stack; - Eterm e; - Eterm *hp; - Uint sz; if (is_atom(key) && key == am_EOT) return DB_ERROR_BADKEY; stack = get_any_stack(tb); - this = find_prev(tb, stack, key); + this = find_prev(tb, stack, key, NULL); release_stack(tb,stack); if (this == NULL) { *ret = am_EOT; return DB_ERROR_NONE; } - e = GETKEY(tb, this->dbterm.tpl); - sz = size_object(e); + *ret = db_copy_key(p, tbl, &this->dbterm); + return DB_ERROR_NONE; +} - hp = HAlloc(p, sz); +static ERTS_INLINE int cmp_key(DbTableTree* tb, Eterm key, Eterm* key_base, + TreeDbTerm* obj) +{ + return cmp_rel(key, key_base, + GETKEY(tb,obj->dbterm.tpl), obj->dbterm.tpl); +} - *ret = copy_struct(e,sz,&hp,&MSO(p)); - - return DB_ERROR_NONE; +static ERTS_INLINE int cmp_key_eq(DbTableTree* tb, Eterm key, Eterm* key_base, + TreeDbTerm* obj) +{ + Eterm obj_key = GETKEY(tb,obj->dbterm.tpl); + return is_same(key, key_base, obj_key, obj->dbterm.tpl) + || cmp_rel(key, key_base, obj_key, obj->dbterm.tpl) == 0; } static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) @@ -622,12 +617,12 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) erts_smp_atomic_dec(&tb->common.nitems); return DB_ERROR_SYSRES; } - *this = get_term(tb, NULL, obj); + *this = new_dbterm(tb, obj); (*this)->balance = 0; (*this)->left = (*this)->right = NULL; break; - } else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { - /* go left */ + } else if ((c = cmp_key(tb, key, NULL, *this)) < 0) { + /* go lefts */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -636,7 +631,7 @@ static int db_put_tree(DbTable *tbl, Eterm obj, int key_clash_fail) tstack[tpos++] = this; this = &((*this)->right); } else if (!key_clash_fail) { /* Equal key and this is a set, replace. */ - *this = get_term(tb, *this, obj); + *this = replace_dbterm(tb, *this, obj); break; } else { return DB_ERROR_BADKEY; /* key already exists */ @@ -714,7 +709,7 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) { DbTableTree *tb = &tbl->tree; Eterm copy; - Eterm *hp; + Eterm *hp, *hend; TreeDbTerm *this; /* @@ -728,11 +723,11 @@ static int db_get_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) *ret = NIL; } else { hp = HAlloc(p, this->dbterm.size + 2); - copy = copy_shallow(DBTERM_BUF(&this->dbterm), - this->dbterm.size, - &hp, - &MSO(p)); + hend = hp + this->dbterm.size + 2; + copy = db_copy_object_from_ets(&tb->common, &this->dbterm, &hp, &MSO(p)); *ret = CONS(hp, copy, NIL); + hp += 2; + HRelease(p,hend,hp); } return DB_ERROR_NONE; } @@ -766,18 +761,10 @@ static int db_get_element_tree(Process *p, DbTable *tbl, if (this == NULL) { return DB_ERROR_BADKEY; } else { - Eterm element; - Uint sz; if (ndex > arityval(this->dbterm.tpl[0])) { return DB_ERROR_BADPARAM; } - element = this->dbterm.tpl[ndex]; - sz = size_object(element); - hp = HAlloc(p, sz); - *ret = copy_struct(element, - sz, - &hp, - &MSO(p)); + *ret = db_copy_element_from_ets(&tb->common, p, &this->dbterm, ndex, &hp, 0); } return DB_ERROR_NONE; } @@ -789,7 +776,7 @@ static int db_erase_tree(DbTable *tbl, Eterm key, Eterm *ret) *ret = am_true; - if ((res = linkout_tree(tb, key)) != NULL) { + if ((res = linkout_tree(tb, key, NULL)) != NULL) { free_term(tb, res); } return DB_ERROR_NONE; @@ -815,7 +802,7 @@ static int db_slot_tree(Process *p, DbTable *tbl, DbTableTree *tb = &tbl->tree; Sint slot; TreeDbTerm *st; - Eterm *hp; + Eterm *hp, *hend; Eterm copy; /* @@ -847,11 +834,11 @@ static int db_slot_tree(Process *p, DbTable *tbl, return DB_ERROR_UNSPEC; } hp = HAlloc(p, st->dbterm.size + 2); - copy = copy_shallow(DBTERM_BUF(&st->dbterm), - st->dbterm.size, - &hp, - &MSO(p)); + hend = hp + st->dbterm.size + 2; + copy = db_copy_object_from_ets(&tb->common, &st->dbterm, &hp, &MSO(p)); *ret = CONS(hp, copy, NIL); + hp += 2; + HRelease(p,hend,hp); return DB_ERROR_NONE; } @@ -981,15 +968,15 @@ static int db_select_continue_tree(Process *p, stack = get_any_stack(tb); if (chunk_size) { if (reverse) { - traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); } else { - traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, NULL, &doit_select_chunk, &sc); } } else { if (reverse) { - traverse_forward(tb, stack, lastkey, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, NULL, &doit_select, &sc); } else { - traverse_backwards(tb, stack, lastkey, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, NULL, &doit_select, &sc); } } release_stack(tb,stack); @@ -1014,10 +1001,9 @@ static int db_select_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); - - sz = size_object(key); + sz = size_object_rel(key,sc.lastobj); hp = HAlloc(p, 9 + sz); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); continuation = TUPLE8 (hp, tptr[1], @@ -1038,8 +1024,8 @@ static int db_select_continue_tree(Process *p, key = GETKEY(tb, sc.lastobj); if (chunk_size) { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key) < 0) || - (reverse && cmp_partly_bound(end_condition,key) > 0))) { + ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0) || + (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0))) { /* done anyway */ if (!sc.got) { RET_TO_BIF(am_EOT, DB_ERROR_NONE); @@ -1051,16 +1037,16 @@ static int db_select_continue_tree(Process *p, } } else { if (end_condition != NIL && - ((!reverse && cmp_partly_bound(end_condition,key) > 0) || - (reverse && cmp_partly_bound(end_condition,key) < 0))) { + ((!reverse && cmp_partly_bound(end_condition,key,sc.lastobj) > 0) || + (reverse && cmp_partly_bound(end_condition,key,sc.lastobj) < 0))) { /* done anyway */ RET_TO_BIF(sc.accum,DB_ERROR_NONE); } } /* Not done yet, let's trap. */ - sz = size_object(key); + sz = size_object_rel(key,sc.lastobj); hp = HAlloc(p, 9 + sz); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); continuation = TUPLE8 (hp, tptr[1], @@ -1087,6 +1073,7 @@ static int db_select_tree(Process *p, DbTable *tbl, struct select_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; + Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1128,7 +1115,7 @@ static int db_select_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - cmp(mpi.least,mpi.most) == 0) { + CMP(mpi.least,mpi.most) == 0) { doit_select(tb,mpi.save_term,&sc,0 /* direction doesn't matter */); RET_TO_BIF(sc.accum,DB_ERROR_NONE); } @@ -1138,20 +1125,20 @@ static int db_select_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - - traverse_forward(tb, stack, lastkey, &doit_select, &sc); + traverse_forward(tb, stack, lastkey, lk_base, &doit_select, &sc); } else { if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - - traverse_backwards(tb, stack, lastkey, &doit_select, &sc); + traverse_backwards(tb, stack, lastkey, lk_base, &doit_select, &sc); } release_stack(tb,stack); #ifdef HARDDEBUG @@ -1164,9 +1151,9 @@ static int db_select_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb=db_make_mp_binary(p,mpi.mp,&hp); @@ -1247,7 +1234,7 @@ static int db_select_count_continue_tree(Process *p, } stack = get_any_stack(tb); - traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, NULL, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); @@ -1257,12 +1244,12 @@ static int db_select_count_continue_tree(Process *p, } key = GETKEY(tb, sc.lastobj); if (end_condition != NIL && - (cmp_partly_bound(end_condition,key) > 0)) { + (cmp_partly_bound(end_condition,key,sc.lastobj) > 0)) { /* done anyway */ RET_TO_BIF(make_small(sc.got),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + 6); egot = make_small(sc.got); @@ -1272,7 +1259,7 @@ static int db_select_count_continue_tree(Process *p, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); continuation = TUPLE5 (hp, tptr[1], @@ -1295,6 +1282,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, struct select_count_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; + Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1335,7 +1323,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - cmp(mpi.least,mpi.most) == 0) { + CMP(mpi.least,mpi.most) == 0) { doit_select_count(tb,mpi.save_term,&sc,0 /* dummy */); RET_TO_BIF(erts_make_integer(sc.got,p),DB_ERROR_NONE); } @@ -1344,11 +1332,12 @@ static int db_select_count_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, stack, lastkey, &doit_select_count, &sc); + traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_count, &sc); release_stack(tb,stack); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1356,7 +1345,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); if (IS_USMALL(0, sc.got)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); egot = make_small(sc.got); @@ -1366,7 +1355,7 @@ static int db_select_count_tree(Process *p, DbTable *tbl, egot = uint_to_big(sc.got, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1397,6 +1386,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, struct select_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; + Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1438,7 +1428,7 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, sc.all_objects = mpi.all_objects; if (!mpi.got_partial && mpi.some_limitation && - cmp(mpi.least,mpi.most) == 0) { + CMP(mpi.least,mpi.most) == 0) { doit_select(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); if (sc.accum != NIL) { hp=HAlloc(p, 3); @@ -1453,20 +1443,20 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - - traverse_backwards(tb, stack, lastkey, &doit_select_chunk, &sc); + traverse_backwards(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); } else { if (mpi.some_limitation) { if ((this = find_prev_from_pb_key(tb, stack, mpi.least)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.most; } - - traverse_forward(tb, stack, lastkey, &doit_select_chunk, &sc); + traverse_forward(tb, stack, lastkey, lk_base, &doit_select_chunk, &sc); } release_stack(tb,stack); @@ -1491,9 +1481,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; mpb = db_make_mp_binary(p,mpi.mp,&hp); @@ -1516,9 +1506,9 @@ static int db_select_chunk_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, sc.lastobj); - sz = size_object(key); + sz = size_object_rel(key, sc.lastobj); hp = HAlloc(p, 9 + sz + PROC_BIN_SIZE); - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastobj, NULL); if (mpi.all_objects) (mpi.mp)->flags |= BIN_FLAG_ALL_OBJECTS; @@ -1594,7 +1584,7 @@ static int db_select_delete_continue_tree(Process *p, sc.keypos = tb->common.keypos; ASSERT(!erts_smp_atomic_read(&tb->is_stack_busy)); - traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, NULL, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); @@ -1603,11 +1593,11 @@ static int db_select_delete_continue_tree(Process *p, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); if (end_condition != NIL && - cmp_partly_bound(end_condition,key) > 0) { /* done anyway */ + cmp_partly_bound(end_condition,key,sc.lastterm->dbterm.tpl) > 0) { /* done anyway */ RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); } /* Not done yet, let's trap. */ - sz = size_object(key); + sz = size_object_rel(key, sc.lastterm->dbterm.tpl); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + 6); eaccsum = make_small(sc.accum); @@ -1617,7 +1607,7 @@ static int db_select_delete_continue_tree(Process *p, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); continuation = TUPLE5 (hp, tptr[1], @@ -1638,6 +1628,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, struct select_delete_context sc; struct mp_info mpi; Eterm lastkey = THE_NON_VALUE; + Eterm* lk_base = NULL; Eterm key; Eterm continuation; unsigned sz; @@ -1681,7 +1672,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, sc.mp = mpi.mp; if (!mpi.got_partial && mpi.some_limitation && - cmp(mpi.least,mpi.most) == 0) { + CMP(mpi.least,mpi.most) == 0) { doit_select_delete(tb,mpi.save_term,&sc, 0 /* direction doesn't matter */); RET_TO_BIF(erts_make_integer(sc.accum,p),DB_ERROR_NONE); @@ -1690,11 +1681,12 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, if (mpi.some_limitation) { if ((this = find_next_from_pb_key(tb, &tb->static_stack, mpi.most)) != NULL) { lastkey = GETKEY(tb, this->dbterm.tpl); + lk_base = this->dbterm.tpl; } sc.end_condition = mpi.least; } - traverse_backwards(tb, &tb->static_stack, lastkey, &doit_select_delete, &sc); + traverse_backwards(tb, &tb->static_stack, lastkey, lk_base, &doit_select_delete, &sc); BUMP_REDS(p, 1000 - sc.max); if (sc.max > 0) { @@ -1702,7 +1694,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, } key = GETKEY(tb, (sc.lastterm)->dbterm.tpl); - sz = size_object(key); + sz = size_object_rel(key, sc.lastterm->dbterm.tpl); if (IS_USMALL(0, sc.accum)) { hp = HAlloc(p, sz + PROC_BIN_SIZE + 6); eaccsum = make_small(sc.accum); @@ -1712,7 +1704,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, eaccsum = uint_to_big(sc.accum, hp); hp += BIG_UINT_HEAP_SIZE; } - key = copy_struct(key, sz, &hp, &MSO(p)); + key = copy_struct_rel(key, sz, &hp, &MSO(p), sc.lastterm->dbterm.tpl, NULL); mpb = db_make_mp_binary(p,mpi.mp,&hp); continuation = TUPLE5 @@ -1738,7 +1730,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, ** Other interface routines (not directly coupled to one bif) */ -/* Display hash table contents (for dump) */ +/* Display tree contents (for dump) */ static void db_print_tree(int to, void *to_arg, int show, DbTable *tbl) @@ -1754,7 +1746,6 @@ static void db_print_tree(int to, void *to_arg, "------------------------------------------------\n"); #else erts_print(to, to_arg, "Ordered set (AVL tree), Elements: %d\n", NITEMS(tb)); - do_dump_tree(to, to_arg, tb->root); #endif } @@ -1830,7 +1821,7 @@ do_db_tree_foreach_offheap(TreeDbTerm *tdbt, } static TreeDbTerm *linkout_tree(DbTableTree *tb, - Eterm key) + Eterm key, Eterm* key_base) { TreeDbTerm **tstack[STACK_NEED]; int tpos = 0; @@ -1853,7 +1844,7 @@ static TreeDbTerm *linkout_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { + } else if ((c = cmp_key(tb, key, key_base, *this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -1917,7 +1908,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp(key,GETKEY(tb,(*this)->dbterm.tpl))) < 0) { + } else if ((c = cmp_key(tb,key,NULL,*this)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -1926,7 +1917,7 @@ static TreeDbTerm *linkout_object_tree(DbTableTree *tb, tstack[tpos++] = this; this = &((*this)->right); } else { /* Equal key, found the only possible matching object*/ - if (!eq(object,make_tuple((*this)->dbterm.tpl))) { + if (!db_eq(&tb->common,object,&(*this)->dbterm)) { return NULL; } q = (*this); @@ -2070,24 +2061,6 @@ static int analyze_pattern(DbTableTree *tb, Eterm pattern, return DB_ERROR_NONE; } -static void do_dump_tree(int to, void *to_arg, TreeDbTerm *t) -{ - if (t != NULL) { - do_dump_tree(to, to_arg, t->left); - erts_print(to, to_arg, "%T\n", make_tuple(t->dbterm.tpl)); - do_dump_tree(to, to_arg, t->right); - } -} - -static void free_term(DbTableTree *tb, TreeDbTerm* p) -{ - db_free_term_data(&(p->dbterm)); - erts_db_free(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - (void *) p, - SIZ_DBTERM(p)*sizeof(Uint)); -} - static int do_free_tree_cont(DbTableTree *tb, int num_left) { TreeDbTerm *root; @@ -2118,17 +2091,6 @@ static int do_free_tree_cont(DbTableTree *tb, int num_left) return 1; } -static TreeDbTerm* get_term(DbTableTree *tb, - TreeDbTerm* old, - Eterm obj) -{ - TreeDbTerm* p = db_get_term((DbTableCommon *) tb, - (old != NULL) ? &(old->dbterm) : NULL, - ((char *) &(old->dbterm)) - ((char *) old), - obj); - return p; -} - /* * Deletion helpers */ @@ -2332,14 +2294,15 @@ done: * Find next and previous in sort order */ -static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) +static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, + Eterm key, Eterm* key_base) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) { + if (!cmp_key_eq(tb,key,key_base,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2349,14 +2312,14 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) < 0) { + if (( c = cmp_key(tb,key,key_base,this) ) > 0) { if (this->right == NULL) /* We are at the previos and the element does not exist */ break; else this = this->right; - } else if (c > 0) { + } else if (c < 0) { if (this->left == NULL) /* Done */ return this; else @@ -2389,14 +2352,15 @@ static TreeDbTerm *find_next(DbTableTree *tb, DbTreeStack* stack, Eterm key) return this; } -static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key) +static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, + Eterm key, Eterm* key_base) { TreeDbTerm *this; TreeDbTerm *tmp; Sint c; if(( this = TOP_NODE(stack)) != NULL) { - if (!CMP_EQ(GETKEY(tb, this->dbterm.tpl),key)) { + if (!cmp_key_eq(tb,key,key_base,this)) { /* Start from the beginning */ stack->pos = stack->slot = 0; } @@ -2406,14 +2370,14 @@ static TreeDbTerm *find_prev(DbTableTree *tb, DbTreeStack* stack, Eterm key) return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp(GETKEY(tb, this->dbterm.tpl),key) ) > 0) { + if (( c = cmp_key(tb,key,key_base,this) ) < 0) { if (this->left == NULL) /* We are at the next and the element does not exist */ break; else this = this->left; - } else if (c < 0) { + } else if (c > 0) { if (this->right == NULL) /* Done */ return this; else @@ -2459,7 +2423,8 @@ static TreeDbTerm *find_next_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) >= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), + this->dbterm.tpl) ) >= 0) { if (this->right == NULL) { do { tmp = POP_NODE(stack); @@ -2492,7 +2457,8 @@ static TreeDbTerm *find_prev_from_pb_key(DbTableTree *tb, DbTreeStack* stack, return NULL; for (;;) { PUSH_NODE(stack, this); - if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl)) ) <= 0) { + if (( c = cmp_partly_bound(key,GETKEY(tb, this->dbterm.tpl), + this->dbterm.tpl) ) <= 0) { if (this->left == NULL) { do { tmp = POP_NODE(stack); @@ -2522,12 +2488,11 @@ static TreeDbTerm *find_node(DbTableTree *tb, Eterm key) Sint res; DbTreeStack* stack = get_static_stack(tb); - if(!stack || EMPTY_NODE(stack) - || !CMP_EQ(GETKEY(tb, ( this = TOP_NODE(stack) )->dbterm.tpl), key)) { + if(!stack || EMPTY_NODE(stack) + || !cmp_key_eq(tb, key, NULL, (this=TOP_NODE(stack)))) { this = tb->root; - while (this != NULL && - ( res = cmp(key, GETKEY(tb, this->dbterm.tpl)) ) != 0) { + while (this != NULL && (res = cmp_key(tb,key,NULL,this)) != 0) { if (res < 0) this = this->left; else @@ -2549,8 +2514,7 @@ static TreeDbTerm **find_node2(DbTableTree *tb, Eterm key) Sint res; this = &tb->root; - while ((*this) != NULL && - ( res = cmp(key, GETKEY(tb, (*this)->dbterm.tpl)) ) != 0) { + while ((*this) != NULL && (res = cmp_key(tb, key, NULL, *this)) != 0) { if (res < 0) this = &((*this)->left); else @@ -2570,46 +2534,24 @@ static int db_lookup_dbterm_tree(DbTable *tbl, Eterm key, DbUpdateHandle* handle handle->tb = tbl; handle->dbterm = &(*pp)->dbterm; + handle->mustResize = 0; handle->bp = (void**) pp; handle->new_size = (*pp)->dbterm.size; - handle->mustResize = 0; +#if HALFWORD_HEAP + handle->abs_vec = NULL; +#endif return 1; } static void db_finalize_dbterm_tree(DbUpdateHandle* handle) { if (handle->mustResize) { - ErlOffHeap tmp_offheap; - Eterm* top; - Eterm copy; - DbTerm* newDbTerm; - DbTableTree *tb = &handle->tb->tree; TreeDbTerm* oldp = (TreeDbTerm*) *handle->bp; - TreeDbTerm* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, - handle->tb, - sizeof(TreeDbTerm)+sizeof(Eterm)*(handle->new_size-1)); - memcpy(newp, oldp, sizeof(TreeDbTerm)-sizeof(DbTerm)); /* copy only tree header */ - *(handle->bp) = newp; - reset_static_stack(tb); - newDbTerm = &newp->dbterm; - - newDbTerm->size = handle->new_size; - tmp_offheap.first = NULL; - tmp_offheap.overhead = 0; - - /* make a flat copy */ - top = DBTERM_BUF(newDbTerm); - copy = copy_struct(make_tuple(handle->dbterm->tpl), - handle->new_size, - &top, &tmp_offheap); - newDbTerm->first_oh = tmp_offheap.first; - DBTERM_SET_TPL(newDbTerm,tuple_val(copy)); - - db_free_term_data(handle->dbterm); - erts_db_free(ERTS_ALC_T_DB_TERM, - handle->tb, - (void *) (((char *) handle->dbterm) - (sizeof(TreeDbTerm) - sizeof(DbTerm))), - sizeof(TreeDbTerm) + sizeof(Eterm)*(handle->dbterm->size-1)); + + db_finalize_resize(handle, offsetof(TreeDbTerm,dbterm)); + reset_static_stack(&handle->tb->tree); + + free_term(&handle->tb->tree, oldp); } #ifdef DEBUG handle->dbterm = 0; @@ -2622,7 +2564,7 @@ static void db_finalize_dbterm_tree(DbUpdateHandle* handle) */ static void traverse_backwards(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, + Eterm lastkey, Eterm* lk_base, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2641,15 +2583,16 @@ static void traverse_backwards(DbTableTree *tb, this = this->right; } this = TOP_NODE(stack); - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), + this->dbterm.tpl); if (!((*doit)(tb, this, context, 0))) return; } else { - next = find_prev(tb, stack, lastkey); + next = find_prev(tb, stack, lastkey, lk_base); } while ((this = next) != NULL) { - next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl)); + next = find_prev(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); if (!((*doit)(tb, this, context, 0))) return; } @@ -2660,7 +2603,7 @@ static void traverse_backwards(DbTableTree *tb, */ static void traverse_forward(DbTableTree *tb, DbTreeStack* stack, - Eterm lastkey, + Eterm lastkey, Eterm* lk_base, int (*doit)(DbTableTree *, TreeDbTerm *, void *, @@ -2679,15 +2622,15 @@ static void traverse_forward(DbTableTree *tb, this = this->left; } this = TOP_NODE(stack); - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); if (!((*doit)(tb, this, context, 1))) return; } else { - next = find_next(tb, stack, lastkey); + next = find_next(tb, stack, lastkey, lk_base); } while ((this = next) != NULL) { - next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl)); + next = find_next(tb, stack, GETKEY(tb, this->dbterm.tpl), this->dbterm.tpl); if (!((*doit)(tb, this, context, 1))) return; } @@ -2713,7 +2656,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, if (( this = find_node(tb, key) ) == NULL) { return -1; } - *ret = this; + *ret = this; return 1; } else if (partly_bound != NULL && key != am_Underscore && db_is_variable(key) < 0) @@ -2724,7 +2667,7 @@ static int key_given(DbTableTree *tb, Eterm pattern, TreeDbTerm **ret, -static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done) +static Sint do_cmp_partly_bound(Eterm a, Eterm b, Eterm* b_base, int *done) { Eterm* aa; Eterm* bb; @@ -2738,44 +2681,44 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done) *done = 1; return 0; } - if (a == b) + if (is_same(a,NULL,b,b_base)) return 0; switch (a & _TAG_PRIMARY_MASK) { case TAG_PRIMARY_LIST: if (!is_list(b)) { - return cmp(a,b); + return cmp_rel(a,NULL,b,b_base); } aa = list_val(a); - bb = list_val(b); + bb = list_val_rel(b,b_base); while (1) { - if ((j = do_cmp_partly_bound(*aa++, *bb++, done)) != 0 || *done) + if ((j = do_cmp_partly_bound(*aa++, *bb++, b_base, done)) != 0 || *done) return j; if (*aa==*bb) return 0; if (is_not_list(*aa) || is_not_list(*bb)) - return do_cmp_partly_bound(*aa, *bb, done); + return do_cmp_partly_bound(*aa, *bb, b_base, done); aa = list_val(*aa); - bb = list_val(*bb); + bb = list_val_rel(*bb,b_base); } case TAG_PRIMARY_BOXED: if ((b & _TAG_PRIMARY_MASK) != TAG_PRIMARY_BOXED) { - return cmp(a,b); + return cmp_rel(a,NULL,b,b_base); } a_hdr = ((*boxed_val(a)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; - b_hdr = ((*boxed_val(b)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; + b_hdr = ((*boxed_val_rel(b,b_base)) & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE; if (a_hdr != b_hdr) { - return cmp(a, b); + return cmp_rel(a, NULL, b, b_base); } if (a_hdr == (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE)) { aa = tuple_val(a); - bb = tuple_val(b); + bb = tuple_val_rel(b, b_base); /* compare the arities */ i = arityval(*aa); /* get the arity*/ if (i < arityval(*bb)) return(-1); if (i > arityval(*bb)) return(1); while (i--) { - if ((j = do_cmp_partly_bound(*++aa, *++bb, done)) != 0 + if ((j = do_cmp_partly_bound(*++aa, *++bb, b_base, done)) != 0 || *done) return j; } @@ -2783,14 +2726,14 @@ static Sint do_cmp_partly_bound(Eterm a, Eterm b, int *done) } /* Drop through */ default: - return cmp(a, b); + return cmp_rel(a, NULL, b, b_base); } } -static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) +static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key, Eterm* bk_base) { int done = 0; - Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, &done); + Sint ret = do_cmp_partly_bound(partly_bound_key, bound_key, bk_base, &done); #ifdef HARDDEBUG erts_fprintf(stderr,"\ncmp_partly_bound: %T", partly_bound_key); if (ret < 0) @@ -2799,7 +2742,7 @@ static Sint cmp_partly_bound(Eterm partly_bound_key, Eterm bound_key) erts_fprintf(stderr," > "); else erts_fprintf(stderr," == "); - erts_fprintf(stderr,"%T\n",bound_key); + erts_fprintf(stderr,"%T\n",bound_key); // HALFWORD BUG: printing rterm #endif return ret; } @@ -2886,7 +2829,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b, if (not_eq_tags(a,b)) { *done = 1; - return (cmp(a, b) < 0) ? 1 : 0; + return (CMP(a, b) < 0) ? 1 : 0; } /* we now know that tags are the same */ @@ -2922,7 +2865,7 @@ static int do_partly_bound_can_match_lesser(Eterm a, Eterm b, bb = list_val(*bb); } default: - if((i = cmp(a, b)) != 0) { + if((i = CMP(a, b)) != 0) { *done = 1; } return (i < 0) ? 1 : 0; @@ -2957,7 +2900,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b, if (not_eq_tags(a,b)) { *done = 1; - return (cmp(a, b) > 0) ? 1 : 0; + return (CMP(a, b) > 0) ? 1 : 0; } /* we now know that tags are the same */ @@ -2993,7 +2936,7 @@ static int do_partly_bound_can_match_greater(Eterm a, Eterm b, bb = list_val(*bb); } default: - if((i = cmp(a, b)) != 0) { + if((i = CMP(a, b)) != 0) { *done = 1; } return (i > 0) ? 1 : 0; @@ -3009,39 +2952,24 @@ static int doit_select(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_context *sc = (struct select_context *) ptr; Eterm ret; - Uint32 dummy; + Eterm* hp; sc->lastobj = this->dbterm.tpl; if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) > 0))) { return 0; } - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - NULL,0, &dummy); + ret = db_match_dbterm(&tb->common,sc->p,sc->mp,sc->all_objects, + &this->dbterm, &hp, 2); if (is_value(ret)) { - Uint sz; - Eterm *hp; - if (sc->all_objects) { - hp = HAlloc(sc->p, this->dbterm.size + 2); - ret = copy_shallow(DBTERM_BUF(&this->dbterm), - this->dbterm.size, - &hp, - &MSO(sc->p)); - } else { - sz = size_object(ret); - hp = HAlloc(sc->p, sz + 2); - ret = copy_struct(ret, sz, - &hp, &MSO(sc->p)); - } sc->accum = CONS(hp, ret, sc->accum); } if (MBUF(sc->p)) { @@ -3062,20 +2990,18 @@ static int doit_select_count(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_count_context *sc = (struct select_count_context *) ptr; Eterm ret; - Uint32 dummy; sc->lastobj = this->dbterm.tpl; /* Always backwards traversing */ if (sc->end_condition != NIL && (cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) > 0)) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) > 0)) { return 0; } - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - NULL,0, &dummy); + ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, + &this->dbterm, NULL, 0); if (ret == am_true) { ++(sc->got); } @@ -3090,41 +3016,26 @@ static int doit_select_chunk(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_context *sc = (struct select_context *) ptr; Eterm ret; - Uint32 dummy; + Eterm* hp; sc->lastobj = this->dbterm.tpl; if (sc->end_condition != NIL && ((forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) < 0) || + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) < 0) || (!forward && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) > 0))) { + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) > 0))) { return 0; } - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - NULL,0, &dummy); + ret = db_match_dbterm(&tb->common, sc->p, sc->mp, sc->all_objects, + &this->dbterm, &hp, 2); if (is_value(ret)) { - Uint sz; - Eterm *hp; - ++(sc->got); - if (sc->all_objects) { - hp = HAlloc(sc->p, this->dbterm.size + 2); - ret = copy_shallow(DBTERM_BUF(&this->dbterm), - this->dbterm.size, - &hp, - &MSO(sc->p)); - } else { - sz = size_object(ret); - hp = HAlloc(sc->p, sz + 2); - ret = copy_struct(ret, sz, &hp, &MSO(sc->p)); - } sc->accum = CONS(hp, ret, sc->accum); } if (MBUF(sc->p)) { @@ -3146,7 +3057,6 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, { struct select_delete_context *sc = (struct select_delete_context *) ptr; Eterm ret; - Uint32 dummy; Eterm key; if (sc->erase_lastterm) @@ -3156,15 +3066,14 @@ static int doit_select_delete(DbTableTree *tb, TreeDbTerm *this, void *ptr, if (sc->end_condition != NIL && cmp_partly_bound(sc->end_condition, - GETKEY_WITH_POS(sc->keypos, - this->dbterm.tpl)) > 0) + GETKEY_WITH_POS(sc->keypos, this->dbterm.tpl), + this->dbterm.tpl) > 0) return 0; - ret = db_prog_match(sc->p, sc->mp, - make_tuple(this->dbterm.tpl), - NULL,0, &dummy); + ret = db_match_dbterm(&tb->common, sc->p, sc->mp, 0, + &this->dbterm, NULL, 0); if (ret == am_true) { key = GETKEY(sc->tb, this->dbterm.tpl); - linkout_tree(sc->tb, key); + linkout_tree(sc->tb, key, this->dbterm.tpl); sc->erase_lastterm = 1; ++sc->accum; } diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 2f34561234..0b63ab9ba0 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,7 +25,6 @@ #ifdef HAVE_CONFIG_H # include "config.h" #endif - #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -58,6 +57,7 @@ DBIF_TABLE_GUARD | DBIF_TABLE_BODY | DBIF_TRACE_GUARD | DBIF_TRACE_BODY +#define HEAP_XTRA 100 /* ** Some convenience macros for stacks (DMC == db_match_compile) @@ -230,6 +230,11 @@ typedef enum { matchCall2, matchCall3, matchPushV, +#if HALFWORD_HEAP + matchPushVGuard, /* First guard-only variable reference */ +#endif + matchPushVResult, /* First variable reference in result, or (if HALFWORD) + in guard if also referenced in result */ matchPushExpr, /* Push the whole expression we're matching ('$_') */ matchPushArrayAsList, /* Only when parameter is an Array and not an erlang term (DCOMP_TRACE) */ @@ -293,11 +298,19 @@ DMC_DECLARE_STACK_TYPE(unsigned); ** Data about the heap during compilation */ +typedef struct DMCVariable { + int is_bound; + int is_in_body; +#if HALFWORD_HEAP + int first_guard_label; /* to maybe change from PushVGuard to PushVResult */ +#endif +} DMCVariable; + typedef struct DMCHeap { int size; - unsigned def[DMC_DEFAULT_SIZE]; - unsigned *data; - int used; + DMCVariable vars_def[DMC_DEFAULT_SIZE]; + DMCVariable* vars; + int vars_used; } DMCHeap; /* @@ -324,7 +337,6 @@ typedef struct dmc_context { Eterm *bodyexpr; int num_match; int current_match; - int eheap_need; Uint cflags; int is_guard; /* 1 if in guard, 0 if in body */ int special; /* 1 if the head in the match was a single expression */ @@ -347,9 +359,22 @@ typedef struct dmc_context { #define ERTS_DEFAULT_MS_HEAP_SIZE 128 +/* Runtime info about a $-variable +*/ +typedef struct MatchVariable { + Eterm term; +#ifdef DEBUG + Process* proc; + Eterm* base; +#endif +} MatchVariable; + typedef struct { Process process; - Eterm *heap; + union { + Eterm* heap; + MatchVariable* variables; /* first on "heap" */ + }u; Eterm default_heap[ERTS_DEFAULT_MS_HEAP_SIZE]; } ErtsMatchPseudoProcess; @@ -372,10 +397,10 @@ cleanup_match_pseudo_process(ErtsMatchPseudoProcess *mpsp, int keep_heap) } #endif if (!keep_heap) { - if (mpsp->heap != &mpsp->default_heap[0]) { + if (mpsp->u.heap != mpsp->default_heap) { /* Have to be done *after* call to erts_cleanup_empty_process() */ - erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->heap); - mpsp->heap = &mpsp->default_heap[0]; + erts_free(ERTS_ALC_T_DB_MS_RUN_HEAP, (void *) mpsp->u.heap); + mpsp->u.heap = mpsp->default_heap; } #ifdef DEBUG else { @@ -399,7 +424,7 @@ create_match_pseudo_process(void) mpsp = (ErtsMatchPseudoProcess *)erts_alloc(ERTS_ALC_T_DB_MS_PSDO_PROC, sizeof(ErtsMatchPseudoProcess)); erts_init_empty_process(&mpsp->process); - mpsp->heap = &mpsp->default_heap[0]; + mpsp->u.heap = mpsp->default_heap; return mpsp; } @@ -423,11 +448,11 @@ get_match_pseudo_process(Process *c_p, Uint heap_size) mpsp = match_pseudo_process; cleanup_match_pseudo_process(mpsp, 0); #endif - if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE) - mpsp->heap = (Eterm *) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, - heap_size*sizeof(Uint)); + if (heap_size > ERTS_DEFAULT_MS_HEAP_SIZE*sizeof(Eterm)) { + mpsp->u.heap = (Eterm*) erts_alloc(ERTS_ALC_T_DB_MS_RUN_HEAP, heap_size); + } else { - ASSERT(mpsp->heap == &mpsp->default_heap[0]); + ASSERT(mpsp->u.heap == mpsp->default_heap); } return mpsp; } @@ -468,23 +493,6 @@ erts_match_set_release_result(Process* c_p) static erts_smp_atomic_t trace_control_word; - -Eterm -erts_ets_copy_object(Eterm obj, Process* to) -{ - Uint size = size_object(obj); - Eterm* hp = HAlloc(to, size); - Eterm res; - - res = copy_struct(obj, size, &hp, &MSO(to)); -#ifdef DEBUG - if (eq(obj, res) == 0) { - erl_exit(1, "copy not equal to source\n"); - } -#endif - return res; -} - /* This needs to be here, before the bif table... */ static Eterm db_set_trace_control_word_fake_1(Process *p, Eterm val); @@ -870,9 +878,9 @@ static DMCRet dmc_one_term(DMCContext *context, #ifdef DMC_DEBUG static int test_disassemble_next = 0; -static void db_match_dis(Binary *prog); +void db_match_dis(Binary *prog); #define TRACE erts_fprintf(stderr,"Trace: %s:%d\n",__FILE__,__LINE__) -#define FENCE_PATTERN_SIZE 1 +#define FENCE_PATTERN_SIZE (1*sizeof(Uint)) #define FENCE_PATTERN 0xDEADBEEFUL #else #define TRACE /* Nothing */ @@ -890,6 +898,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace); static Eterm seq_trace_fake(Process *p, Eterm arg1); +static void db_free_tmp_uncompressed(DbTerm* obj); + /* ** Interface routines. @@ -914,7 +924,7 @@ BIF_RETTYPE db_set_trace_control_word_1(Process *p, Eterm new) if (val != ((Uint32)val)) BIF_ERROR(p, BADARG); - old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (long) val); + old_tcw = (Uint32) erts_smp_atomic_xchg(&trace_control_word, (erts_aint_t) val); BIF_RET(erts_make_integer((Uint) old_tcw, p)); } @@ -1178,14 +1188,14 @@ done: } Eterm erts_match_set_run(Process *p, Binary *mpsp, - Eterm *args, int num_args, + Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, Uint32 *return_flags) { Eterm ret; - ret = db_prog_match(p, mpsp, - NIL, args, - num_args, return_flags); + ret = db_prog_match(p, mpsp, NIL, NULL, args, num_args, + in_flags, return_flags); #if defined(HARDDEBUG) if (is_non_value(ret)) { erts_fprintf(stderr, "Failed\n"); @@ -1209,9 +1219,9 @@ static Eterm erts_match_set_run_ets(Process *p, Binary *mpsp, { Eterm ret; - ret = db_prog_match(p, mpsp, - args, NULL, - num_args, return_flags); + ret = db_prog_match(p, mpsp, args, NULL, NULL, num_args, + ERTS_PAM_CONTIGUOUS_TUPLE | ERTS_PAM_COPY_RESULT, + return_flags); #if defined(HARDDEBUG) if (is_non_value(ret)) { erts_fprintf(stderr, "Failed\n"); @@ -1279,7 +1289,6 @@ Binary *db_match_compile(Eterm *matchexpr, int structure_checked; DMCRet res; int current_try_label; - Uint max_eheap_need; Binary *bp = NULL; unsigned clause_start; @@ -1292,27 +1301,24 @@ Binary *db_match_compile(Eterm *matchexpr, context.matchexpr = matchexpr; context.guardexpr = guards; context.bodyexpr = body; - context.eheap_need = 0; context.err_info = err_info; context.cflags = flags; heap.size = DMC_DEFAULT_SIZE; - heap.data = heap.def; + heap.vars = heap.vars_def; /* ** Compile the match expression */ restart: - heap.used = 0; - max_eheap_need = 0; + heap.vars_used = 0; for (context.current_match = 0; context.current_match < num_progs; ++context.current_match) { /* This loop is long, too long */ - memset(heap.data, 0, heap.size * sizeof(*heap.data)); + memset(heap.vars, 0, heap.size * sizeof(*heap.vars)); t = context.matchexpr[context.current_match]; context.stack_used = 0; - context.eheap_need = 0; structure_checked = 0; if (context.current_match < num_progs - 1) { DMC_PUSH(text,matchTryMeElse); @@ -1484,10 +1490,6 @@ restart: if (current_try_label >= 0) { DMC_POKE(text, current_try_label, DMC_STACK_NUM(text)); } - /* So, how much eheap did this part of the match program need? */ - if (context.eheap_need > max_eheap_need) { - max_eheap_need = context.eheap_need; - } } /* for (context.current_match = 0 ...) */ @@ -1523,16 +1525,13 @@ restart: ret->saved_program_buf = NULL; ret->saved_program = NIL; ret->term_save = context.save; - ret->num_bindings = heap.used; + ret->num_bindings = heap.vars_used; ret->single_variable = context.special; sys_memcpy(ret->text, DMC_STACK_DATA(text), DMC_STACK_NUM(text) * sizeof(UWord)); - ret->heap_size = ((heap.used * sizeof(Eterm)) + - (max_eheap_need * sizeof(Eterm)) + - (context.stack_need * sizeof(Eterm *)) + - (3 * (FENCE_PATTERN_SIZE * sizeof(Eterm *)))); - ret->eheap_offset = heap.used + FENCE_PATTERN_SIZE; - ret->stack_offset = ret->eheap_offset + max_eheap_need + FENCE_PATTERN_SIZE; + ret->stack_offset = heap.vars_used*sizeof(MatchVariable) + FENCE_PATTERN_SIZE; + ret->heap_size = ret->stack_offset + context.stack_need * sizeof(Eterm*) + FENCE_PATTERN_SIZE; + #ifdef DMC_DEBUG ret->prog_end = ret->text + DMC_STACK_NUM(text); #endif @@ -1550,8 +1549,8 @@ error: /* Here is were we land when compilation failed. */ DMC_FREE(text); if (context.copy != NULL) free_message_buffer(context.copy); - if (heap.data != heap.def) - erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.data); + if (heap.vars != heap.vars_def) + erts_free(ERTS_ALC_T_DB_MS_CMPL_HEAP, (void *) heap.vars); return bp; } @@ -1596,7 +1595,7 @@ erts_match_prog_foreach_offheap(Binary *bprog, */ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) { - Eterm *hp = HAlloc(psp, arity * 2); + Eterm *hp = HAllocX(psp, arity * 2, HEAP_XTRA); Eterm ret = NIL; while (--arity >= 0) { ret = CONS(hp, arr[arity], ret); @@ -1604,15 +1603,83 @@ static Eterm dpm_array_to_list(Process *psp, Eterm *arr, int arity) } return ret; } + + +#if HALFWORD_HEAP +struct heap_checkpoint_t +{ + Process *p; + Eterm* htop; + ErlHeapFragment* mbuf; + unsigned used_size; + ErlOffHeap off_heap; +}; + +static void heap_checkpoint_init(Process* p, struct heap_checkpoint_t* hcp) +{ + hcp->p = p; + hcp->htop = HEAP_TOP(p); + hcp->mbuf = MBUF(p); + hcp->used_size = hcp->mbuf ? hcp->mbuf->used_size : 0; + hcp->off_heap = MSO(p); +} + +static void heap_checkpoint_revert(struct heap_checkpoint_t* hcp) +{ + struct erl_off_heap_header* oh = MSO(hcp->p).first; + + if (oh != hcp->off_heap.first) { + ASSERT(oh != NULL); + if (hcp->off_heap.first) { + while (oh->next != hcp->off_heap.first) { + oh = oh->next; + } + oh->next = NULL; + } + erts_cleanup_offheap(&MSO(hcp->p)); + MSO(hcp->p) = hcp->off_heap; + } + if (MBUF(hcp->p) != hcp->mbuf) { + ErlHeapFragment* hf = MBUF(hcp->p); + ASSERT(hf != NULL); + if (hcp->mbuf) { + while (hf->next != hcp->mbuf) { + hf = hf->next; + } + hf->next = NULL; + } + free_message_buffer(MBUF(hcp->p)); + MBUF(hcp->p) = hcp->mbuf; + } + if (hcp->mbuf != NULL && hcp->mbuf->used_size != hcp->used_size) { + hcp->mbuf->used_size = hcp->used_size; + } + HEAP_TOP(hcp->p) = hcp->htop; +} +#endif /* HALFWORD_HEAP */ + +static ERTS_INLINE Eterm copy_object_rel(Process* p, Eterm term, Eterm* base) +{ + if (!is_immed(term)) { + Uint sz = size_object_rel(term, base); + Eterm* top = HAllocX(p, sz, HEAP_XTRA); + return copy_struct_rel(term, sz, &top, &MSO(p), base, NULL); + } + return term; +} + + /* ** Execution of the match program, this is Pam. ** May return THE_NON_VALUE, which is a bailout. -** the para meter 'arity' is only used if 'term' is actually an array, +** the parameter 'arity' is only used if 'term' is actually an array, ** i.e. 'DCOMP_TRACE' was specified */ -Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, +Eterm db_prog_match(Process *c_p, Binary *bprog, + Eterm term, Eterm* base, Eterm *termp, int arity, + enum erts_pam_run_flags in_flags, Uint32 *return_flags) { MatchProg *prog = Binary2MatchProg(bprog); @@ -1621,7 +1688,7 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, Eterm t; Eterm **sp; Eterm *esp; - Eterm *hp; + MatchVariable* variables; BeamInstr *cp; UWord *pc = prog->text; Eterm *ehp; @@ -1631,19 +1698,24 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, unsigned do_catch; ErtsMatchPseudoProcess *mpsp; Process *psp; + Process* build_proc; Process *tmpp; Process *current_scheduled; ErtsSchedulerData *esdp; Eterm (*bif)(Process*, ...); int fail_label; int atomic_trace; +#if HALFWORD_HEAP + struct heap_checkpoint_t c_p_checkpoint = {}; +#endif #ifdef DMC_DEBUG Uint *heap_fence; - Uint *eheap_fence; Uint *stack_fence; Uint save_op; #endif /* DMC_DEBUG */ + ASSERT(base==NULL || HALFWORD_HEAP); + mpsp = get_match_pseudo_process(c_p, prog->heap_size); psp = &mpsp->process; @@ -1653,7 +1725,6 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, esdp = ERTS_GET_SCHEDULER_DATA_FROM_PROC(c_p); ASSERT(esdp != NULL); current_scheduled = esdp->current_process; - esdp->current_process = psp; /* SMP: psp->scheduler_data is set by get_match_pseudo_process */ atomic_trace = 0; @@ -1676,11 +1747,9 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, #ifdef DMC_DEBUG save_op = 0; - heap_fence = (Uint *) mpsp->heap + prog->eheap_offset - 1; - eheap_fence = (Uint *) mpsp->heap + prog->stack_offset - 1; - stack_fence = (Uint *) mpsp->heap + prog->heap_size - 1; + heap_fence = (Eterm*)((char*) mpsp->u.heap + prog->stack_offset) - 1; + stack_fence = (Eterm*)((char*) mpsp->u.heap + prog->heap_size) - 1; *heap_fence = FENCE_PATTERN; - *eheap_fence = FENCE_PATTERN; *stack_fence = FENCE_PATTERN; #endif /* DMC_DEBUG */ @@ -1694,36 +1763,48 @@ Eterm db_prog_match(Process *c_p, Binary *bprog, Eterm term, *return_flags = 0U; + variables = mpsp->u.variables; +#if HALFWORD_HEAP + c_p_checkpoint.p = NULL; +#endif + restart: ep = &term; - esp = mpsp->heap + prog->stack_offset; + esp = (Eterm*)((char*)mpsp->u.heap + prog->stack_offset); sp = (Eterm **) esp; - hp = mpsp->heap; - ehp = mpsp->heap + prog->eheap_offset; ret = am_true; do_catch = 0; fail_label = -1; + build_proc = psp; + esdp->current_process = psp; + ASSERT_HALFWORD(!c_p_checkpoint.p); + +#ifdef DEBUG + ASSERT(variables == mpsp->u.variables); + for (i=0; i<prog->num_bindings; i++) { + variables[i].term = THE_NON_VALUE; + variables[i].proc = NULL; + variables[i].base = base; + } +#endif for (;;) { -#ifdef DMC_DEBUG + + #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { erl_exit(1, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } - if (*eheap_fence != FENCE_PATTERN) { - erl_exit(1, "Eheap fence overwritten in db_prog_match after op " - "0x%08x, overwritten with 0x%08x.", save_op, - *eheap_fence); - } if (*stack_fence != FENCE_PATTERN) { erl_exit(1, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } save_op = *pc; -#endif + #endif switch (*pc++) { case matchTryMeElse: + ASSERT(fail_label == -1); fail_label = *pc++; break; case matchArray: /* only when DCOMP_TRACE, is always first @@ -1734,13 +1815,14 @@ restart: ep = termp; break; case matchArrayBind: /* When the array size is unknown. */ + ASSERT(termp); n = *pc++; - hp[n] = dpm_array_to_list(psp, termp, arity); + variables[n].term = dpm_array_to_list(psp, termp, arity); break; case matchTuple: /* *ep is a tuple of arity n */ - if (!is_tuple(*ep)) + if (!is_tuple_rel(*ep,base)) FAIL(); - ep = tuple_val(*ep); + ep = tuple_val_rel(*ep,base); n = *pc++; if (arityval(*ep) != n) FAIL(); @@ -1748,9 +1830,9 @@ restart: break; case matchPushT: /* *ep is a tuple of arity n, push ptr to first element */ - if (!is_tuple(*ep)) + if (!is_tuple_rel(*ep,base)) FAIL(); - tp = tuple_val(*ep); + tp = tuple_val_rel(*ep,base); n = *pc++; if (arityval(*tp) != n) FAIL(); @@ -1760,12 +1842,12 @@ restart: case matchList: if (!is_list(*ep)) FAIL(); - ep = list_val(*ep); + ep = list_val_rel(*ep,base); break; case matchPushL: if (!is_list(*ep)) FAIL(); - *sp++ = list_val(*ep); + *sp++ = list_val_rel(*ep,base); ++ep; break; case matchPop: @@ -1773,41 +1855,44 @@ restart: break; case matchBind: n = *pc++; - hp[n] = *ep++; + variables[n].term = *ep++; break; case matchCmp: n = *pc++; - if (!eq(hp[n],*ep)) + if (!eq_rel(variables[n].term, base, *ep, base)) FAIL(); ++ep; break; case matchEqBin: t = (Eterm) *pc++; - if (!eq(*ep,t)) + if (!eq_rel(t,NULL,*ep,base)) FAIL(); ++ep; break; case matchEqFloat: - if (!is_float(*ep)) + if (!is_float_rel(*ep,base)) FAIL(); - if (memcmp(float_val(*ep) + 1, pc, sizeof(double))) + if (memcmp(float_val_rel(*ep,base) + 1, pc, sizeof(double))) FAIL(); pc += TermWords(2); ++ep; break; - case matchEqRef: - if (!is_ref(*ep)) + case matchEqRef: { + Eterm* epc = (Eterm*)pc; + if (!is_ref_rel(*ep,base)) FAIL(); - if (!eq(*ep, make_internal_ref((Uint *) pc))) + if (!eq_rel(make_internal_ref_rel(epc, epc), epc, *ep, base)) { FAIL(); - i = thing_arityval(*((Uint *) pc)); + } + i = thing_arityval(*epc); pc += TermWords(i+1); ++ep; break; + } case matchEqBig: - if (!is_big(*ep)) + if (!is_big_rel(*ep,base)) FAIL(); - tp = big_val(*ep); + tp = big_val_rel(*ep,base); { Eterm *epc = (Eterm *) pc; if (*tp != *epc) @@ -1823,7 +1908,8 @@ restart: ++ep; break; case matchEq: - t = (Eterm) *pc++; + t = (Eterm) *pc++; + ASSERT(is_immed(t)); if (t != *ep++) FAIL(); break; @@ -1831,25 +1917,32 @@ restart: ++ep; break; /* - * Here comes guard instructions + * Here comes guard & body instructions */ case matchPushC: /* Push constant */ - *esp++ = *pc++; + if ((in_flags & ERTS_PAM_COPY_RESULT) + && do_catch && !is_immed(*pc)) { + *esp++ = copy_object(*pc++, c_p); + } + else { + *esp++ = *pc++; + } break; case matchConsA: - ehp[1] = *--esp; - ehp[0] = esp[-1]; + ehp = HAllocX(build_proc, 2, HEAP_XTRA); + CDR(ehp) = *--esp; + CAR(ehp) = esp[-1]; esp[-1] = make_list(ehp); - ehp += 2; break; case matchConsB: - ehp[0] = *--esp; - ehp[1] = esp[-1]; + ehp = HAllocX(build_proc, 2, HEAP_XTRA); + CAR(ehp) = *--esp; + CDR(ehp) = esp[-1]; esp[-1] = make_list(ehp); - ehp += 2; break; case matchMkTuple: n = *pc++; + ehp = HAllocX(build_proc, n+1, HEAP_XTRA); t = make_tuple(ehp); *ehp++ = make_arityval(n); while (n--) { @@ -1859,7 +1952,7 @@ restart: break; case matchCall0: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp); + t = (*bif)(build_proc); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1870,7 +1963,7 @@ restart: break; case matchCall1: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp, esp[-1]); + t = (*bif)(build_proc, esp[-1]); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1881,7 +1974,7 @@ restart: break; case matchCall2: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp, esp[-1], esp[-2]); + t = (*bif)(build_proc, esp[-1], esp[-2]); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1893,7 +1986,7 @@ restart: break; case matchCall3: bif = (Eterm (*)(Process*, ...)) *pc++; - t = (*bif)(psp, esp[-1], esp[-2], esp[-3]); + t = (*bif)(build_proc, esp[-1], esp[-2], esp[-3]); if (is_non_value(t)) { if (do_catch) t = FAIL_TERM; @@ -1903,15 +1996,73 @@ restart: esp -= 2; esp[-1] = t; break; + + #if HALFWORD_HEAP + case matchPushVGuard: + if (!base) goto case_matchPushV; + /* Build NULL-based copy on pseudo heap for easy disposal */ + n = *pc++; + ASSERT(is_value(variables[n].term)); + ASSERT(!variables[n].proc); + variables[n].term = copy_object_rel(psp, variables[n].term, base); + *esp++ = variables[n].term; + #ifdef DEBUG + variables[n].proc = psp; + variables[n].base = NULL; + #endif + break; + #endif + case matchPushVResult: + if (!(in_flags & ERTS_PAM_COPY_RESULT)) goto case_matchPushV; + + /* Build (NULL-based) copy on callers heap */ + #if HALFWORD_HEAP + if (!do_catch && !c_p_checkpoint.p) { + heap_checkpoint_init(c_p, &c_p_checkpoint); + } + #endif + n = *pc++; + ASSERT(is_value(variables[n].term)); + ASSERT(!variables[n].proc); + variables[n].term = copy_object_rel(c_p, variables[n].term, base); + *esp++ = variables[n].term; + #ifdef DEBUG + variables[n].proc = c_p; + variables[n].base = NULL; + #endif + break; case matchPushV: - *esp++ = hp[*pc++]; + case_matchPushV: + n = *pc++; + ASSERT(is_value(variables[n].term)); + ASSERT(!variables[n].base); + *esp++ = variables[n].term; break; case matchPushExpr: - *esp++ = term; + if (in_flags & ERTS_PAM_COPY_RESULT) { + Uint sz; + Eterm* top; + sz = size_object_rel(term, base); + top = HAllocX(build_proc, sz, HEAP_XTRA); + if (in_flags & ERTS_PAM_CONTIGUOUS_TUPLE) { + ASSERT(is_tuple_rel(term,base)); + *esp++ = copy_shallow_rel(tuple_val_rel(term,base), sz, + &top, &MSO(build_proc), base); + } + else { + *esp++ = copy_struct_rel(term, sz, &top, &MSO(build_proc), + base, NULL); + } + } + else { + *esp = term; + } break; case matchPushArrayAsList: + ASSERT_HALFWORD(base == NULL); n = arity; /* Only happens when 'term' is an array */ tp = termp; + ehp = HAllocX(build_proc, n*2, HEAP_XTRA); *esp++ = make_list(ehp); while (n--) { *ehp++ = *tp++; @@ -1924,7 +2075,8 @@ restart: break; case matchPushArrayAsListU: /* This instruction is NOT efficient. */ - *esp++ = dpm_array_to_list(psp, termp, arity); + ASSERT_HALFWORD(base == NULL); + *esp++ = dpm_array_to_list(build_proc, termp, arity); break; case matchTrue: if (*--esp != am_true) @@ -2010,7 +2162,8 @@ restart: case matchProcessDump: { erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(0); print_process_info(ERTS_PRINT_DSBUF, (void *) dsbufp, c_p); - *esp++ = new_binary(psp, (byte *)dsbufp->str, (int)dsbufp->str_len); + *esp++ = new_binary(build_proc, (byte *)dsbufp->str, + dsbufp->str_len); erts_destroy_tmp_dsbuf(dsbufp); break; } @@ -2054,29 +2207,24 @@ restart: if (SEQ_TRACE_TOKEN(c_p) == NIL) *esp++ = NIL; else { + Eterm sender = SEQ_TRACE_TOKEN_SENDER(c_p); + Uint sender_sz = is_immed(sender) ? 0 : size_object(sender); + ehp = HAllocX(build_proc, 6 + sender_sz, HEAP_XTRA); + if (sender_sz) { + sender = copy_struct(sender, sender_sz, &ehp, &MSO(build_proc)); + } *esp++ = make_tuple(ehp); ehp[0] = make_arityval(5); ehp[1] = SEQ_TRACE_TOKEN_FLAGS(c_p); ehp[2] = SEQ_TRACE_TOKEN_LABEL(c_p); ehp[3] = SEQ_TRACE_TOKEN_SERIAL(c_p); - ehp[4] = SEQ_TRACE_TOKEN_SENDER(c_p); + ehp[4] = sender; ehp[5] = SEQ_TRACE_TOKEN_LASTCNT(c_p); ASSERT(SEQ_TRACE_TOKEN_ARITY(c_p) == 5); ASSERT(is_immed(ehp[1])); ASSERT(is_immed(ehp[2])); ASSERT(is_immed(ehp[3])); ASSERT(is_immed(ehp[5])); - if(!is_immed(ehp[4])) { - Eterm *sender = &ehp[4]; - ehp += 6; - *sender = copy_struct(*sender, - size_object(*sender), - &ehp, - &MSO(psp)); - } - else - ehp += 6; - } break; case matchEnableTrace: @@ -2125,12 +2273,12 @@ restart: if (!(c_p->cp) || !(cp = find_function_from_pc(c_p->cp))) { *esp++ = am_undefined; } else { + ehp = HAllocX(build_proc, 4, HEAP_XTRA); *esp++ = make_tuple(ehp); ehp[0] = make_arityval(3); ehp[1] = cp[0]; ehp[2] = cp[1]; ehp[3] = make_small((Uint) cp[2]); - ehp += 4; } break; case matchSilent: @@ -2207,8 +2355,12 @@ restart: } } break; - case matchCatch: + case matchCatch: /* Match success, now build result */ do_catch = 1; + if (in_flags & ERTS_PAM_COPY_RESULT) { + build_proc = c_p; + esdp->current_process = c_p; + } break; case matchHalt: goto success; @@ -2217,9 +2369,16 @@ restart: } } fail: +#if HALFWORD_HEAP + if (c_p_checkpoint.p) { + /* Dispose garbage built by guards on caller heap */ + heap_checkpoint_revert(&c_p_checkpoint); + c_p_checkpoint.p = NULL; + } +#endif *return_flags = 0U; - if (fail_label >= 0) { /* We failed during a "TryMeElse", - lets restart, with the next match + if (fail_label >= 0) { /* We failed during a "TryMeElse", + lets restart, with the next match program */ pc = (prog->text) + fail_label; cleanup_match_pseudo_process(mpsp, 1); @@ -2233,11 +2392,6 @@ success: erl_exit(1, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } - if (*eheap_fence != FENCE_PATTERN) { - erl_exit(1, "Eheap fence overwritten in db_prog_match after op " - "0x%08x, overwritten with 0x%08x.", save_op, - *eheap_fence); - } if (*stack_fence != FENCE_PATTERN) { erl_exit(1, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, @@ -2248,6 +2402,7 @@ success: esdp->current_process = current_scheduled; END_ATOMIC_TRACE(c_p); + return ret; #undef FAIL #undef FAIL_TERM @@ -2259,7 +2414,8 @@ success: /* * Convert a match program to a "magic" binary to return up to erlang */ -Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) { +Eterm db_make_mp_binary(Process *p, Binary *mp, Eterm **hpp) +{ return erts_mk_magic_binary_term(hpp, &MSO(p), mp); } @@ -2325,13 +2481,13 @@ void db_free_dmc_err_info(DMCErrInfo *ei){ ** Store bignum in *hpp and increase *hpp accordingly. ** *hpp is assumed to be large enough to hold the result. */ -Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) +Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr) { DeclareTmpHeapNoproc(big_tmp,2); Eterm res; Sint ires; - Eterm arg1; - Eterm arg2; + Wterm arg1; + Wterm arg2; if (is_both_small(counter,incr)) { ires = signed_val(counter) + signed_val(incr); @@ -2372,6 +2528,34 @@ Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr) } } +/* Must be called to read elements after db_lookup_dbterm. +** Will decompress if needed. +** HEALFWORD_HEAP: +** Will convert from relative to Wterm format if needed. +** (but only on top level, tuples and lists will still contain rterms) +*/ +Wterm db_do_read_element(DbUpdateHandle* handle, Sint position) +{ + Eterm elem = handle->dbterm->tpl[position]; + if (!is_header(elem)) { +#if HALFWORD_HEAP + if (!is_immed(elem) + && !handle->tb->common.compress + && !(handle->abs_vec && handle->abs_vec[position])) { + return rterm2wterm(elem, handle->dbterm->tpl); + } +#endif + return elem; + } + + ASSERT(((DbTableCommon*)handle->tb)->compress); + ASSERT(!handle->mustResize); + handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, + handle->dbterm); + handle->mustResize = 1; + return handle->dbterm->tpl[position]; +} + /* ** Update one element: ** handle: Initialized by db_lookup_dbterm() @@ -2388,132 +2572,489 @@ void db_do_update_element(DbUpdateHandle* handle, Eterm* oldp; Uint newval_sz; Uint oldval_sz; +#if HALFWORD_HEAP + Eterm* old_base; +#endif if (is_both_immed(newval,oldval)) { handle->dbterm->tpl[position] = newval; + #ifdef DEBUG_CLONE + if (handle->dbterm->debug_clone) { + handle->dbterm->debug_clone[position] = newval; + } + #endif return; } - else if (!handle->mustResize && is_boxed(newval)) { - newp = boxed_val(newval); - switch (*newp & _TAG_HEADER_MASK) { - case _TAG_HEADER_POS_BIG: - case _TAG_HEADER_NEG_BIG: - case _TAG_HEADER_FLOAT: - case _TAG_HEADER_HEAP_BIN: - newval_sz = header_arity(*newp) + 1; - if (is_boxed(oldval)) { - oldp = boxed_val(oldval); - switch (*oldp & _TAG_HEADER_MASK) { + if (!handle->mustResize) { + if (handle->tb->common.compress) { + handle->dbterm = db_alloc_tmp_uncompressed(&handle->tb->common, + handle->dbterm); + handle->mustResize = 1; + oldval = handle->dbterm->tpl[position]; + #if HALFWORD_HEAP + old_base = NULL; + #endif + } + else { + #if HALFWORD_HEAP + ASSERT(!handle->abs_vec); + old_base = handle->dbterm->tpl; + #endif + if (is_boxed(newval)) { + newp = boxed_val(newval); + switch (*newp & _TAG_HEADER_MASK) { case _TAG_HEADER_POS_BIG: case _TAG_HEADER_NEG_BIG: case _TAG_HEADER_FLOAT: case _TAG_HEADER_HEAP_BIN: - oldval_sz = header_arity(*oldp) + 1; - if (oldval_sz == newval_sz) { - /* "self contained" terms of same size, do memcpy */ - sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm)); - return; + newval_sz = header_arity(*newp) + 1; + if (is_boxed(oldval)) { + oldp = boxed_val_rel(oldval,old_base); + switch (*oldp & _TAG_HEADER_MASK) { + case _TAG_HEADER_POS_BIG: + case _TAG_HEADER_NEG_BIG: + case _TAG_HEADER_FLOAT: + case _TAG_HEADER_HEAP_BIN: + oldval_sz = header_arity(*oldp) + 1; + if (oldval_sz == newval_sz) { + /* "self contained" terms of same size, do memcpy */ + sys_memcpy(oldp, newp, newval_sz*sizeof(Eterm)); + return; + } + goto both_size_set; + } } - goto both_size_set; + goto new_size_set; } } - goto new_size_set; } } +#if HALFWORD_HEAP + else { + old_base = (handle->tb->common.compress + || (handle->abs_vec && handle->abs_vec[position])) ? + NULL : handle->dbterm->tpl; + } +#endif /* Not possible for simple memcpy or dbterm is already non-contiguous, */ /* need to realloc... */ newval_sz = is_immed(newval) ? 0 : size_object(newval); new_size_set: - - oldval_sz = is_immed(oldval) ? 0 : size_object(oldval); + + oldval_sz = is_immed(oldval) ? 0 : size_object_rel(oldval,old_base); both_size_set: handle->new_size = handle->new_size - oldval_sz + newval_sz; - /* write new value in old dbterm, finalize will make a flat copy */ + /* write new value in old dbterm, finalize will make a flat copy */ handle->dbterm->tpl[position] = newval; handle->mustResize = 1; + +#if HALFWORD_HEAP + if (old_base && newval_sz > 0) { + ASSERT(!handle->tb->common.compress); + if (!handle->abs_vec) { + int i = header_arity(handle->dbterm->tpl[0]); + handle->abs_vec = erts_alloc(ERTS_ALC_T_TMP, (i+1)*sizeof(char)); + sys_memset(handle->abs_vec, 0, i+1); + /* abs_vec[0] not used */ + } + handle->abs_vec[position] = 1; + } +#endif +} + +static ERTS_INLINE byte* db_realloc_term(DbTableCommon* tb, void* old, + Uint old_sz, Uint new_sz, Uint offset) +{ + byte* ret; + if (erts_ets_realloc_always_moves) { + ret = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz); + sys_memcpy(ret, old, offset); + erts_db_free(ERTS_ALC_T_DB_TERM, (DbTable*)tb, old, old_sz); + } else { + ret = erts_db_realloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, + old, old_sz, new_sz); + } + return ret; +} + +/* Allocated size of a compressed dbterm +*/ +static ERTS_INLINE Uint db_alloced_size_comp(DbTerm* obj) +{ + return obj->tpl[arityval(*obj->tpl) + 1]; +} + +void db_free_term(DbTable *tb, void* basep, Uint offset) +{ + DbTerm* db = (DbTerm*) ((byte*)basep + offset); + Uint size; + if (tb->common.compress) { + db_cleanup_offheap_comp(db); + size = db_alloced_size_comp(db); + } + else { + ErlOffHeap tmp_oh; + tmp_oh.first = db->first_oh; + erts_cleanup_offheap(&tmp_oh); + size = offset + offsetof(DbTerm,tpl) + db->size*sizeof(Eterm); + } + erts_db_free(ERTS_ALC_T_DB_TERM, tb, basep, size); } +static ERTS_INLINE Uint align_up(Uint value, Uint pow2) +{ + ASSERT((pow2 & (pow2-1)) == 0); + return (value + (pow2-1)) & ~(pow2-1); +} + +/* Compressed size of an uncompressed term +*/ +static Uint db_size_dbterm_comp(DbTableCommon* tb, Eterm obj) +{ + Eterm* tpl = tuple_val(obj); + int i; + Uint size = sizeof(DbTerm) + + arityval(*tpl) * sizeof(Eterm) + + sizeof(Uint); /* "alloc_size" */ + + for (i = arityval(*tpl); i>0; i--) { + if (i != tb->keypos && is_not_immed(tpl[i])) { + size += erts_encode_ext_size_ets(tpl[i]); + } + } + size += size_object(tpl[tb->keypos]) * sizeof(Eterm); + return align_up(size, sizeof(Uint)); +} + +/* Conversion between top tuple element and pointer to compressed data +*/ +static ERTS_INLINE Eterm ext2elem(Eterm* tpl, byte* ext) +{ + return (((Uint)(ext - (byte*)tpl)) << _TAG_PRIMARY_SIZE) | TAG_PRIMARY_HEADER; +} +static ERTS_INLINE byte* elem2ext(Eterm* tpl, Uint ix) +{ + ASSERT(is_header(tpl[ix])); + return (byte*)tpl + (tpl[ix] >> _TAG_PRIMARY_SIZE); +} + +static void* copy_to_comp(DbTableCommon* tb, Eterm obj, DbTerm* dest, + Uint alloc_size) +{ + ErlOffHeap tmp_offheap; + Eterm* src = tuple_val(obj); + Eterm* tpl = dest->tpl; + Eterm key = src[tb->keypos]; + int arity = arityval(src[0]); + union { + Eterm* ep; + byte* cp; + UWord ui; + }top; + int i; + + top.ep = tpl+ 1 + arity + 1; + tpl[0] = src[0]; + tpl[arity + 1] = alloc_size; + + tmp_offheap.first = NULL; + tpl[tb->keypos] = copy_struct_rel(key, size_object(key), &top.ep, &tmp_offheap, NULL, tpl); + dest->first_oh = tmp_offheap.first; + for (i=1; i<=arity; i++) { + if (i != tb->keypos) { + if (is_immed(src[i])) { + tpl[i] = src[i]; + } + else { + tpl[i] = ext2elem(tpl, top.cp); + top.cp = erts_encode_ext_ets(src[i], top.cp, &dest->first_oh); + } + } + } + +#ifdef DEBUG_CLONE + { + Eterm* dbg_top = erts_alloc(ERTS_ALC_T_DB_TERM, dest->size * sizeof(Eterm)); + dest->debug_clone = dbg_top; + tmp_offheap.first = dest->first_oh; + copy_struct_rel(obj, dest->size, &dbg_top, &tmp_offheap, NULL, dbg_top); + dest->first_oh = tmp_offheap.first; + ASSERT(dbg_top == dest->debug_clone + dest->size); + } +#endif + return top.cp; +} /* ** Copy the object into a possibly new DbTerm, ** offset is the offset of the DbTerm from the start -** of the sysAllocaed structure, The possibly realloced and copied +** of the allocated structure, The possibly realloced and copied ** structure is returned. Make sure (((char *) old) - offset) is a ** pointer to a ERTS_ALC_T_DB_TERM allocated data area. */ -void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) +void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) { + byte* basep; + DbTerm* newp; + Eterm* top; int size = size_object(obj); - void *structp = ((char*) old) - offset; - DbTerm* p; - Eterm copy; - Eterm *top; ErlOffHeap tmp_offheap; if (old != 0) { + basep = ((byte*) old) - offset; tmp_offheap.first = old->first_oh; - tmp_offheap.overhead = 0; erts_cleanup_offheap(&tmp_offheap); old->first_oh = tmp_offheap.first; if (size == old->size) { - p = old; - } else { + newp = old; + } + else { Uint new_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1); Uint old_sz = offset + sizeof(DbTerm) + sizeof(Eterm)*(old->size-1); - if (erts_ets_realloc_always_moves) { - void *nstructp = erts_db_alloc(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - new_sz); - memcpy(nstructp,structp,offset); - erts_db_free(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - structp, - old_sz); - structp = nstructp; - } else { - structp = erts_db_realloc(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - structp, - old_sz, - new_sz); - } - p = (DbTerm*) ((void *)(((char *) structp) + offset)); + basep = db_realloc_term(tb, basep, old_sz, new_sz, offset); + newp = (DbTerm*) (basep + offset); } } else { - structp = erts_db_alloc(ERTS_ALC_T_DB_TERM, - (DbTable *) tb, - (offset - + sizeof(DbTerm) - + sizeof(Eterm)*(size-1))); - p = (DbTerm*) ((void *)(((char *) structp) + offset)); - } - p->size = size; + basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable *)tb, + (offset + sizeof(DbTerm) + sizeof(Eterm)*(size-1))); + newp = (DbTerm*) (basep + offset); + } + newp->size = size; + top = newp->tpl; tmp_offheap.first = NULL; - tmp_offheap.overhead = 0; + copy_struct_rel(obj, size, &top, &tmp_offheap, NULL, top); + newp->first_oh = tmp_offheap.first; +#ifdef DEBUG_CLONE + newp->debug_clone = NULL; +#endif + return basep; +} - top = DBTERM_BUF(p); - copy = copy_struct(obj, size, &top, &tmp_offheap); - p->first_oh = tmp_offheap.first; - DBTERM_SET_TPL(p,tuple_val(copy)); - return structp; +void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj) +{ + Uint new_sz = offset + db_size_dbterm_comp(tb, obj); + byte* basep; + DbTerm* newp; + byte* top; + + ASSERT(tb->compress); + if (old != 0) { + Uint old_sz = db_alloced_size_comp(old); + db_cleanup_offheap_comp(old); + + basep = ((byte*) old) - offset; + if (new_sz == old_sz) { + newp = old; + } + else { + basep = db_realloc_term(tb, basep, old_sz, new_sz, offset); + newp = (DbTerm*) (basep + offset); + } + } + else { + basep = erts_db_alloc(ERTS_ALC_T_DB_TERM, (DbTable*)tb, new_sz); + newp = (DbTerm*) (basep + offset); + } + + newp->size = size_object(obj); + top = copy_to_comp(tb, obj, newp, new_sz); + ASSERT(top <= basep + new_sz); + + /* ToDo: Maybe realloc if ((basep+new_sz) - top) > WASTED_SPACE_LIMIT */ + + return basep; +} + + +void db_finalize_resize(DbUpdateHandle* handle, Uint offset) +{ + DbTable* tbl = handle->tb; + DbTerm* newDbTerm; + Uint alloc_sz = offset + + (tbl->common.compress ? + db_size_dbterm_comp(&tbl->common, make_tuple(handle->dbterm->tpl)) : + sizeof(DbTerm)+sizeof(Eterm)*(handle->new_size-1)); + byte* newp = erts_db_alloc(ERTS_ALC_T_DB_TERM, tbl, alloc_sz); + byte* oldp = *(handle->bp); + + sys_memcpy(newp, oldp, offset); /* copy only hash/tree header */ + *(handle->bp) = newp; + newDbTerm = (DbTerm*) (newp + offset); + newDbTerm->size = handle->new_size; +#ifdef DEBUG_CLONE + newDbTerm->debug_clone = NULL; +#endif + + /* make a flat copy */ + + if (tbl->common.compress) { + copy_to_comp(&tbl->common, make_tuple(handle->dbterm->tpl), + newDbTerm, alloc_sz); + db_free_tmp_uncompressed(handle->dbterm); + } + else { + ErlOffHeap tmp_offheap; + Eterm* tpl = handle->dbterm->tpl; + Eterm* top = newDbTerm->tpl; + + tmp_offheap.first = NULL; + + #if HALFWORD_HEAP + if (handle->abs_vec) { + int i, arity = header_arity(handle->dbterm->tpl[0]); + + top[0] = tpl[0]; + top += arity + 1; + for (i=1; i<=arity; i++) { + Eterm* src_base = handle->abs_vec[i] ? NULL : tpl; + + newDbTerm->tpl[i] = copy_struct_rel(tpl[i], + size_object_rel(tpl[i],src_base), + &top, &tmp_offheap, src_base, + newDbTerm->tpl); + } + newDbTerm->first_oh = tmp_offheap.first; + ASSERT((byte*)top <= (newp + alloc_sz)); + erts_free(ERTS_ALC_T_TMP, handle->abs_vec); + } + else + #endif /* HALFWORD_HEAP */ + { + copy_struct_rel(make_tuple_rel(tpl,tpl), handle->new_size, &top, + &tmp_offheap, tpl, top); + newDbTerm->first_oh = tmp_offheap.first; + ASSERT((byte*)top == (newp + alloc_sz)); + } + } } +Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, + ErlOffHeap* off_heap) +{ + Eterm* hp = *hpp; + int i, arity = arityval(bp->tpl[0]); + + hp[0] = bp->tpl[0]; + *hpp += arity + 1; + + hp[tb->keypos] = copy_struct_rel(bp->tpl[tb->keypos], + size_object_rel(bp->tpl[tb->keypos], bp->tpl), + hpp, off_heap, bp->tpl, NULL); + for (i=arity; i>0; i--) { + if (i != tb->keypos) { + if (is_immed(bp->tpl[i])) { + hp[i] = bp->tpl[i]; + } + else { + hp[i] = erts_decode_ext_ets(hpp, off_heap, + elem2ext(bp->tpl, i)); + } + } + } + ASSERT((*hpp - hp) <= bp->size); +#ifdef DEBUG_CLONE + ASSERT(eq_rel(make_tuple(hp),make_tuple(bp->debug_clone),bp->debug_clone)); +#endif + return make_tuple(hp); +} -void db_free_term_data(DbTerm* p) +Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, + DbTerm* obj, Uint pos, + Eterm** hpp, Uint extra) +{ + if (is_immed(obj->tpl[pos])) { + *hpp = HAlloc(p, extra); + return obj->tpl[pos]; + } + if (tb->compress && pos != tb->keypos) { + byte* ext = elem2ext(obj->tpl, pos); + Sint sz = erts_decode_ext_size_ets(ext, db_alloced_size_comp(obj)) + extra; + Eterm* hp = HAlloc(p, sz); + Eterm* endp = hp + sz; + Eterm copy = erts_decode_ext_ets(&hp, &MSO(p), ext); + *hpp = hp; + hp += extra; + HRelease(p, endp, hp); +#ifdef DEBUG_CLONE + ASSERT(eq_rel(copy, obj->debug_clone[pos], obj->debug_clone)); +#endif + return copy; + } + else { + Uint sz = size_object_rel(obj->tpl[pos], obj->tpl); + *hpp = HAlloc(p, sz + extra); + return copy_struct_rel(obj->tpl[pos], sz, hpp, &MSO(p), obj->tpl, NULL); + } +} + + +/* Our own "cleanup_offheap" + * as refc-binaries may be unaligned in compressed terms +*/ +void db_cleanup_offheap_comp(DbTerm* obj) +{ + union erl_off_heap_ptr u; + ProcBin tmp; + + for (u.hdr = obj->first_oh; u.hdr; u.hdr = u.hdr->next) { + if ((UWord)u.voidp % sizeof(Uint) != 0) { /* unaligned ptr */ + sys_memcpy(&tmp, u.voidp, sizeof(tmp)); + /* Warning, must pass (void*)-variable to memcpy. Otherwise it will + cause Bus error on Sparc due to false compile time assumptions + about word aligned memory (type cast is not enough) */ + u.pb = &tmp; + } + switch (thing_subtag(u.hdr->thing_word)) { + case REFC_BINARY_SUBTAG: + if (erts_refc_dectest(&u.pb->val->refc, 0) == 0) { + erts_bin_free(u.pb->val); + } + break; + case FUN_SUBTAG: + ASSERT(u.pb != &tmp); + if (erts_refc_dectest(&u.fun->fe->refc, 0) == 0) { + erts_erase_fun_entry(u.fun->fe); + } + break; + default: + ASSERT(is_external_header(u.hdr->thing_word)); + ASSERT(u.pb != &tmp); + erts_deref_node_entry(u.ext->node); + break; + } + } +#ifdef DEBUG_CLONE + if (obj->debug_clone != NULL) { + erts_free(ERTS_ALC_T_DB_TERM, obj->debug_clone); + obj->debug_clone = NULL; + } +#endif +} + +int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b) { ErlOffHeap tmp_offheap; - tmp_offheap.first = p->first_oh; - tmp_offheap.overhead = 0; + Eterm* allocp; + Eterm* hp; + Eterm tmp_b; + int is_eq; + + ASSERT(tb->compress); + hp = allocp = erts_alloc(ERTS_ALC_T_TMP, b->size*sizeof(Eterm)); + tmp_offheap.first = NULL; + tmp_b = db_copy_from_comp(tb, b, &hp, &tmp_offheap); + is_eq = eq(a,tmp_b); erts_cleanup_offheap(&tmp_offheap); + erts_free(ERTS_ALC_T_TMP, allocp); + return is_eq; } - /* ** Check if object represents a "match" variable ** i.e and atom $N where N is an integer @@ -2658,7 +3199,7 @@ static DMCRet dmc_one_term(DMCContext *context, ** Ouch, big integer in match variable. */ Eterm *save_hp; - ASSERT(heap->data == heap->def); + ASSERT(heap->vars == heap->vars_def); sz = sz2 = sz3 = 0; for (j = 0; j < context->num_match; ++j) { sz += size_object(context->matchexpr[j]); @@ -2696,24 +3237,23 @@ static DMCRet dmc_one_term(DMCContext *context, may be atoms that changed */ context->matchexpr[j] = context->copy->mem[j]; } - heap->data = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP, - heap->size*sizeof(unsigned)); - sys_memset(heap->data, 0, - heap->size * sizeof(unsigned)); + heap->vars = erts_alloc(ERTS_ALC_T_DB_MS_CMPL_HEAP, + heap->size*sizeof(DMCVariable)); + sys_memset(heap->vars, 0, heap->size * sizeof(DMCVariable)); DMC_CLEAR(*stack); /*DMC_PUSH(*stack,NIL);*/ DMC_CLEAR(*text); return retRestart; } - if (heap->data[n]) { /* already bound ? */ + if (heap->vars[n].is_bound) { DMC_PUSH(*text,matchCmp); DMC_PUSH(*text,n); } else { /* Not bound, bind! */ - if (n >= heap->used) - heap->used = n + 1; + if (n >= heap->vars_used) + heap->vars_used = n + 1; DMC_PUSH(*text,matchBind); DMC_PUSH(*text,n); - heap->data[n] = 1; + heap->vars[n].is_bound = 1; } } else if (c == am_Underscore) { DMC_PUSH(*text, matchSkip); @@ -2738,6 +3278,8 @@ static DMCRet dmc_one_term(DMCContext *context, DMC_PUSH(*stack, c); break; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): + { + Eterm* ref_val = internal_ref_val(c); DMC_PUSH(*text, matchEqRef); #if HALFWORD_HEAP { @@ -2745,25 +3287,27 @@ static DMCRet dmc_one_term(DMCContext *context, UWord u; Uint t[2]; } fiddle; - ASSERT(thing_arityval(*internal_ref_val(c)) == 3); - fiddle.t[0] = *internal_ref_val(c); - fiddle.t[1] = (Uint) internal_ref_val(c)[1]; + ASSERT(thing_arityval(ref_val[0]) == 3); + fiddle.t[0] = ref_val[0]; + fiddle.t[1] = ref_val[1]; DMC_PUSH(*text, fiddle.u); - fiddle.t[0] = (Uint) internal_ref_val(c)[2]; - fiddle.t[1] = (Uint) internal_ref_val(c)[3]; + fiddle.t[0] = ref_val[2]; + fiddle.t[1] = ref_val[3]; DMC_PUSH(*text, fiddle.u); } #else - n = thing_arityval(*internal_ref_val(c)); - DMC_PUSH(*text, *internal_ref_val(c)); - for (i = 1; i <= n; ++i) { - DMC_PUSH(*text, (Uint) internal_ref_val(c)[i]); + n = thing_arityval(ref_val[0]); + for (i = 0; i <= n; ++i) { + DMC_PUSH(*text, ref_val[i]); } #endif break; + } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): - n = thing_arityval(*big_val(c)); + { + Eterm* bval = big_val(c); + n = thing_arityval(bval[0]); DMC_PUSH(*text, matchEqBig); #if HALFWORD_HEAP { @@ -2772,13 +3316,13 @@ static DMCRet dmc_one_term(DMCContext *context, Uint t[2]; } fiddle; ASSERT(n >= 1); - fiddle.t[0] = *big_val(c); - fiddle.t[1] = big_val(c)[1]; + fiddle.t[0] = bval[0]; + fiddle.t[1] = bval[1]; DMC_PUSH(*text, fiddle.u); for (i = 2; i <= n; ++i) { - fiddle.t[0] = big_val(c)[i]; + fiddle.t[0] = bval[i]; if (++i <= n) { - fiddle.t[1] = big_val(c)[i]; + fiddle.t[1] = bval[i]; } else { fiddle.t[1] = (Uint) 0; } @@ -2786,12 +3330,12 @@ static DMCRet dmc_one_term(DMCContext *context, } } #else - DMC_PUSH(*text, *big_val(c)); - for (i = 1; i <= n; ++i) { - DMC_PUSH(*text, (Uint) big_val(c)[i]); + for (i = 0; i <= n; ++i) { + DMC_PUSH(*text, (Uint) bval[i]); } #endif break; + } case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): DMC_PUSH(*text,matchEqFloat); #if HALFWORD_HEAP @@ -2924,7 +3468,6 @@ static DMCRet dmc_list(DMCContext *context, DMC_PUSH(*text, matchConsB); } --context->stack_used; /* Two objects on stack becomes one */ - context->eheap_need += 2; return retOk; } @@ -2983,7 +3526,6 @@ static DMCRet dmc_tuple(DMCContext *context, DMC_PUSH(*text, matchMkTuple); DMC_PUSH(*text, nelems); context->stack_used -= (nelems - 1); - context->eheap_need += (nelems + 1); *constant = 0; return retOk; } @@ -3001,9 +3543,6 @@ static DMCRet dmc_whole_expression(DMCContext *context, } else { ASSERT(is_tuple(context->matchexpr [context->current_match])); - context->eheap_need += - arityval(*(tuple_val(context->matchexpr - [context->current_match]))) * 2; DMC_PUSH(*text, matchPushArrayAsList); } } else { @@ -3016,6 +3555,41 @@ static DMCRet dmc_whole_expression(DMCContext *context, return retOk; } +/* Figure out which PushV instruction to use. +*/ +static void dmc_add_pushv_variant(DMCContext *context, DMCHeap *heap, + DMC_STACK_TYPE(UWord) *text, Uint n) +{ + DMCVariable* v = &heap->vars[n]; + MatchOps instr = matchPushV; + + ASSERT(n < heap->vars_used && v->is_bound); + if (context->is_guard) { + #if HALFWORD_HEAP + if (!v->first_guard_label) { + v->first_guard_label = DMC_STACK_NUM(*text); + ASSERT(v->first_guard_label); + instr = matchPushVGuard; /* may be changed to PushVResult below */ + } + #endif + } + else { /* body */ + #if HALFWORD_HEAP + if (v->first_guard_label) { + /* Avoid double-copy, copy to result heap at first encounter in guard */ + DMC_POKE(*text, v->first_guard_label, matchPushVResult); + v->is_in_body = 1; + } + #endif + if (!v->is_in_body) { + instr = matchPushVResult; + v->is_in_body = 1; + } + } + DMC_PUSH(*text, instr); + DMC_PUSH(*text, n); +} + static DMCRet dmc_variable(DMCContext *context, DMCHeap *heap, DMC_STACK_TYPE(UWord) *text, @@ -3023,13 +3597,13 @@ static DMCRet dmc_variable(DMCContext *context, int *constant) { Uint n = db_is_variable(t); - ASSERT(n >= 0); - if (n >= heap->used) - RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant); - if (heap->data[n] == 0U) + + if (n >= heap->vars_used || !heap->vars[n].is_bound) { RETURN_VAR_ERROR("Variable $%d is unbound.", n, context, *constant); - DMC_PUSH(*text, matchPushV); - DMC_PUSH(*text, n); + } + + dmc_add_pushv_variant(context, heap, text, n); + ++context->stack_used; if (context->stack_used > context->stack_need) context->stack_need = context->stack_used; @@ -3048,10 +3622,9 @@ static DMCRet dmc_all_bindings(DMCContext *context, DMC_PUSH(*text, matchPushC); DMC_PUSH(*text, NIL); - for (i = heap->used - 1; i >= 0; --i) { - if (heap->data[i]) { - DMC_PUSH(*text, matchPushV); - DMC_PUSH(*text, i); + for (i = heap->vars_used - 1; i >= 0; --i) { + if (heap->vars[i].is_bound) { + dmc_add_pushv_variant(context, heap, text, i); DMC_PUSH(*text, matchConsB); heap_used += 2; } @@ -3059,7 +3632,6 @@ static DMCRet dmc_all_bindings(DMCContext *context, ++context->stack_used; if ((context->stack_used + 1) > context->stack_need) context->stack_need = (context->stack_used + 1); - context->eheap_need += heap_used; *constant = 0; return retOk; } @@ -3462,10 +4034,6 @@ static DMCRet dmc_get_seq_token(DMCContext *context, *constant = 0; DMC_PUSH(*text, matchGetSeqToken); - context->eheap_need += (6 /* A 5-tuple is built */ - + EXTERNAL_THING_HEAD_SIZE + 2 /* Sender can - be an external - pid */); if (++context->stack_used > context->stack_need) context->stack_need = context->stack_used; return retOk; @@ -3762,7 +4330,6 @@ static DMCRet dmc_caller(DMCContext *context, } *constant = 0; DMC_PUSH(*text, matchCaller); /* Creates binary */ - context->eheap_need += 4; /* A 3-tuple is built */ if (++context->stack_used > context->stack_need) context->stack_need = context->stack_used; return retOk; @@ -4360,7 +4927,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) } save_cp = p->cp; p->cp = NULL; - res = erts_match_set_run(p, mps, arr, n, &ret_flags); + res = erts_match_set_run(p, mps, arr, n, + ERTS_PAM_COPY_RESULT, &ret_flags); p->cp = save_cp; } else { n = 0; @@ -4373,11 +4941,10 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) if (is_non_value(res)) { res = am_false; } - sz = size_object(res); + sz = 0; if (ret_flags & MATCH_SET_EXCEPTION_TRACE) sz += 2; if (ret_flags & MATCH_SET_RETURN_TRACE) sz += 2; hp = HAlloc(p, 5 + sz); - res = copy_struct(res, sz, &hp, &MSO(p)); flg = NIL; if (ret_flags & MATCH_SET_EXCEPTION_TRACE) { flg = CONS(hp, am_exception_trace, flg); @@ -4404,12 +4971,67 @@ static Eterm seq_trace_fake(Process *p, Eterm arg1) } return result; } - + +DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org) +{ + ErlOffHeap tmp_offheap; + DbTerm* res = erts_alloc(ERTS_ALC_T_TMP, + sizeof(DbTerm) + org->size*sizeof(Eterm)); + Eterm* hp = res->tpl; + tmp_offheap.first = NULL; + db_copy_from_comp(tb, org, &hp, &tmp_offheap); + res->first_oh = tmp_offheap.first; + res->size = org->size; +#ifdef DEBUG_CLONE + res->debug_clone = NULL; +#endif + return res; +} + +void db_free_tmp_uncompressed(DbTerm* obj) +{ + ErlOffHeap off_heap; + off_heap.first = obj->first_oh; + erts_cleanup_offheap(&off_heap); +#ifdef DEBUG_CLONE + ASSERT(obj->debug_clone == NULL); +#endif + erts_free(ERTS_ALC_T_TMP, obj); +} + +Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, + int all, DbTerm* obj, Eterm** hpp, Uint extra) +{ + Uint32 dummy; + Eterm* base; + Eterm res; + + if (tb->compress) { + obj = db_alloc_tmp_uncompressed(tb, obj); + base = NULL; + } + else base = HALFWORD_HEAP ? obj->tpl : NULL; + + res = db_prog_match(c_p, bprog, make_tuple_rel(obj->tpl,base), base, NULL, 0, + ERTS_PAM_COPY_RESULT|ERTS_PAM_CONTIGUOUS_TUPLE, &dummy); + + if (is_value(res) && hpp!=NULL) { + *hpp = HAlloc(c_p, extra); + } + + if (tb->compress) { + db_free_tmp_uncompressed(obj); + } + return res; +} + + #ifdef DMC_DEBUG + /* ** Disassemble match program */ -static void db_match_dis(Binary *bp) +void db_match_dis(Binary *bp) { MatchProg *prog = Binary2MatchProg(bp); UWord *t = prog->text; @@ -4624,6 +5246,18 @@ static void db_match_dis(Binary *bp) ++t; erts_printf("PushV\t%bpu\n", n); break; + #if HALFWORD_HEAP + case matchPushVGuard: + n = (Uint) *++t; + ++t; + erts_printf("PushVGuard\t%bpu\n", n); + break; + #endif + case matchPushVResult: + n = (Uint) *++t; + ++t; + erts_printf("PushVResult\t%bpu\n", n); + break; case matchTrue: ++t; erts_printf("True\n"); @@ -4734,7 +5368,6 @@ static void db_match_dis(Binary *bp) erts_printf("}\n"); erts_printf("num_bindings: %d\n", prog->num_bindings); erts_printf("heap_size: %bpu\n", prog->heap_size); - erts_printf("eheap_offset: %bpu\n", prog->eheap_offset); erts_printf("stack_offset: %bpu\n", prog->stack_offset); erts_printf("text: 0x%08x\n", (unsigned long) prog->text); erts_printf("stack_size: %d (words)\n", prog->heap_size-prog->stack_offset); diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 0f333e8b34..bb1751d309 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -52,22 +52,27 @@ is broken.*/ #define DB_ERROR_UNSPEC -10 /* Unspecified error */ +/*#define DEBUG_CLONE*/ /* * A datatype for a database entry stored out of a process heap */ typedef struct db_term { struct erl_off_heap_header* first_oh; /* Off heap data for term. */ - Uint size; /* Size of term in "words" */ - Eterm tpl[1]; /* Untagged "constant pointer" to top tuple */ - /* (assumed to be first in buffer) */ + Uint size; /* Heap size of term in "words" */ +#ifdef DEBUG_CLONE + Eterm* debug_clone; /* An uncompressed copy */ +#endif + Eterm tpl[1]; /* Term data. Top tuple always first */ + + /* Compression: is_immed and key element are uncompressed. + Compressed elements are stored in external format after each other + last in dbterm. The top tuple elements contains byte offsets, to + the start of the data, tagged as headers. + The allocated size of the dbterm in bytes is stored at tpl[arity+1]. + */ } DbTerm; -/* "Assign" a value to DbTerm.tpl */ -#define DBTERM_SET_TPL(dbtermPtr,tplPtr) ASSERT((tplPtr)==(dbtermPtr->tpl)) -/* Get start of term buffer */ -#define DBTERM_BUF(dbtermPtr) ((dbtermPtr)->tpl) - union db_table; typedef union db_table DbTable; @@ -81,6 +86,9 @@ typedef struct { Uint new_size; int mustResize; void* lck; +#if HALFWORD_HEAP + unsigned char* abs_vec; /* [i] true if dbterm->tpl[i] is absolute Eterm */ +#endif } DbUpdateHandle; @@ -186,6 +194,12 @@ typedef struct db_table_method } DbTableMethod; +typedef struct db_fixation { + Eterm pid; + Uint counter; + struct db_fixation *next; +} DbFixation; + /* * This structure contains data for all different types of database * tables. Note that these fields must match the same fields @@ -194,16 +208,8 @@ typedef struct db_table_method * operations may be the same on different types of tables. */ -typedef struct db_fixation { - Eterm pid; - Uint counter; - struct db_fixation *next; -} DbFixation; - - typedef struct db_table_common { - erts_refc_t ref; - erts_refc_t fixref; /* fixation counter */ + erts_refc_t ref; /* fixation counter and delete counter */ #ifdef ERTS_SMP erts_smp_rwmtx_t rwlock; /* rw lock on table */ erts_smp_mtx_t fixlock; /* Protects fixations,megasec,sec,microsec */ @@ -226,6 +232,7 @@ typedef struct db_table_common { Uint32 status; /* bit masks defined below */ int slot; /* slot index in meta_main_tab */ int keypos; /* defaults to 1 */ + int compress; } DbTableCommon; /* These are status bit patterns */ @@ -248,23 +255,70 @@ typedef struct db_table_common { (DB_BAG | DB_SET | DB_DUPLICATE_BAG))) #define IS_TREE_TABLE(Status) (!!((Status) & \ DB_ORDERED_SET)) -#define NFIXED(T) (erts_refc_read(&(T)->common.fixref,0)) +#define NFIXED(T) (erts_refc_read(&(T)->common.ref,0)) #define IS_FIXED(T) (NFIXED(T) != 0) -Eterm erts_ets_copy_object(Eterm, Process*); +/* + * tplp is an untagged pointer to a tuple we know is large enough + * and dth is a pointer to a DbTableHash. + */ +#define GETKEY(dth, tplp) (*((tplp) + ((DbTableCommon*)(dth))->keypos)) + + +ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj); +Eterm db_copy_from_comp(DbTableCommon* tb, DbTerm* bp, Eterm** hpp, + ErlOffHeap* off_heap); +int db_eq_comp(DbTableCommon* tb, Eterm a, DbTerm* b); +DbTerm* db_alloc_tmp_uncompressed(DbTableCommon* tb, DbTerm* org); + +ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp, + Eterm** hpp, ErlOffHeap* off_heap); +ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b); +Wterm db_do_read_element(DbUpdateHandle* handle, Sint position); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE Eterm db_copy_key(Process* p, DbTable* tb, DbTerm* obj) +{ + Eterm key = GETKEY(tb, obj->tpl); + if IS_CONST(key) return key; + else { + Uint size = size_object_rel(key, obj->tpl); + Eterm* hp = HAlloc(p, size); + Eterm res = copy_struct_rel(key, size, &hp, &MSO(p), obj->tpl, NULL); + ASSERT(eq_rel(res,NULL,key,obj->tpl)); + return res; + } +} + +ERTS_GLB_INLINE Eterm db_copy_object_from_ets(DbTableCommon* tb, DbTerm* bp, + Eterm** hpp, ErlOffHeap* off_heap) +{ + if (tb->compress) { + return db_copy_from_comp(tb, bp, hpp, off_heap); + } + else { + return copy_shallow_rel(bp->tpl, bp->size, hpp, off_heap, bp->tpl); + } +} + +ERTS_GLB_INLINE int db_eq(DbTableCommon* tb, Eterm a, DbTerm* b) +{ + if (!tb->compress) { + return eq_rel(a, NULL, make_tuple_rel(b->tpl,b->tpl), b->tpl); + } + else { + return db_eq_comp(tb, a, b); + } +} + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ -/* optimised version of copy_object (normal case? atomic object) */ -#define COPY_OBJECT(obj, p, objp) \ - if (IS_CONST(obj)) { *(objp) = (obj); } \ - else { *objp = erts_ets_copy_object(obj, p); } #define DB_READ (DB_PROTECTED|DB_PUBLIC) #define DB_WRITE DB_PUBLIC #define DB_INFO (DB_PROTECTED|DB_PUBLIC|DB_PRIVATE) -/* tb is an DbTableCommon and obj is an Eterm (tagged) */ -#define TERM_GETKEY(tb, obj) db_getkey((tb)->common.keypos, (obj)) - #define ONLY_WRITER(P,T) (((T)->common.status & (DB_PRIVATE|DB_PROTECTED)) \ && (T)->common.owner == (P)->id) @@ -277,15 +331,19 @@ Eterm db_set_trace_control_word_1(Process *p, Eterm val); void db_initialize_util(void); Eterm db_getkey(int keypos, Eterm obj); -void db_free_term_data(DbTerm* p); -void* db_get_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +void db_cleanup_offheap_comp(DbTerm* p); +void db_free_term(DbTable *tb, void* basep, Uint offset); +void* db_store_term(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +void* db_store_term_comp(DbTableCommon *tb, DbTerm* old, Uint offset, Eterm obj); +Eterm db_copy_element_from_ets(DbTableCommon* tb, Process* p, DbTerm* obj, + Uint pos, Eterm** hpp, Uint extra); int db_has_variable(Eterm obj); int db_is_variable(Eterm obj); void db_do_update_element(DbUpdateHandle* handle, Sint position, Eterm newval); -void db_finalize_update_element(DbUpdateHandle* handle); -Eterm db_add_counter(Eterm** hpp, Eterm counter, Eterm incr); +void db_finalize_resize(DbUpdateHandle* handle, Uint offset); +Eterm db_add_counter(Eterm** hpp, Wterm counter, Eterm incr); Eterm db_match_set_lint(Process *p, Eterm matchexpr, Uint flags); Binary *db_match_set_compile(Process *p, Eterm matchexpr, Uint flags); @@ -302,7 +360,6 @@ typedef struct match_prog { struct erl_heap_fragment *saved_program_buf; Eterm saved_program; Uint heap_size; /* size of: heap + eheap + stack */ - Uint eheap_offset; Uint stack_offset; #ifdef DMC_DEBUG UWord* prog_end; /* End of program */ @@ -367,8 +424,15 @@ Binary *db_match_compile(Eterm *matchexpr, Eterm *guards, Uint flags, DMCErrInfo *err_info); /* Returns newly allocated MatchProg binary with refc == 0*/ -Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm *termp, int arity, + +Eterm db_match_dbterm(DbTableCommon* tb, Process* c_p, Binary* bprog, + int all, DbTerm* obj, Eterm** hpp, Uint extra); + +Eterm db_prog_match(Process *p, Binary *prog, Eterm term, Eterm* base, + Eterm *termp, int arity, + enum erts_pam_run_flags in_flags, Uint32 *return_flags /* Zeroed on enter */); + /* returns DB_ERROR_NONE if matches, 1 if not matches and some db error on error. */ DMCErrInfo *db_new_dmc_err_info(void); diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index 9733c0e5b5..13a73e01bb 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -150,6 +150,27 @@ typedef struct { #define ERL_DRV_FLAG_SOFT_BUSY (1 << 1) /* + * Integer types + */ + +typedef unsigned long ErlDrvTermData; +typedef unsigned long ErlDrvUInt; +typedef signed long ErlDrvSInt; + +#if defined(__WIN32__) +typedef unsigned __int64 ErlDrvUInt64; +typedef __int64 ErlDrvSInt64; +#elif SIZEOF_LONG == 8 +typedef unsigned long ErlDrvUInt64; +typedef long ErlDrvSInt64; +#elif SIZEOF_LONG_LONG == 8 +typedef unsigned long long ErlDrvUInt64; +typedef long long ErlDrvSInt64; +#else +#error No 64-bit integer type +#endif + +/* * A binary as seen in a driver. Note that a binary should never be * altered by the driver when it has been sent to Erlang. */ @@ -179,26 +200,6 @@ struct erl_drv_event_data { #endif typedef struct erl_drv_event_data *ErlDrvEventData; /* Event data */ -/* - * Used in monitors... - */ -typedef unsigned long ErlDrvTermData; -typedef unsigned long ErlDrvUInt; -typedef signed long ErlDrvSInt; - -#if defined(__WIN32__) -typedef unsigned __int64 ErlDrvUInt64; -typedef __int64 ErlDrvSInt64; -#elif SIZEOF_LONG == 8 -typedef unsigned long ErlDrvUInt64; -typedef long ErlDrvSInt64; -#elif SIZEOF_LONG_LONG == 8 -typedef unsigned long long ErlDrvUInt64; -typedef long long ErlDrvSInt64; -#else -#error No 64-bit integer type -#endif - /* * A driver monitor */ @@ -394,9 +395,9 @@ EXTERN int driver_exit (ErlDrvPort port, int err); EXTERN ErlDrvPDL driver_pdl_create(ErlDrvPort); EXTERN void driver_pdl_lock(ErlDrvPDL); EXTERN void driver_pdl_unlock(ErlDrvPDL); -EXTERN long driver_pdl_get_refc(ErlDrvPDL); -EXTERN long driver_pdl_inc_refc(ErlDrvPDL); -EXTERN long driver_pdl_dec_refc(ErlDrvPDL); +EXTERN ErlDrvSInt driver_pdl_get_refc(ErlDrvPDL); +EXTERN ErlDrvSInt driver_pdl_inc_refc(ErlDrvPDL); +EXTERN ErlDrvSInt driver_pdl_dec_refc(ErlDrvPDL); /* * Process monitors @@ -432,9 +433,9 @@ EXTERN ErlDrvBinary* driver_realloc_binary(ErlDrvBinary *bin, int size); EXTERN void driver_free_binary(ErlDrvBinary *bin); /* Referenc count on driver binaries */ -EXTERN long driver_binary_get_refc(ErlDrvBinary *dbp); -EXTERN long driver_binary_inc_refc(ErlDrvBinary *dbp); -EXTERN long driver_binary_dec_refc(ErlDrvBinary *dbp); +EXTERN ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp); +EXTERN ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp); +EXTERN ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp); /* Allocation interface */ EXTERN void *driver_alloc(size_t size); diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index d42820ddf3..39bbe9633b 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * Copyright Ericsson AB 2007-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,6 +24,10 @@ #include "global.h" #include <string.h> +#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) +#define __DARWIN__ 1 +#endif + #define ERL_DRV_THR_OPTS_SIZE(LAST_FIELD) \ (((size_t) &((ErlDrvThreadOpts *) 0)->LAST_FIELD) \ + sizeof(((ErlDrvThreadOpts *) 0)->LAST_FIELD)) @@ -528,7 +532,7 @@ erl_drv_tsd_get(ErlDrvTSDKey key) if (!dtid) return NULL; #endif - if (ERL_DRV_TSD_LEN__ < key) + if (ERL_DRV_TSD_LEN__ <= key) return NULL; return ERL_DRV_TSD__[key]; } @@ -692,3 +696,57 @@ erl_drv_thread_join(ErlDrvTid tid, void **respp) #endif } +#if defined(__DARWIN__) && defined(USE_THREADS) && defined(ERTS_SMP) +extern int erts_darwin_main_thread_pipe[2]; +extern int erts_darwin_main_thread_result_pipe[2]; + + +int +erl_drv_stolen_main_thread_join(ErlDrvTid tid, void **respp) +{ + void *dummy; + void **x; + if (respp == NULL) + x = &dummy; + else + x = respp; + read(erts_darwin_main_thread_result_pipe[0],x,sizeof(void *)); + return 0; +} + +int +erl_drv_steal_main_thread(char *name, + ErlDrvTid *tid, + void* (*func)(void*), + void* arg, + ErlDrvThreadOpts *opts) +{ + char buff[sizeof(void* (*)(void*)) + sizeof(void *)]; + int buff_sz = sizeof(void* (*)(void*)) + sizeof(void *); + /*struct ErlDrvTid_ *dtid; + + dtid = erts_alloc_fnf(ERTS_ALC_T_DRV_TID, + (sizeof(struct ErlDrvTid_) + + (name ? sys_strlen(name) + 1 : 0))); + if (!dtid) + return ENOMEM; + memset(dtid,0,sizeof(ErlDrvTid_)); + dtid->tid = (void * ) -1; + dtid->drv_thr = 1; + dtid->func = func; + dtid->arg = arg; + dtid->tsd = NULL; + dtid->tsd_len = 0; + dtid->name = no_name; + *tid = (ErlDrvTid) dtid; + */ + *tid = NULL; + /* Ignore options and name... */ + + memcpy(buff,&func,sizeof(void* (*)(void*))); + memcpy(buff + sizeof(void* (*)(void*)),&arg,sizeof(void *)); + write(erts_darwin_main_thread_pipe[1],buff,buff_sz); + return 0; +} + +#endif diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index 84869f12d6..88947b5536 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -97,7 +97,7 @@ erts_put_fun_entry(Eterm mod, int uniq, int index) { ErlFunEntry template; ErlFunEntry* fe; - long refc; + erts_aint_t refc; ASSERT(is_atom(mod)); template.old_uniq = uniq; template.old_index = index; @@ -119,7 +119,7 @@ erts_put_fun_entry2(Eterm mod, int old_uniq, int old_index, { ErlFunEntry template; ErlFunEntry* fe; - long refc; + erts_aint_t refc; ASSERT(is_atom(mod)); template.old_uniq = old_uniq; @@ -157,7 +157,7 @@ erts_get_fun_entry(Eterm mod, int uniq, int index) erts_fun_read_lock(); ret = (ErlFunEntry *) hash_get(&erts_fun_table, (void*) &template); if (ret) { - long refc = erts_refc_inctest(&ret->refc, 1); + erts_aint_t refc = erts_refc_inctest(&ret->refc, 1); if (refc < 2) /* Pending delete */ erts_refc_inc(&ret->refc, 1); } @@ -257,7 +257,7 @@ erts_dump_fun_entries(int to, void *to_arg) #ifdef HIPE erts_print(to, to_arg, "Native_address: %p\n", fe->native_address); #endif - erts_print(to, to_arg, "Refc: %d\n", erts_refc_read(&fe->refc, 1)); + erts_print(to, to_arg, "Refc: %ld\n", erts_refc_read(&fe->refc, 1)); b = b->next; } } diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 0f4d2a2ef9..d9150d86fe 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,6 +33,7 @@ #include "erl_gc.h" #if HIPE #include "hipe_stack.h" +#include "hipe_mode_switch.h" #endif #define ERTS_INACT_WR_PB_LEAVE_MUCH_LIMIT 1 @@ -486,6 +487,9 @@ erts_garbage_collect_hibernate(Process* p) htop = heap; n = setup_rootset(p, p->arg_reg, p->arity, &rootset); +#if HIPE + hipe_empty_nstack(p); +#endif src = (char *) p->heap; src_size = (char *) p->htop - src; @@ -2471,7 +2475,7 @@ erts_check_off_heap2(Process *p, Eterm *htop) old = 0; for (u.hdr = MSO(p).first; u.hdr; u.hdr = u.hdr->next) { - long refc; + erts_aint_t refc; switch (thing_subtag(u.hdr->thing_word)) { case REFC_BINARY_SUBTAG: refc = erts_refc_read(&u.pb->val->refc, 1); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 36fefc5cba..0a57eb6d88 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -41,6 +41,7 @@ #include "erl_printf_term.h" #include "erl_misc_utils.h" #include "packet_parser.h" +#include "erl_cpu_topology.h" #ifdef HIPE #include "hipe_mode_switch.h" /* for hipe_mode_switch_init() */ @@ -63,6 +64,8 @@ extern void ConNormalExit(void); extern void ConWaitForExit(void); #endif +static void erl_init(int ncpu); + #define ERTS_MIN_COMPAT_REL 7 #ifdef ERTS_SMP @@ -76,9 +79,6 @@ int erts_initialized = 0; static erts_tid_t main_thread; #endif -erts_cpu_info_t *erts_cpuinfo; - -int erts_reader_groups; int erts_use_sender_punish; /* @@ -100,7 +100,7 @@ int erts_backtrace_depth; /* How many functions to show in a backtrace int erts_async_max_threads; /* number of threads for async support */ int erts_async_thread_suggested_stack_size; -erts_smp_atomic_t erts_max_gen_gcs; +erts_smp_atomic32_t erts_max_gen_gcs; Eterm erts_error_logger_warnings; /* What to map warning logs to, am_error, am_info or am_warning, am_error is @@ -111,7 +111,6 @@ int erts_compat_rel; static int use_multi_run_queue; static int no_schedulers; static int no_schedulers_online; -static int max_reader_groups; #ifdef DEBUG Uint32 verbose; /* See erl_debug.h for information about verbose */ @@ -230,18 +229,18 @@ void erl_error(char *fmt, va_list args) erts_vfprintf(stderr, fmt, args); } -static void early_init(int *argc, char **argv); +static int early_init(int *argc, char **argv); void erts_short_init(void) { - early_init(NULL, NULL); - erl_init(); + int ncpu = early_init(NULL, NULL); + erl_init(ncpu); erts_initialized = 1; } -void -erl_init(void) +static void +erl_init(int ncpu) { init_benchmarking(); @@ -251,12 +250,13 @@ erl_init(void) erts_init_monitors(); erts_init_gc(); - init_time(); - erts_init_process(); + erts_init_time(); + erts_init_sys_common_misc(); + erts_init_process(ncpu); erts_init_scheduling(use_multi_run_queue, no_schedulers, no_schedulers_online); - + erts_init_cpu_topology(); /* Must be after init_scheduling */ H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); BIN_VH_MIN_SIZE = erts_next_heap_size(BIN_VH_MIN_SIZE, 0); @@ -289,7 +289,7 @@ erl_init(void) erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG - erts_mseg_late_init(); /* Must be after timer (init_time()) and thread + erts_mseg_late_init(); /* Must be after timer (erts_init_time()) and thread initializations */ #endif #ifdef HIPE @@ -323,7 +323,7 @@ init_shared_memory(int argc, char **argv) #endif global_gen_gcs = 0; - global_max_gen_gcs = erts_smp_atomic_read(&erts_max_gen_gcs); + global_max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); global_gc_flags = erts_default_process_flags; erts_global_offheap.mso = NULL; @@ -588,7 +588,7 @@ static void ethr_ll_free(void *ptr) #endif -static void +static int early_init(int *argc, char **argv) /* * Only put things here which are * really important initialize @@ -601,6 +601,10 @@ early_init(int *argc, char **argv) /* int ncpuavail; int schdlrs; int schdlrs_onln; + int max_main_threads; + int max_reader_groups; + int reader_groups; + use_multi_run_queue = 1; erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; @@ -616,13 +620,11 @@ early_init(int *argc, char **argv) /* erts_use_sender_punish = 1; - erts_cpuinfo = erts_cpu_info_create(); - -#ifdef ERTS_SMP - ncpu = erts_get_cpu_configured(erts_cpuinfo); - ncpuonln = erts_get_cpu_online(erts_cpuinfo); - ncpuavail = erts_get_cpu_available(erts_cpuinfo); -#else + erts_pre_early_init_cpu_topology(&max_reader_groups, + &ncpu, + &ncpuonln, + &ncpuavail); +#ifndef ERTS_SMP ncpu = 1; ncpuonln = 1; ncpuavail = 1; @@ -649,7 +651,7 @@ early_init(int *argc, char **argv) /* erts_writing_erl_crash_dump = 0; #endif - erts_smp_atomic_init(&erts_max_gen_gcs, (long)((Uint16) -1)); + erts_smp_atomic32_init(&erts_max_gen_gcs, (erts_aint32_t) ((Uint16) -1)); erts_pre_init_process(); #if defined(USE_THREADS) && !defined(ERTS_SMP) @@ -665,15 +667,9 @@ early_init(int *argc, char **argv) /* ? ncpuavail : (ncpuonln > 0 ? ncpuonln : no_schedulers)); -#ifdef ERTS_SMP - erts_max_main_threads = no_schedulers_online; -#endif - schdlrs = no_schedulers; schdlrs_onln = no_schedulers_online; - max_reader_groups = ERTS_MAX_READER_GROUPS; - if (argc && argv) { int i = 1; while (i < *argc) { @@ -769,9 +765,13 @@ early_init(int *argc, char **argv) /* erts_alloc_init(argc, argv, &alloc_opts); /* Handles (and removes) -M flags. */ - - erts_early_init_scheduling(); /* Require allocators */ - erts_init_utils(); /* Require allocators */ + /* Require allocators */ + erts_early_init_scheduling(); + erts_init_utils(); + erts_early_init_cpu_topology(no_schedulers, + &max_main_threads, + max_reader_groups, + &reader_groups); #ifdef USE_THREADS { @@ -785,24 +785,13 @@ early_init(int *argc, char **argv) /* elid.mem.ll.alloc = ethr_ll_alloc; elid.mem.ll.realloc = ethr_ll_realloc; elid.mem.ll.free = ethr_ll_free; - -#ifdef ERTS_SMP - elid.main_threads = erts_max_main_threads; -#else - elid.main_threads = 1; -#endif - elid.reader_groups = (elid.main_threads > 1 - ? elid.main_threads - : 0); - if (max_reader_groups <= 1) - elid.reader_groups = 0; - if (elid.reader_groups > max_reader_groups) - elid.reader_groups = max_reader_groups; - erts_reader_groups = elid.reader_groups; + elid.main_threads = max_main_threads; + elid.reader_groups = reader_groups; erts_thr_late_init(&elid); } #endif + #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_late_init(); #endif @@ -819,7 +808,10 @@ early_init(int *argc, char **argv) /* erl_sys_args(argc, argv); erts_ets_realloc_always_moves = 0; + erts_ets_always_compress = 0; erts_dist_buf_busy_limit = ERTS_DE_BUSY_LIMIT; + + return ncpu; } #ifndef ERTS_SMP @@ -853,8 +845,7 @@ erl_start(int argc, char **argv) char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; int async_max_threads = erts_async_max_threads; - - early_init(&argc, argv); + int ncpu = early_init(&argc, argv); envbufsz = sizeof(envbuf); if (erts_sys_getenv(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) @@ -865,7 +856,7 @@ erl_start(int argc, char **argv) envbufsz = sizeof(envbuf); if (erts_sys_getenv("ERL_FULLSWEEP_AFTER", envbuf, &envbufsz) == 0) { Uint16 max_gen_gcs = atoi(envbuf); - erts_smp_atomic_set(&erts_max_gen_gcs, (long) max_gen_gcs); + erts_smp_atomic32_set(&erts_max_gen_gcs, (erts_aint32_t) max_gen_gcs); } envbufsz = sizeof(envbuf); @@ -917,7 +908,27 @@ erl_start(int argc, char **argv) VERBOSE(DEBUG_SYSTEM, ("using display items %d\n",display_items)); break; - + case 'f': + if (!strncmp(argv[i],"-fn",3)) { + arg = get_arg(argv[i]+3, argv[i+1], &i); + switch (*arg) { + case 'u': + erts_set_user_requested_filename_encoding(ERL_FILENAME_UTF8); + break; + case 'l': + erts_set_user_requested_filename_encoding(ERL_FILENAME_LATIN1); + break; + case 'a': + erts_set_user_requested_filename_encoding(ERL_FILENAME_UNKNOWN); + default: + erts_fprintf(stderr, "bad filename encoding %s, can be (l,u or a)\n", arg); + erts_usage(); + } + break; + } else { + erts_fprintf(stderr, "%s unknown flag %s\n", argv[0], argv[i]); + erts_usage(); + } case 'l': display_loads++; break; @@ -1039,15 +1050,20 @@ erl_start(int argc, char **argv) break; case 'e': - /* set maximum number of ets tables */ - arg = get_arg(argv[i]+2, argv[i+1], &i); - if (( user_requested_db_max_tabs = atoi(arg) ) < 0) { - erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg); - erts_usage(); + if (sys_strcmp("c", argv[i]+2) == 0) { + erts_ets_always_compress = 1; + } + else { + /* set maximum number of ets tables */ + arg = get_arg(argv[i]+2, argv[i+1], &i); + if (( user_requested_db_max_tabs = atoi(arg) ) < 0) { + erts_fprintf(stderr, "bad maximum number of ets tables %s\n", arg); + erts_usage(); + } + VERBOSE(DEBUG_SYSTEM, + ("using maximum number of ets tables %d\n", + user_requested_db_max_tabs)); } - VERBOSE(DEBUG_SYSTEM, - ("using maximum number of ets tables %d\n", - user_requested_db_max_tabs)); break; case 'i': @@ -1111,7 +1127,7 @@ erl_start(int argc, char **argv) char *sub_param = argv[i]+2; if (has_prefix("bt", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); - res = erts_init_scheduler_bind_type(arg); + res = erts_init_scheduler_bind_type_string(arg); if (res != ERTS_INIT_SCHED_BIND_TYPE_SUCCESS) { switch (res) { case ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED: @@ -1136,7 +1152,7 @@ erl_start(int argc, char **argv) } else if (has_prefix("ct", sub_param)) { arg = get_arg(sub_param+2, argv[i+1], &i); - res = erts_init_cpu_topology(arg); + res = erts_init_cpu_topology_string(arg); if (res != ERTS_INIT_CPU_TOPOLOGY_OK) { switch (res) { case ERTS_INIT_CPU_TOPOLOGY_INVALID_ID: @@ -1407,7 +1423,7 @@ erl_start(int argc, char **argv) boot_argc = argc - i; /* Number of arguments to init */ boot_argv = &argv[i]; - erl_init(); + erl_init(ncpu); init_shared_memory(boot_argc, boot_argv); load_preloaded(); diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index d6138fa4e4..9e18997890 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -128,8 +128,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "removed_fd_pre_alloc_lock", NULL }, { "state_prealloc", NULL }, { "schdlr_sspnd", NULL }, - { "cpu_bind", NULL }, { "run_queue", "address" }, + { "cpu_info", NULL }, { "pollset", "address" }, #ifdef __WIN32__ { "pollwaiter", "address" }, @@ -155,7 +155,7 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "alcu_allocator", "index" }, { "alcu_delayed_free", "index" }, { "mseg", NULL }, -#ifdef HALFWORD_HEAP +#if HALFWORD_HEAP { "pmmap", NULL }, #endif #ifdef ERTS_SMP @@ -177,6 +177,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "async_id", NULL }, { "pix_lock", "address" }, { "run_queues_lists", NULL }, + { "misc_aux_work_queue", "index" }, + { "misc_aux_work_pre_alloc_lock", "address" }, { "sched_stat", NULL }, { "run_queue_sleep_list", "address" }, #endif @@ -978,10 +980,10 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, Uint16 op_flags) /* We only force busy if a lock order violation would occur and when on an even millisecond. */ { - erts_thr_timeval_t time; - erts_thr_time_now(&time); + SysTimeval tv; + sys_gettimeofday(&tv); - if ((time.tv_nsec / 1000000) & 1) + if ((tv.tv_usec / 1000) & 1) return 0; } #endif diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h index 0372e6850d..cdb06d4458 100644 --- a/erts/emulator/beam/erl_lock_check.h +++ b/erts/emulator/beam/erl_lock_check.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index 239773f366..a36c53560e 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -159,7 +159,7 @@ static char* lock_opt(Uint16 flag) { } static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action, char *extra) { - long int colls, tries, w_state, r_state; + erts_aint_t colls, tries, w_state, r_state; erts_lcnt_lock_stats_t *stats = NULL; char *type; @@ -385,7 +385,7 @@ void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { /* lock */ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { - long r_state = 0, w_state = 0; + erts_aint_t r_state = 0, w_state = 0; erts_lcnt_thread_data_t *eltd; if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; @@ -418,7 +418,7 @@ void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { } void erts_lcnt_lock(erts_lcnt_lock_t *lock) { - long w_state; + erts_aint_t w_state; erts_lcnt_thread_data_t *eltd; if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; @@ -471,7 +471,7 @@ void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line erts_lcnt_time_t time_wait; erts_lcnt_lock_stats_t *stats; #ifdef DEBUG - long flowstate; + erts_aint_t flowstate; #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; @@ -516,8 +516,8 @@ void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) { void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { #ifdef DEBUG - long w_state; - long flowstate; + erts_aint_t w_state; + erts_aint_t flowstate; #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; #ifdef DEBUG @@ -552,7 +552,7 @@ void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) { void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { /* Determine lock_state via res instead of state */ #ifdef DEBUG - long flowstate; + erts_aint_t flowstate; #endif if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; if (res != EBUSY) { diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index d873c7a701..9751b5d77c 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -85,7 +85,7 @@ static ERTS_INLINE int cmp_mon_ref(Eterm ref1, Eterm ref2) if (is_ref_thing_header(*b2)) { return 1; } - return cmp(ref1,ref2); + return CMP(ref1,ref2); } #define CP_LINK_VAL(To, Hp, From) \ @@ -380,7 +380,7 @@ int erts_add_link(ErtsLink **root, Uint type, Eterm pid) state = 1; *this = create_link(type,pid); break; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { /* go left */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -415,7 +415,7 @@ erts_add_or_lookup_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid) state = 1; res = *this = create_suspend_monitor(pid); break; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { /* go left */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -453,7 +453,7 @@ ErtsLink *erts_add_or_lookup_link(ErtsLink **root, Uint type, Eterm pid) *this = create_link(type,pid); ret = *this; break; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { /* go left */ dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; @@ -663,7 +663,7 @@ ErtsLink *erts_remove_link(ErtsLink **root, Eterm pid) for (;;) { if (!*this) { /* Failure */ return NULL; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -715,7 +715,7 @@ erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid) for (;;) { if (!*this) { /* Nothing found */ return; - } else if ((c = cmp(pid,(*this)->pid)) < 0) { + } else if ((c = CMP(pid,(*this)->pid)) < 0) { dstack[dpos++] = DIR_LEFT; tstack[tpos++] = this; this = &((*this)->left); @@ -771,7 +771,7 @@ ErtsLink *erts_lookup_link(ErtsLink *root, Eterm pid) Sint c; for (;;) { - if (root == NULL || (c = cmp(pid,root->pid)) == 0) { + if (root == NULL || (c = CMP(pid,root->pid)) == 0) { return root; } else if (c < 0) { root = root->left; @@ -787,7 +787,7 @@ erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid) Sint c; for (;;) { - if (root == NULL || (c = cmp(pid,root->pid)) == 0) { + if (root == NULL || (c = CMP(pid,root->pid)) == 0) { return root; } else if (c < 0) { root = root->left; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 1dd9c8bd4a..135c6b0ccc 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -81,7 +81,6 @@ static ERTS_INLINE Eterm* alloc_heap(ErlNifEnv* env, unsigned need) static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp) { - unsigned frag_sz; env->hp = hp; if (env->heap_frag == NULL) { ASSERT(HEAP_LIMIT(env->proc) == env->hp_end); @@ -91,14 +90,24 @@ static Eterm* alloc_heap_heavy(ErlNifEnv* env, unsigned need, Eterm* hp) env->heap_frag->used_size = hp - env->heap_frag->mem; ASSERT(env->heap_frag->used_size <= env->heap_frag->alloc_size); } - frag_sz = need + MIN_HEAP_FRAG_SZ; - hp = erts_heap_alloc(env->proc, frag_sz); - env->hp = hp + need; - env->hp_end = hp + frag_sz; + hp = erts_heap_alloc(env->proc, need, MIN_HEAP_FRAG_SZ); env->heap_frag = MBUF(env->proc); + env->hp = hp + need; + env->hp_end = env->heap_frag->mem + env->heap_frag->alloc_size; + return hp; } +#if SIZEOF_LONG != ERTS_SIZEOF_ETERM +static ERTS_INLINE void ensure_heap(ErlNifEnv* env, unsigned may_need) +{ + if (env->hp + may_need > env->hp_end) { + alloc_heap_heavy(env, may_need, env->hp); + env->hp -= may_need; + } +} +#endif + void erts_pre_nif(ErlNifEnv* env, Process* p, struct erl_module_nif* mod_nif) { env->mod_nif = mod_nif; @@ -564,7 +573,7 @@ int enif_is_identical(Eterm lhs, Eterm rhs) int enif_compare(Eterm lhs, Eterm rhs) { - return cmp(lhs,rhs); + return CMP(lhs,rhs); } int enif_get_tuple(ErlNifEnv* env, Eterm tpl, int* arity, const Eterm** array) @@ -730,9 +739,8 @@ int enif_get_long(ErlNifEnv* env, Eterm term, long* ip) { #if SIZEOF_LONG == ERTS_SIZEOF_ETERM return term_to_Sint(term, ip); -#elif SIZEOF_INT == ERTS_SIZEOF_ETERM - Sint i; - return term_to_Sint(term, &i) ? (*ip = (long) i, 1) : 0; +#elif SIZEOF_LONG == 8 + return term_to_Sint64(term, ip); #else # error Unknown long word size #endif @@ -742,9 +750,8 @@ int enif_get_ulong(ErlNifEnv* env, Eterm term, unsigned long* ip) { #if SIZEOF_LONG == ERTS_SIZEOF_ETERM return term_to_Uint(term, ip); -#elif SIZEOF_INT == ERTS_SIZEOF_ETERM - Uint u; - return term_to_Uint(term, &u) ? (*ip = (unsigned long) u, 1) : 0; +#elif SIZEOF_LONG == 8 + return term_to_Uint64(term, ip); #else # error Unknown long word size #endif @@ -821,12 +828,22 @@ ERL_NIF_TERM enif_make_uint(ErlNifEnv* env, unsigned i) ERL_NIF_TERM enif_make_long(ErlNifEnv* env, long i) { +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return IS_SSMALL(i) ? make_small(i) : small_to_big(i, alloc_heap(env,2)); +#elif SIZEOF_LONG == 8 + ensure_heap(env,3); + return erts_sint64_to_big(i, &env->hp); +#endif } ERL_NIF_TERM enif_make_ulong(ErlNifEnv* env, unsigned long i) { +#if SIZEOF_LONG == ERTS_SIZEOF_ETERM return IS_USMALL(0,i) ? make_small(i) : uint_to_big(i,alloc_heap(env,2)); +#elif SIZEOF_LONG == 8 + ensure_heap(env,3); + return erts_uint64_to_big(i, &env->hp); +#endif } #if HAVE_INT64 && SIZEOF_LONG != 8 @@ -921,21 +938,26 @@ ERL_NIF_TERM enif_make_list_cell(ErlNifEnv* env, Eterm car, Eterm cdr) ERL_NIF_TERM enif_make_list(ErlNifEnv* env, unsigned cnt, ...) { - Eterm* hp = alloc_heap(env,cnt*2); - Eterm ret = make_list(hp); - Eterm* last = &ret; - va_list ap; - - va_start(ap,cnt); - while (cnt--) { - *last = make_list(hp); - *hp = va_arg(ap,Eterm); - last = ++hp; - ++hp; + if (cnt == 0) { + return NIL; + } + else { + Eterm* hp = alloc_heap(env,cnt*2); + Eterm ret = make_list(hp); + Eterm* last = &ret; + va_list ap; + + va_start(ap,cnt); + while (cnt--) { + *last = make_list(hp); + *hp = va_arg(ap,Eterm); + last = ++hp; + ++hp; + } + va_end(ap); + *last = NIL; + return ret; } - va_end(ap); - *last = NIL; - return ret; } ERL_NIF_TERM enif_make_list_from_array(ErlNifEnv* env, const ERL_NIF_TERM arr[], unsigned cnt) @@ -1456,7 +1478,13 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ret = load_nif_error(BIF_P, bad_lib, "Library version (%d.%d) not compatible (with %d.%d).", entry->major, entry->minor, ERL_NIF_MAJOR_VERSION, ERL_NIF_MINOR_VERSION); - } + } + else if (entry->minor >= 1 + && sys_strcmp(entry->vm_variant, ERL_NIF_VM_VARIANT) != 0) { + ret = load_nif_error(BIF_P, bad_lib, "Library (%s) not compiled for " + "this vm variant (%s).", + entry->vm_variant, ERL_NIF_VM_VARIANT); + } else if (!erts_is_atom_str((char*)entry->name, mod_atom)) { ret = load_nif_error(BIF_P, bad_lib, "Library module name '%s' does not" " match calling module '%T'", entry->name, mod_atom); diff --git a/erts/emulator/beam/erl_nif.h b/erts/emulator/beam/erl_nif.h index ee3a7cd5f4..8050b3640a 100644 --- a/erts/emulator/beam/erl_nif.h +++ b/erts/emulator/beam/erl_nif.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -30,9 +30,10 @@ ** 0.1: R13B03 ** 1.0: R13B04 ** 2.0: R14A +** 2.1: R14B02 "vm_variant" */ #define ERL_NIF_MAJOR_VERSION 2 -#define ERL_NIF_MINOR_VERSION 0 +#define ERL_NIF_MINOR_VERSION 1 #include <stdlib.h> @@ -80,8 +81,10 @@ typedef long long ErlNifSInt64; #endif #ifdef HALFWORD_HEAP_EMULATOR +# define ERL_NIF_VM_VARIANT "beam.halfword" typedef unsigned int ERL_NIF_TERM; #else +# define ERL_NIF_VM_VARIANT "beam.vanilla" typedef unsigned long ERL_NIF_TERM; #endif @@ -105,7 +108,8 @@ typedef struct enif_entry_t int (*load) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); int (*reload) (ErlNifEnv*, void** priv_data, ERL_NIF_TERM load_info); int (*upgrade)(ErlNifEnv*, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info); - void (*unload) (ErlNifEnv*, void* priv_data); + void (*unload) (ErlNifEnv*, void* priv_data); + const char* vm_variant; }ErlNifEntry; @@ -198,6 +202,7 @@ extern TWinDynNifCallbacks WinDynNifCallbacks; #define ERL_NIF_INIT(NAME, FUNCS, LOAD, RELOAD, UPGRADE, UNLOAD) \ ERL_NIF_INIT_PROLOGUE \ ERL_NIF_INIT_GLOB \ +ERL_NIF_INIT_DECL(NAME); \ ERL_NIF_INIT_DECL(NAME) \ { \ static ErlNifEntry entry = \ @@ -207,7 +212,8 @@ ERL_NIF_INIT_DECL(NAME) \ #NAME, \ sizeof(FUNCS) / sizeof(*FUNCS), \ FUNCS, \ - LOAD, RELOAD, UPGRADE, UNLOAD \ + LOAD, RELOAD, UPGRADE, UNLOAD, \ + ERL_NIF_VM_VARIANT \ }; \ ERL_NIF_INIT_BODY; \ return &entry; \ diff --git a/erts/emulator/beam/erl_nmgc.c b/erts/emulator/beam/erl_nmgc.c index 626d4e295a..d7bfb2ab12 100644 --- a/erts/emulator/beam/erl_nmgc.c +++ b/erts/emulator/beam/erl_nmgc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,7 +26,6 @@ #include "erl_nmgc.h" #include "erl_debug.h" #if HIPE -#include "hipe_bif0.h" /* for hipe_constants_{start,next} */ #include "hipe_stack.h" #endif diff --git a/erts/emulator/beam/erl_node_container_utils.h b/erts/emulator/beam/erl_node_container_utils.h index ae1316eba2..2c67e781e0 100644 --- a/erts/emulator/beam/erl_node_container_utils.h +++ b/erts/emulator/beam/erl_node_container_utils.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -255,19 +255,32 @@ extern int erts_use_r9_pids_ports; #define internal_ref_no_of_numbers(x) \ (internal_ref_data((x))[0]) +#define internal_thing_ref_no_of_numbers(thing) \ + (internal_thing_ref_data(thing)[0]) #define internal_ref_numbers(x) \ (&internal_ref_data((x))[1]) +#define internal_thing_ref_numbers(thing) \ + (&internal_thing_ref_data(thing)[1]) #define external_ref_no_of_numbers(x) \ (external_ref_data((x))[0]) +#define external_thing_ref_no_of_numbers(thing) \ + (external_thing_ref_data(thing)[0]) #define external_ref_numbers(x) \ (&external_ref_data((x))[1]) +#define external_thing_ref_numbers(thing) \ + (&external_thing_ref_data(thing)[1]) + #else #define internal_ref_no_of_numbers(x) (internal_ref_data_words((x))) +#define internal_thing_ref_no_of_numbers(t) (internal_thing_ref_data_words(t)) #define internal_ref_numbers(x) (internal_ref_data((x))) +#define internal_thing_ref_numbers(t) (internal_thing_ref_data(t)) #define external_ref_no_of_numbers(x) (external_ref_data_words((x))) +#define external_thing_ref_no_of_numbers(t) (external_thing_ref_data_words((t))) #define external_ref_numbers(x) (external_ref_data((x))) +#define external_thing_ref_numbers(t) (external_thing_ref_data((t))) #endif @@ -311,6 +324,8 @@ extern int erts_use_r9_pids_ports; : external_ref_channel_no((x))) #define is_ref(x) (is_internal_ref((x)) \ || is_external_ref((x))) +#define is_ref_rel(x,Base) (is_internal_ref_rel((x),Base) \ + || is_external_ref_rel((x),Base)) #define is_not_ref(x) (!is_ref(x)) #endif diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index d0b08bf72e..6daa127d23 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -107,7 +107,7 @@ dist_table_alloc(void *dep_tmpl) dep->nlinks = NULL; dep->monitors = NULL; - erts_smp_spinlock_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr); + erts_smp_mtx_init_x(&dep->qlock, "dist_entry_out_queue", chnl_nr); dep->qflgs = 0; dep->qsize = 0; dep->out_queue.first = NULL; @@ -172,7 +172,7 @@ dist_table_free(void *vdep) ASSERT(!dep->cache); erts_smp_rwmtx_destroy(&dep->rwmtx); erts_smp_mtx_destroy(&dep->lnk_mtx); - erts_smp_spinlock_destroy(&dep->qlock); + erts_smp_mtx_destroy(&dep->qlock); #ifdef DEBUG sys_memset(vdep, 0x77, sizeof(DistEntry)); @@ -235,7 +235,7 @@ erts_sysname_to_connected_dist_entry(Eterm sysname) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); res_dep = (DistEntry *) hash_get(&erts_dist_table, (void *) &de); if (res_dep) { - long refc = erts_refc_inctest(&res_dep->refc, 1); + erts_aint_t refc = erts_refc_inctest(&res_dep->refc, 1); if (refc < 2) /* Pending delete */ erts_refc_inc(&res_dep->refc, 1); } @@ -257,7 +257,7 @@ DistEntry *erts_find_or_insert_dist_entry(Eterm sysname) { DistEntry *res; DistEntry de; - long refc; + erts_aint_t refc; res = erts_find_dist_entry(sysname); if (res) return res; @@ -279,7 +279,7 @@ DistEntry *erts_find_dist_entry(Eterm sysname) erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); res = hash_get(&erts_dist_table, (void *) &de); if (res) { - long refc = erts_refc_inctest(&res->refc, 1); + erts_aint_t refc = erts_refc_inctest(&res->refc, 1); if (refc < 2) /* Pending delete */ erts_refc_inc(&res->refc, 1); } @@ -586,7 +586,7 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) erts_smp_rwmtx_rlock(&erts_node_table_rwmtx); res = hash_get(&erts_node_table, (void *) &ne); if (res && res != erts_this_node) { - long refc = erts_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_refc_inctest(&res->refc, 0); if (refc < 2) /* New or pending delete */ erts_refc_inc(&res->refc, 1); } @@ -598,7 +598,7 @@ ErlNode *erts_find_or_insert_node(Eterm sysname, Uint creation) res = hash_put(&erts_node_table, (void *) &ne); ASSERT(res); if (res != erts_this_node) { - long refc = erts_refc_inctest(&res->refc, 0); + erts_aint_t refc = erts_refc_inctest(&res->refc, 0); if (refc < 2) /* New or pending delete */ erts_refc_inc(&res->refc, 1); } @@ -755,9 +755,9 @@ void erts_init_node_tables(void) erts_this_dist_entry->nlinks = NULL; erts_this_dist_entry->monitors = NULL; - erts_smp_spinlock_init_x(&erts_this_dist_entry->qlock, - "dist_entry_out_queue", - make_small(ERST_INTERNAL_CHANNEL_NO)); + erts_smp_mtx_init_x(&erts_this_dist_entry->qlock, + "dist_entry_out_queue", + make_small(ERST_INTERNAL_CHANNEL_NO)); erts_this_dist_entry->qflgs = 0; erts_this_dist_entry->qsize = 0; erts_this_dist_entry->out_queue.first = NULL; diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index eb759b87e9..b0a63ae035 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -131,7 +131,7 @@ typedef struct dist_entry_ { ErtsLink *nlinks; /* Link tree with subtrees */ ErtsMonitor *monitors; /* Monitor tree */ - erts_smp_spinlock_t qlock; /* Protects qflgs and out_queue */ + erts_smp_mtx_t qlock; /* Protects qflgs and out_queue */ Uint32 qflgs; Sint qsize; ErtsDistOutputQueue out_queue; diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index c10724b951..1b07024ca1 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -129,7 +129,7 @@ reset_handle(ErtsPortTask *ptp) { if (ptp->handle) { ASSERT(ptp == handle2task(ptp->handle)); - erts_smp_atomic_set(ptp->handle, (long) NULL); + erts_smp_atomic_set(ptp->handle, (erts_aint_t) NULL); } } @@ -138,7 +138,7 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) { ptp->handle = pthp; if (pthp) { - erts_smp_atomic_set(pthp, (long) ptp); + erts_smp_atomic_set(pthp, (erts_aint_t) ptp); ASSERT(ptp == handle2task(ptp->handle)); } } @@ -568,7 +568,7 @@ erts_port_task_schedule(Eterm id, ErtsRunQueue *xrunq = erts_check_emigration_need(runq, ERTS_PORT_PRIO_LEVEL); if (xrunq) { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (long) xrunq); + erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); erts_smp_runq_unlock(runq); runq = xrunq; } @@ -727,7 +727,8 @@ resume_after_block(void *vd) ErtsPortTaskExeBlockData *d = (ErtsPortTaskExeBlockData *) vd; erts_smp_runq_lock(d->runq); if (d->resp) - *d->resp = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; + *d->resp = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); } /* @@ -748,7 +749,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) ErtsPortTask *ptp; int res = 0; int reds = ERTS_PORT_REDS_EXECUTE; - long io_tasks_executed = 0; + erts_aint_t io_tasks_executed = 0; int fpe_was_unmasked; ErtsPortTaskExeBlockData blk_data = {runq, NULL}; @@ -942,7 +943,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) } else { /* Port emigrated ... */ - erts_smp_atomic_set(&pp->run_queue, (long) xrunq); + erts_smp_atomic_set(&pp->run_queue, (erts_aint_t) xrunq); enqueue_port(xrunq, pp); ASSERT(pp->sched.exe_taskq); pp->sched.exe_taskq = NULL; @@ -953,7 +954,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) port_was_enqueued = 1; } - res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; + res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); ERTS_PT_CHK_PRES_PORTQ(runq, pp); @@ -971,7 +973,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_port_release(pp); #else { - long refc; + erts_aint_t refc; erts_smp_mtx_unlock(pp->lock); refc = erts_smp_atomic_dectest(&pp->refc); ASSERT(refc >= 0); @@ -979,7 +981,8 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) erts_smp_runq_unlock(runq); erts_port_cleanup(pp); /* Might aquire runq lock */ erts_smp_runq_lock(runq); - res = erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != (long) 0; + res = (erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) + != (erts_aint_t) 0); } } #endif @@ -1112,7 +1115,7 @@ erts_port_migrate(Port *prt, int *prt_locked, if (!ERTS_PORT_IS_IN_RUNQ(from_rq, prt)) return ERTS_MIGRATE_FAILED_NOT_IN_RUNQ; dequeue_port(from_rq, prt); - erts_smp_atomic_set(&prt->run_queue, (long) to_rq); + erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) to_rq); enqueue_port(to_rq, prt); return ERTS_MIGRATE_SUCCESS; } @@ -1125,7 +1128,7 @@ erts_port_migrate(Port *prt, int *prt_locked, void erts_port_task_init(void) { - erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (long) 0); + erts_smp_atomic_init(&erts_port_task_outstanding_io_tasks, (erts_aint_t) 0); init_port_task_alloc(); init_port_taskq_alloc(); } diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index f12d02da0c..3e2c5f07ab 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -79,7 +79,7 @@ ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void); ERTS_GLB_INLINE void erts_port_task_handle_init(ErtsPortTaskHandle *pthp) { - erts_smp_atomic_init(pthp, (long) NULL); + erts_smp_atomic_init(pthp, (erts_aint_t) NULL); } ERTS_GLB_INLINE int @@ -102,6 +102,7 @@ erts_port_task_init_sched(ErtsPortTaskSched *ptsp) ERTS_GLB_INLINE int erts_port_task_have_outstanding_io_tasks(void) { + ERTS_THR_MEMORY_BARRIER; return erts_smp_atomic_read(&erts_port_task_outstanding_io_tasks) != 0; } diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index d9f132f067..b71404fd27 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -216,233 +216,289 @@ static int print_atom_name(fmtfn_t fn, void* arg, Eterm atom, long *dcount) } +#define PRT_BAR ((Eterm) 0) +#define PRT_COMMA ((Eterm) 1) +#define PRT_CLOSE_LIST ((Eterm) 2) +#define PRT_CLOSE_TUPLE ((Eterm) 3) +#define PRT_TERM ((Eterm) 4) +#define PRT_ONE_CONS ((Eterm) 5) +#define PRT_PATCH_FUN_SIZE ((Eterm) 6) +#define PRT_LAST_ARRAY_ELEMENT ((Eterm) 7) /* Note! Must be last... */ static int print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { + DECLARE_WSTACK(s); int res; int i; + Eterm val; Uint32 *ref_num; Eterm* nobj; +#if HALFWORD_HEAP + UWord wobj; +#endif res = 0; - if ((*dcount)-- <= 0) - return res; - -#ifdef HYBRID___NOT_ACTIVE - /* Color coded output based on memory location */ - if(ptr_val(obj) >= global_heap && ptr_val(obj) < global_hend) - PRINT_STRING(res, fn, arg, "\033[32m"); -#ifdef INCREMENTAL - else if(ptr_val(obj) >= inc_fromspc && ptr_val(obj) < inc_fromend) - PRINT_STRING(res, fn, arg, "\033[33m"); + goto L_jump_start; + + L_outer_loop: + while (!WSTACK_ISEMPTY(s)) { + switch (val = WSTACK_POP(s)) { + case PRT_COMMA: + PRINT_CHAR(res, fn, arg, ','); + goto L_outer_loop; + case PRT_BAR: + PRINT_CHAR(res, fn, arg, '|'); + goto L_outer_loop; + case PRT_CLOSE_LIST: + PRINT_CHAR(res, fn, arg, ']'); + goto L_outer_loop; + case PRT_CLOSE_TUPLE: + PRINT_CHAR(res, fn, arg, '}'); + goto L_outer_loop; + default: +#if HALFWORD_HEAP + obj = (Eterm) (wobj = WSTACK_POP(s)); +#else + obj = WSTACK_POP(s); #endif - else if(IS_CONST(obj)) - PRINT_STRING(res, fn, arg, "\033[34m"); - else - PRINT_STRING(res, fn, arg, "\033[31m"); + switch (val) { + case PRT_TERM: + break; + case PRT_ONE_CONS: + L_print_one_cons: + { + Eterm* cons = list_val(obj); + Eterm tl; + + obj = CAR(cons); + tl = CDR(cons); + if (is_not_nil(tl)) { + if (is_list(tl)) { + WSTACK_PUSH(s, tl); + WSTACK_PUSH(s, PRT_ONE_CONS); + WSTACK_PUSH(s, PRT_COMMA); + } else { + WSTACK_PUSH(s, tl); + WSTACK_PUSH(s, PRT_TERM); + WSTACK_PUSH(s, PRT_BAR); + } + } + } + break; + case PRT_LAST_ARRAY_ELEMENT: + { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else + Eterm* ptr = (Eterm *) obj; #endif + obj = *ptr; + } + break; + default: /* PRT_LAST_ARRAY_ELEMENT+1 and upwards */ + { +#if HALFWORD_HEAP + Eterm* ptr = (Eterm *) wobj; +#else + Eterm* ptr = (Eterm *) obj; +#endif + obj = *ptr++; + WSTACK_PUSH(s, (UWord) ptr); + WSTACK_PUSH(s, val-1); + WSTACK_PUSH(s, PRT_COMMA); + } + break; + } + break; + } - if (is_CP(obj)) { - PRINT_STRING(res, fn, arg, "<cp/header:"); - PRINT_POINTER(res, fn, arg, cp_val(obj)); - PRINT_CHAR(res, fn, arg, '>'); - return res; - } + L_jump_start: - switch (tag_val_def(obj)) { - case NIL_DEF: - PRINT_STRING(res, fn, arg, "[]"); - break; - case ATOM_DEF: { - int tres = print_atom_name(fn, arg, obj, dcount); - if (tres < 0) - return tres; - res += tres; - if (*dcount <= 0) - return res; - break; - } - case SMALL_DEF: - PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj)); - break; - case BIG_DEF: { - int print_res; - char def_buf[64]; - char *buf, *big_str; - Uint sz = (Uint) big_decimal_estimate(obj); - sz++; - if (sz <= 64) - buf = &def_buf[0]; - else - buf = erts_alloc(ERTS_ALC_T_TMP, sz); - big_str = erts_big_to_string(obj, buf, sz); - print_res = erts_printf_string(fn, arg, big_str); - if (buf != &def_buf[0]) - erts_free(ERTS_ALC_T_TMP, (void *) buf); - if (print_res < 0) - return print_res; - res += print_res; - break; - } - case REF_DEF: - case EXTERNAL_REF_DEF: - PRINT_STRING(res, fn, arg, "#Ref<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) ref_channel_no(obj)); - ref_num = ref_numbers(obj); - for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) { - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]); + if ((*dcount)-- <= 0) + goto L_done; + + if (is_CP(obj)) { + PRINT_STRING(res, fn, arg, "<cp/header:"); + PRINT_POINTER(res, fn, arg, cp_val(obj)); + PRINT_CHAR(res, fn, arg, '>'); + goto L_done; } - PRINT_CHAR(res, fn, arg, '>'); - break; - case PID_DEF: - case EXTERNAL_PID_DEF: - PRINT_CHAR(res, fn, arg, '<'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_channel_no(obj)); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_number(obj)); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) pid_serial(obj)); - PRINT_CHAR(res, fn, arg, '>'); - break; - case PORT_DEF: - case EXTERNAL_PORT_DEF: - PRINT_STRING(res, fn, arg, "#Port<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_channel_no(obj)); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, - (unsigned long) port_number(obj)); - PRINT_CHAR(res, fn, arg, '>'); - break; - case LIST_DEF: - if (is_printable_string(obj)) { - int c; - PRINT_CHAR(res, fn, arg, '"'); - nobj = list_val(obj); - while (1) { - if ((*dcount)-- <= 0) - return res; - c = signed_val(*nobj++); - if (c == '\n') - PRINT_STRING(res, fn, arg, "\\n"); - else { - if (c == '"') - PRINT_CHAR(res, fn, arg, '\\'); - PRINT_CHAR(res, fn, arg, (char) c); - } - if (is_not_list(*nobj)) - break; - nobj = list_val(*nobj); - } - PRINT_CHAR(res, fn, arg, '"'); - } else { - PRINT_CHAR(res, fn, arg, '['); - nobj = list_val(obj); - while (1) { - int tres = print_term(fn, arg, *nobj++, dcount); - if (tres < 0) - return tres; - res += tres; - if (*dcount <= 0) - return res; - if (is_not_list(*nobj)) - break; - PRINT_CHAR(res, fn, arg, ','); - nobj = list_val(*nobj); - } - if (is_not_nil(*nobj)) { - int tres; - PRINT_CHAR(res, fn, arg, '|'); - tres = print_term(fn, arg, *nobj, dcount); - if (tres < 0) - return tres; - res += tres; - if (*dcount <= 0) - return res; + + switch (tag_val_def(obj)) { + case NIL_DEF: + PRINT_STRING(res, fn, arg, "[]"); + break; + case ATOM_DEF: { + int tres = print_atom_name(fn, arg, obj, dcount); + if (tres < 0) { + res = tres; + goto L_done; } - PRINT_CHAR(res, fn, arg, ']'); - } - break; - case TUPLE_DEF: - nobj = tuple_val(obj); /* pointer to arity */ - i = arityval(*nobj); /* arity */ - PRINT_CHAR(res, fn, arg, '{'); - while (i--) { - int tres = print_term(fn, arg, *++nobj, dcount); - if (tres < 0) - return tres; res += tres; if (*dcount <= 0) - return res; - if (i >= 1) - PRINT_CHAR(res, fn, arg, ','); - } - PRINT_CHAR(res, fn, arg, '}'); - break; - case FLOAT_DEF: { - FloatDef ff; - GET_DOUBLE(obj, ff); - PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd); + goto L_done; + break; } - break; - case BINARY_DEF: - { - ProcBin* pb = (ProcBin *) binary_val(obj); - if (pb->size == 1) - PRINT_STRING(res, fn, arg, "<<1 byte>>"); - else { - PRINT_STRING(res, fn, arg, "<<"); - PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size); - PRINT_STRING(res, fn, arg, " bytes>>"); + case SMALL_DEF: + PRINT_SLONG(res, fn, arg, 'd', 0, 1, (signed long) signed_val(obj)); + break; + case BIG_DEF: { + int print_res; + char def_buf[64]; + char *buf, *big_str; + Uint sz = (Uint) big_decimal_estimate(obj); + sz++; + if (sz <= 64) + buf = &def_buf[0]; + else + buf = erts_alloc(ERTS_ALC_T_TMP, sz); + big_str = erts_big_to_string(obj, buf, sz); + print_res = erts_printf_string(fn, arg, big_str); + if (buf != &def_buf[0]) + erts_free(ERTS_ALC_T_TMP, (void *) buf); + if (print_res < 0) { + res = print_res; + goto L_done; } + res += print_res; + break; } - break; - case EXPORT_DEF: - { - Export* ep = *((Export **) (export_val(obj) + 1)); - Atom* module = atom_tab(atom_val(ep->code[0])); - Atom* name = atom_tab(atom_val(ep->code[1])); - - PRINT_STRING(res, fn, arg, "#Fun<"); - PRINT_BUF(res, fn, arg, module->name, module->len); + case REF_DEF: + case EXTERNAL_REF_DEF: + PRINT_STRING(res, fn, arg, "#Ref<"); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) ref_channel_no(obj)); + ref_num = ref_numbers(obj); + for (i = ref_no_of_numbers(obj)-1; i >= 0; i--) { + PRINT_CHAR(res, fn, arg, '.'); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) ref_num[i]); + } + PRINT_CHAR(res, fn, arg, '>'); + break; + case PID_DEF: + case EXTERNAL_PID_DEF: + PRINT_CHAR(res, fn, arg, '<'); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) pid_channel_no(obj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_BUF(res, fn, arg, name->name, name->len); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) pid_number(obj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) ep->code[2]); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) pid_serial(obj)); PRINT_CHAR(res, fn, arg, '>'); - } - break; - case FUN_DEF: - { - ErlFunThing *funp = (ErlFunThing *) fun_val(obj); - Atom *ap = atom_tab(atom_val(funp->fe->module)); - - PRINT_STRING(res, fn, arg, "#Fun<"); - PRINT_BUF(res, fn, arg, ap->name, ap->len); - PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) funp->fe->old_index); + break; + case PORT_DEF: + case EXTERNAL_PORT_DEF: + PRINT_STRING(res, fn, arg, "#Port<"); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) port_channel_no(obj)); PRINT_CHAR(res, fn, arg, '.'); - PRINT_SLONG(res, fn, arg, 'd', 0, 1, - (signed long) funp->fe->old_uniq); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, + (unsigned long) port_number(obj)); PRINT_CHAR(res, fn, arg, '>'); + break; + case LIST_DEF: + if (is_printable_string(obj)) { + int c; + PRINT_CHAR(res, fn, arg, '"'); + nobj = list_val(obj); + while (1) { + if ((*dcount)-- <= 0) + goto L_done; + c = signed_val(*nobj++); + if (c == '\n') + PRINT_STRING(res, fn, arg, "\\n"); + else { + if (c == '"') + PRINT_CHAR(res, fn, arg, '\\'); + PRINT_CHAR(res, fn, arg, (char) c); + } + if (is_not_list(*nobj)) + break; + nobj = list_val(*nobj); + } + PRINT_CHAR(res, fn, arg, '"'); + } else { + PRINT_CHAR(res, fn, arg, '['); + WSTACK_PUSH(s,PRT_CLOSE_LIST); + goto L_print_one_cons; + } + break; + case TUPLE_DEF: + nobj = tuple_val(obj); /* pointer to arity */ + i = arityval(*nobj); /* arity */ + PRINT_CHAR(res, fn, arg, '{'); + WSTACK_PUSH(s,PRT_CLOSE_TUPLE); + ++nobj; + if (i > 0) { + WSTACK_PUSH(s, (UWord) nobj); + WSTACK_PUSH(s, PRT_LAST_ARRAY_ELEMENT+i-1); + } + break; + case FLOAT_DEF: { + FloatDef ff; + GET_DOUBLE(obj, ff); + PRINT_DOUBLE(res, fn, arg, 'e', 6, 0, ff.fd); + } + break; + case BINARY_DEF: + { + ProcBin* pb = (ProcBin *) binary_val(obj); + if (pb->size == 1) + PRINT_STRING(res, fn, arg, "<<1 byte>>"); + else { + PRINT_STRING(res, fn, arg, "<<"); + PRINT_ULONG(res, fn, arg, 'u', 0, 1, (unsigned long) pb->size); + PRINT_STRING(res, fn, arg, " bytes>>"); + } + } + break; + case EXPORT_DEF: + { + Export* ep = *((Export **) (export_val(obj) + 1)); + Atom* module = atom_tab(atom_val(ep->code[0])); + Atom* name = atom_tab(atom_val(ep->code[1])); + + PRINT_STRING(res, fn, arg, "#Fun<"); + PRINT_BUF(res, fn, arg, module->name, module->len); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_BUF(res, fn, arg, name->name, name->len); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_SLONG(res, fn, arg, 'd', 0, 1, + (signed long) ep->code[2]); + PRINT_CHAR(res, fn, arg, '>'); + } + break; + case FUN_DEF: + { + ErlFunThing *funp = (ErlFunThing *) fun_val(obj); + Atom *ap = atom_tab(atom_val(funp->fe->module)); + + PRINT_STRING(res, fn, arg, "#Fun<"); + PRINT_BUF(res, fn, arg, ap->name, ap->len); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_SLONG(res, fn, arg, 'd', 0, 1, + (signed long) funp->fe->old_index); + PRINT_CHAR(res, fn, arg, '.'); + PRINT_SLONG(res, fn, arg, 'd', 0, 1, + (signed long) funp->fe->old_uniq); + PRINT_CHAR(res, fn, arg, '>'); + } + break; + default: + PRINT_STRING(res, fn, arg, "<unknown:"); + PRINT_POINTER(res, fn, arg, (UWord) obj); + PRINT_CHAR(res, fn, arg, '>'); + break; } - break; - default: - PRINT_STRING(res, fn, arg, "<unknown:"); - PRINT_POINTER(res, fn, arg, (UWord) obj); - PRINT_CHAR(res, fn, arg, '>'); - break; } + L_done: + + DESTROY_WSTACK(s); return res; } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b47ce97c46..f8997f3c07 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,7 +24,6 @@ #endif #include <stddef.h> /* offsetof() */ -#include <ctype.h> #include "sys.h" #include "erl_vm.h" #include "global.h" @@ -39,6 +38,7 @@ #include "erl_threads.h" #include "erl_binary.h" #include "beam_bp.h" +#include "erl_cpu_topology.h" #define ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED (2000*CONTEXT_REDS) #define ERTS_RUNQ_CALL_CHECK_BALANCE_REDS \ @@ -63,8 +63,6 @@ #define ERTS_WAKEUP_OTHER_DEC 10 #define ERTS_WAKEUP_OTHER_FIXED_INC (CONTEXT_REDS/10) -#define ERTS_MAX_CPU_TOPOLOGY_ID ((int) 0xffff) - #if 0 || defined(DEBUG) #define ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA #endif @@ -119,10 +117,6 @@ Uint erts_process_tab_index_mask; static int wakeup_other_limit; -#ifdef ERTS_SMP -Uint erts_max_main_threads; -#endif - int erts_sched_thread_suggested_stack_size = -1; #ifdef ERTS_ENABLE_LOCK_CHECK @@ -133,21 +127,22 @@ ErtsLcPSDLocks erts_psd_required_locks[ERTS_PSD_SIZE]; int erts_disable_proc_not_running_opt; -#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((long) 1) << 0) -#define ERTS_SCHDLR_SSPND_CHNG_MSB (((long) 1) << 1) -#define ERTS_SCHDLR_SSPND_CHNG_ONLN (((long) 1) << 2) +#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0) +#define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) +#define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) #ifndef DEBUG #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic_set(&schdlr_sspnd.changing, (VAL)) + erts_smp_atomic32_set(&schdlr_sspnd.changing, (VAL)) #else #define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ do { \ - long old_val__ = erts_smp_atomic_xchg(&schdlr_sspnd.changing, \ - (VAL)); \ + erts_aint32_t old_val__; \ + old_val__ = erts_smp_atomic32_xchg(&schdlr_sspnd.changing, \ + (VAL)); \ ASSERT(old_val__ == (OLD_VAL)); \ } while (0) @@ -160,10 +155,10 @@ static struct { int online; int curr_online; int wait_curr_online; - erts_smp_atomic_t changing; - erts_smp_atomic_t active; + erts_smp_atomic32_t changing; + erts_smp_atomic32_t active; struct { - erts_smp_atomic_t ongoing; + erts_smp_atomic32_t ongoing; long wait_active; ErtsProcList *procs; } msb; /* Multi Scheduling Block */ @@ -171,11 +166,11 @@ static struct { static struct { erts_smp_mtx_t update_mtx; - erts_smp_atomic_t active_runqs; + erts_smp_atomic32_t active_runqs; int last_active_runqs; - erts_smp_atomic_t used_runqs; + erts_smp_atomic32_t used_runqs; int forced_check_balance; - erts_smp_atomic_t checking_balance; + erts_smp_atomic32_t checking_balance; int halftime; int full_reds_history_index; struct { @@ -195,48 +190,6 @@ do { \ #endif -/* - * Cpu topology hierarchy. - */ -#define ERTS_TOPOLOGY_NODE 0 -#define ERTS_TOPOLOGY_PROCESSOR 1 -#define ERTS_TOPOLOGY_PROCESSOR_NODE 2 -#define ERTS_TOPOLOGY_CORE 3 -#define ERTS_TOPOLOGY_THREAD 4 -#define ERTS_TOPOLOGY_LOGICAL 5 - -#define ERTS_TOPOLOGY_MAX_DEPTH 6 - -typedef struct { - int bind_id; - int bound_id; -} ErtsCpuBindData; - -static ErtsCpuBindData *scheduler2cpu_map; -erts_smp_rwmtx_t erts_cpu_bind_rwmtx; - -typedef enum { - ERTS_CPU_BIND_UNDEFINED, - ERTS_CPU_BIND_SPREAD, - ERTS_CPU_BIND_PROCESSOR_SPREAD, - ERTS_CPU_BIND_THREAD_SPREAD, - ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD, - ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD, - ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD, - ERTS_CPU_BIND_NO_SPREAD, - ERTS_CPU_BIND_NONE -} ErtsCpuBindOrder; - -#define ERTS_CPU_BIND_DEFAULT_BIND \ - ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD - -ErtsCpuBindOrder cpu_bind_order; - -static erts_cpu_topology_t *user_cpudata; -static int user_cpudata_size; -static erts_cpu_topology_t *system_cpudata; -static int system_cpudata_size; - erts_sched_stat_t erts_sched_stat; ErtsRunQueue *erts_common_run_queue; @@ -247,11 +200,11 @@ static erts_tsd_key_t sched_data_key; static erts_smp_mtx_t proc_tab_mtx; -static erts_smp_atomic_t function_calls; +static erts_smp_atomic32_t function_calls; #ifdef ERTS_SMP -static erts_smp_atomic_t doing_sys_schedule; -static erts_smp_atomic_t no_empty_run_queues; +static erts_smp_atomic32_t doing_sys_schedule; +static erts_smp_atomic32_t no_empty_run_queues; #else /* !ERTS_SMP */ ErtsSchedulerData *erts_scheduler_data; #endif @@ -259,11 +212,6 @@ ErtsSchedulerData *erts_scheduler_data; ErtsAlignedRunQueue *erts_aligned_run_queues; Uint erts_no_run_queues; -typedef union { - ErtsSchedulerData esd; - char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; -} ErtsAlignedSchedulerData; - ErtsAlignedSchedulerData *erts_aligned_scheduler_data; #ifdef ERTS_SMP @@ -300,7 +248,10 @@ Uint erts_num_active_procs; Process** erts_active_procs; #endif -static erts_smp_atomic_t process_count; +#if ERTS_MAX_PROCESSES > 0x7fffffff +#error "Need to store process_count in another type" +#endif +static erts_smp_atomic32_t process_count; typedef struct ErtsTermProcElement_ ErtsTermProcElement; struct ErtsTermProcElement_ { @@ -334,12 +285,6 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(proclist, 200, ERTS_ALC_T_PROC_LIST) -#define ERTS_RUNQ_IX(IX) \ - (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues), \ - &erts_aligned_run_queues[(IX)].runq) -#define ERTS_SCHEDULER_IX(IX) \ - (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ - &erts_aligned_scheduler_data[(IX)].esd) #define ERTS_SCHED_SLEEP_INFO_IX(IX) \ (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ &aligned_sched_sleep_info[(IX)].ssi) @@ -398,22 +343,8 @@ static int stack_element_dump(int to, void *to_arg, Process* p, Eterm* sp, #ifdef ERTS_SMP static void handle_pending_exiters(ErtsProcList *); -static void cpu_bind_order_sort(erts_cpu_topology_t *cpudata, - int size, - ErtsCpuBindOrder bind_order, - int mk_seq); -static void signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size); - #endif -static int reader_group_lookup(int logical); -static void create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, - int *cpudata_size); -static void destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata); - -static void early_cpu_bind_init(void); -static void late_cpu_bind_init(void); - #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_smp_lc_runq_is_locked(ErtsRunQueue *runq) @@ -469,18 +400,18 @@ erts_pre_init_process(void) /* initialize the scheduler */ void -erts_init_process(void) +erts_init_process(int ncpu) { Uint proc_bits = ERTS_PROC_BITS; #ifdef ERTS_SMP erts_disable_proc_not_running_opt = 0; - erts_init_proc_lock(); + erts_init_proc_lock(ncpu); #endif init_proclist_alloc(); - erts_smp_atomic_init(&process_count, 0); + erts_smp_atomic32_init(&process_count, 0); if (erts_use_r9_pids_ports) { proc_bits = ERTS_R9_PROC_BITS; @@ -641,7 +572,7 @@ erts_psd_set_init(Process *p, ErtsProcLocks plocks, int ix, void *data) #ifdef ERTS_SMP void -erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, long flags) +erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) { switch (flags & ERTS_SSI_FLGS_SLEEP_TYPE) { case ERTS_SSI_FLG_POLL_SLEEPING: @@ -659,6 +590,118 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, long flags) } } +typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; +struct erts_misc_aux_work_t_ { + erts_misc_aux_work_t *next; + void (*func)(void *); + void *arg; +}; + +typedef struct { + erts_smp_mtx_t mtx; + erts_misc_aux_work_t *first; + erts_misc_aux_work_t *last; +} erts_misc_aux_work_q_t; + +typedef union { + erts_misc_aux_work_q_t data; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_misc_aux_work_q_t))]; +} erts_algnd_misc_aux_work_q_t; + +static erts_algnd_misc_aux_work_q_t *misc_aux_work_queues; + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(misc_aux_work, + erts_misc_aux_work_t, + 200, + ERTS_ALC_T_MISC_AUX_WORK) + +static void +init_misc_aux_work(void) +{ + int ix; + + init_misc_aux_work_alloc(); + + misc_aux_work_queues = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_MISC_AUX_WORK_Q, + erts_no_schedulers * + sizeof(erts_algnd_misc_aux_work_q_t)); + + for (ix = 0; ix < erts_no_schedulers; ix++) { + erts_smp_mtx_init_x(&misc_aux_work_queues[ix].data.mtx, + "misc_aux_work_queue", + make_small(ix + 1)); + misc_aux_work_queues[ix].data.first = NULL; + misc_aux_work_queues[ix].data.last = NULL; + } +} + +static void +handle_misc_aux_work(ErtsSchedulerData *esdp) +{ + int ix = (int) esdp->no - 1; + erts_misc_aux_work_t *mawp; + + erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); + mawp = misc_aux_work_queues[ix].data.first; + misc_aux_work_queues[ix].data.first = NULL; + misc_aux_work_queues[ix].data.last = NULL; + erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); + + while (mawp) { + erts_misc_aux_work_t *free_mawp; + mawp->func(mawp->arg); + free_mawp = mawp; + mawp = mawp->next; + misc_aux_work_free(free_mawp); + } +} + +void +erts_smp_schedule_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg) +{ + int ix, ignore_ix = -1; + + if (ignore_self) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + if (esdp) + ignore_ix = (int) esdp->no - 1; + } + + ASSERT(0 <= max_sched && max_sched <= erts_no_schedulers); + + for (ix = 0; ix < max_sched; ix++) { + erts_aint32_t aux_work; + erts_misc_aux_work_t *mawp; + ErtsSchedulerSleepInfo *ssi; + if (ix == ignore_ix) + continue; + + mawp = misc_aux_work_alloc(); + + mawp->func = func; + mawp->arg = arg; + mawp->next = NULL; + + erts_smp_mtx_lock(&misc_aux_work_queues[ix].data.mtx); + if (!misc_aux_work_queues[ix].data.last) + misc_aux_work_queues[ix].data.first = mawp; + else + misc_aux_work_queues[ix].data.last->next = mawp; + misc_aux_work_queues[ix].data.last = mawp; + erts_smp_mtx_unlock(&misc_aux_work_queues[ix].data.mtx); + + ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); + aux_work = erts_smp_atomic32_bor(&ssi->aux_work, + ERTS_SSI_AUX_WORK_MISC); + if ((aux_work & ERTS_SSI_AUX_WORK_MISC) == 0) + erts_sched_poke(ssi); + } +} + #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN void erts_smp_notify_check_children_needed(void) @@ -666,11 +709,11 @@ erts_smp_notify_check_children_needed(void) int i; for (i = 0; i < erts_no_schedulers; i++) { - long aux_work; + erts_aint32_t aux_work; ErtsSchedulerSleepInfo *ssi; ssi = ERTS_SCHED_SLEEP_INFO_IX(i); - aux_work = erts_smp_atomic_bor(&ssi->aux_work, - ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + aux_work = erts_smp_atomic32_bor(&ssi->aux_work, + ERTS_SSI_AUX_WORK_CHECK_CHILDREN); if (!(aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN)) erts_sched_poke(ssi); } @@ -678,16 +721,22 @@ erts_smp_notify_check_children_needed(void) #endif #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK -static ERTS_INLINE long +static ERTS_INLINE erts_aint32_t blockable_aux_work(ErtsSchedulerData *esdp, ErtsSchedulerSleepInfo *ssi, - long aux_work) + erts_aint32_t aux_work) { if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { + if (aux_work & ERTS_SSI_AUX_WORK_MISC) { + aux_work = erts_smp_atomic32_band(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_MISC); + aux_work &= ~ERTS_SSI_AUX_WORK_MISC; + handle_misc_aux_work(esdp); + } #ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { - aux_work = erts_smp_atomic_band(&ssi->aux_work, - ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); + aux_work = erts_smp_atomic32_band(&ssi->aux_work, + ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN); aux_work &= ~ERTS_SSI_AUX_WORK_CHECK_CHILDREN; erts_check_children(); } @@ -699,10 +748,10 @@ blockable_aux_work(ErtsSchedulerData *esdp, #endif #ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK -static ERTS_INLINE long +static ERTS_INLINE erts_aint32_t nonblockable_aux_work(ErtsSchedulerData *esdp, ErtsSchedulerSleepInfo *ssi, - long aux_work) + erts_aint32_t aux_work) { if (aux_work & ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK) { @@ -762,15 +811,31 @@ erts_active_schedulers(void) return as; } +#ifdef ERTS_SMP + +static ERTS_INLINE void +clear_sys_scheduling(void) +{ + erts_smp_atomic32_set_relb(&doing_sys_schedule, 0); +} + +static ERTS_INLINE int +try_set_sys_scheduling(void) +{ + return 0 == erts_smp_atomic32_cmpxchg_acqb(&doing_sys_schedule, 1, 0); +} + +#endif + static ERTS_INLINE int prepare_for_sys_schedule(void) { #ifdef ERTS_SMP while (!erts_port_task_have_outstanding_io_tasks() - && !erts_smp_atomic_xchg(&doing_sys_schedule, 1)) { + && try_set_sys_scheduling()) { if (!erts_port_task_have_outstanding_io_tasks()) return 1; - erts_smp_atomic_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); } return 0; #else @@ -818,53 +883,55 @@ sched_active(Uint no, ErtsRunQueue *rq) static int ERTS_INLINE ongoing_multi_scheduling_block(void) { - return erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing) != 0; + return erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing) != 0; } static ERTS_INLINE void empty_runq(ErtsRunQueue *rq) { - long oifls = erts_smp_atomic_band(&rq->info_flags, ~ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_band(&rq->info_flags, + ~ERTS_RUNQ_IFLG_NONEMPTY); if (oifls & ERTS_RUNQ_IFLG_NONEMPTY) { #ifdef DEBUG - long empty = erts_smp_atomic_read(&no_empty_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); /* * For a short period of time no_empty_run_queues may have * been increased twice for a specific run queue. */ ASSERT(0 <= empty && empty < 2*erts_no_run_queues); #endif - erts_smp_atomic_inc(&no_empty_run_queues); + erts_smp_atomic32_inc(&no_empty_run_queues); } } static ERTS_INLINE void non_empty_runq(ErtsRunQueue *rq) { - long oifls = erts_smp_atomic_bor(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + erts_aint32_t oifls = erts_smp_atomic32_bor(&rq->info_flags, + ERTS_RUNQ_IFLG_NONEMPTY); if (!(oifls & ERTS_RUNQ_IFLG_NONEMPTY)) { #ifdef DEBUG - long empty = erts_smp_atomic_read(&no_empty_run_queues); + erts_aint32_t empty = erts_smp_atomic32_read(&no_empty_run_queues); /* * For a short period of time no_empty_run_queues may have * been increased twice for a specific run queue. */ ASSERT(0 < empty && empty <= 2*erts_no_run_queues); #endif - erts_smp_atomic_dec(&no_empty_run_queues); + erts_smp_atomic32_dec(&no_empty_run_queues); } } -static long +static erts_aint32_t sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) { - long oflgs; - long nflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING); - long xflgs = 0; + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + erts_aint32_t xflgs = 0; do { - oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -872,16 +939,16 @@ sched_prep_spin_wait(ErtsSchedulerSleepInfo *ssi) return oflgs; } -static long +static erts_aint32_t sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) { - long oflgs; - long nflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING); - long xflgs = ERTS_SSI_FLG_WAITING; + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING); + erts_aint32_t xflgs = ERTS_SSI_FLG_WAITING; do { - oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -890,15 +957,15 @@ sched_prep_cont_spin_wait(ErtsSchedulerSleepInfo *ssi) return oflgs; } -static long +static erts_aint32_t sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) { - long until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; + int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; int sc = spincount; - long flgs; + erts_aint32_t flgs; do { - flgs = erts_smp_atomic_read(&ssi->flags); + flgs = erts_smp_atomic32_read(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) != (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) { break; @@ -912,18 +979,18 @@ sched_spin_wait(ErtsSchedulerSleepInfo *ssi, int spincount) return flgs; } -static long -sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, long sleep_type) +static erts_aint32_t +sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, erts_aint32_t sleep_type) { - long oflgs; - long nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type; - long xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + erts_aint32_t oflgs; + erts_aint32_t nflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING|sleep_type; + erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; if (sleep_type == ERTS_SSI_FLG_TSE_SLEEPING) erts_tse_reset(ssi->event); while (1) { - oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; if ((oflgs & (ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING)) @@ -940,14 +1007,14 @@ sched_set_sleeptype(ErtsSchedulerSleepInfo *ssi, long sleep_type) != ERTS_SSI_FLG_WAITING) static void -scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) +scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) { ErtsSchedulerSleepInfo *ssi = esdp->ssi; int spincount; - long flgs; + erts_aint32_t flgs; #if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) - long aux_work; + erts_aint32_t aux_work; #endif ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); @@ -983,7 +1050,7 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) tse_wait: #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read(&ssi->aux_work); tse_blockable_aux_work: aux_work = blockable_aux_work(esdp, ssi, aux_work); #endif @@ -993,7 +1060,7 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) #ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK #ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read(&ssi->aux_work); #endif nonblockable_aux_work(esdp, ssi, aux_work); #endif @@ -1026,7 +1093,7 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) } #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read(&ssi->aux_work); if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); goto tse_blockable_aux_work; @@ -1038,16 +1105,16 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); erts_smp_runq_lock(rq); sched_active(esdp->no, rq); } else { - long dt; + erts_aint_t dt; - erts_smp_atomic_set(&function_calls, 0); + erts_smp_atomic32_set(&function_calls, 0); *fcalls = 0; sched_waiting_sys(esdp->no, rq); @@ -1060,25 +1127,27 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sys_poll_aux_work: + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + erl_sys_schedule(1); /* Might give us something to do */ - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); sys_aux_work: #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read(&ssi->aux_work); aux_work = blockable_aux_work(esdp, ssi, aux_work); #endif #ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK #ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read(&ssi->aux_work); #endif nonblockable_aux_work(esdp, ssi, aux_work); #endif - flgs = erts_smp_atomic_read(&ssi->flags); + flgs = erts_smp_atomic32_read(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_WAITING)) { ASSERT(!(flgs & ERTS_SSI_FLG_SLEEPING)); goto sys_woken; @@ -1096,7 +1165,7 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * call erl_sys_schedule() until it is handled. */ if (erts_port_task_have_outstanding_io_tasks()) { - erts_smp_atomic_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); /* * Got to check that we still got I/O tasks; otherwise * we have to continue checking for I/O... @@ -1115,7 +1184,7 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) * sleep in erl_sys_schedule(). */ if (erts_port_task_have_outstanding_io_tasks()) { - erts_smp_atomic_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); /* * Got to check that we still got I/O tasks; otherwise @@ -1155,10 +1224,12 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) erts_smp_runq_unlock(rq); + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + erl_sys_schedule(0); - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); flgs = sched_prep_cont_spin_wait(ssi); if (flgs & ERTS_SSI_FLG_WAITING) @@ -1167,9 +1238,9 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) sys_woken: erts_smp_runq_lock(rq); sys_locked_woken: - erts_smp_atomic_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); if (flgs & ~ERTS_SSI_FLG_SUSPENDED) - erts_smp_atomic_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_band(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); sched_active_sys(esdp->no, rq); } } @@ -1177,15 +1248,15 @@ scheduler_wait(long *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(rq)); } -static ERTS_INLINE long +static ERTS_INLINE erts_aint32_t ssi_flags_set_wake(ErtsSchedulerSleepInfo *ssi) { /* reset all flags but suspended */ - long oflgs; - long nflgs = 0; - long xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; + erts_aint32_t oflgs; + erts_aint32_t nflgs = 0; + erts_aint32_t xflgs = ERTS_SSI_FLG_SLEEPING|ERTS_SSI_FLG_WAITING; while (1) { - oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return oflgs; nflgs = oflgs & ERTS_SSI_FLG_SUSPENDED; @@ -1217,7 +1288,7 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) if (!ssi) erts_smp_spin_unlock(&sl->lock); else if (one) { - long flgs; + erts_aint32_t flgs; if (ssi->prev) ssi->prev->next = ssi->next; else { @@ -1230,6 +1301,7 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) res = sl->list != NULL; erts_smp_spin_unlock(&sl->lock); + ERTS_THR_MEMORY_BARRIER; flgs = ssi_flags_set_wake(ssi); erts_sched_finish_poke(ssi, flgs); @@ -1239,10 +1311,12 @@ wake_scheduler(ErtsRunQueue *rq, int incq, int one) else { sl->list = NULL; erts_smp_spin_unlock(&sl->lock); + + ERTS_THR_MEMORY_BARRIER; do { ErtsSchedulerSleepInfo *wake_ssi = ssi; ssi = ssi->next; - erts_sched_finish_poke(ssi, ssi_flags_set_wake(wake_ssi)); + erts_sched_finish_poke(wake_ssi, ssi_flags_set_wake(wake_ssi)); } while (ssi); } } @@ -1264,15 +1338,17 @@ wake_all_schedulers(void) static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { - long iflgs; + erts_aint32_t iflgs; ErtsRunQueue *wrq; if (crq->ix == ix) return 0; wrq = ERTS_RUNQ_IX(ix); - iflgs = erts_smp_atomic_read(&wrq->info_flags); + iflgs = erts_smp_atomic32_read(&wrq->info_flags); if (!(iflgs & (ERTS_RUNQ_IFLG_SUSPENDED|ERTS_RUNQ_IFLG_NONEMPTY))) { if (activate) { - if (ix == erts_smp_atomic_cmpxchg(&balance_info.active_runqs, ix+1, ix)) { + if (ix == erts_smp_atomic32_cmpxchg(&balance_info.active_runqs, + ix+1, + ix)) { erts_smp_xrunq_lock(crq, wrq); wrq->flags &= ~ERTS_RUNQ_FLG_INACTIVE; erts_smp_xrunq_unlock(crq, wrq); @@ -1289,8 +1365,8 @@ wake_scheduler_on_empty_runq(ErtsRunQueue *crq) { int ix = crq->ix; int stop_ix = ix; - int active_ix = erts_smp_atomic_read(&balance_info.active_runqs); - int balance_ix = erts_smp_atomic_read(&balance_info.used_runqs); + int active_ix = erts_smp_atomic32_read(&balance_info.active_runqs); + int balance_ix = erts_smp_atomic32_read(&balance_info.used_runqs); if (active_ix > balance_ix) active_ix = balance_ix; @@ -1335,6 +1411,31 @@ erts_smp_notify_inc_runq(ErtsRunQueue *runq) smp_notify_inc_runq(runq); } +void +erts_sched_notify_check_cpu_bind(void) +{ +#ifdef ERTS_SMP + int ix; + if (erts_common_run_queue) { + for (ix = 0; ix < erts_no_schedulers; ix++) + erts_smp_atomic32_set(&ERTS_SCHEDULER_IX(ix)->chk_cpu_bind, 1); + wake_all_schedulers(); + } + else { + for (ix = 0; ix < erts_no_run_queues; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + erts_smp_runq_lock(rq); + rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + erts_smp_runq_unlock(rq); + wake_scheduler(rq, 0, 1); + }; + } +#else + erts_sched_check_cpu_bind(erts_get_scheduler_data()); +#endif +} + + #ifdef ERTS_SMP ErtsRunQueue * @@ -1485,14 +1586,15 @@ evacuate_run_queue(ErtsRunQueue *evac_rq, ErtsRunQueue *rq) erts_smp_runq_lock(evac_rq); - erts_smp_atomic_bor(&evac_rq->scheduler->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_bor(&evac_rq->scheduler->ssi->flags, + ERTS_SSI_FLG_SUSPENDED); evac_rq->flags &= ~ERTS_RUNQ_FLGS_IMMIGRATE_QMASK; evac_rq->flags |= (ERTS_RUNQ_FLGS_EMIGRATE_QMASK | ERTS_RUNQ_FLGS_EVACUATE_QMASK | ERTS_RUNQ_FLG_SUSPENDED); - erts_smp_atomic_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); + erts_smp_atomic32_bor(&evac_rq->info_flags, ERTS_RUNQ_IFLG_SUSPENDED); /* * Need to set up evacuation paths first since we * may release the run queue lock on evac_rq @@ -1741,7 +1843,7 @@ static ERTS_INLINE int check_possible_steal_victim(ErtsRunQueue *rq, int *rq_lockedp, int vix) { ErtsRunQueue *vrq = ERTS_RUNQ_IX(vix); - long iflgs = erts_smp_atomic_read(&vrq->info_flags); + erts_aint32_t iflgs = erts_smp_atomic32_read(&vrq->info_flags); if (iflgs & ERTS_RUNQ_IFLG_NONEMPTY) return try_steal_task_from_victim(rq, rq_lockedp, vrq); else @@ -1771,8 +1873,8 @@ try_steal_task(ErtsRunQueue *rq) ERTS_SMP_LC_CHK_RUNQ_LOCK(rq, rq_locked); - active_rqs = erts_smp_atomic_read(&balance_info.active_runqs); - blnc_rqs = erts_smp_atomic_read(&balance_info.used_runqs); + active_rqs = erts_smp_atomic32_read(&balance_info.active_runqs); + blnc_rqs = erts_smp_atomic32_read(&balance_info.used_runqs); if (active_rqs > blnc_rqs) active_rqs = blnc_rqs; @@ -1783,7 +1885,7 @@ try_steal_task(ErtsRunQueue *rq) if (active_rqs < blnc_rqs) { int no = blnc_rqs - active_rqs; int stop_ix = vix = active_rqs + rq->ix % no; - while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { res = check_possible_steal_victim(rq, &rq_locked, vix); if (res) goto done; @@ -1798,7 +1900,7 @@ try_steal_task(ErtsRunQueue *rq) vix = rq->ix; /* ... then try to steal a job from another active queue... */ - while (erts_smp_atomic_read(&no_empty_run_queues) < blnc_rqs) { + while (erts_smp_atomic32_read(&no_empty_run_queues) < blnc_rqs) { vix++; if (vix >= active_rqs) vix = 0; @@ -1886,20 +1988,23 @@ do { \ static void check_balance(ErtsRunQueue *c_rq) { +#if ERTS_MAX_PROCESSES >= (1 << 27) +# error check_balance() assumes ERTS_MAX_PROCESS < (1 << 27) +#endif ErtsRunQueueBalance avg = {0}; Sint64 scheds_reds, full_scheds_reds; int forced, active, current_active, oowc, half_full_scheds, full_scheds, mmax_len, blnc_no_rqs, qix, pix, freds_hist_ix; - if (erts_smp_atomic_xchg(&balance_info.checking_balance, 1)) { + if (erts_smp_atomic32_xchg(&balance_info.checking_balance, 1)) { c_rq->check_balance_reds = INT_MAX; return; } - blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs); + blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); if (blnc_no_rqs == 1) { c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); return; } @@ -1907,7 +2012,7 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.halftime) { balance_info.halftime = 0; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); ERTS_FOREACH_RUNQ(rq, { if (rq->waiting) @@ -1935,12 +2040,12 @@ check_balance(ErtsRunQueue *c_rq) forced = balance_info.forced_check_balance; balance_info.forced_check_balance = 0; - blnc_no_rqs = (int) erts_smp_atomic_read(&balance_info.used_runqs); + blnc_no_rqs = (int) erts_smp_atomic32_read(&balance_info.used_runqs); if (blnc_no_rqs == 1) { erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_runq_lock(c_rq); c_rq->check_balance_reds = INT_MAX; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); return; } @@ -1949,7 +2054,7 @@ check_balance(ErtsRunQueue *c_rq) if (balance_info.full_reds_history_index >= ERTS_FULL_REDS_HISTORY_SIZE) balance_info.full_reds_history_index = 0; - current_active = erts_smp_atomic_read(&balance_info.active_runqs); + current_active = erts_smp_atomic32_read(&balance_info.active_runqs); /* Read balance information for all run queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { @@ -2009,12 +2114,14 @@ check_balance(ErtsRunQueue *c_rq) run_queue_info[qix].prio[pix].avail = 0; } else { - int xreds = 0; - int procreds = treds; - procreds -= run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds; + Sint64 xreds = 0; + Sint64 procreds = treds; + procreds -= + ((Sint64) + run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].reds); for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - int av; + Sint64 av; if (xreds == 0) av = 100; @@ -2025,9 +2132,10 @@ check_balance(ErtsRunQueue *c_rq) if (av == 0) av = 1; } - run_queue_info[qix].prio[pix].avail = av; + run_queue_info[qix].prio[pix].avail = (int) av; + ASSERT(run_queue_info[qix].prio[pix].avail >= 0); if (pix < PRIORITY_NORMAL) /* ie., max or high */ - xreds += run_queue_info[qix].prio[pix].reds; + xreds += (Sint64) run_queue_info[qix].prio[pix].reds; } run_queue_info[qix].prio[ERTS_PORT_PRIO_LEVEL].avail = 100; } @@ -2132,7 +2240,8 @@ check_balance(ErtsRunQueue *c_rq) if (max_len != 0) { int avail = avg.prio[pix].avail; if (avail != 0) { - max_len = ((100*max_len - 1) / avail) + 1; + max_len = (int) ((100*((Sint64) max_len) - 1) + / ((Sint64) avail)) + 1; avg.prio[pix].max_len = max_len; ASSERT(max_len >= 0); } @@ -2149,9 +2258,10 @@ check_balance(ErtsRunQueue *c_rq) || run_queue_info[qix].prio[pix].avail == 0) limit = 0; else - limit = (((avg.prio[pix].max_len - * run_queue_info[qix].prio[pix].avail) - 1) - / 100 + 1); + limit = (int) (((((Sint64) avg.prio[pix].max_len) + * ((Sint64) run_queue_info[qix].prio[pix].avail)) + - 1) + / 100 + 1); run_queue_info[qix].prio[pix].migration_limit = limit; } } @@ -2279,10 +2389,10 @@ erts_fprintf(stderr, "--------------------------------\n"); } balance_info.last_active_runqs = active; - erts_smp_atomic_set(&balance_info.active_runqs, active); + erts_smp_atomic32_set(&balance_info.active_runqs, active); balance_info.halftime = 1; - erts_smp_atomic_set(&balance_info.checking_balance, 0); + erts_smp_atomic32_set(&balance_info.checking_balance, 0); /* Write migration paths and reset balance statistics in all queues */ for (qix = 0; qix < blnc_no_rqs; qix++) { @@ -2379,7 +2489,6 @@ erts_debug_nbalance(void) void erts_early_init_scheduling(void) { - early_cpu_bind_init(); wakeup_other_limit = ERTS_WAKEUP_OTHER_LIMIT_MEDIUM; } @@ -2421,18 +2530,11 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) n = (int) (mrq ? no_schedulers : 1); - erts_aligned_run_queues = erts_alloc(ERTS_ALC_T_RUNQS, - (sizeof(ErtsAlignedRunQueue)*(n+1))); - if ((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) != 0) - erts_aligned_run_queues = ((ErtsAlignedRunQueue *) - ((((UWord) erts_aligned_run_queues) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); - - ASSERT((((UWord) erts_aligned_run_queues) & ERTS_CACHE_LINE_MASK) == 0); - + erts_aligned_run_queues = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_RUNQS, + sizeof(ErtsAlignedRunQueue) * n); #ifdef ERTS_SMP - erts_smp_atomic_init(&no_empty_run_queues, 0); + erts_smp_atomic32_init(&no_empty_run_queues, 0); #endif erts_no_run_queues = n; @@ -2442,7 +2544,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); rq->ix = ix; - erts_smp_atomic_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); + erts_smp_atomic32_init(&rq->info_flags, ERTS_RUNQ_IFLG_NONEMPTY); /* make sure that the "extra" id correponds to the schedulers * id if the esdp->no <-> ix+1 mapping change. @@ -2525,38 +2627,27 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) #ifdef ERTS_SMP /* Create and initialize scheduler sleep info */ - aligned_sched_sleep_info = erts_alloc(ERTS_ALC_T_SCHDLR_SLP_INFO, - (sizeof(ErtsAlignedSchedulerSleepInfo) - *(n+1))); - if ((((UWord) aligned_sched_sleep_info) & ERTS_CACHE_LINE_MASK) == 0) - aligned_sched_sleep_info = ((ErtsAlignedSchedulerSleepInfo *) - ((((UWord) aligned_sched_sleep_info) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); + aligned_sched_sleep_info = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_SLP_INFO, + n * sizeof(ErtsAlignedSchedulerSleepInfo)); + for (ix = 0; ix < n; ix++) { ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); #if 0 /* no need to initialize these... */ ssi->next = NULL; ssi->prev = NULL; #endif - erts_smp_atomic_init(&ssi->flags, 0); + erts_smp_atomic32_init(&ssi->flags, 0); ssi->event = NULL; /* initialized in sched_thread_func */ - erts_smp_atomic_init(&ssi->aux_work, 0); + erts_smp_atomic32_init(&ssi->aux_work, 0); } #endif /* Create and initialize scheduler specific data */ - erts_aligned_scheduler_data = erts_alloc(ERTS_ALC_T_SCHDLR_DATA, - (sizeof(ErtsAlignedSchedulerData) - *(n+1))); - if ((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) != 0) - erts_aligned_scheduler_data = ((ErtsAlignedSchedulerData *) - ((((UWord) erts_aligned_scheduler_data) - & ~ERTS_CACHE_LINE_MASK) - + ERTS_CACHE_LINE_SIZE)); - - ASSERT((((UWord) erts_aligned_scheduler_data) & ERTS_CACHE_LINE_MASK) == 0); + erts_aligned_scheduler_data = + erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, + n*sizeof(ErtsAlignedSchedulerData)); for (ix = 0; ix < n; ix++) { ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); @@ -2592,7 +2683,7 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) } #ifdef ERTS_SMP - erts_smp_atomic_init(&esdp->chk_cpu_bind, 0); + erts_smp_atomic32_init(&esdp->chk_cpu_bind, 0); #endif } @@ -2600,21 +2691,21 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); erts_smp_cnd_init(&schdlr_sspnd.cnd); - erts_smp_atomic_init(&schdlr_sspnd.changing, 0); + erts_smp_atomic32_init(&schdlr_sspnd.changing, 0); schdlr_sspnd.online = no_schedulers_online; schdlr_sspnd.curr_online = no_schedulers; - erts_smp_atomic_init(&schdlr_sspnd.msb.ongoing, 0); - erts_smp_atomic_init(&schdlr_sspnd.active, no_schedulers); + erts_smp_atomic32_init(&schdlr_sspnd.msb.ongoing, 0); + erts_smp_atomic32_init(&schdlr_sspnd.active, no_schedulers); schdlr_sspnd.msb.procs = NULL; - erts_smp_atomic_set(&balance_info.used_runqs, - erts_common_run_queue ? 1 : no_schedulers_online); - erts_smp_atomic_init(&balance_info.active_runqs, no_schedulers); + erts_smp_atomic32_set(&balance_info.used_runqs, + erts_common_run_queue ? 1 : no_schedulers_online); + erts_smp_atomic32_init(&balance_info.active_runqs, no_schedulers); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); balance_info.forced_check_balance = 0; balance_info.halftime = 1; balance_info.full_reds_history_index = 0; - erts_smp_atomic_init(&balance_info.checking_balance, 0); + erts_smp_atomic32_init(&balance_info.checking_balance, 0); balance_info.prev_rise.active_runqs = 0; balance_info.prev_rise.max_len = 0; balance_info.prev_rise.reds = 0; @@ -2623,8 +2714,8 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) if (no_schedulers_online < no_schedulers) { if (erts_common_run_queue) { for (ix = no_schedulers_online; ix < no_schedulers; ix++) - erts_smp_atomic_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); } else { for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) @@ -2638,7 +2729,9 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - erts_smp_atomic_init(&doing_sys_schedule, 0); + erts_smp_atomic32_init(&doing_sys_schedule, 0); + + init_misc_aux_work(); #else /* !ERTS_SMP */ { @@ -2652,12 +2745,19 @@ erts_init_scheduling(int mrq, int no_schedulers, int no_schedulers_online) erts_no_schedulers = 1; #endif - erts_smp_atomic_init(&function_calls, 0); + erts_smp_atomic32_init(&function_calls, 0); /* init port tasks */ erts_port_task_init(); - late_cpu_bind_init(); +#ifndef ERTS_SMP +#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC + erts_scheduler_data->verify_unused_temp_alloc + = erts_alloc_get_verify_unused_temp_alloc( + &erts_scheduler_data->verify_unused_temp_alloc_data); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); +#endif +#endif } ErtsRunQueue * @@ -2769,6 +2869,19 @@ resume_process(Process *p) p->rstatus = P_FREE; } +int +erts_get_max_no_executing_schedulers(void) +{ +#ifdef ERTS_SMP + if (erts_smp_atomic32_read(&schdlr_sspnd.changing)) + return (int) erts_no_schedulers; + ERTS_THR_MEMORY_BARRIER; + return (int) erts_smp_atomic32_read(&schdlr_sspnd.active); +#else + return 1; +#endif +} + #ifdef ERTS_SMP static void @@ -2787,13 +2900,13 @@ static void scheduler_ix_resume_wake(Uint ix) { ErtsSchedulerSleepInfo *ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - long xflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED); - long oflgs; + erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t oflgs; do { - oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, 0, xflgs); + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, 0, xflgs); if (oflgs == xflgs) { erts_sched_finish_poke(ssi, oflgs); break; @@ -2802,17 +2915,17 @@ scheduler_ix_resume_wake(Uint ix) } while (oflgs & ERTS_SSI_FLG_SUSPENDED); } -static long -sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, long xpct) +static erts_aint32_t +sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, erts_aint32_t xpct) { - long oflgs; - long nflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED); - long xflgs = xpct; + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t xflgs = xpct; do { - oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; xflgs = oflgs; @@ -2821,15 +2934,15 @@ sched_prep_spin_suspended(ErtsSchedulerSleepInfo *ssi, long xpct) return oflgs; } -static long +static erts_aint32_t sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) { int until_yield = ERTS_SCHED_SPIN_UNTIL_YIELD; int sc = spincount; - long flgs; + erts_aint32_t flgs; do { - flgs = erts_smp_atomic_read(&ssi->flags); + flgs = erts_smp_atomic32_read(&ssi->flags); if ((flgs & (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) @@ -2847,22 +2960,22 @@ sched_spin_suspended(ErtsSchedulerSleepInfo *ssi, int spincount) return flgs; } -static long +static erts_aint32_t sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) { - long oflgs; - long nflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED); - long xflgs = (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t oflgs; + erts_aint32_t nflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_TSE_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); + erts_aint32_t xflgs = (ERTS_SSI_FLG_SLEEPING + | ERTS_SSI_FLG_WAITING + | ERTS_SSI_FLG_SUSPENDED); erts_tse_reset(ssi->event); while (1) { - oflgs = erts_smp_atomic_cmpxchg(&ssi->flags, nflgs, xflgs); + oflgs = erts_smp_atomic32_cmpxchg(&ssi->flags, nflgs, xflgs); if (oflgs == xflgs) return nflgs; if ((oflgs & (ERTS_SSI_FLG_SLEEPING @@ -2880,18 +2993,16 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) static void suspend_scheduler(ErtsSchedulerData *esdp) { - long flgs; - int changing; + erts_aint32_t flgs; + erts_aint32_t changing; long no = (long) esdp->no; - ErtsRunQueue *rq = esdp->run_queue; ErtsSchedulerSleepInfo *ssi = esdp->ssi; long active_schedulers; int curr_online = 1; int wake = 0; - int reset_read_group = 0; #if defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) \ || defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) - long aux_work; + erts_aint32_t aux_work; #endif /* @@ -2909,20 +3020,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_runq_unlock(esdp->run_queue); - /* Unbind from cpu */ - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - if (scheduler2cpu_map[esdp->no].bound_id >= 0 - && erts_unbind_from_cpu(erts_cpuinfo) == 0) { - esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; - reset_read_group = 1; - } - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - - if (reset_read_group) - erts_smp_rwmtx_set_reader_group(0); - - if (esdp->no <= erts_max_main_threads) - erts_thr_set_main_status(0, 0); + erts_sched_check_cpu_bind_prep_suspend(esdp); if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_inactive); @@ -2932,15 +3030,15 @@ suspend_scheduler(ErtsSchedulerData *esdp) flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); if (flgs & ERTS_SSI_FLG_SUSPENDED) { - active_schedulers = erts_smp_atomic_dectest(&schdlr_sspnd.active); + active_schedulers = erts_smp_atomic32_dectest(&schdlr_sspnd.active); ASSERT(active_schedulers >= 1); - changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { if (active_schedulers == schdlr_sspnd.msb.wait_active) wake = 1; if (active_schedulers == 1) { - changing = erts_smp_atomic_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); + changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; } } @@ -2962,8 +3060,8 @@ suspend_scheduler(ErtsSchedulerData *esdp) && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) wake = 1; if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { - changing = erts_smp_atomic_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + changing = erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; } } @@ -2973,29 +3071,30 @@ suspend_scheduler(ErtsSchedulerData *esdp) wake = 0; } - flgs = erts_smp_atomic_read(&ssi->flags); + flgs = erts_smp_atomic32_read(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read(&ssi->aux_work); blockable_aux_work: blockable_aux_work(esdp, ssi, aux_work); #endif erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); while (1) { - long flgs; + erts_aint32_t flgs; #ifdef ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK #ifndef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read(&ssi->aux_work); #endif nonblockable_aux_work(esdp, ssi, aux_work); #endif - flgs = sched_spin_suspended(ssi, ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); + flgs = sched_spin_suspended(ssi, + ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); if (flgs == (ERTS_SSI_FLG_SLEEPING | ERTS_SSI_FLG_WAITING | ERTS_SSI_FLG_SUSPENDED)) { @@ -3015,13 +3114,13 @@ suspend_scheduler(ErtsSchedulerData *esdp) | ERTS_SSI_FLG_SUSPENDED)); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; - changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) break; #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK - aux_work = erts_smp_atomic_read(&ssi->aux_work); + aux_work = erts_smp_atomic32_read(&ssi->aux_work); if (aux_work & ERTS_SSI_BLOCKABLE_AUX_WORK_MASK) { erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); goto blockable_aux_work; @@ -3033,19 +3132,19 @@ suspend_scheduler(ErtsSchedulerData *esdp) erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); } - active_schedulers = erts_smp_atomic_inctest(&schdlr_sspnd.active); - changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + active_schedulers = erts_smp_atomic32_inctest(&schdlr_sspnd.active); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) && schdlr_sspnd.online == active_schedulers) { - erts_smp_atomic_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_MSB); } ASSERT(no <= schdlr_sspnd.online); - ASSERT(!erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)); + ASSERT(!erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); } @@ -3056,17 +3155,10 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (erts_system_profile_flags.scheduler) profile_scheduler(make_small(esdp->no), am_active); - if (esdp->no <= erts_max_main_threads) - erts_thr_set_main_status(1, (int) esdp->no); - erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); - /* Make sure we check if we should bind to a cpu or not... */ - if (rq->flags & ERTS_RUNQ_FLG_SHARED_RUNQ) - erts_smp_atomic_set(&esdp->chk_cpu_bind, 1); - else - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; + erts_sched_check_cpu_bind_post_suspend(esdp); } #define ERTS_RUNQ_RESET_SUSPEND_INFO(RQ, DBG_ID) \ @@ -3081,7 +3173,7 @@ do { \ (RQ)->flags |= (ERTS_RUNQ_FLG_OUT_OF_WORK \ | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK); \ (RQ)->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; \ - erts_smp_atomic_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED); \ + erts_smp_atomic32_band(&(RQ)->info_flags, ~ERTS_RUNQ_IFLG_SUSPENDED);\ for (pix__ = 0; pix__ < ERTS_NO_PROC_PRIO_LEVELS; pix__++) { \ (RQ)->procs.prio_info[pix__].max_len = 0; \ (RQ)->procs.prio_info[pix__].reds = 0; \ @@ -3123,9 +3215,9 @@ erts_schedulers_state(Uint *total, int yield_allowed) { int res; - long changing; + erts_aint32_t changing; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) res = ERTS_SCHDLR_SSPND_YIELD_RESTART; else { @@ -3146,7 +3238,7 @@ erts_set_schedulers_online(Process *p, Sint *old_no) { int ix, res, no, have_unlocked_plocks; - long changing; + erts_aint32_t changing; if (new_no < 1 || erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; @@ -3156,7 +3248,7 @@ erts_set_schedulers_online(Process *p, have_unlocked_plocks = 0; no = (int) new_no; - changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; } @@ -3203,7 +3295,7 @@ erts_set_schedulers_online(Process *p, ErtsRunQueue *to_rq = ERTS_RUNQ_IX(ix % no); evacuate_run_queue(from_rq, to_rq); } - erts_smp_atomic_set(&balance_info.used_runqs, no); + erts_smp_atomic32_set(&balance_info.used_runqs, no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); } @@ -3231,8 +3323,8 @@ erts_set_schedulers_online(Process *p, for (ix = no; ix < online; ix++) { ErtsSchedulerSleepInfo *ssi; ssi = ERTS_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic_bor(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_bor(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); } wake_all_schedulers(); } @@ -3257,7 +3349,7 @@ erts_set_schedulers_online(Process *p, for (ix = erts_no_run_queues-1; ix >= no; ix--) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % no)); - erts_smp_atomic_set(&balance_info.used_runqs, no); + erts_smp_atomic32_set(&balance_info.used_runqs, no); erts_smp_mtx_unlock(&balance_info.update_mtx); erts_smp_mtx_lock(&schdlr_sspnd.mtx); for (ix = no; ix < online; ix++) { @@ -3279,10 +3371,11 @@ erts_set_schedulers_online(Process *p, NULL); ASSERT(res != ERTS_SCHDLR_SSPND_DONE ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic_read(&schdlr_sspnd.changing)) + & erts_smp_atomic32_read(&schdlr_sspnd.changing)) : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic_read(&schdlr_sspnd.changing))); - erts_smp_atomic_band(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + == erts_smp_atomic32_read(&schdlr_sspnd.changing))); + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } } @@ -3297,11 +3390,11 @@ ErtsSchedSuspendResult erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) { int ix, res, have_unlocked_plocks = 0; - long changing; + erts_aint32_t changing; ErtsProcList *plp; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic_read(&schdlr_sspnd.changing); + changing = erts_smp_atomic32_read(&schdlr_sspnd.changing); if (changing) { res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ } @@ -3311,7 +3404,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) plp->next = schdlr_sspnd.msb.procs; schdlr_sspnd.msb.procs = plp; p->flags |= F_HAVE_BLCKD_MSCHED; - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; } @@ -3322,11 +3415,11 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - ASSERT(0 == erts_smp_atomic_read(&schdlr_sspnd.msb.ongoing)); - erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 1); + ASSERT(0 == erts_smp_atomic32_read(&schdlr_sspnd.msb.ongoing)); + erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 1); if (online == 1) { res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); ASSERT(p->scheduler_data->no == 1); } else { @@ -3346,14 +3439,14 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) } if (erts_common_run_queue) { for (ix = 1; ix < online; ix++) - erts_smp_atomic_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_bor(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { erts_smp_mtx_unlock(&schdlr_sspnd.mtx); erts_smp_mtx_lock(&balance_info.update_mtx); - erts_smp_atomic_set(&balance_info.used_runqs, 1); + erts_smp_atomic32_set(&balance_info.used_runqs, 1); for (ix = 0; ix < online; ix++) { ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); erts_smp_runq_lock(rq); @@ -3375,7 +3468,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) susp_sched_prep_block, susp_sched_resume_block, NULL); - while (erts_smp_atomic_read(&schdlr_sspnd.active) + while (erts_smp_atomic32_read(&schdlr_sspnd.active) != schdlr_sspnd.msb.wait_active) erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); erts_smp_activity_end(ERTS_ACTIVITY_WAIT, @@ -3384,11 +3477,11 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) NULL); ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic_read(&schdlr_sspnd.changing)) + & erts_smp_atomic32_read(&schdlr_sspnd.changing)) : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic_read(&schdlr_sspnd.changing))); - erts_smp_atomic_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + == erts_smp_atomic32_read(&schdlr_sspnd.changing))); + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_WAITER); } plp = proclist_create(p); plp->next = schdlr_sspnd.msb.procs; @@ -3455,16 +3548,16 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) }); #endif p->flags &= ~F_HAVE_BLCKD_MSCHED; - erts_smp_atomic_set(&schdlr_sspnd.msb.ongoing, 0); + erts_smp_atomic32_set(&schdlr_sspnd.msb.ongoing, 0); if (schdlr_sspnd.online == 1) { /* No schedulers to resume */ - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.active) == 1); + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.active) == 1); ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); } else if (erts_common_run_queue) { for (ix = 1; ix < schdlr_sspnd.online; ix++) - erts_smp_atomic_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, - ~ERTS_SSI_FLG_SUSPENDED); + erts_smp_atomic32_band(&ERTS_SCHED_SLEEP_INFO_IX(ix)->flags, + ~ERTS_SSI_FLG_SUSPENDED); wake_all_schedulers(); } else { @@ -3490,7 +3583,7 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) evacuate_run_queue(ERTS_RUNQ_IX(ix), ERTS_RUNQ_IX(ix % online)); - erts_smp_atomic_set(&balance_info.used_runqs, online); + erts_smp_atomic32_set(&balance_info.used_runqs, online); /* Make sure that we balance soon... */ balance_info.forced_check_balance = 1; erts_smp_runq_lock(ERTS_RUNQ_IX(0)); @@ -3514,7 +3607,7 @@ void erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) { if (return_value == am_blocked) { - long active = erts_smp_atomic_read(&schdlr_sspnd.active); + erts_aint32_t active = erts_smp_atomic32_read(&schdlr_sspnd.active); ASSERT(1 <= active && active <= 2); ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1); } @@ -3583,15 +3676,7 @@ sched_thread_func(void *vesdp) erts_tsd_set(sched_data_key, vesdp); #ifdef ERTS_SMP - if (no <= erts_max_main_threads) { - erts_thr_set_main_status(1, (int) no); - if (erts_reader_groups) { - int rg = (int) no; - if (rg > erts_reader_groups) - rg = (((int) no) - 1) % erts_reader_groups + 1; - erts_smp_rwmtx_set_reader_group(rg); - } - } + erts_sched_init_check_cpu_bind((ErtsSchedulerData *) vesdp); erts_proc_lock_prepare_proc_lock_waiter(); ERTS_SCHED_SLEEP_INFO_IX(no - 1)->event = erts_tse_fetch(); @@ -3605,12 +3690,12 @@ sched_thread_func(void *vesdp) erts_thread_init_float(); erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(erts_smp_atomic_read(&schdlr_sspnd.changing) + ASSERT(erts_smp_atomic32_read(&schdlr_sspnd.changing) & ERTS_SCHDLR_SSPND_CHNG_ONLN); if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { - erts_smp_atomic_band(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); + erts_smp_atomic32_band(&schdlr_sspnd.changing, + ~ERTS_SCHDLR_SSPND_CHNG_ONLN); if (((ErtsSchedulerData *) vesdp)->no != 1) erts_smp_cnd_signal(&schdlr_sspnd.cnd); } @@ -3632,6 +3717,13 @@ sched_thread_func(void *vesdp) } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); +#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC + ((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc + = erts_alloc_get_verify_unused_temp_alloc( + &((ErtsSchedulerData *) vesdp)->verify_unused_temp_alloc_data); + ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); +#endif + process_main(); /* No schedulers should *ever* terminate */ erl_exit(ERTS_ABORT_EXIT, "Scheduler thread number %bpu terminated\n", @@ -3693,1907 +3785,6 @@ erts_start_schedulers(void) #endif /* ERTS_SMP */ -static int -int_cmp(const void *vx, const void *vy) -{ - return *((int *) vx) - *((int *) vy); -} - -static int -cpu_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->core != y->core) - return x->core - y->core; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->node != y->node) - return x->node - y->node; - return 0; -} - -static int -cpu_processor_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - if (x->node != y->node) - return x->node - y->node; - if (x->processor != y->processor) - return x->processor - y->processor; - return 0; -} - -static int -cpu_thread_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->node != y->node) - return x->node - y->node; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - return 0; -} - -static int -cpu_thread_no_node_processor_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->node != y->node) - return x->node - y->node; - if (x->core != y->core) - return x->core - y->core; - if (x->processor != y->processor) - return x->processor - y->processor; - return 0; -} - -static int -cpu_no_node_processor_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->node != y->node) - return x->node - y->node; - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->core != y->core) - return x->core - y->core; - if (x->processor != y->processor) - return x->processor - y->processor; - return 0; -} - -static int -cpu_no_node_thread_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->node != y->node) - return x->node - y->node; - if (x->thread != y->thread) - return x->thread - y->thread; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->core != y->core) - return x->core - y->core; - return 0; -} - -static int -cpu_no_spread_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->node != y->node) - return x->node - y->node; - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - if (x->thread != y->thread) - return x->thread - y->thread; - return 0; -} - -static ERTS_INLINE void -make_cpudata_id_seq(erts_cpu_topology_t *cpudata, int size, int no_node) -{ - int ix; - int node = -1; - int processor = -1; - int processor_node = -1; - int processor_node_node = -1; - int core = -1; - int thread = -1; - int old_node = -1; - int old_processor = -1; - int old_processor_node = -1; - int old_core = -1; - int old_thread = -1; - - for (ix = 0; ix < size; ix++) { - if (!no_node || cpudata[ix].node >= 0) { - if (old_node == cpudata[ix].node) - cpudata[ix].node = node; - else { - old_node = cpudata[ix].node; - old_processor = processor = -1; - if (!no_node) - old_processor_node = processor_node = -1; - old_core = core = -1; - old_thread = thread = -1; - if (no_node || cpudata[ix].node >= 0) - cpudata[ix].node = ++node; - } - } - if (old_processor == cpudata[ix].processor) - cpudata[ix].processor = processor; - else { - old_processor = cpudata[ix].processor; - if (!no_node) - processor_node_node = old_processor_node = processor_node = -1; - old_core = core = -1; - old_thread = thread = -1; - cpudata[ix].processor = ++processor; - } - if (no_node && cpudata[ix].processor_node < 0) - old_processor_node = -1; - else { - if (old_processor_node == cpudata[ix].processor_node) { - if (no_node) - cpudata[ix].node = cpudata[ix].processor_node = node; - else { - if (processor_node_node >= 0) - cpudata[ix].node = processor_node_node; - cpudata[ix].processor_node = processor_node; - } - } - else { - old_processor_node = cpudata[ix].processor_node; - old_core = core = -1; - old_thread = thread = -1; - if (no_node) - cpudata[ix].node = cpudata[ix].processor_node = ++node; - else { - cpudata[ix].node = processor_node_node = ++node; - cpudata[ix].processor_node = ++processor_node; - } - } - } - if (!no_node && cpudata[ix].processor_node < 0) - cpudata[ix].processor_node = 0; - if (old_core == cpudata[ix].core) - cpudata[ix].core = core; - else { - old_core = cpudata[ix].core; - old_thread = thread = -1; - cpudata[ix].core = ++core; - } - if (old_thread == cpudata[ix].thread) - cpudata[ix].thread = thread; - else - old_thread = cpudata[ix].thread = ++thread; - } -} - -static void -cpu_bind_order_sort(erts_cpu_topology_t *cpudata, - int size, - ErtsCpuBindOrder bind_order, - int mk_seq) -{ - if (size > 1) { - int no_node = 0; - int (*cmp_func)(const void *, const void *); - switch (bind_order) { - case ERTS_CPU_BIND_SPREAD: - cmp_func = cpu_spread_order_cmp; - break; - case ERTS_CPU_BIND_PROCESSOR_SPREAD: - cmp_func = cpu_processor_spread_order_cmp; - break; - case ERTS_CPU_BIND_THREAD_SPREAD: - cmp_func = cpu_thread_spread_order_cmp; - break; - case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: - no_node = 1; - cmp_func = cpu_thread_no_node_processor_spread_order_cmp; - break; - case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: - no_node = 1; - cmp_func = cpu_no_node_processor_spread_order_cmp; - break; - case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: - no_node = 1; - cmp_func = cpu_no_node_thread_spread_order_cmp; - break; - case ERTS_CPU_BIND_NO_SPREAD: - cmp_func = cpu_no_spread_order_cmp; - break; - default: - cmp_func = NULL; - erl_exit(ERTS_ABORT_EXIT, - "Bad cpu bind type: %d\n", - (int) cpu_bind_order); - break; - } - - if (mk_seq) - make_cpudata_id_seq(cpudata, size, no_node); - - qsort(cpudata, size, sizeof(erts_cpu_topology_t), cmp_func); - } -} - -static int -processor_order_cmp(const void *vx, const void *vy) -{ - erts_cpu_topology_t *x = (erts_cpu_topology_t *) vx; - erts_cpu_topology_t *y = (erts_cpu_topology_t *) vy; - - if (x->processor != y->processor) - return x->processor - y->processor; - if (x->node != y->node) - return x->node - y->node; - if (x->processor_node != y->processor_node) - return x->processor_node - y->processor_node; - if (x->core != y->core) - return x->core - y->core; - if (x->thread != y->thread) - return x->thread - y->thread; - return 0; -} - -static void -check_cpu_bind(ErtsSchedulerData *esdp) -{ - int rg = 0; - int res; - int cpu_id; - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - cpu_id = scheduler2cpu_map[esdp->no].bind_id; - if (cpu_id >= 0 && cpu_id != scheduler2cpu_map[esdp->no].bound_id) { - res = erts_bind_to_cpu(erts_cpuinfo, cpu_id); - if (res == 0) - esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = cpu_id; - else { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Scheduler %d failed to bind to cpu %d: %s\n", - (int) esdp->no, cpu_id, erl_errno_id(-res)); - erts_send_error_to_logger_nogl(dsbufp); - if (scheduler2cpu_map[esdp->no].bound_id >= 0) - goto unbind; - } - } - else if (cpu_id < 0) { - unbind: - /* Get rid of old binding */ - res = erts_unbind_from_cpu(erts_cpuinfo); - if (res == 0) - esdp->cpu_id = scheduler2cpu_map[esdp->no].bound_id = -1; - else if (res != -ENOTSUP) { - erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, "Scheduler %d failed to unbind from cpu %d: %s\n", - (int) esdp->no, cpu_id, erl_errno_id(-res)); - erts_send_error_to_logger_nogl(dsbufp); - } - } - if (erts_reader_groups) { - if (esdp->cpu_id >= 0) - rg = reader_group_lookup(esdp->cpu_id); - else - rg = (((int) esdp->no) - 1) % erts_reader_groups + 1; - } - erts_smp_runq_lock(esdp->run_queue); -#ifdef ERTS_SMP - if (erts_common_run_queue) - erts_smp_atomic_set(&esdp->chk_cpu_bind, 0); - else { - esdp->run_queue->flags &= ~ERTS_RUNQ_FLG_CHK_CPU_BIND; - } -#endif - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - - if (erts_reader_groups) - erts_smp_rwmtx_set_reader_group(rg); -} - -static void -signal_schedulers_bind_change(erts_cpu_topology_t *cpudata, int size) -{ - int s_ix = 1; - int cpu_ix; - - if (cpu_bind_order != ERTS_CPU_BIND_NONE && size) { - - cpu_bind_order_sort(cpudata, size, cpu_bind_order, 1); - - for (cpu_ix = 0; cpu_ix < size && cpu_ix < erts_no_schedulers; cpu_ix++) - if (erts_is_cpu_available(erts_cpuinfo, cpudata[cpu_ix].logical)) - scheduler2cpu_map[s_ix++].bind_id = cpudata[cpu_ix].logical; - } - - if (s_ix <= erts_no_schedulers) - for (; s_ix <= erts_no_schedulers; s_ix++) - scheduler2cpu_map[s_ix].bind_id = -1; - -#ifdef ERTS_SMP - if (erts_common_run_queue) { - for (s_ix = 0; s_ix < erts_no_schedulers; s_ix++) - erts_smp_atomic_set(&ERTS_SCHEDULER_IX(s_ix)->chk_cpu_bind, 1); - wake_all_schedulers(); - } - else { - for (s_ix = 0; s_ix < erts_no_run_queues; s_ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(s_ix); - erts_smp_runq_lock(rq); - rq->flags |= ERTS_RUNQ_FLG_CHK_CPU_BIND; - erts_smp_runq_unlock(rq); - wake_scheduler(rq, 0, 1); - }; - } -#else - check_cpu_bind(erts_get_scheduler_data()); -#endif -} - -int -erts_init_scheduler_bind_type(char *how) -{ - if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) - return ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED; - - if (!system_cpudata && !user_cpudata) - return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY; - - if (sys_strcmp(how, "db") == 0) - cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; - else if (sys_strcmp(how, "s") == 0) - cpu_bind_order = ERTS_CPU_BIND_SPREAD; - else if (sys_strcmp(how, "ps") == 0) - cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; - else if (sys_strcmp(how, "ts") == 0) - cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (sys_strcmp(how, "tnnps") == 0) - cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; - else if (sys_strcmp(how, "nnps") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; - else if (sys_strcmp(how, "nnts") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; - else if (sys_strcmp(how, "ns") == 0) - cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; - else if (sys_strcmp(how, "u") == 0) - cpu_bind_order = ERTS_CPU_BIND_NONE; - else - return ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE; - - return ERTS_INIT_SCHED_BIND_TYPE_SUCCESS; -} - -/* - * reader groups map - */ - -typedef struct { - int level[ERTS_TOPOLOGY_MAX_DEPTH+1]; -} erts_avail_cput; - -typedef struct { - int *map; - int size; - int groups; -} erts_reader_groups_map_test; - -typedef struct { - int id; - int sub_levels; - int reader_groups; -} erts_rg_count_t; - -typedef struct { - int logical; - int reader_group; -} erts_reader_groups_map_t; - -typedef struct { - erts_reader_groups_map_t *map; - int map_size; - int logical_processors; - int groups; -} erts_make_reader_groups_map_test; - -static int reader_groups_available_cpu_check; -static int reader_groups_logical_processors; -static int reader_groups_map_size; -static erts_reader_groups_map_t *reader_groups_map; - -#define ERTS_TOPOLOGY_RG ERTS_TOPOLOGY_MAX_DEPTH - -static void -make_reader_groups_map(erts_make_reader_groups_map_test *test); - -static Eterm -get_reader_groups_map(Process *c_p, - erts_reader_groups_map_t *map, - int map_size, - int logical_processors) -{ -#ifdef DEBUG - Eterm *endp; -#endif - Eterm res = NIL, tuple; - Eterm *hp; - int i; - - hp = HAlloc(c_p, logical_processors*(2+3)); -#ifdef DEBUG - endp = hp + logical_processors*(2+3); -#endif - for (i = map_size - 1; i >= 0; i--) { - if (map[i].logical >= 0) { - tuple = TUPLE2(hp, - make_small(map[i].logical), - make_small(map[i].reader_group)); - hp += 3; - res = CONS(hp, tuple, res); - hp += 2; - } - } - ASSERT(hp == endp); - return res; -} - -Eterm -erts_debug_reader_groups_map(Process *c_p, int groups) -{ - Eterm res; - erts_make_reader_groups_map_test test; - - test.groups = groups; - make_reader_groups_map(&test); - if (!test.map) - res = NIL; - else { - res = get_reader_groups_map(c_p, - test.map, - test.map_size, - test.logical_processors); - erts_free(ERTS_ALC_T_TMP, test.map); - } - return res; -} - - -Eterm -erts_get_reader_groups_map(Process *c_p) -{ - Eterm res; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - res = get_reader_groups_map(c_p, - reader_groups_map, - reader_groups_map_size, - reader_groups_logical_processors); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return res; -} - -static void -make_available_cpu_topology(erts_avail_cput *no, - erts_avail_cput *avail, - erts_cpu_topology_t *cpudata, - int *size, - int test) -{ - int len = *size; - erts_cpu_topology_t last; - int a, i, j; - - no->level[ERTS_TOPOLOGY_NODE] = -1; - no->level[ERTS_TOPOLOGY_PROCESSOR] = -1; - no->level[ERTS_TOPOLOGY_PROCESSOR_NODE] = -1; - no->level[ERTS_TOPOLOGY_CORE] = -1; - no->level[ERTS_TOPOLOGY_THREAD] = -1; - no->level[ERTS_TOPOLOGY_LOGICAL] = -1; - - last.node = INT_MIN; - last.processor = INT_MIN; - last.processor_node = INT_MIN; - last.core = INT_MIN; - last.thread = INT_MIN; - last.logical = INT_MIN; - - a = 0; - - for (i = 0; i < len; i++) { - - if (!test && !erts_is_cpu_available(erts_cpuinfo, cpudata[i].logical)) - continue; - - if (last.node != cpudata[i].node) - goto node; - if (last.processor != cpudata[i].processor) - goto processor; - if (last.processor_node != cpudata[i].processor_node) - goto processor_node; - if (last.core != cpudata[i].core) - goto core; - ASSERT(last.thread != cpudata[i].thread); - goto thread; - - node: - no->level[ERTS_TOPOLOGY_NODE]++; - processor: - no->level[ERTS_TOPOLOGY_PROCESSOR]++; - processor_node: - no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; - core: - no->level[ERTS_TOPOLOGY_CORE]++; - thread: - no->level[ERTS_TOPOLOGY_THREAD]++; - - no->level[ERTS_TOPOLOGY_LOGICAL]++; - - for (j = 0; j < ERTS_TOPOLOGY_LOGICAL; j++) - avail[a].level[j] = no->level[j]; - - avail[a].level[ERTS_TOPOLOGY_LOGICAL] = cpudata[i].logical; - avail[a].level[ERTS_TOPOLOGY_RG] = 0; - - ASSERT(last.logical != cpudata[a].logical); - - last = cpudata[i]; - a++; - } - - no->level[ERTS_TOPOLOGY_NODE]++; - no->level[ERTS_TOPOLOGY_PROCESSOR]++; - no->level[ERTS_TOPOLOGY_PROCESSOR_NODE]++; - no->level[ERTS_TOPOLOGY_CORE]++; - no->level[ERTS_TOPOLOGY_THREAD]++; - no->level[ERTS_TOPOLOGY_LOGICAL]++; - - *size = a; -} - -static int -reader_group_lookup(int logical) -{ - int start = logical % reader_groups_map_size; - int ix = start; - - do { - if (reader_groups_map[ix].logical == logical) { - ASSERT(reader_groups_map[ix].reader_group > 0); - return reader_groups_map[ix].reader_group; - } - ix++; - if (ix == reader_groups_map_size) - ix = 0; - } while (ix != start); - - erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); -} - -static void -reader_group_insert(erts_reader_groups_map_t *map, int map_size, - int logical, int reader_group) -{ - int start = logical % map_size; - int ix = start; - - do { - if (map[ix].logical < 0) { - map[ix].logical = logical; - map[ix].reader_group = reader_group; - return; - } - ix++; - if (ix == map_size) - ix = 0; - } while (ix != start); - - erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); -} - - -static int -sub_levels(erts_rg_count_t *rgc, int level, int aix, int avail_sz, erts_avail_cput *avail) -{ - int sub_level = level+1; - int last = -1; - rgc->sub_levels = 0; - - do { - if (last != avail[aix].level[sub_level]) { - rgc->sub_levels++; - last = avail[aix].level[sub_level]; - } - aix++; - } - while (aix < avail_sz && rgc->id == avail[aix].level[level]); - rgc->reader_groups = 0; - return aix; -} - -static int -write_reader_groups(int *rgp, erts_rg_count_t *rgcp, - int level, int a, - int avail_sz, erts_avail_cput *avail) -{ - int rg = *rgp; - int sub_level = level+1; - int sl_per_gr = rgcp->sub_levels / rgcp->reader_groups; - int xsl = rgcp->sub_levels % rgcp->reader_groups; - int sls = 0; - int last = -1; - int xsl_rg_lim = (rgcp->reader_groups - xsl) + rg + 1; - - ASSERT(level < 0 || avail[a].level[level] == rgcp->id) - - do { - if (last != avail[a].level[sub_level]) { - if (!sls) { - sls = sl_per_gr; - rg++; - if (rg >= xsl_rg_lim) - sls++; - } - last = avail[a].level[sub_level]; - sls--; - } - avail[a].level[ERTS_TOPOLOGY_RG] = rg; - a++; - } while (a < avail_sz && (level < 0 - || avail[a].level[level] == rgcp->id)); - - ASSERT(rgcp->reader_groups == rg - *rgp); - - *rgp = rg; - - return a; -} - -static int -rg_count_sub_levels_compare(const void *vx, const void *vy) -{ - erts_rg_count_t *x = (erts_rg_count_t *) vx; - erts_rg_count_t *y = (erts_rg_count_t *) vy; - if (x->sub_levels != y->sub_levels) - return y->sub_levels - x->sub_levels; - return x->id - y->id; -} - -static int -rg_count_id_compare(const void *vx, const void *vy) -{ - erts_rg_count_t *x = (erts_rg_count_t *) vx; - erts_rg_count_t *y = (erts_rg_count_t *) vy; - return x->id - y->id; -} - -static void -make_reader_groups_map(erts_make_reader_groups_map_test *test) -{ - int i, spread_level, avail_sz; - erts_avail_cput no, *avail; - erts_cpu_topology_t *cpudata; - erts_reader_groups_map_t *map; - int map_sz; - int groups = erts_reader_groups; - - if (test) { - test->map = NULL; - test->map_size = 0; - groups = test->groups; - } - - if (!groups) - return; - - if (!test) { - if (reader_groups_map) - erts_free(ERTS_ALC_T_RDR_GRPS_MAP, reader_groups_map); - - reader_groups_logical_processors = 0; - reader_groups_map_size = 0; - reader_groups_map = NULL; - } - - create_tmp_cpu_topology_copy(&cpudata, &avail_sz); - - if (!cpudata) - return; - - cpu_bind_order_sort(cpudata, - avail_sz, - ERTS_CPU_BIND_NO_SPREAD, - 1); - - avail = erts_alloc(ERTS_ALC_T_TMP, - sizeof(erts_avail_cput)*avail_sz); - - make_available_cpu_topology(&no, avail, cpudata, - &avail_sz, test != NULL); - - destroy_tmp_cpu_topology_copy(cpudata); - - map_sz = avail_sz*2+1; - - if (test) { - map = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_reader_groups_map_t) - * map_sz)); - test->map = map; - test->map_size = map_sz; - test->logical_processors = avail_sz; - } - else { - map = erts_alloc(ERTS_ALC_T_RDR_GRPS_MAP, - (sizeof(erts_reader_groups_map_t) - * map_sz)); - reader_groups_map = map; - reader_groups_logical_processors = avail_sz; - reader_groups_map_size = map_sz; - - } - - for (i = 0; i < map_sz; i++) { - map[i].logical = -1; - map[i].reader_group = 0; - } - - spread_level = ERTS_TOPOLOGY_CORE; - for (i = ERTS_TOPOLOGY_NODE; i < ERTS_TOPOLOGY_THREAD; i++) { - if (no.level[i] > groups) { - spread_level = i; - break; - } - } - - if (no.level[spread_level] <= groups) { - int a, rg, last = -1; - rg = 0; - ASSERT(spread_level == ERTS_TOPOLOGY_CORE); - for (a = 0; a < avail_sz; a++) { - if (last != avail[a].level[spread_level]) { - rg++; - last = avail[a].level[spread_level]; - } - reader_group_insert(map, - map_sz, - avail[a].level[ERTS_TOPOLOGY_LOGICAL], - rg); - } - } - else { /* groups < no.level[spread_level] */ - erts_rg_count_t *rg_count; - int a, rg, tl, toplevels; - - tl = spread_level-1; - - if (spread_level == ERTS_TOPOLOGY_NODE) - toplevels = 1; - else - toplevels = no.level[tl]; - - rg_count = erts_alloc(ERTS_ALC_T_TMP, - toplevels*sizeof(erts_rg_count_t)); - - if (toplevels == 1) { - rg_count[0].id = 0; - rg_count[0].sub_levels = no.level[spread_level]; - rg_count[0].reader_groups = groups; - } - else { - int rgs_per_tl, rgs; - rgs = groups; - rgs_per_tl = rgs / toplevels; - - a = 0; - for (i = 0; i < toplevels; i++) { - rg_count[i].id = avail[a].level[tl]; - a = sub_levels(&rg_count[i], tl, a, avail_sz, avail); - } - - qsort(rg_count, - toplevels, - sizeof(erts_rg_count_t), - rg_count_sub_levels_compare); - - for (i = 0; i < toplevels; i++) { - if (rg_count[i].sub_levels < rgs_per_tl) { - rg_count[i].reader_groups = rg_count[i].sub_levels; - rgs -= rg_count[i].sub_levels; - } - else { - rg_count[i].reader_groups = rgs_per_tl; - rgs -= rgs_per_tl; - } - } - - while (rgs > 0) { - for (i = 0; i < toplevels; i++) { - if (rg_count[i].sub_levels == rg_count[i].reader_groups) - break; - else { - rg_count[i].reader_groups++; - if (--rgs == 0) - break; - } - } - } - - qsort(rg_count, - toplevels, - sizeof(erts_rg_count_t), - rg_count_id_compare); - } - - a = i = rg = 0; - while (a < avail_sz) { - a = write_reader_groups(&rg, &rg_count[i], tl, - a, avail_sz, avail); - i++; - } - - ASSERT(groups == rg); - - for (a = 0; a < avail_sz; a++) - reader_group_insert(map, - map_sz, - avail[a].level[ERTS_TOPOLOGY_LOGICAL], - avail[a].level[ERTS_TOPOLOGY_RG]); - - erts_free(ERTS_ALC_T_TMP, rg_count); - } - - erts_free(ERTS_ALC_T_TMP, avail); -} - -/* - * CPU topology - */ - -typedef struct { - int *id; - int used; - int size; -} ErtsCpuTopIdSeq; - -typedef struct { - ErtsCpuTopIdSeq logical; - ErtsCpuTopIdSeq thread; - ErtsCpuTopIdSeq core; - ErtsCpuTopIdSeq processor_node; - ErtsCpuTopIdSeq processor; - ErtsCpuTopIdSeq node; -} ErtsCpuTopEntry; - -static void -init_cpu_top_entry(ErtsCpuTopEntry *cte) -{ - int size = 10; - cte->logical.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->logical.size = size; - cte->thread.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->thread.size = size; - cte->core.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->core.size = size; - cte->processor_node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->processor_node.size = size; - cte->processor.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->processor.size = size; - cte->node.id = erts_alloc(ERTS_ALC_T_TMP_CPU_IDS, - sizeof(int)*size); - cte->node.size = size; -} - -static void -destroy_cpu_top_entry(ErtsCpuTopEntry *cte) -{ - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->logical.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->thread.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->core.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor_node.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->processor.id); - erts_free(ERTS_ALC_T_TMP_CPU_IDS, cte->node.id); -} - -static int -get_cput_value_or_range(int *v, int *vr, char **str) -{ - long l; - char *c = *str; - errno = 0; - if (!isdigit((unsigned char)*c)) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; - l = strtol(c, &c, 10); - if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID; - *v = (int) l; - if (*c == '-') { - c++; - if (!isdigit((unsigned char)*c)) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - l = strtol(c, &c, 10); - if (errno != 0 || l < 0 || ERTS_MAX_CPU_TOPOLOGY_ID < l) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - *vr = (int) l; - } - *str = c; - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -static int -get_cput_id_seq(ErtsCpuTopIdSeq *idseq, char **str) -{ - int ix = 0; - int need_size = 0; - char *c = *str; - - while (1) { - int res; - int val; - int nids; - int val_range = -1; - res = get_cput_value_or_range(&val, &val_range, &c); - if (res != ERTS_INIT_CPU_TOPOLOGY_OK) - return res; - if (val_range < 0 || val_range == val) - nids = 1; - else { - if (val_range > val) - nids = val_range - val + 1; - else - nids = val - val_range + 1; - } - need_size += nids; - if (need_size > idseq->size) { - idseq->size = need_size + 10; - idseq->id = erts_realloc(ERTS_ALC_T_TMP_CPU_IDS, - idseq->id, - sizeof(int)*idseq->size); - } - if (nids == 1) - idseq->id[ix++] = val; - else if (val_range > val) { - for (; val <= val_range; val++) - idseq->id[ix++] = val; - } - else { - for (; val >= val_range; val--) - idseq->id[ix++] = val; - } - if (*c != ',') - break; - c++; - } - *str = c; - idseq->used = ix; - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -static int -get_cput_entry(ErtsCpuTopEntry *cput, char **str) -{ - int h; - char *c = *str; - - cput->logical.used = 0; - cput->thread.id[0] = 0; - cput->thread.used = 1; - cput->core.id[0] = 0; - cput->core.used = 1; - cput->processor_node.id[0] = -1; - cput->processor_node.used = 1; - cput->processor.id[0] = 0; - cput->processor.used = 1; - cput->node.id[0] = -1; - cput->node.used = 1; - - h = ERTS_TOPOLOGY_MAX_DEPTH; - while (*c != ':' && *c != '\0') { - int res; - ErtsCpuTopIdSeq *idseqp; - switch (*c++) { - case 'L': - if (h <= ERTS_TOPOLOGY_LOGICAL) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->logical; - h = ERTS_TOPOLOGY_LOGICAL; - break; - case 't': - case 'T': - if (h <= ERTS_TOPOLOGY_THREAD) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->thread; - h = ERTS_TOPOLOGY_THREAD; - break; - case 'c': - case 'C': - if (h <= ERTS_TOPOLOGY_CORE) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->core; - h = ERTS_TOPOLOGY_CORE; - break; - case 'p': - case 'P': - if (h <= ERTS_TOPOLOGY_PROCESSOR) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->processor; - h = ERTS_TOPOLOGY_PROCESSOR; - break; - case 'n': - case 'N': - if (h <= ERTS_TOPOLOGY_PROCESSOR) { - do_node: - if (h <= ERTS_TOPOLOGY_NODE) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->node; - h = ERTS_TOPOLOGY_NODE; - } - else { - int p_node = 0; - char *p_chk = c; - while (*p_chk != '\0' && *p_chk != ':') { - if (*p_chk == 'p' || *p_chk == 'P') { - p_node = 1; - break; - } - p_chk++; - } - if (!p_node) - goto do_node; - if (h <= ERTS_TOPOLOGY_PROCESSOR_NODE) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY; - idseqp = &cput->processor_node; - h = ERTS_TOPOLOGY_PROCESSOR_NODE; - } - break; - default: - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE; - } - res = get_cput_id_seq(idseqp, &c); - if (res != ERTS_INIT_CPU_TOPOLOGY_OK) - return res; - } - - if (cput->logical.used < 1) - return ERTS_INIT_CPU_TOPOLOGY_MISSING_LID; - - if (*c == ':') { - c++; - } - - if (cput->thread.used != 1 - && cput->thread.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->core.used != 1 - && cput->core.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->processor_node.used != 1 - && cput->processor_node.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->processor.used != 1 - && cput->processor.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - if (cput->node.used != 1 - && cput->node.used != cput->logical.used) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE; - - *str = c; - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -static int -verify_topology(erts_cpu_topology_t *cpudata, int size) -{ - if (size > 0) { - int *logical; - int node, processor, no_nodes, i; - - /* Verify logical ids */ - logical = erts_alloc(ERTS_ALC_T_TMP, sizeof(int)*size); - - for (i = 0; i < size; i++) - logical[i] = cpudata[i].logical; - - qsort(logical, size, sizeof(int), int_cmp); - for (i = 0; i < size-1; i++) { - if (logical[i] == logical[i+1]) { - erts_free(ERTS_ALC_T_TMP, logical); - return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS; - } - } - - erts_free(ERTS_ALC_T_TMP, logical); - - qsort(cpudata, size, sizeof(erts_cpu_topology_t), processor_order_cmp); - - /* Verify unique entities */ - - for (i = 1; i < size; i++) { - if (cpudata[i-1].processor == cpudata[i].processor - && cpudata[i-1].node == cpudata[i].node - && (cpudata[i-1].processor_node - == cpudata[i].processor_node) - && cpudata[i-1].core == cpudata[i].core - && cpudata[i-1].thread == cpudata[i].thread) { - return ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES; - } - } - - /* Verify numa nodes */ - node = cpudata[0].node; - processor = cpudata[0].processor; - no_nodes = cpudata[0].node < 0 && cpudata[0].processor_node < 0; - for (i = 1; i < size; i++) { - if (no_nodes) { - if (cpudata[i].node >= 0 || cpudata[i].processor_node >= 0) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - } - else { - if (cpudata[i].processor == processor && cpudata[i].node != node) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - node = cpudata[i].node; - processor = cpudata[i].processor; - if (node >= 0 && cpudata[i].processor_node >= 0) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - if (node < 0 && cpudata[i].processor_node < 0) - return ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES; - } - } - } - - return ERTS_INIT_CPU_TOPOLOGY_OK; -} - -int -erts_init_cpu_topology(char *topology_str) -{ - ErtsCpuTopEntry cput; - int need_size; - char *c; - int ix; - int error = ERTS_INIT_CPU_TOPOLOGY_OK; - - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata_size = 10; - - user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(erts_cpu_topology_t) - * user_cpudata_size)); - - init_cpu_top_entry(&cput); - - ix = 0; - need_size = 0; - - c = topology_str; - if (*c == '\0') { - error = ERTS_INIT_CPU_TOPOLOGY_MISSING; - goto fail; - } - do { - int r; - error = get_cput_entry(&cput, &c); - if (error != ERTS_INIT_CPU_TOPOLOGY_OK) - goto fail; - need_size += cput.logical.used; - if (user_cpudata_size < need_size) { - user_cpudata_size = need_size + 10; - user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, - user_cpudata, - (sizeof(erts_cpu_topology_t) - * user_cpudata_size)); - } - - ASSERT(cput.thread.used == 1 - || cput.thread.used == cput.logical.used); - ASSERT(cput.core.used == 1 - || cput.core.used == cput.logical.used); - ASSERT(cput.processor_node.used == 1 - || cput.processor_node.used == cput.logical.used); - ASSERT(cput.processor.used == 1 - || cput.processor.used == cput.logical.used); - ASSERT(cput.node.used == 1 - || cput.node.used == cput.logical.used); - - for (r = 0; r < cput.logical.used; r++) { - user_cpudata[ix].logical = cput.logical.id[r]; - user_cpudata[ix].thread = - cput.thread.id[cput.thread.used == 1 ? 0 : r]; - user_cpudata[ix].core = - cput.core.id[cput.core.used == 1 ? 0 : r]; - user_cpudata[ix].processor_node = - cput.processor_node.id[cput.processor_node.used == 1 ? 0 : r]; - user_cpudata[ix].processor = - cput.processor.id[cput.processor.used == 1 ? 0 : r]; - user_cpudata[ix].node = - cput.node.id[cput.node.used == 1 ? 0 : r]; - ix++; - } - } while (*c != '\0'); - - if (user_cpudata_size != ix) { - user_cpudata_size = ix; - user_cpudata = erts_realloc(ERTS_ALC_T_CPUDATA, - user_cpudata, - (sizeof(erts_cpu_topology_t) - * user_cpudata_size)); - } - - error = verify_topology(user_cpudata, user_cpudata_size); - if (error == ERTS_INIT_CPU_TOPOLOGY_OK) { - destroy_cpu_top_entry(&cput); - return ERTS_INIT_CPU_TOPOLOGY_OK; - } - - fail: - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata_size = 0; - destroy_cpu_top_entry(&cput); - return error; -} - -#define ERTS_GET_CPU_TOPOLOGY_ERROR -1 -#define ERTS_GET_USED_CPU_TOPOLOGY 0 -#define ERTS_GET_DETECTED_CPU_TOPOLOGY 1 -#define ERTS_GET_DEFINED_CPU_TOPOLOGY 2 - -static Eterm get_cpu_topology_term(Process *c_p, int type); - -Eterm -erts_set_cpu_topology(Process *c_p, Eterm term) -{ - erts_cpu_topology_t *cpudata = NULL; - int cpudata_size = 0; - Eterm res; - - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - res = get_cpu_topology_term(c_p, ERTS_GET_USED_CPU_TOPOLOGY); - if (term == am_undefined) { - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata = NULL; - user_cpudata_size = 0; - - if (cpu_bind_order != ERTS_CPU_BIND_NONE && system_cpudata) { - cpudata_size = system_cpudata_size; - cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * cpudata_size)); - - sys_memcpy((void *) cpudata, - (void *) system_cpudata, - sizeof(erts_cpu_topology_t)*cpudata_size); - } - } - else if (is_not_list(term)) { - error: - res = THE_NON_VALUE; - goto done; - } - else { - Eterm list = term; - int ix = 0; - - cpudata_size = 100; - cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * cpudata_size)); - - while (is_list(list)) { - Eterm *lp = list_val(list); - Eterm cpu = CAR(lp); - Eterm* tp; - Sint id; - - if (is_not_tuple(cpu)) - goto error; - - tp = tuple_val(cpu); - - if (arityval(tp[0]) != 7 || tp[1] != am_cpu) - goto error; - - if (ix >= cpudata_size) { - cpudata_size += 100; - cpudata = erts_realloc(ERTS_ALC_T_TMP, - cpudata, - (sizeof(erts_cpu_topology_t) - * cpudata_size)); - } - - id = signed_val(tp[2]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].node = (int) id; - - id = signed_val(tp[3]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].processor = (int) id; - - id = signed_val(tp[4]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].processor_node = (int) id; - - id = signed_val(tp[5]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].core = (int) id; - - id = signed_val(tp[6]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].thread = (int) id; - - id = signed_val(tp[7]); - if (id < -1 || ERTS_MAX_CPU_TOPOLOGY_ID < id) - goto error; - cpudata[ix].logical = (int) id; - - list = CDR(lp); - ix++; - } - - if (is_not_nil(list)) - goto error; - - cpudata_size = ix; - - if (ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(cpudata, cpudata_size)) - goto error; - - if (user_cpudata_size != cpudata_size) { - if (user_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, user_cpudata); - user_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - sizeof(erts_cpu_topology_t)*cpudata_size); - user_cpudata_size = cpudata_size; - } - - sys_memcpy((void *) user_cpudata, - (void *) cpudata, - sizeof(erts_cpu_topology_t)*cpudata_size); - } - - make_reader_groups_map(NULL); - - signal_schedulers_bind_change(cpudata, cpudata_size); - - done: - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - - if (cpudata) - erts_free(ERTS_ALC_T_TMP, cpudata); - - return res; -} - -static Eterm -bound_schedulers_term(ErtsCpuBindOrder order) -{ - switch (order) { - case ERTS_CPU_BIND_SPREAD: { - ERTS_DECL_AM(spread); - return AM_spread; - } - case ERTS_CPU_BIND_PROCESSOR_SPREAD: { - ERTS_DECL_AM(processor_spread); - return AM_processor_spread; - } - case ERTS_CPU_BIND_THREAD_SPREAD: { - ERTS_DECL_AM(thread_spread); - return AM_thread_spread; - } - case ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD: { - ERTS_DECL_AM(thread_no_node_processor_spread); - return AM_thread_no_node_processor_spread; - } - case ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD: { - ERTS_DECL_AM(no_node_processor_spread); - return AM_no_node_processor_spread; - } - case ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD: { - ERTS_DECL_AM(no_node_thread_spread); - return AM_no_node_thread_spread; - } - case ERTS_CPU_BIND_NO_SPREAD: { - ERTS_DECL_AM(no_spread); - return AM_no_spread; - } - case ERTS_CPU_BIND_NONE: { - ERTS_DECL_AM(unbound); - return AM_unbound; - } - default: - ASSERT(0); - return THE_NON_VALUE; - } -} - -Eterm -erts_bound_schedulers_term(Process *c_p) -{ - ErtsCpuBindOrder order; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - order = cpu_bind_order; - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return bound_schedulers_term(order); -} - -static void -create_tmp_cpu_topology_copy(erts_cpu_topology_t **cpudata, int *cpudata_size) -{ - if (user_cpudata) { - *cpudata_size = user_cpudata_size; - *cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * (*cpudata_size))); - sys_memcpy((void *) *cpudata, - (void *) user_cpudata, - sizeof(erts_cpu_topology_t)*(*cpudata_size)); - } - else if (system_cpudata) { - *cpudata_size = system_cpudata_size; - *cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * (*cpudata_size))); - sys_memcpy((void *) *cpudata, - (void *) system_cpudata, - sizeof(erts_cpu_topology_t)*(*cpudata_size)); - } - else { - *cpudata = NULL; - *cpudata_size = 0; - } -} - -static void -destroy_tmp_cpu_topology_copy(erts_cpu_topology_t *cpudata) -{ - if (cpudata) - erts_free(ERTS_ALC_T_TMP, cpudata); -} - -Eterm -erts_bind_schedulers(Process *c_p, Eterm how) -{ - Eterm res; - erts_cpu_topology_t *cpudata; - int cpudata_size; - ErtsCpuBindOrder old_cpu_bind_order; - - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - - if (erts_bind_to_cpu(erts_cpuinfo, -1) == -ENOTSUP) { - ERTS_BIF_PREP_ERROR(res, c_p, EXC_NOTSUP); - } - else { - - old_cpu_bind_order = cpu_bind_order; - - if (ERTS_IS_ATOM_STR("default_bind", how)) - cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; - else if (ERTS_IS_ATOM_STR("spread", how)) - cpu_bind_order = ERTS_CPU_BIND_SPREAD; - else if (ERTS_IS_ATOM_STR("processor_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("thread_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("no_spread", how)) - cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; - else if (ERTS_IS_ATOM_STR("unbound", how)) - cpu_bind_order = ERTS_CPU_BIND_NONE; - else { - cpu_bind_order = old_cpu_bind_order; - ERTS_BIF_PREP_ERROR(res, c_p, BADARG); - goto done; - } - - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - - if (!cpudata) { - cpu_bind_order = old_cpu_bind_order; - ERTS_BIF_PREP_ERROR(res, c_p, BADARG); - goto done; - } - - signal_schedulers_bind_change(cpudata, cpudata_size); - - destroy_tmp_cpu_topology_copy(cpudata); - - res = bound_schedulers_term(old_cpu_bind_order); - } - - done: - - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - - return res; -} - -Eterm -erts_fake_scheduler_bindings(Process *p, Eterm how) -{ - ErtsCpuBindOrder fake_cpu_bind_order; - erts_cpu_topology_t *cpudata; - int cpudata_size; - Eterm res; - - if (ERTS_IS_ATOM_STR("default_bind", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_DEFAULT_BIND; - else if (ERTS_IS_ATOM_STR("spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_SPREAD; - else if (ERTS_IS_ATOM_STR("processor_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("thread_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("thread_no_node_processor_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_processor_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD; - else if (ERTS_IS_ATOM_STR("no_node_thread_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD; - else if (ERTS_IS_ATOM_STR("no_spread", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NO_SPREAD; - else if (ERTS_IS_ATOM_STR("unbound", how)) - fake_cpu_bind_order = ERTS_CPU_BIND_NONE; - else { - ERTS_BIF_PREP_ERROR(res, p, BADARG); - return res; - } - - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - - if (!cpudata || fake_cpu_bind_order == ERTS_CPU_BIND_NONE) - ERTS_BIF_PREP_RET(res, am_false); - else { - int i; - Eterm *hp; - - cpu_bind_order_sort(cpudata, cpudata_size, fake_cpu_bind_order, 1); - -#ifdef ERTS_FAKE_SCHED_BIND_PRINT_SORTED_CPU_DATA - - erts_fprintf(stderr, "node: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].node); - erts_fprintf(stderr, "\n"); - erts_fprintf(stderr, "processor: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].processor); - erts_fprintf(stderr, "\n"); - if (fake_cpu_bind_order != ERTS_CPU_BIND_THREAD_NO_NODE_PROCESSOR_SPREAD - && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_PROCESSOR_SPREAD - && fake_cpu_bind_order != ERTS_CPU_BIND_NO_NODE_THREAD_SPREAD) { - erts_fprintf(stderr, "processor_node:"); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].processor_node); - erts_fprintf(stderr, "\n"); - } - erts_fprintf(stderr, "core: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].core); - erts_fprintf(stderr, "\n"); - erts_fprintf(stderr, "thread: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].thread); - erts_fprintf(stderr, "\n"); - erts_fprintf(stderr, "logical: "); - for (i = 0; i < cpudata_size; i++) - erts_fprintf(stderr, " %2d", cpudata[i].logical); - erts_fprintf(stderr, "\n"); -#endif - - hp = HAlloc(p, cpudata_size+1); - ERTS_BIF_PREP_RET(res, make_tuple(hp)); - *hp++ = make_arityval((Uint) cpudata_size); - for (i = 0; i < cpudata_size; i++) - *hp++ = make_small((Uint) cpudata[i].logical); - } - - destroy_tmp_cpu_topology_copy(cpudata); - - return res; -} - -Eterm -erts_get_schedulers_binds(Process *c_p) -{ - int ix; - ERTS_DECL_AM(unbound); - Eterm *hp = HAlloc(c_p, erts_no_schedulers+1); - Eterm res = make_tuple(hp); - - *(hp++) = make_arityval(erts_no_schedulers); - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - for (ix = 1; ix <= erts_no_schedulers; ix++) - *(hp++) = (scheduler2cpu_map[ix].bound_id >= 0 - ? make_small(scheduler2cpu_map[ix].bound_id) - : AM_unbound); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return res; -} - -static Eterm -bld_topology_term(Eterm **hpp, - Uint *hszp, - erts_cpu_topology_t *cpudata, - int size) -{ - Eterm res = NIL; - int i; - - if (size == 0) - return am_undefined; - - for (i = size-1; i >= 0; i--) { - res = erts_bld_cons(hpp, - hszp, - erts_bld_tuple(hpp, - hszp, - 7, - am_cpu, - make_small(cpudata[i].node), - make_small(cpudata[i].processor), - make_small(cpudata[i].processor_node), - make_small(cpudata[i].core), - make_small(cpudata[i].thread), - make_small(cpudata[i].logical)), - res); - } - return res; -} - -static Eterm -get_cpu_topology_term(Process *c_p, int type) -{ -#ifdef DEBUG - Eterm *hp_end; -#endif - Eterm *hp; - Uint hsz; - Eterm res = THE_NON_VALUE; - erts_cpu_topology_t *cpudata = NULL; - int size = 0; - - switch (type) { - case ERTS_GET_USED_CPU_TOPOLOGY: - if (user_cpudata) - goto defined; - else - goto detected; - case ERTS_GET_DETECTED_CPU_TOPOLOGY: - detected: - if (!system_cpudata) - res = am_undefined; - else { - size = system_cpudata_size; - cpudata = erts_alloc(ERTS_ALC_T_TMP, - (sizeof(erts_cpu_topology_t) - * size)); - sys_memcpy((void *) cpudata, - (void *) system_cpudata, - sizeof(erts_cpu_topology_t)*size); - } - break; - case ERTS_GET_DEFINED_CPU_TOPOLOGY: - defined: - if (!user_cpudata) - res = am_undefined; - else { - size = user_cpudata_size; - cpudata = user_cpudata; - } - break; - default: - erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); - break; - } - - if (res == am_undefined) { - ASSERT(!cpudata); - return res; - } - - hsz = 0; - - bld_topology_term(NULL, &hsz, - cpudata, size); - - hp = HAlloc(c_p, hsz); - -#ifdef DEBUG - hp_end = hp + hsz; -#endif - - res = bld_topology_term(&hp, NULL, - cpudata, size); - - ASSERT(hp_end == hp); - - if (cpudata && cpudata != system_cpudata && cpudata != user_cpudata) - erts_free(ERTS_ALC_T_TMP, cpudata); - - return res; -} - -Eterm -erts_get_cpu_topology_term(Process *c_p, Eterm which) -{ - Eterm res; - int type; - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - if (ERTS_IS_ATOM_STR("used", which)) - type = ERTS_GET_USED_CPU_TOPOLOGY; - else if (ERTS_IS_ATOM_STR("detected", which)) - type = ERTS_GET_DETECTED_CPU_TOPOLOGY; - else if (ERTS_IS_ATOM_STR("defined", which)) - type = ERTS_GET_DEFINED_CPU_TOPOLOGY; - else - type = ERTS_GET_CPU_TOPOLOGY_ERROR; - if (type == ERTS_GET_CPU_TOPOLOGY_ERROR) - res = THE_NON_VALUE; - else - res = get_cpu_topology_term(c_p, type); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - return res; -} - -static void -early_cpu_bind_init(void) -{ - user_cpudata = NULL; - user_cpudata_size = 0; - - system_cpudata_size = erts_get_cpu_topology_size(erts_cpuinfo); - system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(erts_cpu_topology_t) - * system_cpudata_size)); - - cpu_bind_order = ERTS_CPU_BIND_UNDEFINED; - - reader_groups_available_cpu_check = 1; - reader_groups_logical_processors = 0; - reader_groups_map_size = 0; - reader_groups_map = NULL; - - if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata) - || ERTS_INIT_CPU_TOPOLOGY_OK != verify_topology(system_cpudata, - system_cpudata_size)) { - erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); - system_cpudata = NULL; - system_cpudata_size = 0; - } -} - -static void -late_cpu_bind_init(void) -{ - int ix; - - erts_smp_rwmtx_init(&erts_cpu_bind_rwmtx, "cpu_bind"); - - scheduler2cpu_map = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(ErtsCpuBindData) - * (erts_no_schedulers+1))); - for (ix = 1; ix <= erts_no_schedulers; ix++) { - scheduler2cpu_map[ix].bind_id = -1; - scheduler2cpu_map[ix].bound_id = -1; - } - - if (cpu_bind_order == ERTS_CPU_BIND_UNDEFINED) { - int ncpus = erts_get_cpu_configured(erts_cpuinfo); - if (ncpus < 1 || erts_no_schedulers < ncpus) - cpu_bind_order = ERTS_CPU_BIND_NONE; - else - cpu_bind_order = ((system_cpudata || user_cpudata) - && (erts_bind_to_cpu(erts_cpuinfo, -1) != -ENOTSUP) - ? ERTS_CPU_BIND_DEFAULT_BIND - : ERTS_CPU_BIND_NONE); - } - - make_reader_groups_map(NULL); - - if (cpu_bind_order != ERTS_CPU_BIND_NONE) { - erts_cpu_topology_t *cpudata; - int cpudata_size; - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - signal_schedulers_bind_change(cpudata, cpudata_size); - destroy_tmp_cpu_topology_copy(cpudata); - } -} - -int -erts_update_cpu_info(void) -{ - int changed; - erts_smp_rwmtx_rwlock(&erts_cpu_bind_rwmtx); - changed = erts_cpu_info_update(erts_cpuinfo); - if (changed) { - erts_cpu_topology_t *cpudata; - int cpudata_size; - - if (system_cpudata) - erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); - - system_cpudata_size = erts_get_cpu_topology_size(erts_cpuinfo); - if (!system_cpudata_size) - system_cpudata = NULL; - else { - system_cpudata = erts_alloc(ERTS_ALC_T_CPUDATA, - (sizeof(erts_cpu_topology_t) - * system_cpudata_size)); - - if (!erts_get_cpu_topology(erts_cpuinfo, system_cpudata) - || (ERTS_INIT_CPU_TOPOLOGY_OK - != verify_topology(system_cpudata, - system_cpudata_size))) { - erts_free(ERTS_ALC_T_CPUDATA, system_cpudata); - system_cpudata = NULL; - system_cpudata_size = 0; - } - } - - create_tmp_cpu_topology_copy(&cpudata, &cpudata_size); - signal_schedulers_bind_change(cpudata, cpudata_size); - destroy_tmp_cpu_topology_copy(cpudata); - } - erts_smp_rwmtx_rwunlock(&erts_cpu_bind_rwmtx); - return changed; -} - #ifdef ERTS_SMP static void @@ -6884,10 +5075,10 @@ Process *schedule(Process *p, int calls) { ErtsRunQueue *rq; ErtsRunPrioQueue *rpq; - long dt; + erts_aint_t dt; ErtsSchedulerData *esdp; int context_reds; - long fcalls; + int fcalls; int input_reductions; int actual_reds; int reds; @@ -6910,7 +5101,7 @@ Process *schedule(Process *p, int calls) esdp = erts_get_scheduler_data(); rq = erts_get_runq_current(esdp); ASSERT(esdp); - fcalls = erts_smp_atomic_read(&function_calls); + fcalls = (int) erts_smp_atomic32_read(&function_calls); actual_reds = reds = 0; erts_smp_runq_lock(rq); } else { @@ -6928,7 +5119,7 @@ Process *schedule(Process *p, int calls) reds = ERTS_PROC_MIN_CONTEXT_SWITCH_REDS_COST; esdp->virtual_reds = 0; - fcalls = erts_smp_atomic_addtest(&function_calls, reds); + fcalls = (int) erts_smp_atomic32_addtest(&function_calls, reds); ASSERT(esdp && esdp == erts_get_scheduler_data()); rq = erts_get_runq_current(esdp); @@ -7029,10 +5220,10 @@ Process *schedule(Process *p, int calls) ERTS_SMP_CHK_NO_PROC_LOCKS; - dt = do_time_read_and_reset(); + dt = erts_do_time_read_and_reset(); if (dt) { erts_smp_runq_unlock(rq); - bump_timer(dt); + erts_bump_timer(dt); erts_smp_runq_lock(rq); } BM_STOP_TIMER(system); @@ -7061,15 +5252,15 @@ Process *schedule(Process *p, int calls) | ERTS_RUNQ_FLG_CHK_CPU_BIND | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic_read(&esdp->ssi->flags) + || (erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { - ASSERT(erts_smp_atomic_read(&esdp->ssi->flags) + ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); suspend_scheduler(esdp); } if ((rq->flags & ERTS_RUNQ_FLG_CHK_CPU_BIND) - || erts_smp_atomic_read(&esdp->chk_cpu_bind)) { - check_cpu_bind(esdp); + || erts_smp_atomic32_read(&esdp->chk_cpu_bind)) { + erts_sched_check_cpu_bind(esdp); } } @@ -7077,7 +5268,7 @@ Process *schedule(Process *p, int calls) || defined(ERTS_SCHED_NEED_NONBLOCKABLE_AUX_WORK) { ErtsSchedulerSleepInfo *ssi = esdp->ssi; - long aux_work = erts_smp_atomic_read(&ssi->aux_work); + erts_aint32_t aux_work = erts_smp_atomic32_read(&ssi->aux_work); if (aux_work) { erts_smp_runq_unlock(rq); #ifdef ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK @@ -7119,9 +5310,9 @@ Process *schedule(Process *p, int calls) if (rq->flags & (ERTS_RUNQ_FLG_SHARED_RUNQ | ERTS_RUNQ_FLG_SUSPENDED)) { if ((rq->flags & ERTS_RUNQ_FLG_SUSPENDED) - || (erts_smp_atomic_read(&esdp->ssi->flags) + || (erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED)) { - ASSERT(erts_smp_atomic_read(&esdp->ssi->flags) + ASSERT(erts_smp_atomic32_read(&esdp->ssi->flags) & ERTS_SSI_FLG_SUSPENDED); non_empty_runq(rq); goto continue_check_activities_to_run; @@ -7163,19 +5354,21 @@ Process *schedule(Process *p, int calls) * Schedule system-level activities. */ - erts_smp_atomic_set(&function_calls, 0); + erts_smp_atomic32_set(&function_calls, 0); fcalls = 0; + ASSERT(!erts_port_task_have_outstanding_io_tasks()); + #ifdef ERTS_SMP /* erts_sys_schedule_interrupt(0); */ #endif erts_smp_runq_unlock(rq); erl_sys_schedule(runnable); - dt = do_time_read_and_reset(); - if (dt) bump_timer(dt); + dt = erts_do_time_read_and_reset(); + if (dt) erts_bump_timer(dt); #ifdef ERTS_SMP erts_smp_runq_lock(rq); - erts_smp_atomic_set(&doing_sys_schedule, 0); + clear_sys_scheduling(); goto continue_check_activities_to_run; #else if (!runnable) @@ -7203,7 +5396,7 @@ Process *schedule(Process *p, int calls) if (erts_common_run_queue->waiting) wake_scheduler(erts_common_run_queue, 0, 1); } - else if (erts_smp_atomic_read(&no_empty_run_queues) != 0) { + else if (erts_smp_atomic32_read(&no_empty_run_queues) != 0) { wake_scheduler_on_empty_runq(rq); rq->wakeup_other = 0; } @@ -7497,6 +5690,15 @@ erts_schedule_misc_op(void (*func)(void *), void *arg) ErtsRunQueue *rq = erts_get_runq_current(NULL); ErtsMiscOpList *molp = misc_op_list_alloc(); + if (!rq) { + /* + * This can only happen when the sys msg dispatcher + * thread schedules misc ops (this happens *very* + * seldom; only when trace drivers are unloaded). + */ + rq = ERTS_RUNQ_IX(0); + } + erts_smp_runq_lock(rq); while (rq->misc.evac_runq) { @@ -7651,7 +5853,7 @@ erts_test_next_pid(int set, Uint next) Uint erts_process_count(void) { - long res = erts_smp_atomic_read(&process_count); + erts_aint32_t res = erts_smp_atomic32_read(&process_count); ASSERT(res >= 0); return (Uint) res; } @@ -7700,7 +5902,7 @@ alloc_process(void) ASSERT(!process_tab[p_next]); process_tab[p_next] = p; - erts_smp_atomic_inc(&process_count); + erts_smp_atomic32_inc(&process_count); p->id = make_internal_pid(p_serial << p_serial_shift | p_next); if (p->id == ERTS_INVALID_PID) { /* Do not use the invalid pid; change serial */ @@ -7826,7 +6028,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->min_heap_size = H_MIN_SIZE; p->min_vheap_size = BIN_VH_MIN_SIZE; p->prio = PRIORITY_NORMAL; - p->max_gen_gcs = (Uint16) erts_smp_atomic_read(&erts_max_gen_gcs); + p->max_gen_gcs = (Uint16) erts_smp_atomic32_read(&erts_max_gen_gcs); } p->skipped = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); @@ -8879,11 +7081,11 @@ static void doit_exit_monitor(ErtsMonitor *mon, void *vpcontext) DeclareTmpHeapNoproc(lhp,3); ErtsProcLocks rp_locks = (ERTS_PROC_LOCK_LINK | ERTS_PROC_LOCKS_MSG_SEND); - UseTmpHeapNoproc(3); rp = erts_pid2proc(NULL, 0, mon->pid, rp_locks); if (rp == NULL) { goto done; } + UseTmpHeapNoproc(3); rmon = erts_remove_monitor(&(rp->monitors),mon->ref); if (rmon) { erts_destroy_monitor(rmon); @@ -9283,8 +7485,8 @@ continue_exit_process(Process *p p->status_flags = 0; #endif process_tab[pix] = NULL; /* Time of death! */ - ASSERT(erts_smp_atomic_read(&process_count) > 0); - erts_smp_atomic_dec(&process_count); + ASSERT(erts_smp_atomic32_read(&process_count) > 0); + erts_smp_atomic32_dec(&process_count); #ifdef ERTS_SMP erts_pix_unlock(pix_lock); @@ -9424,7 +7626,7 @@ cancel_timer(Process* p) #ifdef ERTS_SMP erts_cancel_smp_ptimer(p->u.ptimer); #else - erl_cancel_timer(&p->u.tm); + erts_cancel_timer(&p->u.tm); #endif } @@ -9450,7 +7652,7 @@ set_timer(Process* p, Uint timeout) (ErlTimeoutProc) timeout_proc, timeout); #else - erl_set_timer(&p->u.tm, + erts_set_timer(&p->u.tm, (ErlTimeoutProc) timeout_proc, NULL, (void*) p, @@ -9498,7 +7700,7 @@ erts_program_counter_info(int to, void *to_arg, Process *p) * only cause problems. */ for (i = 0; i < p->arity; i++) - erts_print(to, to_arg, " %T\n", p->arg_reg[i]); + erts_print(to, to_arg, " %.*T\n", INT_MAX, p->arg_reg[i]); } } } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 4365e409e5..8f78a7d76e 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,6 +28,12 @@ #define ERTS_INCLUDE_SCHEDULER_INTERNALS #endif +/* #define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC */ + +#if !defined(ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC) && defined(DEBUG) +# define ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC +#endif + typedef struct process Process; #include "sys.h" @@ -89,7 +95,6 @@ extern int erts_sched_thread_suggested_stack_size; #define ERTS_SCHED_THREAD_MAX_STACK_SIZE 8192 /* Kilo words */ #ifdef ERTS_SMP -extern Uint erts_max_main_threads; #include "erl_bits.h" #endif @@ -175,8 +180,8 @@ extern Uint erts_max_main_threads; #define ERTS_UNSET_RUNQ_FLG_EVACUATE(FLGS, PRIO) \ ((FLGS) &= ~ERTS_RUNQ_FLG_EVACUATE((PRIO))) -#define ERTS_RUNQ_IFLG_SUSPENDED (((long) 1) << 0) -#define ERTS_RUNQ_IFLG_NONEMPTY (((long) 1) << 1) +#define ERTS_RUNQ_IFLG_SUSPENDED (((erts_aint32_t) 1) << 0) +#define ERTS_RUNQ_IFLG_NONEMPTY (((erts_aint32_t) 1) << 1) #ifdef DEBUG @@ -220,11 +225,11 @@ typedef enum { ERTS_MIGRATE_FAILED_RUNQ_SUSPENDED } ErtsMigrateResult; -#define ERTS_SSI_FLG_SLEEPING (((long) 1) << 0) -#define ERTS_SSI_FLG_POLL_SLEEPING (((long) 1) << 1) -#define ERTS_SSI_FLG_TSE_SLEEPING (((long) 1) << 2) -#define ERTS_SSI_FLG_WAITING (((long) 1) << 3) -#define ERTS_SSI_FLG_SUSPENDED (((long) 1) << 4) +#define ERTS_SSI_FLG_SLEEPING (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_FLG_POLL_SLEEPING (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_FLG_TSE_SLEEPING (((erts_aint32_t) 1) << 2) +#define ERTS_SSI_FLG_WAITING (((erts_aint32_t) 1) << 3) +#define ERTS_SSI_FLG_SUSPENDED (((erts_aint32_t) 1) << 4) #define ERTS_SSI_FLGS_SLEEP_TYPE \ (ERTS_SSI_FLG_TSE_SLEEPING|ERTS_SSI_FLG_POLL_SLEEPING) @@ -237,16 +242,14 @@ typedef enum { | ERTS_SSI_FLG_WAITING \ | ERTS_SSI_FLG_SUSPENDED) - -#if !defined(ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK) \ - && defined(ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN) #define ERTS_SCHED_NEED_BLOCKABLE_AUX_WORK -#endif -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((long) 1) << 0) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 1) #define ERTS_SSI_BLOCKABLE_AUX_WORK_MASK \ - (ERTS_SSI_AUX_WORK_CHECK_CHILDREN) + (ERTS_SSI_AUX_WORK_CHECK_CHILDREN \ + | ERTS_SSI_AUX_WORK_MISC) #define ERTS_SSI_NONBLOCKABLE_AUX_WORK_MASK \ (0) @@ -260,9 +263,9 @@ typedef struct { struct ErtsSchedulerSleepInfo_ { ErtsSchedulerSleepInfo *next; ErtsSchedulerSleepInfo *prev; - erts_smp_atomic_t flags; + erts_smp_atomic32_t flags; erts_tse_t *event; - erts_smp_atomic_t aux_work; + erts_smp_atomic32_t aux_work; }; /* times to reschedule low prio process before running */ @@ -312,7 +315,7 @@ typedef struct { struct ErtsRunQueue_ { int ix; - erts_smp_atomic_t info_flags; + erts_smp_atomic32_t info_flags; erts_smp_mtx_t mtx; erts_smp_cnd_t cnd; @@ -422,10 +425,22 @@ struct ErtsSchedulerData_ { #ifdef ERTS_SMP /* NOTE: These fields are modified under held mutexes by other threads */ - erts_smp_atomic_t chk_cpu_bind; /* Only used when common run queue */ + erts_smp_atomic32_t chk_cpu_bind; /* Only used when common run queue */ +#endif + +#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC + erts_alloc_verify_func_t verify_unused_temp_alloc; + Allctr_t *verify_unused_temp_alloc_data; #endif }; +typedef union { + ErtsSchedulerData esd; + char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsSchedulerData))]; +} ErtsAlignedSchedulerData; + +extern ErtsAlignedSchedulerData *erts_aligned_scheduler_data; + #ifndef ERTS_SMP extern ErtsSchedulerData *erts_scheduler_data; #endif @@ -820,7 +835,7 @@ ERTS_GLB_INLINE void erts_heap_frag_shrink(Process* p, Eterm* hp) } #endif /* inline */ -Eterm* erts_heap_alloc(Process* p, Uint need); +Eterm* erts_heap_alloc(Process* p, Uint need, Uint xtra); #ifdef CHECK_FOR_HOLES Eterm* erts_set_hole_marker(Eterm* ptr, Uint sz); #endif @@ -1007,27 +1022,12 @@ extern struct erts_system_profile_flags_t erts_system_profile_flags; (p)->flags &= ~F_TIMO; \ } while (0) - -#define ERTS_INIT_SCHED_BIND_TYPE_SUCCESS 0 -#define ERTS_INIT_SCHED_BIND_TYPE_NOT_SUPPORTED 1 -#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_CPU_TOPOLOGY 2 -#define ERTS_INIT_SCHED_BIND_TYPE_ERROR_NO_BAD_TYPE 3 - -int erts_init_scheduler_bind_type(char *how); - -#define ERTS_INIT_CPU_TOPOLOGY_OK 0 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID 1 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_RANGE 2 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_HIERARCHY 3 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_ID_TYPE 4 -#define ERTS_INIT_CPU_TOPOLOGY_INVALID_NODES 5 -#define ERTS_INIT_CPU_TOPOLOGY_MISSING_LID 6 -#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_LIDS 7 -#define ERTS_INIT_CPU_TOPOLOGY_NOT_UNIQUE_ENTITIES 8 -#define ERTS_INIT_CPU_TOPOLOGY_MISSING 9 - -int erts_init_cpu_topology(char *topology_str); -int erts_update_cpu_info(void); +#define ERTS_RUNQ_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_run_queues), \ + &erts_aligned_run_queues[(IX)].runq) +#define ERTS_SCHEDULER_IX(IX) \ + (ASSERT_EXPR(0 <= (IX) && (IX) < erts_no_schedulers), \ + &erts_aligned_scheduler_data[(IX)].esd) void erts_pre_init_process(void); void erts_late_init_process(void); @@ -1043,6 +1043,7 @@ int erts_sched_set_wakeup_limit(char *str); #ifdef DEBUG void erts_dbg_multi_scheduling_return_trap(Process *, Eterm); #endif +int erts_get_max_no_executing_schedulers(void); #ifdef ERTS_SMP ErtsSchedSuspendResult erts_schedulers_state(Uint *, Uint *, Uint *, int); @@ -1057,9 +1058,15 @@ int erts_is_multi_scheduling_blocked(void); Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); void erts_smp_notify_check_children_needed(void); +void +erts_smp_schedule_misc_aux_work(int ignore_self, + int max_sched, + void (*func)(void *), + void *arg); #endif +void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); -void erts_init_process(void); +void erts_init_process(int); Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm); Uint erts_run_queues_len(Uint *); void erts_add_to_runq(Process *); @@ -1149,6 +1156,20 @@ Uint erts_debug_nbalance(void); # define ERTS_PROC_GET_SCHDATA(PROC) (erts_scheduler_data) #endif +#ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC +# define ERTS_VERIFY_UNUSED_TEMP_ALLOC(P) \ +do { \ + ErtsSchedulerData *esdp__ = ((P) \ + ? ERTS_PROC_GET_SCHDATA((Process *) (P)) \ + : erts_get_scheduler_data()); \ + if (esdp__) \ + esdp__->verify_unused_temp_alloc( \ + esdp__->verify_unused_temp_alloc_data); \ +} while (0) +#else +# define ERTS_VERIFY_UNUSED_TEMP_ALLOC(ESDP) +#endif + #if defined(ERTS_SMP) || defined(USE_THREADS) ErtsSchedulerData *erts_get_scheduler_data(void); #else @@ -1563,7 +1584,7 @@ extern int erts_disable_proc_not_running_opt; void erts_smp_notify_inc_runq(ErtsRunQueue *runq); #ifdef ERTS_SMP -void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, long); +void erts_sched_finish_poke(ErtsSchedulerSleepInfo *, erts_aint32_t); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -1571,11 +1592,13 @@ ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi); ERTS_GLB_INLINE void erts_sched_poke(ErtsSchedulerSleepInfo *ssi) { - long flags = erts_smp_atomic_read(&ssi->flags); + erts_aint32_t flags; + ERTS_THR_MEMORY_BARRIER; + flags = erts_smp_atomic32_read(&ssi->flags); ASSERT(!(flags & ERTS_SSI_FLG_SLEEPING) || (flags & ERTS_SSI_FLG_WAITING)); if (flags & ERTS_SSI_FLG_SLEEPING) { - flags = erts_smp_atomic_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); + flags = erts_smp_atomic32_band(&ssi->flags, ~ERTS_SSI_FLGS_SLEEP); erts_sched_finish_poke(ssi, flags); } } diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 7a7042abe4..68fda01597 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * Copyright Ericsson AB 2003-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -261,139 +261,139 @@ print_function_from_pc(int to, void *to_arg, BeamInstr* x) static void heap_dump(int to, void *to_arg, Eterm x) { + DeclareTmpHeapNoproc(last,1); + Eterm* next = last; Eterm* ptr; - Eterm last = OUR_NIL; - Eterm* next = &last; if (is_immed(x) || is_CP(x)) { return; } - - again: - if (x == OUR_NIL) { /* We are done. */ - return; - } if (is_CP(x)) { - next = (Eterm *) EXPAND_POINTER(x); - } else if (is_list(x)) { - ptr = list_val(x); - if (ptr[0] != OUR_NIL) { - erts_print(to, to_arg, ADDR_FMT ":l", ptr); - dump_element(to, to_arg, ptr[0]); - erts_putc(to, to_arg, '|'); - dump_element(to, to_arg, ptr[1]); - erts_putc(to, to_arg, '\n'); - if (is_immed(ptr[1])) { - ptr[1] = make_small(0); - } - x = ptr[0]; - ptr[0] = (Eterm) COMPRESS_POINTER(next); - next = ptr + 1; - goto again; - } - } else if (is_boxed(x)) { - Eterm hdr; - - ptr = boxed_val(x); - hdr = *ptr; - if (hdr != OUR_NIL) { /* If not visited */ - erts_print(to, to_arg, ADDR_FMT ":", ptr); - if (is_arity_value(hdr)) { - Uint i; - Uint arity = arityval(hdr); - - erts_print(to, to_arg, "t" WORD_FMT ":", arity); - for (i = 1; i <= arity; i++) { - dump_element(to, to_arg, ptr[i]); - if (is_immed(ptr[i])) { - ptr[i] = make_small(0); - } - if (i < arity) { - erts_putc(to, to_arg, ','); - } - } + UseTmpHeapNoproc(1); + *last = OUR_NIL; + + while (x != OUR_NIL) { + if (is_CP(x)) { + next = (Eterm *) EXPAND_POINTER(x); + } else if (is_list(x)) { + ptr = list_val(x); + if (ptr[0] != OUR_NIL) { + erts_print(to, to_arg, ADDR_FMT ":l", ptr); + dump_element(to, to_arg, ptr[0]); + erts_putc(to, to_arg, '|'); + dump_element(to, to_arg, ptr[1]); erts_putc(to, to_arg, '\n'); - if (arity == 0) { - ptr[0] = OUR_NIL; - } else { - x = ptr[arity]; - ptr[0] = (Eterm) COMPRESS_POINTER(next); - next = ptr + arity - 1; - goto again; + if (is_immed(ptr[1])) { + ptr[1] = make_small(0); } - } else if (hdr == HEADER_FLONUM) { - FloatDef f; - char sbuf[31]; - int i; - - GET_DOUBLE_DATA((ptr+1), f); - i = sys_double_to_chars(f.fd, (char*) sbuf); - sys_memset(sbuf+i, 0, 31-i); - erts_print(to, to_arg, "F%X:%s\n", i, sbuf); - *ptr = OUR_NIL; - } else if (_is_bignum_header(hdr)) { - erts_print(to, to_arg, "B%T\n", x); - *ptr = OUR_NIL; - } else if (is_binary_header(hdr)) { - Uint tag = thing_subtag(hdr); - Uint size = binary_size(x); - Uint i; - - if (tag == HEAP_BINARY_SUBTAG) { - byte* p; - - erts_print(to, to_arg, "Yh%X:", size); - p = binary_bytes(x); - for (i = 0; i < size; i++) { - erts_print(to, to_arg, "%02X", p[i]); + x = ptr[0]; + ptr[0] = (Eterm) COMPRESS_POINTER(next); + next = ptr + 1; + continue; + } + } else if (is_boxed(x)) { + Eterm hdr; + + ptr = boxed_val(x); + hdr = *ptr; + if (hdr != OUR_NIL) { /* If not visited */ + erts_print(to, to_arg, ADDR_FMT ":", ptr); + if (is_arity_value(hdr)) { + Uint i; + Uint arity = arityval(hdr); + + erts_print(to, to_arg, "t" WORD_FMT ":", arity); + for (i = 1; i <= arity; i++) { + dump_element(to, to_arg, ptr[i]); + if (is_immed(ptr[i])) { + ptr[i] = make_small(0); + } + if (i < arity) { + erts_putc(to, to_arg, ','); + } } - } else if (tag == REFC_BINARY_SUBTAG) { - ProcBin* pb = (ProcBin *) binary_val(x); - Binary* val = pb->val; - - if (erts_smp_atomic_xchg(&val->refc, 0) != 0) { - val->flags = (UWord) all_binaries; - all_binaries = val; + erts_putc(to, to_arg, '\n'); + if (arity == 0) { + ptr[0] = OUR_NIL; + } else { + x = ptr[arity]; + ptr[0] = (Eterm) COMPRESS_POINTER(next); + next = ptr + arity - 1; + continue; } - erts_print(to, to_arg, "Yc%X:%X:%X", val, - pb->bytes - (byte *)val->orig_bytes, - size); - } else if (tag == SUB_BINARY_SUBTAG) { - ErlSubBin* Sb = (ErlSubBin *) binary_val(x); - Eterm* real_bin = binary_val(Sb->orig); - void* val; - - if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) { - ProcBin* pb = (ProcBin *) real_bin; - val = pb->val; - } else { /* Heap binary */ - val = real_bin; + } else if (hdr == HEADER_FLONUM) { + FloatDef f; + char sbuf[31]; + int i; + + GET_DOUBLE_DATA((ptr+1), f); + i = sys_double_to_chars(f.fd, (char*) sbuf); + sys_memset(sbuf+i, 0, 31-i); + erts_print(to, to_arg, "F%X:%s\n", i, sbuf); + *ptr = OUR_NIL; + } else if (_is_bignum_header(hdr)) { + erts_print(to, to_arg, "B%T\n", x); + *ptr = OUR_NIL; + } else if (is_binary_header(hdr)) { + Uint tag = thing_subtag(hdr); + Uint size = binary_size(x); + Uint i; + + if (tag == HEAP_BINARY_SUBTAG) { + byte* p; + + erts_print(to, to_arg, "Yh%X:", size); + p = binary_bytes(x); + for (i = 0; i < size; i++) { + erts_print(to, to_arg, "%02X", p[i]); + } + } else if (tag == REFC_BINARY_SUBTAG) { + ProcBin* pb = (ProcBin *) binary_val(x); + Binary* val = pb->val; + + if (erts_smp_atomic_xchg(&val->refc, 0) != 0) { + val->flags = (UWord) all_binaries; + all_binaries = val; + } + erts_print(to, to_arg, "Yc%X:%X:%X", val, + pb->bytes - (byte *)val->orig_bytes, + size); + } else if (tag == SUB_BINARY_SUBTAG) { + ErlSubBin* Sb = (ErlSubBin *) binary_val(x); + Eterm* real_bin = binary_val(Sb->orig); + void* val; + + if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) { + ProcBin* pb = (ProcBin *) real_bin; + val = pb->val; + } else { /* Heap binary */ + val = real_bin; + } + erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size); } - erts_print(to, to_arg, "Ys%X:%X:%X", val, Sb->offs, size); + erts_putc(to, to_arg, '\n'); + *ptr = OUR_NIL; + } else if (is_external_pid_header(hdr)) { + erts_print(to, to_arg, "P%T\n", x); + *ptr = OUR_NIL; + } else if (is_external_port_header(hdr)) { + erts_print(to, to_arg, "p<%bpu.%bpu>\n", + port_channel_no(x), port_number(x)); + *ptr = OUR_NIL; + } else { + /* + * All other we dump in the external term format. + */ + dump_externally(to, to_arg, x); + erts_putc(to, to_arg, '\n'); + *ptr = OUR_NIL; } - erts_putc(to, to_arg, '\n'); - *ptr = OUR_NIL; - } else if (is_external_pid_header(hdr)) { - erts_print(to, to_arg, "P%T\n", x); - *ptr = OUR_NIL; - } else if (is_external_port_header(hdr)) { - erts_print(to, to_arg, "p<%bpu.%bpu>\n", - port_channel_no(x), port_number(x)); - *ptr = OUR_NIL; - } else { - /* - * All other we dump in the external term format. - */ - dump_externally(to, to_arg, x); - erts_putc(to, to_arg, '\n'); - *ptr = OUR_NIL; } } + x = *next; + *next = OUR_NIL; + next--; } - - x = *next; - *next = OUR_NIL; - next--; - goto again; + UnUseTmpHeapNoproc(1); } static void diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index a4d12139e9..72560aa124 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -117,15 +117,14 @@ static int aux_thr_proc_lock_spin_count; static void cleanup_tse(void); void -erts_init_proc_lock(void) +erts_init_proc_lock(int cpus) { int i; - int cpus; erts_smp_spinlock_init(&qs_lock, "proc_lck_qs_alloc"); for (i = 0; i < ERTS_NO_OF_PIX_LOCKS; i++) { #ifdef ERTS_ENABLE_LOCK_COUNT erts_smp_spinlock_init_x(&erts_pix_locks[i].u.spnlck, - "pix_lock", make_small(i)); + "pix_lock", make_small(i)); #else erts_smp_spinlock_init(&erts_pix_locks[i].u.spnlck, "pix_lock"); #endif @@ -138,7 +137,6 @@ erts_init_proc_lock(void) lc_id.proc_lock_msgq = erts_lc_get_lock_order_id("proc_msgq"); lc_id.proc_lock_status = erts_lc_get_lock_order_id("proc_status"); #endif - cpus = erts_get_cpu_configured(erts_cpuinfo); if (cpus > 1) { proc_lock_spin_count = ERTS_PROC_LOCK_SPIN_COUNT_BASE; proc_lock_spin_count += (ERTS_PROC_LOCK_SPIN_COUNT_SCHED_INC @@ -415,7 +413,7 @@ transfer_locks(Process *p, do { erts_tse_t *tmp = wake; wake = wake->next; - erts_atomic_set(&tmp->uaflgs, 0); + erts_atomic32_set(&tmp->uaflgs, 0); erts_tse_set(tmp); } while (wake); @@ -511,14 +509,14 @@ wait_for_locks(Process *p, ASSERT((wtr->uflgs & ~ERTS_PROC_LOCKS_ALL) == 0); - erts_atomic_set(&wtr->uaflgs, 1); + erts_atomic32_set(&wtr->uaflgs, 1); erts_pix_unlock(pix_lock); while (1) { int res; erts_tse_reset(wtr); - if (erts_atomic_read(&wtr->uaflgs) == 0) + if (erts_atomic32_read(&wtr->uaflgs) == 0) break; /* @@ -957,7 +955,7 @@ erts_proc_lock_init(Process *p) { /* We always start with all locks locked */ #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic_init(&p->lock.flags, (long) ERTS_PROC_LOCKS_ALL); + erts_smp_atomic32_init(&p->lock.flags, (erts_aint32_t) ERTS_PROC_LOCKS_ALL); #else p->lock.flags = ERTS_PROC_LOCKS_ALL; #endif @@ -976,7 +974,7 @@ erts_proc_lock_init(Process *p) { int i; for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) - erts_smp_atomic_init(&p->lock.locked[i], (long) 1); + erts_smp_atomic32_init(&p->lock.locked[i], (erts_aint32_t) 1); } #endif } diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 7cfc9893fa..355179f084 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2009. All Rights Reserved. + * Copyright Ericsson AB 2007-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -54,20 +54,20 @@ #define ERTS_PROC_LOCK_MAX_BIT 3 -typedef Uint32 ErtsProcLocks; +typedef erts_aint32_t ErtsProcLocks; typedef struct erts_proc_lock_queues_t_ erts_proc_lock_queues_t; typedef struct erts_proc_lock_t_ { #if ERTS_PROC_LOCK_ATOMIC_IMPL - erts_smp_atomic_t flags; + erts_smp_atomic32_t flags; #else ErtsProcLocks flags; #endif erts_proc_lock_queues_t *queues; - long refc; + Sint32 refc; #ifdef ERTS_PROC_LOCK_DEBUG - erts_smp_atomic_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; + erts_smp_atomic32_t locked[ERTS_PROC_LOCK_MAX_BIT+1]; #endif #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_lock_t lcnt_main; @@ -270,17 +270,19 @@ typedef struct { #if ERTS_PROC_LOCK_ATOMIC_IMPL #define ERTS_PROC_LOCK_FLGS_BAND_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic_band(&(L)->flags, (long) (MSK))) + ((ErtsProcLocks) erts_smp_atomic32_band(&(L)->flags, (erts_aint32_t) (MSK))) #define ERTS_PROC_LOCK_FLGS_BOR_(L, MSK) \ - ((ErtsProcLocks) erts_smp_atomic_bor(&(L)->flags, (long) (MSK))) + ((ErtsProcLocks) erts_smp_atomic32_bor(&(L)->flags, (erts_aint32_t) (MSK))) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_ACQB_(L, NEW, EXPECTED) \ - ((ErtsProcLocks) erts_smp_atomic_cmpxchg_acqb(&(L)->flags, \ - (long) (NEW), (long) (EXPECTED))) + ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_acqb(&(L)->flags, \ + (erts_aint32_t) (NEW), \ + (erts_aint32_t) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_CMPXCHG_RELB_(L, NEW, EXPECTED) \ - ((ErtsProcLocks) erts_smp_atomic_cmpxchg_relb(&(L)->flags, \ - (long) (NEW), (long) (EXPECTED))) + ((ErtsProcLocks) erts_smp_atomic32_cmpxchg_relb(&(L)->flags, \ + (erts_aint32_t) (NEW), \ + (erts_aint32_t) (EXPECTED))) #define ERTS_PROC_LOCK_FLGS_READ_(L) \ - ((ErtsProcLocks) erts_smp_atomic_read(&(L)->flags)) + ((ErtsProcLocks) erts_smp_atomic32_read(&(L)->flags)) #else /* no opt atomic ops */ @@ -334,7 +336,7 @@ erts_proc_lock_flags_cmpxchg(erts_proc_lock_t *lck, ErtsProcLocks new, extern erts_pix_lock_t erts_pix_locks[ERTS_NO_OF_PIX_LOCKS]; -void erts_init_proc_lock(void); +void erts_init_proc_lock(int cpus); void erts_proc_lock_prepare_proc_lock_waiter(void); void erts_proc_lock_failed(Process *, erts_pix_lock_t *, @@ -619,13 +621,13 @@ erts_proc_lock_op_debug(Process *p, ErtsProcLocks locks, int locked) for (i = 0; i <= ERTS_PROC_LOCK_MAX_BIT; i++) { ErtsProcLocks lock = ((ErtsProcLocks) 1) << i; if (locks & lock) { - long lock_count; + erts_aint32_t lock_count; if (locked) { - lock_count = erts_smp_atomic_inctest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_inctest(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 1); } else { - lock_count = erts_smp_atomic_dectest(&p->lock.locked[i]); + lock_count = erts_smp_atomic32_dectest(&p->lock.locked[i]); ERTS_LC_ASSERT(lock_count == 0); } } diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index b41fa70476..287327bfe1 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -54,10 +54,10 @@ typedef erts_cnd_t erts_smp_cnd_t; typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t; typedef erts_rwmtx_t erts_smp_rwmtx_t; typedef erts_tsd_key_t erts_smp_tsd_key_t; -typedef ethr_atomic_t erts_smp_atomic_t; +typedef erts_atomic_t erts_smp_atomic_t; +typedef erts_atomic32_t erts_smp_atomic32_t; typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; -typedef erts_thr_timeval_t erts_smp_thr_timeval_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ #else /* #ifdef ERTS_SMP */ @@ -83,7 +83,8 @@ typedef struct { } erts_smp_rwmtx_opt_t; typedef int erts_smp_rwmtx_t; typedef int erts_smp_tsd_key_t; -typedef long erts_smp_atomic_t; +typedef SWord erts_smp_atomic_t; +typedef Uint32 erts_smp_atomic32_t; #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -92,11 +93,6 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t; typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t; #endif -typedef struct { - long tv_sec; - long tv_nsec; -} erts_smp_thr_timeval_t; - #endif /* #ifdef ERTS_SMP */ ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id); @@ -164,33 +160,82 @@ ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, long i); -ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, long i); -ERTS_GLB_INLINE long erts_smp_atomic_read(erts_smp_atomic_t *var); -ERTS_GLB_INLINE long erts_smp_atomic_inctest(erts_smp_atomic_t *incp); -ERTS_GLB_INLINE long erts_smp_atomic_dectest(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var, + erts_aint_t i); +ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp); ERTS_GLB_INLINE void erts_smp_atomic_inc(erts_smp_atomic_t *incp); ERTS_GLB_INLINE void erts_smp_atomic_dec(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE long erts_smp_atomic_addtest(erts_smp_atomic_t *addp, - long i); -ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, long i); -ERTS_GLB_INLINE long erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, - long new); -ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, - long new, - long expected); -ERTS_GLB_INLINE long erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask); -ERTS_GLB_INLINE long erts_smp_atomic_band(erts_smp_atomic_t *var, long mask); -ERTS_GLB_INLINE long erts_smp_atomic_read_acqb(erts_smp_atomic_t *var); -ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var, long i); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_addtest(erts_smp_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, + erts_aint_t new); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_bor(erts_smp_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_band(erts_smp_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read_acqb(erts_smp_atomic_t *var); +ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var, + erts_aint_t i); ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE long erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp); -ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, - long new, - long exp); -ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, - long new, - long exp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE void +erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read(erts_smp_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE void +erts_smp_atomic32_inc(erts_smp_atomic32_t *incp); +ERTS_GLB_INLINE void +erts_smp_atomic32_dec(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var); +ERTS_GLB_INLINE void +erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void +erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra); @@ -221,7 +266,6 @@ ERTS_GLB_INLINE void erts_smp_write_lock(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE void erts_smp_write_unlock(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rlocked(erts_smp_rwlock_t *lock); ERTS_GLB_INLINE int erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock); -ERTS_GLB_INLINE void erts_smp_thr_time_now(erts_smp_thr_timeval_t *time); ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_smp_tsd_key_delete(erts_smp_tsd_key_t key); ERTS_GLB_INLINE void erts_smp_tsd_set(erts_smp_tsd_key_t key, void *value); @@ -611,7 +655,7 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx) } ERTS_GLB_INLINE void -erts_smp_atomic_init(erts_smp_atomic_t *var, long i) +erts_smp_atomic_init(erts_smp_atomic_t *var, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_init(var, i); @@ -621,7 +665,7 @@ erts_smp_atomic_init(erts_smp_atomic_t *var, long i) } ERTS_GLB_INLINE void -erts_smp_atomic_set(erts_smp_atomic_t *var, long i) +erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_set(var, i); @@ -630,7 +674,7 @@ erts_smp_atomic_set(erts_smp_atomic_t *var, long i) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var) { #ifdef ERTS_SMP @@ -640,7 +684,7 @@ erts_smp_atomic_read(erts_smp_atomic_t *var) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp) { #ifdef ERTS_SMP @@ -650,7 +694,7 @@ erts_smp_atomic_inctest(erts_smp_atomic_t *incp) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp) { #ifdef ERTS_SMP @@ -680,8 +724,8 @@ erts_smp_atomic_dec(erts_smp_atomic_t *decp) #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_addtest(erts_smp_atomic_t *addp, long i) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_addtest(erts_smp_atomic_t *addp, erts_aint_t i) { #ifdef ERTS_SMP return erts_atomic_addtest(addp, i); @@ -691,7 +735,7 @@ erts_smp_atomic_addtest(erts_smp_atomic_t *addp, long i) } ERTS_GLB_INLINE void -erts_smp_atomic_add(erts_smp_atomic_t *addp, long i) +erts_smp_atomic_add(erts_smp_atomic_t *addp, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_add(addp, i); @@ -700,59 +744,61 @@ erts_smp_atomic_add(erts_smp_atomic_t *addp, long i) #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, long new) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, erts_aint_t new) { #ifdef ERTS_SMP return erts_atomic_xchg(xchgp, new); #else - long old; + erts_aint_t old; old = *xchgp; *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, long new, long expected) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected) { #ifdef ERTS_SMP return erts_atomic_cmpxchg(xchgp, new, expected); #else - long old = *xchgp; + erts_aint_t old = *xchgp; if (old == expected) *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_bor(erts_smp_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_bor(erts_smp_atomic_t *var, erts_aint_t mask) { #ifdef ERTS_SMP return erts_atomic_bor(var, mask); #else - long old; + erts_aint_t old; old = *var; *var |= mask; return old; #endif } -ERTS_GLB_INLINE long -erts_smp_atomic_band(erts_smp_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_band(erts_smp_atomic_t *var, erts_aint_t mask) { #ifdef ERTS_SMP return erts_atomic_band(var, mask); #else - long old; + erts_aint_t old; old = *var; *var &= mask; return old; #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read_acqb(erts_smp_atomic_t *var) { #ifdef ERTS_SMP @@ -763,7 +809,7 @@ erts_smp_atomic_read_acqb(erts_smp_atomic_t *var) } ERTS_GLB_INLINE void -erts_smp_atomic_set_relb(erts_smp_atomic_t *var, long i) +erts_smp_atomic_set_relb(erts_smp_atomic_t *var, erts_aint_t i) { #ifdef ERTS_SMP erts_atomic_set_relb(var, i); @@ -772,7 +818,8 @@ erts_smp_atomic_set_relb(erts_smp_atomic_t *var, long i) #endif } -ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) +ERTS_GLB_INLINE void +erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) { #ifdef ERTS_SMP erts_atomic_dec_relb(decp); @@ -781,7 +828,7 @@ ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) { #ifdef ERTS_SMP @@ -791,28 +838,244 @@ erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp) #endif } -ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, - long new, - long exp) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) { #ifdef ERTS_SMP return erts_atomic_cmpxchg_acqb(xchgp, new, exp); #else - long old = *xchgp; + erts_aint_t old = *xchgp; if (old == exp) *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, - long new, - long exp) +ERTS_GLB_INLINE erts_aint_t +erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) { #ifdef ERTS_SMP return erts_atomic_cmpxchg_relb(xchgp, new, exp); #else - long old = *xchgp; + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_init(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_set(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read(erts_smp_atomic32_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic32_read(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp) +{ +#ifdef ERTS_SMP + return erts_atomic32_inctest(incp); +#else + return ++(*incp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic32_dectest(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_inc(erts_smp_atomic32_t *incp) +{ +#ifdef ERTS_SMP + erts_atomic32_inc(incp); +#else + ++(*incp); +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_dec(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic32_dec(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef ERTS_SMP + return erts_atomic32_addtest(addp, i); +#else + return *addp += i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_add(addp, i); +#else + *addp += i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new) +{ +#ifdef ERTS_SMP + return erts_atomic32_xchg(xchgp, new); +#else + erts_aint32_t old; + old = *xchgp; + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg(xchgp, new, expected); +#else + erts_aint32_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef ERTS_SMP + return erts_atomic32_bor(var, mask); +#else + erts_aint32_t old; + old = *var; + *var |= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef ERTS_SMP + return erts_atomic32_band(var, mask); +#else + erts_aint32_t old; + old = *var; + *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var) +{ +#ifdef ERTS_SMP + return erts_atomic32_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i) +{ +#ifdef ERTS_SMP + erts_atomic32_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + erts_atomic32_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp) +{ +#ifdef ERTS_SMP + return erts_atomic32_dectest_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef ERTS_SMP + return erts_atomic32_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; if (old == exp) *xchgp = new; return old; @@ -988,14 +1251,6 @@ erts_smp_lc_rwlock_is_rwlocked(erts_smp_rwlock_t *lock) } ERTS_GLB_INLINE void -erts_smp_thr_time_now(erts_smp_thr_timeval_t *time) -{ -#ifdef ERTS_SMP - erts_thr_time_now(time); -#endif -} - -ERTS_GLB_INLINE void erts_smp_tsd_key_create(erts_smp_tsd_key_t *keyp) { #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index c6458a0e45..f77e8b798f 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -58,9 +58,9 @@ do { \ #endif #if ET_DEBUG -unsigned tag_val_def_debug(Eterm x, const char *file, unsigned line) +unsigned tag_val_def_debug(Wterm x, const char *file, unsigned line) #else -unsigned tag_val_def(Eterm x) +unsigned tag_val_def(Wterm x) #define file __FILE__ #define line __LINE__ #endif @@ -125,10 +125,10 @@ FUNTY checked_##FUN(ARGTY x, const char *file, unsigned line) \ ET_DEFINE_CHECKED(Eterm,make_boxed,Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_boxed,Eterm,!is_header); -ET_DEFINE_CHECKED(Eterm*,boxed_val,Eterm,_boxed_precond); +ET_DEFINE_CHECKED(Eterm*,boxed_val,Wterm,_boxed_precond); ET_DEFINE_CHECKED(Eterm,make_list,Eterm*,_is_taggable_pointer); ET_DEFINE_CHECKED(int,is_not_list,Eterm,!is_header); -ET_DEFINE_CHECKED(Eterm*,list_val,Eterm,_list_precond); +ET_DEFINE_CHECKED(Eterm*,list_val,Wterm,_list_precond); ET_DEFINE_CHECKED(Uint,unsigned_val,Eterm,is_small); ET_DEFINE_CHECKED(Sint,signed_val,Eterm,is_small); ET_DEFINE_CHECKED(Uint,atom_val,Eterm,is_atom); @@ -136,34 +136,35 @@ ET_DEFINE_CHECKED(Uint,header_arity,Eterm,is_header); ET_DEFINE_CHECKED(Uint,arityval,Eterm,is_arity_value); ET_DEFINE_CHECKED(Uint,thing_arityval,Eterm,is_thing); ET_DEFINE_CHECKED(Uint,thing_subtag,Eterm,is_thing); -ET_DEFINE_CHECKED(Eterm*,binary_val,Eterm,is_binary); -ET_DEFINE_CHECKED(Eterm*,fun_val,Eterm,is_fun); +ET_DEFINE_CHECKED(Eterm*,binary_val,Wterm,is_binary); +ET_DEFINE_CHECKED(Eterm*,fun_val,Wterm,is_fun); ET_DEFINE_CHECKED(int,bignum_header_is_neg,Eterm,_is_bignum_header); ET_DEFINE_CHECKED(Eterm,bignum_header_neg,Eterm,_is_bignum_header); ET_DEFINE_CHECKED(Uint,bignum_header_arity,Eterm,_is_bignum_header); -ET_DEFINE_CHECKED(Eterm*,big_val,Eterm,is_big); -ET_DEFINE_CHECKED(Eterm*,float_val,Eterm,is_float); -ET_DEFINE_CHECKED(Eterm*,tuple_val,Eterm,is_tuple); +ET_DEFINE_CHECKED(Eterm*,big_val,Wterm,is_big); +ET_DEFINE_CHECKED(Eterm*,float_val,Wterm,is_float); +ET_DEFINE_CHECKED(Eterm*,tuple_val,Wterm,is_tuple); ET_DEFINE_CHECKED(Uint,internal_pid_data,Eterm,is_internal_pid); ET_DEFINE_CHECKED(struct erl_node_*,internal_pid_node,Eterm,is_internal_pid); ET_DEFINE_CHECKED(Uint,internal_port_data,Eterm,is_internal_port); ET_DEFINE_CHECKED(struct erl_node_*,internal_port_node,Eterm,is_internal_port); -ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Eterm,is_internal_ref); -ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Eterm,is_internal_ref); -ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Eterm,is_internal_ref); +ET_DEFINE_CHECKED(Eterm*,internal_ref_val,Wterm,is_internal_ref); +ET_DEFINE_CHECKED(Uint,internal_ref_data_words,Wterm,is_internal_ref); +ET_DEFINE_CHECKED(Uint32*,internal_ref_data,Wterm,is_internal_ref); ET_DEFINE_CHECKED(struct erl_node_*,internal_ref_node,Eterm,is_internal_ref); -ET_DEFINE_CHECKED(Eterm*,external_val,Eterm,is_external); -ET_DEFINE_CHECKED(Uint,external_data_words,Eterm,is_external); -ET_DEFINE_CHECKED(Uint,external_pid_data_words,Eterm,is_external_pid); -ET_DEFINE_CHECKED(Uint,external_pid_data,Eterm,is_external_pid); -ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Eterm,is_external_pid); -ET_DEFINE_CHECKED(Uint,external_port_data_words,Eterm,is_external_port); -ET_DEFINE_CHECKED(Uint,external_port_data,Eterm,is_external_port); -ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Eterm,is_external_port); -ET_DEFINE_CHECKED(Uint,external_ref_data_words,Eterm,is_external_ref); -ET_DEFINE_CHECKED(Uint32*,external_ref_data,Eterm,is_external_ref); +ET_DEFINE_CHECKED(Eterm*,external_val,Wterm,is_external); +ET_DEFINE_CHECKED(Uint,external_data_words,Wterm,is_external); +ET_DEFINE_CHECKED(Uint,external_pid_data_words,Wterm,is_external_pid); +ET_DEFINE_CHECKED(Uint,external_pid_data,Wterm,is_external_pid); +ET_DEFINE_CHECKED(struct erl_node_*,external_pid_node,Wterm,is_external_pid); +ET_DEFINE_CHECKED(Uint,external_port_data_words,Wterm,is_external_port); +ET_DEFINE_CHECKED(Uint,external_port_data,Wterm,is_external_port); +ET_DEFINE_CHECKED(struct erl_node_*,external_port_node,Wterm,is_external_port); +ET_DEFINE_CHECKED(Uint,external_ref_data_words,Wterm,is_external_ref); +ET_DEFINE_CHECKED(Uint32*,external_ref_data,Wterm,is_external_ref); ET_DEFINE_CHECKED(struct erl_node_*,external_ref_node,Eterm,is_external_ref); -ET_DEFINE_CHECKED(Eterm*,export_val,Eterm,is_export); +ET_DEFINE_CHECKED(Eterm*,export_val,Wterm,is_export); +ET_DEFINE_CHECKED(Uint,external_thing_data_words,ExternalThing*,is_thing_ptr); ET_DEFINE_CHECKED(Eterm,make_cp,UWord *,_is_taggable_pointer); ET_DEFINE_CHECKED(UWord *,cp_val,Eterm,is_CP); diff --git a/erts/emulator/beam/erl_term.h b/erts/emulator/beam/erl_term.h index b8e4473141..1d75fa313c 100644 --- a/erts/emulator/beam/erl_term.h +++ b/erts/emulator/beam/erl_term.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2010. All Rights Reserved. + * Copyright Ericsson AB 2000-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -22,6 +22,8 @@ #include "sys.h" /* defines HALFWORD_HEAP */ +typedef UWord Wterm; /* Full word terms */ + #if HALFWORD_HEAP # define HEAP_ON_C_STACK 0 # if HALFWORD_ASSERT @@ -193,7 +195,7 @@ struct erl_node_; /* Declared in erl_node_tables.h */ #endif #define _is_aligned(x) (((Uint)(x) & 0x3) == 0) #define _unchecked_make_boxed(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_BOXED) -_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*); +_ET_DECLARE_CHECKED(Eterm,make_boxed,Eterm*) #define make_boxed(x) _ET_APPLY(make_boxed,(x)) #if 1 #define _is_not_boxed(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_BOXED)) @@ -204,12 +206,12 @@ _ET_DECLARE_CHECKED(int,is_boxed,Eterm) #define is_boxed(x) (((x) & _TAG_PRIMARY_MASK) == TAG_PRIMARY_BOXED) #endif #define _unchecked_boxed_val(x) ((Eterm*) EXPAND_POINTER(((x) - TAG_PRIMARY_BOXED))) -_ET_DECLARE_CHECKED(Eterm*,boxed_val,Eterm); +_ET_DECLARE_CHECKED(Eterm*,boxed_val,Wterm) #define boxed_val(x) _ET_APPLY(boxed_val,(x)) /* cons cell ("list") access methods */ #define _unchecked_make_list(x) ((Uint) COMPRESS_POINTER(x) + TAG_PRIMARY_LIST) -_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*); +_ET_DECLARE_CHECKED(Eterm,make_list,Eterm*) #define make_list(x) _ET_APPLY(make_list,(x)) #if 1 #define _unchecked_is_not_list(x) ((x) & (_TAG_PRIMARY_MASK-TAG_PRIMARY_LIST)) @@ -226,7 +228,7 @@ _ET_DECLARE_CHECKED(int,is_not_list,Eterm) #define _list_precond(x) (is_list(x)) #endif #define _unchecked_list_val(x) ((Eterm*) EXPAND_POINTER((x) - TAG_PRIMARY_LIST)) -_ET_DECLARE_CHECKED(Eterm*,list_val,Eterm); +_ET_DECLARE_CHECKED(Eterm*,list_val,Wterm) #define list_val(x) _ET_APPLY(list_val,(x)) #define CONS(hp, car, cdr) \ @@ -240,6 +242,8 @@ _ET_DECLARE_CHECKED(Eterm*,list_val,Eterm); #define ptr_val(x) _unchecked_ptr_val((x)) /*XXX*/ #define _unchecked_offset_ptr(x,offs) ((x)+((offs)*sizeof(Eterm))) #define offset_ptr(x,offs) _unchecked_offset_ptr(x,offs) /*XXX*/ +#define _unchecked_byte_offset_ptr(x,byte_offs) ((x)+(offs)) +#define byte_offset_ptr(x,offs) _unchecked_byte_offset_ptr(x,offs) /*XXX*/ /* fixnum ("small") access methods */ #if defined(ARCH_64) && !HALFWORD_HEAP @@ -305,6 +309,7 @@ _ET_DECLARE_CHECKED(Uint,arityval,Eterm) /* thing access methods */ #define is_thing(x) (is_header((x)) && header_is_thing((x))) +#define is_thing_ptr(t) (is_thing((t)->header)) #define _unchecked_thing_arityval(x) _unchecked_header_arity((x)) _ET_DECLARE_CHECKED(Uint,thing_arityval,Eterm) #define thing_arityval(x) _ET_APPLY(thing_arityval,(x)) @@ -339,7 +344,7 @@ _ET_DECLARE_CHECKED(Uint,thing_subtag,Eterm) #define is_binary(x) (is_boxed((x)) && is_binary_header(*boxed_val((x)))) #define is_not_binary(x) (!is_binary((x))) #define _unchecked_binary_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,binary_val,Wterm) #define binary_val(x) _ET_APPLY(binary_val,(x)) /* process binaries stuff (special case of binaries) */ @@ -356,7 +361,7 @@ _ET_DECLARE_CHECKED(Eterm*,binary_val,Eterm) #define is_fun(x) (is_boxed((x)) && is_fun_header(*boxed_val((x)))) #define is_not_fun(x) (!is_fun((x))) #define _unchecked_fun_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,fun_val,Wterm) #define fun_val(x) _ET_APPLY(fun_val,(x)) /* export access methods */ @@ -364,7 +369,7 @@ _ET_DECLARE_CHECKED(Eterm*,fun_val,Eterm) #define is_export(x) (is_boxed((x)) && is_export_header(*boxed_val((x)))) #define is_not_export(x) (!is_export((x))) #define _unchecked_export_val(x) _unchecked_boxed_val(x) -_ET_DECLARE_CHECKED(Eterm*,export_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,export_val,Wterm) #define export_val(x) _ET_APPLY(export_val,(x)) #define is_export_header(x) ((x) == HEADER_EXPORT) #if HALFWORD_HEAP @@ -391,7 +396,7 @@ _ET_DECLARE_CHECKED(Uint,bignum_header_arity,Eterm) #define is_big(x) (is_boxed((x)) && _is_bignum_header(*boxed_val((x)))) #define is_not_big(x) (!is_big((x))) #define _unchecked_big_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,big_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,big_val,Wterm) #define big_val(x) _ET_APPLY(big_val,(x)) /* flonum ("float") access methods */ @@ -404,7 +409,7 @@ _ET_DECLARE_CHECKED(Eterm*,big_val,Eterm) #define is_float(x) (is_boxed((x)) && *boxed_val((x)) == HEADER_FLONUM) #define is_not_float(x) (!is_float(x)) #define _unchecked_float_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,float_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,float_val,Wterm) #define float_val(x) _ET_APPLY(float_val,(x)) /* Float definition for byte and word access */ @@ -422,15 +427,16 @@ typedef union float_def } FloatDef; #if defined(ARCH_64) && !HALFWORD_HEAP -#define GET_DOUBLE(x, f) (f).fdw = *(float_val(x)+1) + +#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fdw = *((fval)+1) #define PUT_DOUBLE(f, x) *(x) = HEADER_FLONUM, \ *((x)+1) = (f).fdw #define GET_DOUBLE_DATA(p, f) (f).fdw = *((Uint *) (p)) #define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fdw #else -#define GET_DOUBLE(x, f) (f).fw[0] = *(float_val(x)+1), \ - (f).fw[1] = *(float_val(x)+2) +#define FLOAT_VAL_GET_DOUBLE(fval, f) (f).fw[0] = *((fval)+1), \ + (f).fw[1] = *((fval)+2) #define PUT_DOUBLE(f, x) *(x) = HEADER_FLONUM, \ *((x)+1) = (f).fw[0], \ @@ -440,6 +446,9 @@ typedef union float_def #define PUT_DOUBLE_DATA(f,p) *((Uint *) (p)) = (f).fw[0],\ *(((Uint *) (p))+1) = (f).fw[1] #endif + +#define GET_DOUBLE(x, f) FLOAT_VAL_GET_DOUBLE(float_val(x), f) + #define DOUBLE_DATA_WORDS (sizeof(ieee754_8)/sizeof(Eterm)) #define FLOAT_SIZE_OBJECT (DOUBLE_DATA_WORDS+1) @@ -451,7 +460,7 @@ typedef union float_def (is_boxed((x)) && *boxed_val((x)) == make_arityval((a))) #define is_not_tuple_arity(x, a) (!is_tuple_arity((x),(a))) #define _unchecked_tuple_val(x) _unchecked_boxed_val(x) -_ET_DECLARE_CHECKED(Eterm*,tuple_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,tuple_val,Wterm) #define tuple_val(x) _ET_APPLY(tuple_val,(x)) #define TUPLE0(t) \ @@ -790,21 +799,24 @@ do { \ ((RefThing*) internal_ref_val(x)) #define is_internal_ref(x) \ - (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x)))) + (_unchecked_is_boxed((x)) && is_ref_thing_header(*boxed_val((x)))) + #define is_not_internal_ref(x) \ (!is_internal_ref((x))) #define _unchecked_internal_ref_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,internal_ref_val,Wterm) #define internal_ref_val(x) _ET_APPLY(internal_ref_val,(x)) +#define internal_thing_ref_data_words(t) (thing_arityval(*(Eterm*)(t))) #define _unchecked_internal_ref_data_words(x) \ (_unchecked_thing_arityval(*_unchecked_internal_ref_val(x))) -_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Eterm) +_ET_DECLARE_CHECKED(Uint,internal_ref_data_words,Wterm) #define internal_ref_data_words(x) _ET_APPLY(internal_ref_data_words,(x)) -#define _unchecked_internal_ref_data(x) (_unchecked_ref_thing_ptr(x)->data.ui32) -_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Eterm) +#define internal_thing_ref_data(thing) ((thing)->data.ui32) +#define _unchecked_internal_ref_data(x) (internal_thing_ref_data(_unchecked_ref_thing_ptr(x))) +_ET_DECLARE_CHECKED(Uint32*,internal_ref_data,Wterm) #define internal_ref_data(x) _ET_APPLY(internal_ref_data,(x)) #define _unchecked_internal_ref_node(x) erts_this_node @@ -889,14 +901,14 @@ typedef struct external_thing_ { #define is_external_header(x) \ (((x) & (_TAG_HEADER_MASK-_BINARY_XXX_MASK)) == _TAG_HEADER_EXTERNAL_PID) -#define is_external(x) \ - (is_boxed((x)) && is_external_header(*boxed_val((x)))) +#define is_external(x) (is_boxed((x)) && is_external_header(*boxed_val((x)))) + #define is_external_pid(x) \ (is_boxed((x)) && is_external_pid_header(*boxed_val((x)))) #define is_external_port(x) \ - (is_boxed((x)) && is_external_port_header(*boxed_val((x)))) -#define is_external_ref(x) \ - (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x)))) + (is_boxed((x)) && is_external_port_header(*boxed_val((x)))) + +#define is_external_ref(x) (_unchecked_is_boxed((x)) && is_external_ref_header(*boxed_val((x)))) #define _unchecked_is_external(x) \ (_unchecked_is_boxed((x)) && is_external_header(*_unchecked_boxed_val((x)))) @@ -914,17 +926,21 @@ typedef struct external_thing_ { #define make_external_ref make_external #define _unchecked_external_val(x) _unchecked_boxed_val((x)) -_ET_DECLARE_CHECKED(Eterm*,external_val,Eterm) +_ET_DECLARE_CHECKED(Eterm*,external_val,Wterm) #define external_val(x) _ET_APPLY(external_val,(x)) #define external_thing_ptr(x) ((ExternalThing *) external_val((x))) #define _unchecked_external_thing_ptr(x) \ ((ExternalThing *) _unchecked_external_val((x))) +#define _unchecked_external_thing_data_words(thing) \ + (_unchecked_thing_arityval((thing)->header) + (1 - EXTERNAL_THING_HEAD_SIZE)) +_ET_DECLARE_CHECKED(Uint,external_thing_data_words,ExternalThing*) +#define external_thing_data_words(thing) _ET_APPLY(external_thing_data_words,(thing)) + #define _unchecked_external_data_words(x) \ - (_unchecked_thing_arityval(_unchecked_external_thing_ptr((x))->header) \ - + (1 - EXTERNAL_THING_HEAD_SIZE)) -_ET_DECLARE_CHECKED(Uint,external_data_words,Eterm) + _unchecked_external_thing_data_words(_unchecked_external_thing_ptr((x))) +_ET_DECLARE_CHECKED(Uint,external_data_words,Wterm) #define external_data_words(x) _ET_APPLY(external_data_words,(x)) #define _unchecked_external_data(x) (_unchecked_external_thing_ptr((x))->data.ui) @@ -935,15 +951,15 @@ _ET_DECLARE_CHECKED(Uint,external_data_words,Eterm) #define _unchecked_external_pid_data_words(x) \ _unchecked_external_data_words((x)) -_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Eterm) +_ET_DECLARE_CHECKED(Uint,external_pid_data_words,Wterm) #define external_pid_data_words(x) _ET_APPLY(external_pid_data_words,(x)) #define _unchecked_external_pid_data(x) _unchecked_external_data((x))[0] -_ET_DECLARE_CHECKED(Uint,external_pid_data,Eterm) +_ET_DECLARE_CHECKED(Uint,external_pid_data,Wterm) #define external_pid_data(x) _ET_APPLY(external_pid_data,(x)) #define _unchecked_external_pid_node(x) _unchecked_external_node((x)) -_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm) +_ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Wterm) #define external_pid_node(x) _ET_APPLY(external_pid_node,(x)) #define external_pid_number(x) _GET_PID_NUM(external_pid_data((x))) @@ -951,27 +967,29 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_pid_node,Eterm) #define _unchecked_external_port_data_words(x) \ _unchecked_external_data_words((x)) -_ET_DECLARE_CHECKED(Uint,external_port_data_words,Eterm) +_ET_DECLARE_CHECKED(Uint,external_port_data_words,Wterm) #define external_port_data_words(x) _ET_APPLY(external_port_data_words,(x)) #define _unchecked_external_port_data(x) _unchecked_external_data((x))[0] -_ET_DECLARE_CHECKED(Uint,external_port_data,Eterm) +_ET_DECLARE_CHECKED(Uint,external_port_data,Wterm) #define external_port_data(x) _ET_APPLY(external_port_data,(x)) #define _unchecked_external_port_node(x) _unchecked_external_node((x)) -_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Eterm) +_ET_DECLARE_CHECKED(struct erl_node_*,external_port_node,Wterm) #define external_port_node(x) _ET_APPLY(external_port_node,(x)) #define external_port_number(x) _GET_PORT_NUM(external_port_data((x))) #define _unchecked_external_ref_data_words(x) \ _unchecked_external_data_words((x)) -_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Eterm) +_ET_DECLARE_CHECKED(Uint,external_ref_data_words,Wterm) #define external_ref_data_words(x) _ET_APPLY(external_ref_data_words,(x)) +#define external_thing_ref_data_words(thing) external_thing_data_words(thing) #define _unchecked_external_ref_data(x) (_unchecked_external_thing_ptr((x))->data.ui32) -_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Eterm) +_ET_DECLARE_CHECKED(Uint32*,external_ref_data,Wterm) #define external_ref_data(x) _ET_APPLY(external_ref_data,(x)) +#define external_thing_ref_data(thing) ((thing)->data.ui32) #define _unchecked_external_ref_node(x) _unchecked_external_node((x)) _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) @@ -995,14 +1013,14 @@ _ET_DECLARE_CHECKED(struct erl_node_*,external_ref_node,Eterm) #endif #define _unchecked_make_cp(x) ((Eterm) COMPRESS_POINTER(x)) -_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*); +_ET_DECLARE_CHECKED(Eterm,make_cp,BeamInstr*) #define make_cp(x) _ET_APPLY(make_cp,(x)) #define is_not_CP(x) ((x) & _CPMASK) #define is_CP(x) (!is_not_CP(x)) #define _unchecked_cp_val(x) ((BeamInstr*) EXPAND_POINTER(x)) -_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm); +_ET_DECLARE_CHECKED(BeamInstr*,cp_val,Eterm) #define cp_val(x) _ET_APPLY(cp_val,(x)) #define make_catch(x) (((x) << _TAG_IMMED2_SIZE) | _TAG_IMMED2_CATCH) @@ -1083,10 +1101,10 @@ _ET_DECLARE_CHECKED(Uint,y_reg_index,Uint) #define SMALL_DEF 0xf #if ET_DEBUG -extern unsigned tag_val_def_debug(Eterm, const char*, unsigned); +extern unsigned tag_val_def_debug(Wterm, const char*, unsigned); #define tag_val_def(x) tag_val_def_debug((x),__FILE__,__LINE__) #else -extern unsigned tag_val_def(Eterm); +extern unsigned tag_val_def(Wterm); #endif #define not_eq_tags(X,Y) (tag_val_def((X)) ^ tag_val_def((Y))) @@ -1102,5 +1120,81 @@ extern unsigned tag_val_def(Eterm); #define FLOAT_BIG _NUMBER_CODE(FLOAT_DEF,BIG_DEF) #define FLOAT_FLOAT _NUMBER_CODE(FLOAT_DEF,FLOAT_DEF) +#if HALFWORD_HEAP +#define ptr2rel(PTR,BASE) ((Eterm*)((char*)(PTR) - (char*)(BASE))) +#define rterm2wterm(REL,BASE) ((Wterm)(REL) + (Wterm)(BASE)) + +#else /* HALFWORD_HEAP */ + +#define ptr2rel(PTR,BASE) (PTR) +#define rterm2wterm(REL,BASE) (REL) + +#endif /* !HALFWORD_HEAP */ + +#define make_list_rel(PTR, BASE) make_list(ptr2rel(PTR,BASE)) +#define make_boxed_rel(PTR, BASE) make_boxed(ptr2rel(PTR,BASE)) +#define make_fun_rel make_boxed_rel +#define make_binary_rel make_boxed_rel +#define make_tuple_rel make_boxed_rel +#define make_external_rel make_boxed_rel +#define make_internal_ref_rel make_boxed_rel + +#define binary_val_rel(RTERM, BASE) binary_val(rterm2wterm(RTERM, BASE)) +#define list_val_rel(RTERM, BASE) list_val(rterm2wterm(RTERM, BASE)) +#define boxed_val_rel(RTERM, BASE) boxed_val(rterm2wterm(RTERM, BASE)) +#define tuple_val_rel(RTERM, BASE) tuple_val(rterm2wterm(RTERM, BASE)) +#define export_val_rel(RTERM, BASE) export_val(rterm2wterm(RTERM, BASE)) +#define fun_val_rel(RTERM, BASE) fun_val(rterm2wterm(RTERM, BASE)) +#define big_val_rel(RTERM,BASE) big_val(rterm2wterm(RTERM,BASE)) +#define float_val_rel(RTERM,BASE) float_val(rterm2wterm(RTERM,BASE)) +#define internal_ref_val_rel(RTERM,BASE) internal_ref_val(rterm2wterm(RTERM,BASE)) + +#define external_thing_ptr_rel(RTERM, BASE) external_thing_ptr(rterm2wterm(RTERM, BASE)) +#define external_data_words_rel(RTERM,BASE) external_data_words(rterm2wterm(RTERM,BASE)) + +#define external_port_node_rel(RTERM,BASE) external_port_node(rterm2wterm(RTERM,BASE)) +#define external_port_data_rel(RTERM,BASE) external_port_data(rterm2wterm(RTERM,BASE)) + +#define is_external_pid_rel(RTERM,BASE) is_external_pid(rterm2wterm(RTERM,BASE)) +#define external_pid_node_rel(RTERM,BASE) external_pid_node(rterm2wterm(RTERM,BASE)) +#define external_pid_data_rel(RTERM,BASE) external_pid_data(rterm2wterm(RTERM,BASE)) + +#define is_binary_rel(RTERM,BASE) is_binary(rterm2wterm(RTERM,BASE)) +#define is_float_rel(RTERM,BASE) is_float(rterm2wterm(RTERM,BASE)) +#define is_fun_rel(RTERM,BASE) is_fun(rterm2wterm(RTERM,BASE)) +#define is_big_rel(RTERM,BASE) is_big(rterm2wterm(RTERM,BASE)) +#define is_export_rel(RTERM,BASE) is_export(rterm2wterm(RTERM,BASE)) +#define is_tuple_rel(RTERM,BASE) is_tuple(rterm2wterm(RTERM,BASE)) + +#define GET_DOUBLE_REL(RTERM, f, BASE) GET_DOUBLE(rterm2wterm(RTERM,BASE), f) + +#define ref_thing_ptr_rel(RTERM,BASE) ref_thing_ptr(rterm2wterm(RTERM,BASE)) +#define is_internal_ref_rel(RTERM,BASE) is_internal_ref(rterm2wterm(RTERM,BASE)) +#define is_external_rel(RTERM,BASE) is_external(rterm2wterm(RTERM,BASE)) +#define is_external_port_rel(RTERM,BASE) is_external_port(rterm2wterm(RTERM,BASE)) +#define is_external_ref_rel(RTERM,BASE) is_external_ref(rterm2wterm(RTERM,BASE)) + +#define external_node_rel(RTERM,BASE) external_node(rterm2wterm(RTERM,BASE)) + + +#if HALFWORD_HEAP +ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE int is_same(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) +{ + /* If bases differ, assume a and b are on different "heaps", + ie can only be same if immed */ + ASSERT(a_base == b_base || is_immed(a) || is_immed(b) + || rterm2wterm(a,a_base) != rterm2wterm(b,b_base)); + + return a == b && (a_base == b_base || is_immed(a)); +} +#endif + +#else /* !HALFWORD_HEAP */ +#define is_same(A,A_BASE,B,B_BASE) ((A)==(B)) +#endif + #endif /* __ERL_TERM_H */ diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 0b7269262e..8c9cace0c5 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -27,9 +27,6 @@ #define ERTS_SPIN_BODY ETHR_SPIN_BODY -#define ERTS_MAX_READER_GROUPS 8 -extern int erts_reader_groups; - #include "sys.h" #ifdef USE_THREADS @@ -39,6 +36,16 @@ extern int erts_reader_groups; #include "erl_lock_count.h" #include "erl_term.h" +#if defined(__GLIBC__) && (__GLIBC__ << 16) + __GLIBC_MINOR__ < (2 << 16) + 4 +/* + * pthread_mutex_destroy() may return EBUSY when it shouldn't :( We have + * only seen this bug in glibc versions before 2.4. Note that condition + * variables, rwmutexes, spinlocks, and rwspinlocks also may be effected by + * this bug since these implementations may use mutexes internally. + */ +# define ERTS_THR_HAVE_BUSY_DESTROY_BUG +#endif + #define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER #ifdef ERTS_ENABLE_LOCK_COUNT @@ -92,7 +99,10 @@ typedef ethr_rwmutex_opt erts_rwmtx_opt_t; typedef ethr_tsd_key erts_tsd_key_t; typedef ethr_ts_event erts_tse_t; +typedef ethr_sint_t erts_aint_t; typedef ethr_atomic_t erts_atomic_t; +typedef ethr_sint32_t erts_aint32_t; +typedef ethr_atomic32_t erts_atomic32_t; /* spinlock */ typedef struct { @@ -116,7 +126,6 @@ typedef struct { #endif } erts_rwlock_t; -typedef ethr_timeval erts_thr_timeval_t; __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ @@ -155,7 +164,10 @@ typedef struct { typedef int erts_rwmtx_t; typedef int erts_tsd_key_t; typedef int erts_tse_t; -typedef long erts_atomic_t; +typedef SWord erts_aint_t; +typedef SWord erts_atomic_t; +typedef SWord erts_aint32_t; +typedef SWord erts_atomic32_t; #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; typedef struct { } erts_rwlock_t; @@ -163,10 +175,6 @@ typedef struct { } erts_rwlock_t; typedef struct { int gcc_is_buggy; } erts_spinlock_t; typedef struct { int gcc_is_buggy; } erts_rwlock_t; #endif -typedef struct { - long tv_sec; - long tv_nsec; -} erts_thr_timeval_t; #define ERTS_MTX_INITER 0 #define ERTS_CND_INITER 0 @@ -176,6 +184,11 @@ typedef struct { #endif /* #ifdef USE_THREADS */ +#define ERTS_AINT_T_MAX (~(((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) +#define ERTS_AINT_T_MIN ((((erts_aint_t) 1) << (sizeof(erts_aint_t)*8-1))) +#define ERTS_AINT32_T_MAX (~(((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1))) +#define ERTS_AINT32_T_MIN ((((erts_aint32_t) 1) << (sizeof(erts_aint32_t)*8-1))) + ERTS_GLB_INLINE void erts_thr_init(erts_thr_init_data_t *id); ERTS_GLB_INLINE void erts_thr_late_init(erts_thr_late_init_data_t *id); ERTS_GLB_INLINE void erts_thr_create(erts_tid_t *tid, void * (*func)(void *), @@ -234,33 +247,65 @@ ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx); ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx); -ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, long i); -ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, long i); -ERTS_GLB_INLINE long erts_atomic_read(erts_atomic_t *var); -ERTS_GLB_INLINE long erts_atomic_inctest(erts_atomic_t *incp); -ERTS_GLB_INLINE long erts_atomic_dectest(erts_atomic_t *decp); +ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var); +ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp); ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp); ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp); -ERTS_GLB_INLINE long erts_atomic_addtest(erts_atomic_t *addp, - long i); -ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, long i); -ERTS_GLB_INLINE long erts_atomic_xchg(erts_atomic_t *xchgp, - long new); -ERTS_GLB_INLINE long erts_atomic_cmpxchg(erts_atomic_t *xchgp, - long new, - long expected); -ERTS_GLB_INLINE long erts_atomic_bor(erts_atomic_t *var, long mask); -ERTS_GLB_INLINE long erts_atomic_band(erts_atomic_t *var, long mask); -ERTS_GLB_INLINE long erts_atomic_read_acqb(erts_atomic_t *var); -ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, long i); +ERTS_GLB_INLINE erts_aint_t erts_atomic_addtest(erts_atomic_t *addp, + erts_aint_t i); +ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, erts_aint_t i); +ERTS_GLB_INLINE erts_aint_t erts_atomic_xchg(erts_atomic_t *xchgp, + erts_aint_t new); +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t expected); +ERTS_GLB_INLINE erts_aint_t erts_atomic_bor(erts_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_atomic_band(erts_atomic_t *var, + erts_aint_t mask); +ERTS_GLB_INLINE erts_aint_t erts_atomic_read_acqb(erts_atomic_t *var); +ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i); ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp); -ERTS_GLB_INLINE long erts_atomic_dectest_relb(erts_atomic_t *decp); -ERTS_GLB_INLINE long erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, - long new, - long exp); -ERTS_GLB_INLINE long erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, - long new, - long exp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest_relb(erts_atomic_t *decp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp); +ERTS_GLB_INLINE void erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read(erts_atomic32_t *var); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_inctest(erts_atomic32_t *incp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest(erts_atomic32_t *decp); +ERTS_GLB_INLINE void erts_atomic32_inc(erts_atomic32_t *incp); +ERTS_GLB_INLINE void erts_atomic32_dec(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_addtest(erts_atomic32_t *addp, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_xchg(erts_atomic32_t *xchgp, + erts_aint32_t new); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_bor(erts_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_band(erts_atomic32_t *var, + erts_aint32_t mask); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_acqb(erts_atomic32_t *var); +ERTS_GLB_INLINE void erts_atomic32_set_relb(erts_atomic32_t *var, + erts_aint32_t i); +ERTS_GLB_INLINE void erts_atomic32_dec_relb(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest_relb(erts_atomic32_t *decp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); +ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp); ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, @@ -295,7 +340,6 @@ ERTS_GLB_INLINE void erts_write_lock(erts_rwlock_t *lock); ERTS_GLB_INLINE void erts_write_unlock(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rlocked(erts_rwlock_t *lock); ERTS_GLB_INLINE int erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock); -ERTS_GLB_INLINE void erts_thr_time_now(erts_thr_timeval_t *time); ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp); ERTS_GLB_INLINE void erts_tsd_key_delete(erts_tsd_key_t key); ERTS_GLB_INLINE void erts_tsd_set(erts_tsd_key_t key, void *value); @@ -520,8 +564,16 @@ erts_mtx_destroy(erts_mtx_t *mtx) erts_lcnt_destroy_lock(&mtx->lcnt); #endif res = ethr_mutex_destroy(&mtx->mtx); - if (res) + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy mutex destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } +#endif erts_thr_fatal_error(res, "destroy mutex"); + } #endif } @@ -616,8 +668,16 @@ erts_cnd_destroy(erts_cnd_t *cnd) { #ifdef USE_THREADS int res = ethr_cond_destroy(cnd); - if (res) + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy cond destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } +#endif erts_thr_fatal_error(res, "destroy condition variable"); + } #endif } @@ -743,8 +803,16 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) erts_lcnt_destroy_lock(&rwmtx->lcnt); #endif res = ethr_rwmutex_destroy(&rwmtx->rwmtx); - if (res != 0) + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy rwmutex destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } +#endif erts_thr_fatal_error(res, "destroy rwmutex"); + } #endif } @@ -928,7 +996,7 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx) } ERTS_GLB_INLINE void -erts_atomic_init(erts_atomic_t *var, long i) +erts_atomic_init(erts_atomic_t *var, erts_aint_t i) { #ifdef USE_THREADS ethr_atomic_init(var, i); @@ -938,7 +1006,7 @@ erts_atomic_init(erts_atomic_t *var, long i) } ERTS_GLB_INLINE void -erts_atomic_set(erts_atomic_t *var, long i) +erts_atomic_set(erts_atomic_t *var, erts_aint_t i) { #ifdef USE_THREADS ethr_atomic_set(var, i); @@ -947,7 +1015,7 @@ erts_atomic_set(erts_atomic_t *var, long i) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var) { #ifdef USE_THREADS @@ -957,7 +1025,7 @@ erts_atomic_read(erts_atomic_t *var) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp) { #ifdef USE_THREADS @@ -967,7 +1035,7 @@ erts_atomic_inctest(erts_atomic_t *incp) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp) { #ifdef USE_THREADS @@ -997,8 +1065,8 @@ erts_atomic_dec(erts_atomic_t *decp) #endif } -ERTS_GLB_INLINE long -erts_atomic_addtest(erts_atomic_t *addp, long i) +ERTS_GLB_INLINE erts_aint_t +erts_atomic_addtest(erts_atomic_t *addp, erts_aint_t i) { #ifdef USE_THREADS return ethr_atomic_add_read(addp, i); @@ -1008,7 +1076,7 @@ erts_atomic_addtest(erts_atomic_t *addp, long i) } ERTS_GLB_INLINE void -erts_atomic_add(erts_atomic_t *addp, long i) +erts_atomic_add(erts_atomic_t *addp, erts_aint_t i) { #ifdef USE_THREADS ethr_atomic_add(addp, i); @@ -1017,59 +1085,58 @@ erts_atomic_add(erts_atomic_t *addp, long i) #endif } -ERTS_GLB_INLINE long -erts_atomic_xchg(erts_atomic_t *xchgp, long new) +ERTS_GLB_INLINE erts_aint_t +erts_atomic_xchg(erts_atomic_t *xchgp, erts_aint_t new) { - long old; #ifdef USE_THREADS return ethr_atomic_xchg(xchgp, new); #else - old = *xchgp; + erts_aint_t old = *xchgp; *xchgp = new; -#endif return old; +#endif } -ERTS_GLB_INLINE long -erts_atomic_cmpxchg(erts_atomic_t *xchgp, long new, long expected) +ERTS_GLB_INLINE erts_aint_t +erts_atomic_cmpxchg(erts_atomic_t *xchgp, erts_aint_t new, erts_aint_t expected) { #ifdef USE_THREADS return ethr_atomic_cmpxchg(xchgp, new, expected); #else - long old = *xchgp; + erts_aint_t old = *xchgp; if (old == expected) *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long -erts_atomic_bor(erts_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint_t +erts_atomic_bor(erts_atomic_t *var, erts_aint_t mask) { #ifdef USE_THREADS return ethr_atomic_read_bor(var, mask); #else - long old; + erts_aint_t old; old = *var; *var |= mask; return old; #endif } -ERTS_GLB_INLINE long -erts_atomic_band(erts_atomic_t *var, long mask) +ERTS_GLB_INLINE erts_aint_t +erts_atomic_band(erts_atomic_t *var, erts_aint_t mask) { #ifdef USE_THREADS return ethr_atomic_read_band(var, mask); #else - long old; + erts_aint_t old; old = *var; *var &= mask; return old; #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_read_acqb(erts_atomic_t *var) { #ifdef USE_THREADS @@ -1080,7 +1147,7 @@ erts_atomic_read_acqb(erts_atomic_t *var) } ERTS_GLB_INLINE void -erts_atomic_set_relb(erts_atomic_t *var, long i) +erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i) { #ifdef USE_THREADS ethr_atomic_set_relb(var, i); @@ -1099,7 +1166,7 @@ erts_atomic_dec_relb(erts_atomic_t *decp) #endif } -ERTS_GLB_INLINE long +ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest_relb(erts_atomic_t *decp) { #ifdef USE_THREADS @@ -1109,28 +1176,243 @@ erts_atomic_dectest_relb(erts_atomic_t *decp) #endif } -ERTS_GLB_INLINE long erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, - long new, - long exp) +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) { #ifdef USE_THREADS return ethr_atomic_cmpxchg_acqb(xchgp, new, exp); #else - long old = *xchgp; + erts_aint_t old = *xchgp; if (old == exp) *xchgp = new; return old; #endif } -ERTS_GLB_INLINE long erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, - long new, - long exp) +ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp, + erts_aint_t new, + erts_aint_t exp) { #ifdef USE_THREADS return ethr_atomic_cmpxchg_relb(xchgp, new, exp); #else - long old = *xchgp; + erts_aint_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +/* atomic32 */ + +ERTS_GLB_INLINE void +erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_init(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_set(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read(erts_atomic32_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic32_read(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_inctest(erts_atomic32_t *incp) +{ +#ifdef USE_THREADS + return ethr_atomic32_inc_read(incp); +#else + return ++(*incp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_dectest(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic32_dec_read(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_inc(erts_atomic32_t *incp) +{ +#ifdef USE_THREADS + ethr_atomic32_inc(incp); +#else + ++(*incp); +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_dec(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic32_dec(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_addtest(erts_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef USE_THREADS + return ethr_atomic32_add_read(addp, i); +#else + return *addp += i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_add(addp, i); +#else + *addp += i; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_xchg(erts_atomic32_t *xchgp, erts_aint32_t new) +{ +#ifdef USE_THREADS + return ethr_atomic32_xchg(xchgp, new); +#else + erts_aint32_t old = *xchgp; + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t expected) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg(xchgp, new, expected); +#else + erts_aint32_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_bor(erts_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef USE_THREADS + return ethr_atomic32_read_bor(var, mask); +#else + erts_aint32_t old; + old = *var; + *var |= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_band(erts_atomic32_t *var, erts_aint32_t mask) +{ +#ifdef USE_THREADS + return ethr_atomic32_read_band(var, mask); +#else + erts_aint32_t old; + old = *var; + *var &= mask; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_read_acqb(erts_atomic32_t *var) +{ +#ifdef USE_THREADS + return ethr_atomic32_read_acqb(var); +#else + return *var; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_set_relb(erts_atomic32_t *var, erts_aint32_t i) +{ +#ifdef USE_THREADS + ethr_atomic32_set_relb(var, i); +#else + *var = i; +#endif +} + +ERTS_GLB_INLINE void +erts_atomic32_dec_relb(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + ethr_atomic32_dec_relb(decp); +#else + --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_dectest_relb(erts_atomic32_t *decp) +{ +#ifdef USE_THREADS + return ethr_atomic32_dec_read_relb(decp); +#else + return --(*decp); +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg_acqb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; + if (old == exp) + *xchgp = new; + return old; +#endif +} + +ERTS_GLB_INLINE erts_aint32_t +erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp, + erts_aint32_t new, + erts_aint32_t exp) +{ +#ifdef USE_THREADS + return ethr_atomic32_cmpxchg_relb(xchgp, new, exp); +#else + erts_aint32_t old = *xchgp; if (old == exp) *xchgp = new; return old; @@ -1207,8 +1489,16 @@ erts_spinlock_destroy(erts_spinlock_t *lock) erts_lcnt_destroy_lock(&lock->lcnt); #endif res = ethr_spinlock_destroy(&lock->slck); - if (res) - erts_thr_fatal_error(res, "destroy spinlock"); + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy spinlock destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } +#endif + erts_thr_fatal_error(res, "destroy rwlock"); + } #else (void)lock; #endif @@ -1317,8 +1607,16 @@ erts_rwlock_destroy(erts_rwlock_t *lock) erts_lcnt_destroy_lock(&lock->lcnt); #endif res = ethr_rwlock_destroy(&lock->rwlck); - if (res) + if (res != 0) { +#ifdef ERTS_THR_HAVE_BUSY_DESTROY_BUG + if (res == EBUSY) { + char *warn = "Ignoring busy rwlock destroy. " + "Most likely a bug in pthread implementation."; + erts_send_warning_to_logger_str_nogl(warn); + } +#endif erts_thr_fatal_error(res, "destroy rwlock"); + } #else (void)lock; #endif @@ -1431,16 +1729,6 @@ erts_lc_rwlock_is_rwlocked(erts_rwlock_t *lock) } ERTS_GLB_INLINE void -erts_thr_time_now(erts_thr_timeval_t *time) -{ -#ifdef USE_THREADS - int res = ethr_time_now(time); - if (res) - erts_thr_fatal_error(res, "get current time"); -#endif -} - -ERTS_GLB_INLINE void erts_tsd_key_create(erts_tsd_key_t *keyp) { #ifdef USE_THREADS diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 6f6b971d34..d0ad73cd81 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -20,11 +20,15 @@ #ifndef ERL_TIME_H__ #define ERL_TIME_H__ +extern erts_smp_atomic_t do_time; /* set at clock interrupt */ +extern SysTimeval erts_first_emu_time; + /* ** Timer entry: */ typedef struct erl_timer { struct erl_timer* next; /* next entry tiw slot or chain */ + struct erl_timer* prev; /* prev entry tiw slot or chain */ Uint slot; /* slot in timer wheel */ Uint count; /* number of loops remaining */ int active; /* 1=activated, 0=deactivated */ @@ -39,7 +43,6 @@ typedef void (*ErlTimeoutProc)(void*); typedef void (*ErlCancelProc)(void*); #ifdef ERTS_SMP - /* * Process and port timer */ @@ -61,7 +64,66 @@ void erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, ErlTimeoutProc timeout_func, Uint timeout); void erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer); +#endif + +/* timer-wheel api */ +void erts_init_time(void); +void erts_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); +void erts_cancel_timer(ErlTimer*); +void erts_bump_timer(erts_aint_t); +Uint erts_timer_wheel_memory_size(void); +Uint erts_time_left(ErlTimer *); +erts_aint_t erts_next_time(void); + +#ifdef DEBUG +void erts_p_slpq(void); #endif +ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void); +ERTS_GLB_INLINE void erts_do_time_add(long); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE erts_aint_t erts_do_time_read_and_reset(void) { return erts_smp_atomic_xchg(&do_time, 0L); } +ERTS_GLB_INLINE void erts_do_time_add(long elapsed) { erts_smp_atomic_add(&do_time, elapsed); } + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ + + +/* time_sup */ + +#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME)) +# ifndef HAVE_ERTS_NOW_CPU +# define HAVE_ERTS_NOW_CPU +# ifdef HAVE_GETHRVTIME +# define erts_start_now_cpu() sys_start_hrvtime() +# define erts_stop_now_cpu() sys_stop_hrvtime() +# endif +# endif +void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); #endif + +void erts_get_timeval(SysTimeval *tv); +long erts_get_time(void); +void erts_get_emu_time(SysTimeval *); + +ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE int +erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p) +{ + if (t1p->tv_sec == t2p->tv_sec) { + if (t1p->tv_usec < t2p->tv_usec) + return -1; + else if (t1p->tv_usec > t2p->tv_usec) + return 1; + return 0; + } + return t1p->tv_sec < t2p->tv_sec ? -1 : 1; +} + +#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ +#endif /* ERL_TIME_H__ */ diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 7b8706ea13..ca4b54188e 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -358,10 +358,6 @@ static int clock_resolution; ** instead of something like select. */ -#if defined(ERTS_TIMER_THREAD) -static ERTS_INLINE void init_erts_deliver_time(const SysTimeval *inittv) { } -static ERTS_INLINE void do_erts_deliver_time(const SysTimeval *current) { } -#else static SysTimeval last_delivered; static void init_erts_deliver_time(const SysTimeval *inittv) @@ -389,11 +385,10 @@ static void do_erts_deliver_time(const SysTimeval *current) this by simply pretend as if the time stood still. :) */ if (elapsed > 0) { - do_time_add(elapsed); + erts_do_time_add(elapsed); last_delivered = cur_time; } } -#endif int erts_init_time_sup(void) @@ -786,7 +781,6 @@ get_sys_now(Uint* megasec, Uint* sec, Uint* microsec) to a struct timeval representing current time (to save a gettimeofday() where possible) or NULL */ -#if !defined(ERTS_TIMER_THREAD) void erts_deliver_time(void) { SysTimeval now; @@ -797,7 +791,6 @@ void erts_deliver_time(void) { erts_smp_mtx_unlock(&erts_timeofday_mtx); } -#endif /* get *real* time (not ticks) remaining until next timeout - if there isn't one, give a "long" time, that is guaranteed @@ -806,14 +799,12 @@ void erts_deliver_time(void) { void erts_time_remaining(SysTimeval *rem_time) { int ticks; -#if !defined(ERTS_TIMER_THREAD) SysTimeval cur_time; -#endif long elapsed; - /* next_time() returns no of ticks to next timeout or -1 if none */ + /* erts_next_time() returns no of ticks to next timeout or -1 if none */ - if ((ticks = next_time()) == -1) { + if ((ticks = erts_next_time()) == -1) { /* timer queue empty */ /* this will cause at most 100000000 ticks */ rem_time->tv_sec = 100000; @@ -822,9 +813,6 @@ void erts_time_remaining(SysTimeval *rem_time) /* next timeout after ticks ticks */ ticks *= CLOCK_RESOLUTION; -#if defined(ERTS_TIMER_THREAD) - elapsed = 0; -#else erts_smp_mtx_lock(&erts_timeofday_mtx); get_tolerant_timeofday(&cur_time); @@ -839,7 +827,6 @@ void erts_time_remaining(SysTimeval *rem_time) rem_time->tv_sec = rem_time->tv_usec = 0; return; } -#endif rem_time->tv_sec = (ticks - elapsed) / 1000; rem_time->tv_usec = 1000 * ((ticks - elapsed) % 1000); } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 3043bb1e8c..c0397ca6c3 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2010. All Rights Reserved. + * Copyright Ericsson AB 1999-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1668,7 +1668,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, return_flags = 0; if (match_spec) { pam_result = erts_match_set_run(p, match_spec, args, arity, - &return_flags); + ERTS_PAM_TMP_RESULT, &return_flags); if (is_non_value(pam_result)) { erts_match_set_release_result(p); #if !HEAP_ON_C_STACK @@ -1815,7 +1815,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, return_flags = 0; if (match_spec) { pam_result = erts_match_set_run(p, match_spec, args, arity, - &return_flags); + ERTS_PAM_TMP_RESULT, &return_flags); if (is_non_value(pam_result)) { erts_match_set_release_result(p); UnUseTmpHeap(ERL_SUB_BIN_SIZE,p); diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index d01a3661f9..545b345a71 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -30,6 +30,8 @@ #include "big.h" #include "erl_unicode.h" +#include "erl_unicode_normalize.h" + typedef struct _restart_context { byte *bytes; @@ -54,13 +56,6 @@ static BIF_RETTYPE finalize_list_to_list(Process *p, Uint num_resulting_chars, int state, int left, Eterm tail); -static int analyze_utf8(byte *source, Uint size, - byte **err_pos, Uint *num_chars, int *left); -#define UTF8_OK 0 -#define UTF8_INCOMPLETE 1 -#define UTF8_ERROR 2 -#define UTF8_ANALYZE_MORE 3 - static BIF_RETTYPE characters_to_utf8_trap(BIF_ALIST_3); static BIF_RETTYPE characters_to_list_trap_1(BIF_ALIST_3); static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3); @@ -463,7 +458,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ } objp = list_val(ioterm); obj = CAR(objp); - if (!is_byte(obj)) + if (!is_small(obj)) break; } } else if (is_nil(obj)) { @@ -970,11 +965,11 @@ static int is_valid_utf8(Eterm orig_bin) bytes = erts_get_aligned_binary_bytes(orig_bin, &temp_alloc); } size = binary_size(orig_bin); - ret = analyze_utf8(bytes, + ret = erts_analyze_utf8(bytes, size, &endpos,&numchar,NULL); erts_free_aligned_binary_bytes(temp_alloc); - return (ret == UTF8_OK); + return (ret == ERTS_UTF8_OK); } BIF_RETTYPE unicode_characters_to_binary_2(BIF_ALIST_2) @@ -1084,14 +1079,14 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char hp += 2; rest_term = CONS(hp,leftover_bin,rest_term); } - BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, UTF8_ERROR, left, NIL)); + BIF_RET(finalize_list_to_list(p, bytes, rest_term, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL)); } else if (rest_term == NIL && num_leftovers != 0) { Eterm leftover_bin = new_binary(p, leftover, num_leftovers); if (check_leftovers(leftover,num_leftovers) != 0) { - BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_ERROR, + BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_ERROR, left, NIL)); } else { - BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, UTF8_INCOMPLETE, + BIF_RET(finalize_list_to_list(p, bytes, leftover_bin, 0U, pos, characters, ERTS_UTF8_INCOMPLETE, left, NIL)); } } else { /* All OK */ @@ -1107,11 +1102,11 @@ static BIF_RETTYPE build_list_return(Process *p, byte *bytes, int pos, Uint char rc.num_processed_bytes = 0; /* not used */ rc.num_bytes_to_process = pos; rc.num_resulting_chars = characters; - rc.state = UTF8_OK; /* not used */ + rc.state = ERTS_UTF8_OK; /* not used */ BIF_TRAP3(&characters_to_list_trap_1_exp, p, make_magic_bin_for_restart(p,&rc), rest_term, latin1); } else { /* Success */ - BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, UTF8_OK, left, NIL)); + BIF_RET(finalize_list_to_list(p, bytes, NIL, 0U, pos, characters, ERTS_UTF8_OK, left, NIL)); } } } @@ -1205,7 +1200,7 @@ BIF_RETTYPE unicode_characters_to_list_2(BIF_ALIST_2) * When input to characters_to_list is a plain binary and the format is 'unicode', we do * a faster analyze and size count with this function. */ -static int analyze_utf8(byte *source, Uint size, +int erts_analyze_utf8(byte *source, Uint size, byte **err_pos, Uint *num_chars, int *left) { *err_pos = source; @@ -1216,60 +1211,60 @@ static int analyze_utf8(byte *source, Uint size, --size; } else if (((*source) & ((byte) 0xE0)) == 0xC0) { if (size < 2) { - return UTF8_INCOMPLETE; + return ERTS_UTF8_INCOMPLETE; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((*source) < 0xC2) /* overlong */) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } source += 2; size -= 2; } else if (((*source) & ((byte) 0xF0)) == 0xE0) { if (size < 3) { - return UTF8_INCOMPLETE; + return ERTS_UTF8_INCOMPLETE; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((source[2] & ((byte) 0xC0)) != 0x80) || (((*source) == 0xE0) && (source[1] < 0xA0)) /* overlong */ ) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } if ((((*source) & ((byte) 0xF)) == 0xD) && ((source[1] & 0x20) != 0)) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } if (((*source) == 0xEF) && (source[1] == 0xBF) && ((source[2] == 0xBE) || (source[2] == 0xBF))) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } source += 3; size -= 3; } else if (((*source) & ((byte) 0xF8)) == 0xF0) { if (size < 4) { - return UTF8_INCOMPLETE; + return ERTS_UTF8_INCOMPLETE; } if (((source[1] & ((byte) 0xC0)) != 0x80) || ((source[2] & ((byte) 0xC0)) != 0x80) || ((source[3] & ((byte) 0xC0)) != 0x80) || (((*source) == 0xF0) && (source[1] < 0x90)) /* overlong */) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } if ((((*source) & ((byte)0x7)) > 0x4U) || ((((*source) & ((byte)0x7)) == 0x4U) && ((source[1] & ((byte)0x3F)) > 0xFU))) { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } source += 4; size -= 4; } else { - return UTF8_ERROR; + return ERTS_UTF8_ERROR; } ++(*num_chars); *err_pos = source; if (left && --(*left) <= 0) { - return UTF8_ANALYZE_MORE; + return ERTS_UTF8_ANALYZE_MORE; } } - return UTF8_OK; + return ERTS_UTF8_OK; } /* @@ -1304,7 +1299,7 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, } else if (((*source) & ((byte) 0xE0)) == 0xC0) { unipoint = (((Uint) ((*source) & ((byte) 0x1F))) << 6) | - ((Uint) (source[1] & ((byte) 0x3F))); + ((Uint) (source[1] & ((byte) 0x3F))); } else if (((*source) & ((byte) 0xF0)) == 0xE0) { unipoint = (((Uint) ((*source) & ((byte) 0xF))) << 12) | @@ -1330,6 +1325,216 @@ static Eterm do_utf8_to_list(Process *p, Uint num, byte *bytes, Uint sz, return ret; } +static int is_candidate(Uint cp) +{ + int index,pos; + if (cp < 768) return 0; + if (cp > 4023) { + if (cp == 12441 || cp == 12442) return 1; + return 0; + } + index = cp / 32 - COMP_CANDIDATE_MAP_OFFSET; + pos = cp % 32; + return !!(comp_candidate_map[index] & (1UL << pos)); +} + +static int hashsearch(int *htab, int htab_size, CompEntry *cv, Uint16 c) +{ + int bucket = c % htab_size; + while (htab[bucket] != -1 && cv[htab[bucket]].c != c) + bucket = (bucket + 1) % htab_size; + return htab[bucket]; +} + +#define TRANSLATE_NO 0 +#define TRANSLATE_MAYBE -1 + +/* The s array is reversed */ +static int translate(Uint16 *s, int slen, Uint16 *res) +{ + /* Go backwards through buffer and match against tree */ + int pos = 0; + CompEntry *cv = compose_tab; + int *hc = hash_compose_tab; + int cvs = compose_tab_size; + int x; + while (pos < slen) { + x = hashsearch(hc,cvs*HASH_SIZE_FACTOR,cv,s[pos]); + if (x < 0) { + return TRANSLATE_NO; + } + if (cv[x].res) { + *res = cv[x].res; + return pos; + } + cvs = cv[x].num_subs; + hc = cv[x].hash; + cv = cv[x].subs; + ++pos; + } + return TRANSLATE_MAYBE; +} + +static void handle_first_norm(Uint16 *savepoints, int *numpointsp, Uint unipoint) +{ + /*erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) *numpointsp);*/ + *numpointsp = 1; + savepoints[0] = (Uint16) unipoint; +} + +static void cleanup_norm(Eterm **hpp, Uint16 *savepoints, int numpoints, Eterm *retp) +{ + Eterm *hp = *hpp; + int res,i; + Uint16 newpoint; + Eterm ret = *retp; + + ret = CONS(hp,make_small((Uint) savepoints[0]),ret); + hp += 2; + + for (i = 1;i < numpoints;) { + if(!is_candidate(savepoints[i]) || + ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) { + ret = CONS(hp,make_small((Uint) savepoints[i]),ret); + hp += 2; + ++i; + } else { + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + i += res; + } + } + *retp = ret; +} + +static void handle_potential_norm(Eterm **hpp, Uint16 *savepoints, int *numpointsp, Uint unipoint, Eterm *retp) +{ + Eterm *hp = *hpp; + int numpoints = *numpointsp; + int res,i; + Uint16 newpoint; + Eterm ret = *retp; + + /* erts_fprintf(stderr,"CP = %d, numpoints = %d\n",(int) unipoint,(int) numpoints);*/ + if ((unipoint >> 16) == 0) { /* otherwise we're done here */ + savepoints[numpoints++] = (Uint16) unipoint; + res = translate(savepoints,numpoints,&newpoint); + if (res == TRANSLATE_NO) { + ret = CONS(hp,make_small((Uint) savepoints[0]),ret); + hp += 2; + for (i = 1;i < numpoints;) { + if(!is_candidate(savepoints[i]) || + ((res = translate(savepoints+i,numpoints - i, &newpoint)) == 0)) { + ret = CONS(hp,make_small((Uint) savepoints[i]),ret); + hp += 2; + ++i; + } else if (res > 0) { + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + i += res; + } else { /* res < 0 */ + /* A "maybe", means we are not done yet */ + int j = 0; + while (i < numpoints) { + savepoints[j++] = savepoints[i++]; + } + numpoints = j; + goto breakaway; + } + } + numpoints = 0; + breakaway: + ; + } else if (res > 0) { + numpoints = 0; + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + } /* < 0 means go on */ + } else { + /* Unconditional rollup, this character is larger than 16 bit */ + ret = CONS(hp,make_small((Uint) savepoints[0]),ret); + hp += 2; + + for (i = 1;i < numpoints;) { + if(!is_candidate(savepoints[i]) || + ((res = translate(savepoints+i,numpoints - i, &newpoint)) <= 0)) { + ret = CONS(hp,make_small((Uint) savepoints[i]),ret); + hp += 2; + ++i; + } else { + ret = CONS(hp,make_small((Uint) newpoint),ret); + hp += 2; + i += res; + } + } + ret = CONS(hp,make_small(unipoint),ret); + hp += 2; + numpoints = 0; + } + *hpp = hp; + *numpointsp = numpoints; + *retp = ret; +} + +static Eterm do_utf8_to_list_normalize(Process *p, Uint num, byte *bytes, Uint sz) +{ + Eterm *hp,*hp_end; + Eterm ret; + byte *source; + Uint unipoint; + Uint16 savepoints[4]; + int numpoints = 0; + + ASSERT(num > 0); + + hp = HAlloc(p,num * 2); /* May be to much */ + hp_end = hp + num * 2; + ret = NIL; + source = bytes + sz; + while(--source >= bytes) { + if (((*source) & ((byte) 0x80)) == 0) { + unipoint = (Uint) *source; + } else if (((*source) & ((byte) 0xE0)) == 0xC0) { + unipoint = + (((Uint) ((*source) & ((byte) 0x1F))) << 6) | + ((Uint) (source[1] & ((byte) 0x3F))); + } else if (((*source) & ((byte) 0xF0)) == 0xE0) { + unipoint = + (((Uint) ((*source) & ((byte) 0xF))) << 12) | + (((Uint) (source[1] & ((byte) 0x3F))) << 6) | + ((Uint) (source[2] & ((byte) 0x3F))); + } else if (((*source) & ((byte) 0xF8)) == 0xF0) { + unipoint = + (((Uint) ((*source) & ((byte) 0x7))) << 18) | + (((Uint) (source[1] & ((byte) 0x3F))) << 12) | + (((Uint) (source[2] & ((byte) 0x3F))) << 6) | + ((Uint) (source[3] & ((byte) 0x3F))); + } else { + /* ignore 2#10XXXXXX */ + continue; + } + if (numpoints) { + handle_potential_norm(&hp,savepoints,&numpoints,unipoint,&ret); + continue; + } + /* We are not building up any normalizations yet, look that we shouldn't start... */ + if (is_candidate(unipoint)) { + handle_first_norm(savepoints,&numpoints,unipoint); + continue; + } + ret = CONS(hp,make_small(unipoint),ret); + hp += 2; + } + /* so, we'we looped to the beginning, do we have anything saved? */ + if (numpoints) { + cleanup_norm(&hp,savepoints,numpoints,&ret); + } + if (hp_end != hp) { + HRelease(p,hp_end,hp); + } + return ret; +} + /* * The last step of characters_to_list, build a list from the buffer 'bytes' (created in the same way * as for characters_to_utf8). All sizes are known in advance and most data will be held in a @@ -1378,10 +1583,10 @@ static BIF_RETTYPE finalize_list_to_list(Process *p, */ free_restart(bytes); - if (state == UTF8_INCOMPLETE) { + if (state == ERTS_UTF8_INCOMPLETE) { hp = HAlloc(p,4); ret = TUPLE3(hp,am_incomplete,converted,rest); - } else if (state == UTF8_ERROR) { + } else if (state == ERTS_UTF8_ERROR) { hp = HAlloc(p,4); ret = TUPLE3(hp,am_error,converted,rest); } else { @@ -1408,7 +1613,7 @@ static BIF_RETTYPE characters_to_list_trap_2(BIF_ALIST_3) /* * Hooks into the process of decoding a binary depending on state. - * If last_state is UTF8_ANALYZE_MORE, num_bytes_to_process + * If last_state is ERTS_UTF8_ANALYZE_MORE, num_bytes_to_process * and num_resulting_chars will grow * until we're done analyzing the binary. Then we'll eat * the bytes to process, lowering num_bytes_to_process and num_resulting_chars, @@ -1465,14 +1670,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, left = allowed_iterations(p); - if (state == UTF8_ANALYZE_MORE) { - state = analyze_utf8(bytes + num_bytes_to_process, + if (state == ERTS_UTF8_ANALYZE_MORE) { + state = erts_analyze_utf8(bytes + num_bytes_to_process, size - num_bytes_to_process, &endpos,&numchar,&left); cost_to_proc(p,numchar); num_resulting_chars += numchar; num_bytes_to_process = endpos - bytes; - if (state == UTF8_ANALYZE_MORE) { + if (state == ERTS_UTF8_ANALYZE_MORE) { Eterm epos = erts_make_integer(num_bytes_to_process,p); Eterm enumchar = erts_make_integer(num_resulting_chars,p); erts_free_aligned_binary_bytes(temp_alloc); @@ -1528,7 +1733,7 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, ErlSubBin *sb; Eterm orig; Uint offset; - ASSERT(state != UTF8_OK); + ASSERT(state != ERTS_UTF8_OK); hp = HAlloc(p, ERL_SUB_BIN_SIZE); sb = (ErlSubBin *) hp; ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize); @@ -1544,14 +1749,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, /* Done */ - if (state == UTF8_INCOMPLETE) { + if (state == ERTS_UTF8_INCOMPLETE) { if (check_leftovers(bytes + num_bytes_to_process + num_processed_bytes, b_sz) != 0) { goto error_return; } hp = HAlloc(p,4); ret = TUPLE3(hp,am_incomplete,converted,rest); - } else if (state == UTF8_ERROR) { + } else if (state == ERTS_UTF8_ERROR) { error_return: hp = HAlloc(p,4); ret = TUPLE3(hp,am_error,converted,rest); @@ -1589,7 +1794,7 @@ static BIF_RETTYPE characters_to_list_trap_3(BIF_ALIST_3) 0U, /* nothing processed yet */ num_bytes_to_process, num_resulting_chars, - UTF8_ANALYZE_MORE, /* always this state here */ + ERTS_UTF8_ANALYZE_MORE, /* always this state here */ NIL); /* Nothing built -> no tail yet */ } @@ -1642,7 +1847,7 @@ static BIF_RETTYPE utf8_to_list(BIF_ALIST_1) BIF_ERROR(BIF_P,BADARG); } return do_bif_utf8_to_list(BIF_P, BIF_ARG_1, 0U, 0U, 0U, - UTF8_ANALYZE_MORE,NIL); + ERTS_UTF8_ANALYZE_MORE,NIL); } @@ -1728,8 +1933,8 @@ binary_to_atom(Process* p, Eterm bin, Eterm enc, int must_exist) Uint n; int reds_left = bin_size+1; /* Number of reductions left. */ - if (analyze_utf8(bytes, bin_size, &err_pos, - &n, &reds_left) == UTF8_OK) { + if (erts_analyze_utf8(bytes, bin_size, &err_pos, + &n, &reds_left) == ERTS_UTF8_OK) { /* * Correct UTF-8 encoding, but too many characters to * fit in an atom. @@ -1813,3 +2018,616 @@ BIF_RETTYPE binary_to_existing_atom_2(BIF_ALIST_2) { return binary_to_atom(BIF_P, BIF_ARG_1, BIF_ARG_2, 1); } + +/********************************************************** + * Simpler non-interruptable routines for UTF-8 and + * Windowish UTF-16 (restricted) + **********************************************************/ +/* + * This function is the heart of the Unicode support for + * open_port - spawn_executable. It converts both the name + * of the executable and the arguments according to the same rules + * as for filename conversion. That means as if your arguments are + * to be raw, you supply binaries, else unicode characters are allowed up to + * the encoding maximum (256 of the unicode max). + * Depending on the filename encoding standard, the vector is then + * converted to whatever is used, which might mean win_utf16 if on windows. + * Do not peek into the argument vector or filenam with ordinary + * string routines, that will certainly fail on some OS. + */ + +char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty) +{ + int encoding = erts_get_native_filename_encoding(); + char* name_buf = NULL; + + if (is_atom(name) || is_list(name) || (allow_empty && is_nil(name))) { + Sint need; + if ((need = erts_native_filename_need(name,encoding)) < 0) { + return NULL; + } + if (encoding == ERL_FILENAME_WIN_WCHAR) { + need += 2; + } else { + ++need; + } + name_buf = (char *) erts_alloc(alloc_type, need); + erts_native_filename_put(name,encoding,(byte *)name_buf); + name_buf[need-1] = 0; + if (encoding == ERL_FILENAME_WIN_WCHAR) { + name_buf[need-2] = 0; + } + } else if (is_binary(name)) { + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + Uint size,num_chars; + + size = binary_size(name); + bytes = erts_get_aligned_binary_bytes(name, &temp_alloc); + if (encoding != ERL_FILENAME_WIN_WCHAR) { + /*Add 0 termination only*/ + name_buf = (char *) erts_alloc(alloc_type, size+1); + memcpy(name_buf,bytes,size); + name_buf[size]=0; + } else if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || + erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + byte *p; + /* What to do now? Maybe latin1, so just take byte for byte instead */ + name_buf = (char *) erts_alloc(alloc_type, (size+1)*2); + p = (byte *) name_buf; + while (size--) { + *p++ = *bytes++; + *p++ = 0; + } + *p++ = 0; + *p++ = 0; + } else { /* WIN_WCHAR and valid UTF8 */ + name_buf = (char *) erts_alloc(alloc_type, (num_chars+1)*2); + erts_copy_utf8_to_utf16_little((byte *) name_buf, bytes, num_chars); + name_buf[num_chars*2] = 0; + name_buf[num_chars*2+1] = 0; + } + erts_free_aligned_binary_bytes(temp_alloc); + } else { + return NULL; + } + return name_buf; +} + + +Sint erts_native_filename_need(Eterm ioterm, int encoding) +{ + Eterm *objp; + Eterm obj; + DECLARE_ESTACK(stack); + Sint need = 0; + + if (is_atom(ioterm)) { + Atom* ap; + int i; + ap = atom_tab(atom_val(ioterm)); + switch (encoding) { + case ERL_FILENAME_LATIN1: + need = ap->len; + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + for (i = 0; i < ap->len; i++) { + need += (ap->name[i] >= 0x80) ? 2 : 1; + } + break; + case ERL_FILENAME_WIN_WCHAR: + need = 2*(ap->len); + break; + default: + need = -1; + } + DESTROY_ESTACK(stack); + return need; + } + + if (is_nil(ioterm)) { + DESTROY_ESTACK(stack); + return need; + } + if (!is_list(ioterm)) { + DESTROY_ESTACK(stack); + return (Sint) -1; + } + /* OK a list, needs to be processed in order, handling each flat list-level + as they occur, just like io_list_to_binary would */ + ESTACK_PUSH(stack,ioterm); + while (!ESTACK_ISEMPTY(stack)) { + ioterm = ESTACK_POP(stack); + if (is_nil(ioterm)) { + /* ignore empty lists */ + continue; + } + if(is_list(ioterm)) { +L_Again: /* Restart with sublist, old listend was pushed on stack */ + objp = list_val(ioterm); + obj = CAR(objp); + for(;;) { /* loop over one flat list of bytes and binaries + until sublist or list end is encountered */ + if (is_small(obj)) { /* Always small */ + for(;;) { + Uint x = unsigned_val(obj); + switch (encoding) { + case ERL_FILENAME_LATIN1: + if (x > 255) { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + need += 1; + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + if (x < 0x80) { + need +=1; + } else if (x < 0x800) { + need += 2; + } else if (x < 0x10000) { + if ((x >= 0xD800 && x <= 0xDFFF) || + (x == 0xFFFE) || + (x == 0xFFFF)) { /* Invalid unicode range */ + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + need += 3; + } else if (x < 0x110000) { + need += 4; + } else { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + break; + case ERL_FILENAME_WIN_WCHAR: + if (x <= 0xffff) { + need += 2; + break; + } /* else fall throug to error */ + default: + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + + /* everything else will give badarg later + in the process, so we dont check */ + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + if (!is_small(obj)) + break; + } + } else if (is_nil(obj)) { + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + } else if (is_list(obj)) { + /* push rest of list for later processing, start + again with sublist */ + ESTACK_PUSH(stack,CDR(objp)); + ioterm = obj; + goto L_Again; + } else { + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + if (is_nil(ioterm) || !is_list(ioterm)) { + break; + } + } /* for(;;) */ + } /* is_list(ioterm) */ + + if (!is_list(ioterm) && !is_nil(ioterm)) { + /* inproper list end */ + DESTROY_ESTACK(stack); + return ((Sint) -1); + } + } /* while not estack empty */ + DESTROY_ESTACK(stack); + return need; +} + +void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) +{ + Eterm *objp; + Eterm obj; + DECLARE_ESTACK(stack); + + if (is_atom(ioterm)) { + Atom* ap; + int i; + ap = atom_tab(atom_val(ioterm)); + switch (encoding) { + case ERL_FILENAME_LATIN1: + for (i = 0; i < ap->len; i++) { + *p++ = ap->name[i]; + } + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + for (i = 0; i < ap->len; i++) { + if(ap->name[i] < 0x80) { + *p++ = ap->name[i]; + } else { + *p++ = (((ap->name[i]) >> 6) | ((byte) 0xC0)); + *p++ = (((ap->name[i]) & 0x3F) | ((byte) 0x80)); + } + } + break; + case ERL_FILENAME_WIN_WCHAR: + for (i = 0; i < ap->len; i++) { + /* Little endian */ + *p++ = ap->name[i]; + *p++ = 0; + } + break; + default: + ASSERT(0); + } + DESTROY_ESTACK(stack); + return; + } + + if (is_nil(ioterm)) { + DESTROY_ESTACK(stack); + return; + } + ASSERT(is_list(ioterm)); + /* OK a list, needs to be processed in order, handling each flat list-level + as they occur, just like io_list_to_binary would */ + ESTACK_PUSH(stack,ioterm); + while (!ESTACK_ISEMPTY(stack)) { + ioterm = ESTACK_POP(stack); + if (is_nil(ioterm)) { + /* ignore empty lists */ + continue; + } + if(is_list(ioterm)) { +L_Again: /* Restart with sublist, old listend was pushed on stack */ + objp = list_val(ioterm); + obj = CAR(objp); + for(;;) { /* loop over one flat list of bytes and binaries + until sublist or list end is encountered */ + if (is_small(obj)) { /* Always small */ + for(;;) { + Uint x = unsigned_val(obj); + switch (encoding) { + case ERL_FILENAME_LATIN1: + ASSERT( x < 256); + *p++ = (byte) x; + break; + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + if (x < 0x80) { + *p++ = (byte) x; + } + else if (x < 0x800) { + *p++ = (((byte) (x >> 6)) | + ((byte) 0xC0)); + *p++ = (((byte) (x & 0x3F)) | + ((byte) 0x80)); + } else if (x < 0x10000) { + ASSERT(!((x >= 0xD800 && x <= 0xDFFF) || + (x == 0xFFFE) || + (x == 0xFFFF))); + *p++ = (((byte) (x >> 12)) | + ((byte) 0xE0)); + *p++ = ((((byte) (x >> 6)) & 0x3F) | + ((byte) 0x80)); + *p++ = (((byte) (x & 0x3F)) | + ((byte) 0x80)); + } else { + ASSERT(x < 0x110000); + *p++ = (((byte) (x >> 18)) | + ((byte) 0xF0)); + *p++ = ((((byte) (x >> 12)) & 0x3F) | + ((byte) 0x80)); + *p++ = ((((byte) (x >> 6)) & 0x3F) | + ((byte) 0x80)); + *p++ = (((byte) (x & 0x3F)) | + ((byte) 0x80)); + } + break; + case ERL_FILENAME_WIN_WCHAR: + ASSERT(x <= 0xFFFF); + *p++ = (byte) (x & 0xFFU); + *p++ = (byte) ((x >> 8) & 0xFFU); + break; + default: + ASSERT(0); + } + + /* everything else will give badarg later + in the process, so we dont check */ + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + if (!is_small(obj)) + break; + } + } else if (is_nil(obj)) { + ioterm = CDR(objp); + if (!is_list(ioterm)) { + break; + } + objp = list_val(ioterm); + obj = CAR(objp); + } else if (is_list(obj)) { + /* push rest of list for later processing, start + again with sublist */ + ESTACK_PUSH(stack,CDR(objp)); + ioterm = obj; + goto L_Again; + } else { + ASSERT(0); + } + if (is_nil(ioterm) || !is_list(ioterm)) { + break; + } + } /* for(;;) */ + } /* is_list(ioterm) */ + + ASSERT(is_list(ioterm) || is_nil(ioterm)); + } /* while not estack empty */ + DESTROY_ESTACK(stack); + return; +} +void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars) +{ + Uint unipoint; + + while (num_chars--) { + if (((*bytes) & ((byte) 0x80)) == 0) { + unipoint = (Uint) *bytes; + ++bytes; + } else if (((*bytes) & ((byte) 0xE0)) == 0xC0) { + unipoint = + (((Uint) ((*bytes) & ((byte) 0x1F))) << 6) | + ((Uint) (bytes[1] & ((byte) 0x3F))); + bytes += 2; + } else if (((*bytes) & ((byte) 0xF0)) == 0xE0) { + unipoint = + (((Uint) ((*bytes) & ((byte) 0xF))) << 12) | + (((Uint) (bytes[1] & ((byte) 0x3F))) << 6) | + ((Uint) (bytes[2] & ((byte) 0x3F))); + bytes +=3; + } else if (((*bytes) & ((byte) 0xF8)) == 0xF0) { + unipoint = + (((Uint) ((*bytes) & ((byte) 0x7))) << 18) | + (((Uint) (bytes[1] & ((byte) 0x3F))) << 12) | + (((Uint) (bytes[2] & ((byte) 0x3F))) << 6) | + ((Uint) (bytes[3] & ((byte) 0x3F))); + bytes += 4; + } else { + erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1"); + } + *target++ = (byte) (unipoint & 0xFF); + *target++ = (byte) ((unipoint >> 8) & 0xFF); + } +} + +/* + * This internal bif converts a filename to whatever format is suitable for the file driver + * It also adds zero termination so that prim_file needn't bother with the character encoding + * of the file driver + */ +BIF_RETTYPE prim_file_internal_name2native_1(BIF_ALIST_1) +{ + int encoding = erts_get_native_filename_encoding(); + Sint need; + Eterm bin_term; + byte* bin_p; + /* Prim file explicitly does not allow atoms, although we could + very well cope with it. Instead of letting 'file' handle them, + it would probably be more efficient to handle them here. Subject to + change in R15. */ + if (is_atom(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + if (is_binary(BIF_ARG_1)) { + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + Uint size,num_chars; + /* Uninterpreted encoding except if windows widechar, in case we convert from + utf8 to win_wchar */ + size = binary_size(BIF_ARG_1); + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if (encoding != ERL_FILENAME_WIN_WCHAR) { + /*Add 0 termination only*/ + bin_term = new_binary(BIF_P, NULL, size+1); + bin_p = binary_bytes(bin_term); + memcpy(bin_p,bytes,size); + bin_p[size]=0; + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(bin_term); + } + /* In a wchar world, the emulator flags only affect how + binaries are interpreted when sent from the user. */ + /* Determine real length and create a new binary */ + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK || + erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + /* What to do now? Maybe latin1, so just take byte for byte instead */ + bin_term = new_binary(BIF_P, 0, (size+1)*2); + bin_p = binary_bytes(bin_term); + while (size--) { + *bin_p++ = *bytes++; + *bin_p++ = 0; + } + *bin_p++ = 0; + *bin_p++ = 0; + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(bin_term); + } + /* OK, UTF8 ok, number of characters is in num_chars */ + bin_term = new_binary(BIF_P, 0, (num_chars+1)*2); + bin_p = binary_bytes(bin_term); + erts_copy_utf8_to_utf16_little(bin_p, bytes, num_chars); + /* zero termination */ + bin_p[num_chars*2] = 0; + bin_p[num_chars*2+1] = 0; + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(bin_term); + } /* binary */ + + + if ((need = erts_native_filename_need(BIF_ARG_1,encoding)) < 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (encoding == ERL_FILENAME_WIN_WCHAR) { + need += 2; + } else { + ++need; + } + + bin_term = new_binary(BIF_P, 0, need); + bin_p = binary_bytes(bin_term); + erts_native_filename_put(BIF_ARG_1,encoding,bin_p); + bin_p[need-1] = 0; + if (encoding == ERL_FILENAME_WIN_WCHAR) { + bin_p[need-2] = 0; + } + BIF_RET(bin_term); +} + +BIF_RETTYPE prim_file_internal_native2name_1(BIF_ALIST_1) +{ + Eterm real_bin; + Uint offset; + Uint size,num_chars; + Uint bitsize; + Uint bitoffs; + Eterm *hp; + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + Uint num_built; /* characters */ + Uint num_eaten; /* bytes */ + Eterm ret; + int mac = 0; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + size = binary_size(BIF_ARG_1); + ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (size == 0) { + BIF_RET(NIL); + } + switch (erts_get_native_filename_encoding()) { + case ERL_FILENAME_LATIN1: + hp = HAlloc(BIF_P, 2 * size); + bytes = binary_bytes(real_bin)+offset; + + BIF_RET(erts_bin_bytes_to_list(NIL, hp, bytes, size, bitoffs)); + case ERL_FILENAME_UTF8_MAC: + mac = 1; + case ERL_FILENAME_UTF8: + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { + erts_free_aligned_binary_bytes(temp_alloc); + goto noconvert; + } + num_built = 0; + num_eaten = 0; + if (mac) { + ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size); + } else { + ret = do_utf8_to_list(BIF_P, num_chars, bytes, size, num_chars, &num_built, &num_eaten, NIL); + } + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(ret); + case ERL_FILENAME_WIN_WCHAR: + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if ((size % 2) != 0) { /* Panic fixup to avoid crashing the emulator */ + size--; + hp = HAlloc(BIF_P, size+2); + ret = CONS(hp,make_small((Uint) bytes[size]),NIL); + hp += 2; + } else { + hp = HAlloc(BIF_P, size); + ret = NIL; + } + bytes += size-1; + while (size > 0) { + Uint x = ((Uint) *bytes--) << 8; + x |= ((Uint) *bytes--); + size -= 2; + ret = CONS(hp,make_small(x),ret); + hp += 2; + } + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(ret); + default: + goto noconvert; + } + noconvert: + BIF_RET(BIF_ARG_1); +} + +BIF_RETTYPE prim_file_internal_normalize_utf8_1(BIF_ALIST_1) +{ + Eterm real_bin; + Uint offset; + Uint size,num_chars; + Uint bitsize; + Uint bitoffs; + Eterm ret; + byte *temp_alloc = NULL; + byte *bytes; + byte *err_pos; + + if (is_not_binary(BIF_ARG_1)) { + BIF_ERROR(BIF_P,BADARG); + } + size = binary_size(BIF_ARG_1); + ERTS_GET_REAL_BIN(BIF_ARG_1, real_bin, offset, bitoffs, bitsize); + if (bitsize != 0) { + BIF_ERROR(BIF_P,BADARG); + } + if (size == 0) { + BIF_RET(NIL); + } + bytes = erts_get_aligned_binary_bytes(BIF_ARG_1, &temp_alloc); + if (erts_analyze_utf8(bytes,size,&err_pos,&num_chars,NULL) != ERTS_UTF8_OK) { + erts_free_aligned_binary_bytes(temp_alloc); + BIF_ERROR(BIF_P,BADARG); + } + ret = do_utf8_to_list_normalize(BIF_P, num_chars, bytes, size); + erts_free_aligned_binary_bytes(temp_alloc); + BIF_RET(ret); +} + +BIF_RETTYPE file_native_name_encoding_0(BIF_ALIST_0) +{ + switch (erts_get_native_filename_encoding()) { + case ERL_FILENAME_LATIN1: + BIF_RET(am_latin1); + case ERL_FILENAME_UTF8_MAC: + case ERL_FILENAME_UTF8: + BIF_RET(am_utf8); + case ERL_FILENAME_WIN_WCHAR: + if (erts_get_user_requested_filename_encoding() == ERL_FILENAME_LATIN1) { + BIF_RET(am_latin1); + } else { + BIF_RET(am_utf8); + } + default: + BIF_RET(am_undefined); + } +} diff --git a/erts/emulator/beam/erl_unicode_normalize.h b/erts/emulator/beam/erl_unicode_normalize.h new file mode 100644 index 0000000000..fb0a111ca2 --- /dev/null +++ b/erts/emulator/beam/erl_unicode_normalize.h @@ -0,0 +1,1687 @@ +/* +* %CopyrightBegin% +* +* Copyright Ericsson AB 1999-2010. All Rights Reserved. +* +* The contents of this file are subject to the Erlang Public License, +* Version 1.1, (the "License"); you may not use this file except in +* compliance with the License. You should have received a copy of the +* Erlang Public License along with this software. If not, it can be +* retrieved online at http://www.erlang.org/. +* +* Software distributed under the License is distributed on an "AS IS" +* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +* the License for the specific language governing rights and limitations +* under the License. +* +* %CopyrightEnd% +*/ +/* +* This file is automatically generated by dec.erl, do not edit manually +*/ +#define HASH_SIZE_FACTOR 2 +typedef struct _compose_entry { + Uint16 c; + Uint16 res; + Uint16 num_subs; + struct _compose_entry *subs; + int *hash; +} CompEntry; + +static int compose_tab_size = 61; +static int hash_compose_tab_0_15[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_0_15 */ +static CompEntry compose_tab_0_15[] = { +{65, 7846, 0, NULL, NULL}, +{69, 7872, 0, NULL, NULL}, +{79, 7890, 0, NULL, NULL}, +{97, 7847, 0, NULL, NULL}, +{101, 7873, 0, NULL, NULL}, +{111, 7891, 0, NULL, NULL} +}; /* compose_tab_0_15 */ +static int hash_compose_tab_0_16[8] = +{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_0_16 */ +static CompEntry compose_tab_0_16[] = { +{69, 7700, 0, NULL, NULL}, +{79, 7760, 0, NULL, NULL}, +{101, 7701, 0, NULL, NULL}, +{111, 7761, 0, NULL, NULL} +}; /* compose_tab_0_16 */ +static int hash_compose_tab_0_17[4] = +{-1,0,1,-1}; /* hash_compose_tab_0_17 */ +static CompEntry compose_tab_0_17[] = { +{65, 7856, 0, NULL, NULL}, +{97, 7857, 0, NULL, NULL} +}; /* compose_tab_0_17 */ +static int hash_compose_tab_0_18[8] = +{-1,2,-1,-1,-1,0,1,3}; /* hash_compose_tab_0_18 */ +static CompEntry compose_tab_0_18[] = { +{85, 475, 0, NULL, NULL}, +{117, 476, 0, NULL, NULL}, +{953, 8146, 0, NULL, NULL}, +{965, 8162, 0, NULL, NULL} +}; /* compose_tab_0_18 */ +static int hash_compose_tab_0_19_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_19_0 */ +static CompEntry compose_tab_0_19_0[] = { +{913, 8074, 0, NULL, NULL}, +{919, 8090, 0, NULL, NULL}, +{937, 8106, 0, NULL, NULL}, +{945, 8066, 0, NULL, NULL}, +{951, 8082, 0, NULL, NULL}, +{969, 8098, 0, NULL, NULL} +}; /* compose_tab_0_19_0 */ +static int hash_compose_tab_0_19[28] = +{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_0_19 */ +static CompEntry compose_tab_0_19[] = { +{837, 0, 6, compose_tab_0_19_0, hash_compose_tab_0_19_0}, +{913, 7946, 0, NULL, NULL}, +{917, 7962, 0, NULL, NULL}, +{919, 7978, 0, NULL, NULL}, +{921, 7994, 0, NULL, NULL}, +{927, 8010, 0, NULL, NULL}, +{937, 8042, 0, NULL, NULL}, +{945, 7938, 0, NULL, NULL}, +{949, 7954, 0, NULL, NULL}, +{951, 7970, 0, NULL, NULL}, +{953, 7986, 0, NULL, NULL}, +{959, 8002, 0, NULL, NULL}, +{965, 8018, 0, NULL, NULL}, +{969, 8034, 0, NULL, NULL} +}; /* compose_tab_0_19 */ +static int hash_compose_tab_0_20_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_0_20_0 */ +static CompEntry compose_tab_0_20_0[] = { +{913, 8075, 0, NULL, NULL}, +{919, 8091, 0, NULL, NULL}, +{937, 8107, 0, NULL, NULL}, +{945, 8067, 0, NULL, NULL}, +{951, 8083, 0, NULL, NULL}, +{969, 8099, 0, NULL, NULL} +}; /* compose_tab_0_20_0 */ +static int hash_compose_tab_0_20[30] = +{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5, + 12}; /* hash_compose_tab_0_20 */ +static CompEntry compose_tab_0_20[] = { +{837, 0, 6, compose_tab_0_20_0, hash_compose_tab_0_20_0}, +{913, 7947, 0, NULL, NULL}, +{917, 7963, 0, NULL, NULL}, +{919, 7979, 0, NULL, NULL}, +{921, 7995, 0, NULL, NULL}, +{927, 8011, 0, NULL, NULL}, +{933, 8027, 0, NULL, NULL}, +{937, 8043, 0, NULL, NULL}, +{945, 7939, 0, NULL, NULL}, +{949, 7955, 0, NULL, NULL}, +{951, 7971, 0, NULL, NULL}, +{953, 7987, 0, NULL, NULL}, +{959, 8003, 0, NULL, NULL}, +{965, 8019, 0, NULL, NULL}, +{969, 8035, 0, NULL, NULL} +}; /* compose_tab_0_20 */ +static int hash_compose_tab_0_21[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_0_21 */ +static CompEntry compose_tab_0_21[] = { +{79, 7900, 0, NULL, NULL}, +{85, 7914, 0, NULL, NULL}, +{111, 7901, 0, NULL, NULL}, +{117, 7915, 0, NULL, NULL} +}; /* compose_tab_0_21 */ +static int hash_compose_tab_0_22[6] = +{-1,-1,-1,0,1,2}; /* hash_compose_tab_0_22 */ +static CompEntry compose_tab_0_22[] = { +{945, 8114, 0, NULL, NULL}, +{951, 8130, 0, NULL, NULL}, +{969, 8178, 0, NULL, NULL} +}; /* compose_tab_0_22 */ +static int hash_compose_tab_0[78] = +{38,3,29,-1,-1,-1,-1,4,19,5,20,6,14,30,31,21,32,33,37,7,-1,-1,-1,8,34,-1,-1,9, + -1,35,-1,-1,-1,10,36,-1,-1,-1,-1,11,-1,12,-1,13,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,23,-1,22,-1,24,-1,25,-1,26,-1,0,-1,-1,15,1,16,27,17,2,18,28,-1,-1}; /* hash_compose_tab_0 */ +static CompEntry compose_tab_0[] = { +{65, 192, 0, NULL, NULL}, +{69, 200, 0, NULL, NULL}, +{73, 204, 0, NULL, NULL}, +{79, 210, 0, NULL, NULL}, +{85, 217, 0, NULL, NULL}, +{87, 7808, 0, NULL, NULL}, +{89, 7922, 0, NULL, NULL}, +{97, 224, 0, NULL, NULL}, +{101, 232, 0, NULL, NULL}, +{105, 236, 0, NULL, NULL}, +{111, 242, 0, NULL, NULL}, +{117, 249, 0, NULL, NULL}, +{119, 7809, 0, NULL, NULL}, +{121, 7923, 0, NULL, NULL}, +{168, 8173, 0, NULL, NULL}, +{770, 0, 6, compose_tab_0_15, hash_compose_tab_0_15}, +{772, 0, 4, compose_tab_0_16, hash_compose_tab_0_16}, +{774, 0, 2, compose_tab_0_17, hash_compose_tab_0_17}, +{776, 0, 4, compose_tab_0_18, hash_compose_tab_0_18}, +{787, 0, 14, compose_tab_0_19, hash_compose_tab_0_19}, +{788, 0, 15, compose_tab_0_20, hash_compose_tab_0_20}, +{795, 0, 4, compose_tab_0_21, hash_compose_tab_0_21}, +{837, 0, 3, compose_tab_0_22, hash_compose_tab_0_22}, +{913, 8122, 0, NULL, NULL}, +{917, 8136, 0, NULL, NULL}, +{919, 8138, 0, NULL, NULL}, +{921, 8154, 0, NULL, NULL}, +{927, 8184, 0, NULL, NULL}, +{933, 8170, 0, NULL, NULL}, +{937, 8186, 0, NULL, NULL}, +{945, 8048, 0, NULL, NULL}, +{949, 8050, 0, NULL, NULL}, +{951, 8052, 0, NULL, NULL}, +{953, 8054, 0, NULL, NULL}, +{959, 8056, 0, NULL, NULL}, +{965, 8058, 0, NULL, NULL}, +{969, 8060, 0, NULL, NULL}, +{8127, 8141, 0, NULL, NULL}, +{8190, 8157, 0, NULL, NULL} +}; /* compose_tab_0 */ +static int hash_compose_tab_1_39[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_1_39 */ +static CompEntry compose_tab_1_39[] = { +{65, 7844, 0, NULL, NULL}, +{69, 7870, 0, NULL, NULL}, +{79, 7888, 0, NULL, NULL}, +{97, 7845, 0, NULL, NULL}, +{101, 7871, 0, NULL, NULL}, +{111, 7889, 0, NULL, NULL} +}; /* compose_tab_1_39 */ +static int hash_compose_tab_1_40[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_40 */ +static CompEntry compose_tab_1_40[] = { +{79, 7756, 0, NULL, NULL}, +{85, 7800, 0, NULL, NULL}, +{111, 7757, 0, NULL, NULL}, +{117, 7801, 0, NULL, NULL} +}; /* compose_tab_1_40 */ +static int hash_compose_tab_1_41[8] = +{3,-1,-1,-1,-1,0,2,1}; /* hash_compose_tab_1_41 */ +static CompEntry compose_tab_1_41[] = { +{69, 7702, 0, NULL, NULL}, +{79, 7762, 0, NULL, NULL}, +{101, 7703, 0, NULL, NULL}, +{111, 7763, 0, NULL, NULL} +}; /* compose_tab_1_41 */ +static int hash_compose_tab_1_42[4] = +{-1,0,1,-1}; /* hash_compose_tab_1_42 */ +static CompEntry compose_tab_1_42[] = { +{65, 7854, 0, NULL, NULL}, +{97, 7855, 0, NULL, NULL} +}; /* compose_tab_1_42 */ +static int hash_compose_tab_1_43[12] = +{-1,0,1,-1,-1,4,5,-1,-1,2,3,-1}; /* hash_compose_tab_1_43 */ +static CompEntry compose_tab_1_43[] = { +{73, 7726, 0, NULL, NULL}, +{85, 471, 0, NULL, NULL}, +{105, 7727, 0, NULL, NULL}, +{117, 472, 0, NULL, NULL}, +{953, 8147, 0, NULL, NULL}, +{965, 8163, 0, NULL, NULL} +}; /* compose_tab_1_43 */ +static int hash_compose_tab_1_44[4] = +{-1,0,1,-1}; /* hash_compose_tab_1_44 */ +static CompEntry compose_tab_1_44[] = { +{65, 506, 0, NULL, NULL}, +{97, 507, 0, NULL, NULL} +}; /* compose_tab_1_44 */ +static int hash_compose_tab_1_45_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_45_0 */ +static CompEntry compose_tab_1_45_0[] = { +{913, 8076, 0, NULL, NULL}, +{919, 8092, 0, NULL, NULL}, +{937, 8108, 0, NULL, NULL}, +{945, 8068, 0, NULL, NULL}, +{951, 8084, 0, NULL, NULL}, +{969, 8100, 0, NULL, NULL} +}; /* compose_tab_1_45_0 */ +static int hash_compose_tab_1_45[28] = +{9,10,-1,5,-1,-1,-1,11,-1,-1,-1,-1,-1,6,12,-1,-1,1,13,-1,-1,2,7,3,-1,0,4,8}; /* hash_compose_tab_1_45 */ +static CompEntry compose_tab_1_45[] = { +{837, 0, 6, compose_tab_1_45_0, hash_compose_tab_1_45_0}, +{913, 7948, 0, NULL, NULL}, +{917, 7964, 0, NULL, NULL}, +{919, 7980, 0, NULL, NULL}, +{921, 7996, 0, NULL, NULL}, +{927, 8012, 0, NULL, NULL}, +{937, 8044, 0, NULL, NULL}, +{945, 7940, 0, NULL, NULL}, +{949, 7956, 0, NULL, NULL}, +{951, 7972, 0, NULL, NULL}, +{953, 7988, 0, NULL, NULL}, +{959, 8004, 0, NULL, NULL}, +{965, 8020, 0, NULL, NULL}, +{969, 8036, 0, NULL, NULL} +}; /* compose_tab_1_45 */ +static int hash_compose_tab_1_46_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_1_46_0 */ +static CompEntry compose_tab_1_46_0[] = { +{913, 8077, 0, NULL, NULL}, +{919, 8093, 0, NULL, NULL}, +{937, 8109, 0, NULL, NULL}, +{945, 8069, 0, NULL, NULL}, +{951, 8085, 0, NULL, NULL}, +{969, 8101, 0, NULL, NULL} +}; /* compose_tab_1_46_0 */ +static int hash_compose_tab_1_46[30] = +{-1,-1,-1,6,-1,13,-1,7,-1,14,-1,-1,-1,1,-1,8,-1,2,-1,3,9,4,10,11,-1,-1,-1,0,5, + 12}; /* hash_compose_tab_1_46 */ +static CompEntry compose_tab_1_46[] = { +{837, 0, 6, compose_tab_1_46_0, hash_compose_tab_1_46_0}, +{913, 7949, 0, NULL, NULL}, +{917, 7965, 0, NULL, NULL}, +{919, 7981, 0, NULL, NULL}, +{921, 7997, 0, NULL, NULL}, +{927, 8013, 0, NULL, NULL}, +{933, 8029, 0, NULL, NULL}, +{937, 8045, 0, NULL, NULL}, +{945, 7941, 0, NULL, NULL}, +{949, 7957, 0, NULL, NULL}, +{951, 7973, 0, NULL, NULL}, +{953, 7989, 0, NULL, NULL}, +{959, 8005, 0, NULL, NULL}, +{965, 8021, 0, NULL, NULL}, +{969, 8037, 0, NULL, NULL} +}; /* compose_tab_1_46 */ +static int hash_compose_tab_1_47[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_1_47 */ +static CompEntry compose_tab_1_47[] = { +{79, 7898, 0, NULL, NULL}, +{85, 7912, 0, NULL, NULL}, +{111, 7899, 0, NULL, NULL}, +{117, 7913, 0, NULL, NULL} +}; /* compose_tab_1_47 */ +static int hash_compose_tab_1_48[4] = +{1,-1,-1,0}; /* hash_compose_tab_1_48 */ +static CompEntry compose_tab_1_48[] = { +{67, 7688, 0, NULL, NULL}, +{99, 7689, 0, NULL, NULL} +}; /* compose_tab_1_48 */ +static int hash_compose_tab_1_49[6] = +{-1,-1,-1,0,1,2}; /* hash_compose_tab_1_49 */ +static CompEntry compose_tab_1_49[] = { +{945, 8116, 0, NULL, NULL}, +{951, 8132, 0, NULL, NULL}, +{959, 8180, 0, NULL, NULL} +}; /* compose_tab_1_49 */ +static int hash_compose_tab_1[140] = +{-1,-1,-1,-1,-1,-1,-1,68,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,34,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,35,-1,-1,-1,-1,64,-1,0,-1,1,-1,2,39,3,40,4,41,5,6,7, + 8,9,10,36,11,12,42,13,43,14,44,15,16,37,45,46,50,47,51,17,52,18,53,19,54,20, + 55,21,56,22,23,24,25,26,27,38,28,29,48,30,57,31,58,32,33,59,60,61,62,65,66, + 63,67,69,-1,-1,-1,-1,-1,49,-1,-1}; /* hash_compose_tab_1 */ +static CompEntry compose_tab_1[] = { +{65, 193, 0, NULL, NULL}, +{67, 262, 0, NULL, NULL}, +{69, 201, 0, NULL, NULL}, +{71, 500, 0, NULL, NULL}, +{73, 205, 0, NULL, NULL}, +{75, 7728, 0, NULL, NULL}, +{76, 313, 0, NULL, NULL}, +{77, 7742, 0, NULL, NULL}, +{78, 323, 0, NULL, NULL}, +{79, 211, 0, NULL, NULL}, +{80, 7764, 0, NULL, NULL}, +{82, 340, 0, NULL, NULL}, +{83, 346, 0, NULL, NULL}, +{85, 218, 0, NULL, NULL}, +{87, 7810, 0, NULL, NULL}, +{89, 221, 0, NULL, NULL}, +{90, 377, 0, NULL, NULL}, +{97, 225, 0, NULL, NULL}, +{99, 263, 0, NULL, NULL}, +{101, 233, 0, NULL, NULL}, +{103, 501, 0, NULL, NULL}, +{105, 237, 0, NULL, NULL}, +{107, 7729, 0, NULL, NULL}, +{108, 314, 0, NULL, NULL}, +{109, 7743, 0, NULL, NULL}, +{110, 324, 0, NULL, NULL}, +{111, 243, 0, NULL, NULL}, +{112, 7765, 0, NULL, NULL}, +{114, 341, 0, NULL, NULL}, +{115, 347, 0, NULL, NULL}, +{117, 250, 0, NULL, NULL}, +{119, 7811, 0, NULL, NULL}, +{121, 253, 0, NULL, NULL}, +{122, 378, 0, NULL, NULL}, +{168, 8174, 0, NULL, NULL}, +{198, 508, 0, NULL, NULL}, +{216, 510, 0, NULL, NULL}, +{230, 509, 0, NULL, NULL}, +{248, 511, 0, NULL, NULL}, +{770, 0, 6, compose_tab_1_39, hash_compose_tab_1_39}, +{771, 0, 4, compose_tab_1_40, hash_compose_tab_1_40}, +{772, 0, 4, compose_tab_1_41, hash_compose_tab_1_41}, +{774, 0, 2, compose_tab_1_42, hash_compose_tab_1_42}, +{776, 0, 6, compose_tab_1_43, hash_compose_tab_1_43}, +{778, 0, 2, compose_tab_1_44, hash_compose_tab_1_44}, +{787, 0, 14, compose_tab_1_45, hash_compose_tab_1_45}, +{788, 0, 15, compose_tab_1_46, hash_compose_tab_1_46}, +{795, 0, 4, compose_tab_1_47, hash_compose_tab_1_47}, +{807, 0, 2, compose_tab_1_48, hash_compose_tab_1_48}, +{837, 0, 3, compose_tab_1_49, hash_compose_tab_1_49}, +{913, 8123, 0, NULL, NULL}, +{917, 8137, 0, NULL, NULL}, +{919, 8139, 0, NULL, NULL}, +{921, 8155, 0, NULL, NULL}, +{927, 8185, 0, NULL, NULL}, +{933, 8171, 0, NULL, NULL}, +{937, 8187, 0, NULL, NULL}, +{945, 8049, 0, NULL, NULL}, +{949, 8051, 0, NULL, NULL}, +{951, 8053, 0, NULL, NULL}, +{953, 8055, 0, NULL, NULL}, +{959, 8057, 0, NULL, NULL}, +{965, 8059, 0, NULL, NULL}, +{969, 8061, 0, NULL, NULL}, +{1043, 1027, 0, NULL, NULL}, +{1050, 1036, 0, NULL, NULL}, +{1075, 1107, 0, NULL, NULL}, +{1082, 1116, 0, NULL, NULL}, +{8127, 8142, 0, NULL, NULL}, +{8190, 8158, 0, NULL, NULL} +}; /* compose_tab_1 */ +static int hash_compose_tab_2_26[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_2_26 */ +static CompEntry compose_tab_2_26[] = { +{65, 7852, 0, NULL, NULL}, +{69, 7878, 0, NULL, NULL}, +{79, 7896, 0, NULL, NULL}, +{97, 7853, 0, NULL, NULL}, +{101, 7879, 0, NULL, NULL}, +{111, 7897, 0, NULL, NULL} +}; /* compose_tab_2_26 */ +static int hash_compose_tab_2[54] = +{-1,-1,-1,20,-1,-1,-1,21,-1,22,-1,0,23,1,24,2,25,3,4,5,6,-1,-1,-1,-1,7,-1,-1, + -1,8,-1,9,-1,10,-1,11,12,-1,-1,-1,-1,-1,-1,13,-1,14,-1,15,26,16,17,18,19,-1}; /* hash_compose_tab_2 */ +static CompEntry compose_tab_2[] = { +{65, 194, 0, NULL, NULL}, +{67, 264, 0, NULL, NULL}, +{69, 202, 0, NULL, NULL}, +{71, 284, 0, NULL, NULL}, +{72, 292, 0, NULL, NULL}, +{73, 206, 0, NULL, NULL}, +{74, 308, 0, NULL, NULL}, +{79, 212, 0, NULL, NULL}, +{83, 348, 0, NULL, NULL}, +{85, 219, 0, NULL, NULL}, +{87, 372, 0, NULL, NULL}, +{89, 374, 0, NULL, NULL}, +{90, 7824, 0, NULL, NULL}, +{97, 226, 0, NULL, NULL}, +{99, 265, 0, NULL, NULL}, +{101, 234, 0, NULL, NULL}, +{103, 285, 0, NULL, NULL}, +{104, 293, 0, NULL, NULL}, +{105, 238, 0, NULL, NULL}, +{106, 309, 0, NULL, NULL}, +{111, 244, 0, NULL, NULL}, +{115, 349, 0, NULL, NULL}, +{117, 251, 0, NULL, NULL}, +{119, 373, 0, NULL, NULL}, +{121, 375, 0, NULL, NULL}, +{122, 7825, 0, NULL, NULL}, +{803, 0, 6, compose_tab_2_26, hash_compose_tab_2_26} +}; /* compose_tab_2 */ +static int hash_compose_tab_3_16[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_3_16 */ +static CompEntry compose_tab_3_16[] = { +{65, 7850, 0, NULL, NULL}, +{69, 7876, 0, NULL, NULL}, +{79, 7894, 0, NULL, NULL}, +{97, 7851, 0, NULL, NULL}, +{101, 7877, 0, NULL, NULL}, +{111, 7895, 0, NULL, NULL} +}; /* compose_tab_3_16 */ +static int hash_compose_tab_3_17[4] = +{-1,0,1,-1}; /* hash_compose_tab_3_17 */ +static CompEntry compose_tab_3_17[] = { +{65, 7860, 0, NULL, NULL}, +{97, 7861, 0, NULL, NULL} +}; /* compose_tab_3_17 */ +static int hash_compose_tab_3_18[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_3_18 */ +static CompEntry compose_tab_3_18[] = { +{79, 7904, 0, NULL, NULL}, +{85, 7918, 0, NULL, NULL}, +{111, 7905, 0, NULL, NULL}, +{117, 7919, 0, NULL, NULL} +}; /* compose_tab_3_18 */ +static int hash_compose_tab_3[38] = +{-1,-1,3,4,13,14,-1,15,-1,5,6,16,-1,7,17,-1,-1,-1,-1,-1,-1,8,-1,-1,-1,9,-1,0, + -1,10,-1,1,-1,-1,11,2,12,18}; /* hash_compose_tab_3 */ +static CompEntry compose_tab_3[] = { +{65, 195, 0, NULL, NULL}, +{69, 7868, 0, NULL, NULL}, +{73, 296, 0, NULL, NULL}, +{78, 209, 0, NULL, NULL}, +{79, 213, 0, NULL, NULL}, +{85, 360, 0, NULL, NULL}, +{86, 7804, 0, NULL, NULL}, +{89, 7928, 0, NULL, NULL}, +{97, 227, 0, NULL, NULL}, +{101, 7869, 0, NULL, NULL}, +{105, 297, 0, NULL, NULL}, +{110, 241, 0, NULL, NULL}, +{111, 245, 0, NULL, NULL}, +{117, 361, 0, NULL, NULL}, +{118, 7805, 0, NULL, NULL}, +{121, 7929, 0, NULL, NULL}, +{770, 0, 6, compose_tab_3_16, hash_compose_tab_3_16}, +{774, 0, 2, compose_tab_3_17, hash_compose_tab_3_17}, +{795, 0, 4, compose_tab_3_18, hash_compose_tab_3_18} +}; /* compose_tab_3 */ +static int hash_compose_tab_4_14[4] = +{-1,0,1,-1}; /* hash_compose_tab_4_14 */ +static CompEntry compose_tab_4_14[] = { +{65, 480, 0, NULL, NULL}, +{97, 481, 0, NULL, NULL} +}; /* compose_tab_4_14 */ +static int hash_compose_tab_4_15[8] = +{-1,0,2,-1,-1,1,3,-1}; /* hash_compose_tab_4_15 */ +static CompEntry compose_tab_4_15[] = { +{65, 478, 0, NULL, NULL}, +{85, 469, 0, NULL, NULL}, +{97, 479, 0, NULL, NULL}, +{117, 470, 0, NULL, NULL} +}; /* compose_tab_4_15 */ +static int hash_compose_tab_4_16[8] = +{-1,-1,1,3,0,2,-1,-1}; /* hash_compose_tab_4_16 */ +static CompEntry compose_tab_4_16[] = { +{76, 7736, 0, NULL, NULL}, +{82, 7772, 0, NULL, NULL}, +{108, 7737, 0, NULL, NULL}, +{114, 7773, 0, NULL, NULL} +}; /* compose_tab_4_16 */ +static int hash_compose_tab_4_17[4] = +{1,-1,-1,0}; /* hash_compose_tab_4_17 */ +static CompEntry compose_tab_4_17[] = { +{79, 492, 0, NULL, NULL}, +{111, 493, 0, NULL, NULL} +}; /* compose_tab_4_17 */ +static int hash_compose_tab_4[56] = +{-1,22,-1,-1,-1,11,13,-1,-1,0,-1,-1,-1,1,23,2,26,3,18,16,-1,-1,-1,4,17,19,-1, + 27,-1,5,12,-1,-1,-1,-1,-1,-1,20,-1,-1,24,6,-1,-1,-1,7,-1,8,14,9,15,21,25,-1, + -1,10}; /* hash_compose_tab_4 */ +static CompEntry compose_tab_4[] = { +{65, 256, 0, NULL, NULL}, +{69, 274, 0, NULL, NULL}, +{71, 7712, 0, NULL, NULL}, +{73, 298, 0, NULL, NULL}, +{79, 332, 0, NULL, NULL}, +{85, 362, 0, NULL, NULL}, +{97, 257, 0, NULL, NULL}, +{101, 275, 0, NULL, NULL}, +{103, 7713, 0, NULL, NULL}, +{105, 299, 0, NULL, NULL}, +{111, 333, 0, NULL, NULL}, +{117, 363, 0, NULL, NULL}, +{198, 482, 0, NULL, NULL}, +{230, 483, 0, NULL, NULL}, +{775, 0, 2, compose_tab_4_14, hash_compose_tab_4_14}, +{776, 0, 4, compose_tab_4_15, hash_compose_tab_4_15}, +{803, 0, 4, compose_tab_4_16, hash_compose_tab_4_16}, +{808, 0, 2, compose_tab_4_17, hash_compose_tab_4_17}, +{913, 8121, 0, NULL, NULL}, +{921, 8153, 0, NULL, NULL}, +{933, 8169, 0, NULL, NULL}, +{945, 8113, 0, NULL, NULL}, +{953, 8145, 0, NULL, NULL}, +{965, 8161, 0, NULL, NULL}, +{1048, 1250, 0, NULL, NULL}, +{1059, 1262, 0, NULL, NULL}, +{1080, 1251, 0, NULL, NULL}, +{1091, 1263, 0, NULL, NULL} +}; /* compose_tab_4 */ +static int hash_compose_tab_5_12[4] = +{-1,0,1,-1}; /* hash_compose_tab_5_12 */ +static CompEntry compose_tab_5_12[] = { +{65, 7862, 0, NULL, NULL}, +{97, 7863, 0, NULL, NULL} +}; /* compose_tab_5_12 */ +static int hash_compose_tab_5_13[4] = +{-1,0,1,-1}; /* hash_compose_tab_5_13 */ +static CompEntry compose_tab_5_13[] = { +{69, 7708, 0, NULL, NULL}, +{101, 7709, 0, NULL, NULL} +}; /* compose_tab_5_13 */ +static int hash_compose_tab_5[60] = +{28,-1,-1,-1,-1,0,19,-1,-1,1,-1,2,29,3,14,-1,-1,-1,-1,4,20,15,-1,12,-1,5,21, + 13,22,23,-1,-1,-1,16,-1,-1,-1,6,-1,24,-1,7,-1,8,-1,9,17,-1,-1,-1,-1,10,25,18, + -1,-1,-1,11,26,27}; /* hash_compose_tab_5 */ +static CompEntry compose_tab_5[] = { +{65, 258, 0, NULL, NULL}, +{69, 276, 0, NULL, NULL}, +{71, 286, 0, NULL, NULL}, +{73, 300, 0, NULL, NULL}, +{79, 334, 0, NULL, NULL}, +{85, 364, 0, NULL, NULL}, +{97, 259, 0, NULL, NULL}, +{101, 277, 0, NULL, NULL}, +{103, 287, 0, NULL, NULL}, +{105, 301, 0, NULL, NULL}, +{111, 335, 0, NULL, NULL}, +{117, 365, 0, NULL, NULL}, +{803, 0, 2, compose_tab_5_12, hash_compose_tab_5_12}, +{807, 0, 2, compose_tab_5_13, hash_compose_tab_5_13}, +{913, 8120, 0, NULL, NULL}, +{921, 8152, 0, NULL, NULL}, +{933, 8168, 0, NULL, NULL}, +{945, 8112, 0, NULL, NULL}, +{953, 8144, 0, NULL, NULL}, +{965, 8160, 0, NULL, NULL}, +{1040, 1232, 0, NULL, NULL}, +{1045, 1238, 0, NULL, NULL}, +{1046, 1217, 0, NULL, NULL}, +{1048, 1049, 0, NULL, NULL}, +{1059, 1038, 0, NULL, NULL}, +{1072, 1233, 0, NULL, NULL}, +{1077, 1239, 0, NULL, NULL}, +{1078, 1218, 0, NULL, NULL}, +{1080, 1081, 0, NULL, NULL}, +{1091, 1118, 0, NULL, NULL} +}; /* compose_tab_5 */ +static int hash_compose_tab_6_36[4] = +{1,-1,-1,0}; /* hash_compose_tab_6_36 */ +static CompEntry compose_tab_6_36[] = { +{83, 7780, 0, NULL, NULL}, +{115, 7781, 0, NULL, NULL} +}; /* compose_tab_6_36 */ +static int hash_compose_tab_6_38[4] = +{1,-1,-1,0}; /* hash_compose_tab_6_38 */ +static CompEntry compose_tab_6_38[] = { +{83, 7782, 0, NULL, NULL}, +{115, 7783, 0, NULL, NULL} +}; /* compose_tab_6_38 */ +static int hash_compose_tab_6_39[4] = +{1,-1,-1,0}; /* hash_compose_tab_6_39 */ +static CompEntry compose_tab_6_39[] = { +{83, 7784, 0, NULL, NULL}, +{115, 7785, 0, NULL, NULL} +}; /* compose_tab_6_39 */ +static int hash_compose_tab_6[80] = +{10,-1,11,12,13,39,-1,14,15,16,17,-1,-1,-1,-1,-1,-1,-1,18,19,20,21,22,23,24, + -1,-1,-1,-1,25,26,-1,27,-1,28,29,30,-1,-1,31,32,33,34,-1,-1,-1,-1,-1,-1,36, + -1,-1,-1,-1,37,-1,-1,-1,-1,-1,38,-1,-1,35,-1,-1,0,1,2,3,4,5,6,7,-1,-1,-1,8,9, + -1}; /* hash_compose_tab_6 */ +static CompEntry compose_tab_6[] = { +{66, 7682, 0, NULL, NULL}, +{67, 266, 0, NULL, NULL}, +{68, 7690, 0, NULL, NULL}, +{69, 278, 0, NULL, NULL}, +{70, 7710, 0, NULL, NULL}, +{71, 288, 0, NULL, NULL}, +{72, 7714, 0, NULL, NULL}, +{73, 304, 0, NULL, NULL}, +{77, 7744, 0, NULL, NULL}, +{78, 7748, 0, NULL, NULL}, +{80, 7766, 0, NULL, NULL}, +{82, 7768, 0, NULL, NULL}, +{83, 7776, 0, NULL, NULL}, +{84, 7786, 0, NULL, NULL}, +{87, 7814, 0, NULL, NULL}, +{88, 7818, 0, NULL, NULL}, +{89, 7822, 0, NULL, NULL}, +{90, 379, 0, NULL, NULL}, +{98, 7683, 0, NULL, NULL}, +{99, 267, 0, NULL, NULL}, +{100, 7691, 0, NULL, NULL}, +{101, 279, 0, NULL, NULL}, +{102, 7711, 0, NULL, NULL}, +{103, 289, 0, NULL, NULL}, +{104, 7715, 0, NULL, NULL}, +{109, 7745, 0, NULL, NULL}, +{110, 7749, 0, NULL, NULL}, +{112, 7767, 0, NULL, NULL}, +{114, 7769, 0, NULL, NULL}, +{115, 7777, 0, NULL, NULL}, +{116, 7787, 0, NULL, NULL}, +{119, 7815, 0, NULL, NULL}, +{120, 7819, 0, NULL, NULL}, +{121, 7823, 0, NULL, NULL}, +{122, 380, 0, NULL, NULL}, +{383, 7835, 0, NULL, NULL}, +{769, 0, 2, compose_tab_6_36, hash_compose_tab_6_36}, +{774, 784, 0, NULL, NULL}, +{780, 0, 2, compose_tab_6_38, hash_compose_tab_6_38}, +{803, 0, 2, compose_tab_6_39, hash_compose_tab_6_39} +}; /* compose_tab_6 */ +static int hash_compose_tab_7_23[4] = +{1,-1,-1,0}; /* hash_compose_tab_7_23 */ +static CompEntry compose_tab_7_23[] = { +{79, 7758, 0, NULL, NULL}, +{111, 7759, 0, NULL, NULL} +}; /* compose_tab_7_23 */ +static int hash_compose_tab_7_24[4] = +{-1,0,1,-1}; /* hash_compose_tab_7_24 */ +static CompEntry compose_tab_7_24[] = { +{85, 7802, 0, NULL, NULL}, +{117, 7803, 0, NULL, NULL} +}; /* compose_tab_7_24 */ +static int hash_compose_tab_7[100] = +{48,10,21,-1,11,12,-1,-1,-1,-1,49,13,-1,-1,-1,20,14,15,-1,16,17,18,25,-1,-1, + -1,-1,-1,-1,22,30,-1,-1,26,-1,-1,-1,-1,-1,-1,31,-1,-1,-1,-1,32,33,34,35,-1, + -1,-1,-1,27,36,-1,-1,-1,-1,37,-1,-1,-1,38,-1,0,28,39,-1,1,-1,23,2,3,24,40,-1, + 41,29,4,42,43,44,-1,-1,5,45,6,7,8,-1,46,-1,-1,-1,47,-1,9,-1,19}; /* hash_compose_tab_7 */ +static CompEntry compose_tab_7[] = { +{65, 196, 0, NULL, NULL}, +{69, 203, 0, NULL, NULL}, +{72, 7718, 0, NULL, NULL}, +{73, 207, 0, NULL, NULL}, +{79, 214, 0, NULL, NULL}, +{85, 220, 0, NULL, NULL}, +{87, 7812, 0, NULL, NULL}, +{88, 7820, 0, NULL, NULL}, +{89, 376, 0, NULL, NULL}, +{97, 228, 0, NULL, NULL}, +{101, 235, 0, NULL, NULL}, +{104, 7719, 0, NULL, NULL}, +{105, 239, 0, NULL, NULL}, +{111, 246, 0, NULL, NULL}, +{116, 7831, 0, NULL, NULL}, +{117, 252, 0, NULL, NULL}, +{119, 7813, 0, NULL, NULL}, +{120, 7821, 0, NULL, NULL}, +{121, 255, 0, NULL, NULL}, +{399, 1242, 0, NULL, NULL}, +{415, 1258, 0, NULL, NULL}, +{601, 1243, 0, NULL, NULL}, +{629, 1259, 0, NULL, NULL}, +{771, 0, 2, compose_tab_7_23, hash_compose_tab_7_23}, +{772, 0, 2, compose_tab_7_24, hash_compose_tab_7_24}, +{921, 938, 0, NULL, NULL}, +{933, 939, 0, NULL, NULL}, +{953, 970, 0, NULL, NULL}, +{965, 971, 0, NULL, NULL}, +{978, 980, 0, NULL, NULL}, +{1030, 1031, 0, NULL, NULL}, +{1040, 1234, 0, NULL, NULL}, +{1045, 1025, 0, NULL, NULL}, +{1046, 1244, 0, NULL, NULL}, +{1047, 1246, 0, NULL, NULL}, +{1048, 1252, 0, NULL, NULL}, +{1054, 1254, 0, NULL, NULL}, +{1059, 1264, 0, NULL, NULL}, +{1063, 1268, 0, NULL, NULL}, +{1067, 1272, 0, NULL, NULL}, +{1072, 1235, 0, NULL, NULL}, +{1077, 1105, 0, NULL, NULL}, +{1078, 1245, 0, NULL, NULL}, +{1079, 1247, 0, NULL, NULL}, +{1080, 1253, 0, NULL, NULL}, +{1086, 1255, 0, NULL, NULL}, +{1091, 1265, 0, NULL, NULL}, +{1095, 1269, 0, NULL, NULL}, +{1099, 1273, 0, NULL, NULL}, +{1110, 1111, 0, NULL, NULL} +}; /* compose_tab_7 */ +static int hash_compose_tab_8_12[12] = +{-1,3,-1,5,-1,0,4,2,-1,1,-1,-1}; /* hash_compose_tab_8_12 */ +static CompEntry compose_tab_8_12[] = { +{65, 7848, 0, NULL, NULL}, +{69, 7874, 0, NULL, NULL}, +{79, 7892, 0, NULL, NULL}, +{97, 7849, 0, NULL, NULL}, +{101, 7875, 0, NULL, NULL}, +{111, 7893, 0, NULL, NULL} +}; /* compose_tab_8_12 */ +static int hash_compose_tab_8_13[4] = +{-1,0,1,-1}; /* hash_compose_tab_8_13 */ +static CompEntry compose_tab_8_13[] = { +{65, 7858, 0, NULL, NULL}, +{97, 7859, 0, NULL, NULL} +}; /* compose_tab_8_13 */ +static int hash_compose_tab_8_14[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_8_14 */ +static CompEntry compose_tab_8_14[] = { +{79, 7902, 0, NULL, NULL}, +{85, 7916, 0, NULL, NULL}, +{111, 7903, 0, NULL, NULL}, +{117, 7917, 0, NULL, NULL} +}; /* compose_tab_8_14 */ +static int hash_compose_tab_8[30] = +{-1,11,-1,-1,-1,0,-1,6,-1,1,-1,7,-1,2,-1,8,14,-1,-1,3,12,9,-1,-1,13,4,-1,10, + -1,5}; /* hash_compose_tab_8 */ +static CompEntry compose_tab_8[] = { +{65, 7842, 0, NULL, NULL}, +{69, 7866, 0, NULL, NULL}, +{73, 7880, 0, NULL, NULL}, +{79, 7886, 0, NULL, NULL}, +{85, 7910, 0, NULL, NULL}, +{89, 7926, 0, NULL, NULL}, +{97, 7843, 0, NULL, NULL}, +{101, 7867, 0, NULL, NULL}, +{105, 7881, 0, NULL, NULL}, +{111, 7887, 0, NULL, NULL}, +{117, 7911, 0, NULL, NULL}, +{121, 7927, 0, NULL, NULL}, +{770, 0, 6, compose_tab_8_12, hash_compose_tab_8_12}, +{774, 0, 2, compose_tab_8_13, hash_compose_tab_8_13}, +{795, 0, 4, compose_tab_8_14, hash_compose_tab_8_14} +}; /* compose_tab_8 */ +static int hash_compose_tab_9[12] = +{-1,1,2,5,-1,0,-1,-1,-1,3,-1,4}; /* hash_compose_tab_9 */ +static CompEntry compose_tab_9[] = { +{65, 197, 0, NULL, NULL}, +{85, 366, 0, NULL, NULL}, +{97, 229, 0, NULL, NULL}, +{117, 367, 0, NULL, NULL}, +{119, 7832, 0, NULL, NULL}, +{121, 7833, 0, NULL, NULL} +}; /* compose_tab_9 */ +static int hash_compose_tab_10[12] = +{-1,1,-1,2,4,-1,-1,0,-1,3,-1,5}; /* hash_compose_tab_10 */ +static CompEntry compose_tab_10[] = { +{79, 336, 0, NULL, NULL}, +{85, 368, 0, NULL, NULL}, +{111, 337, 0, NULL, NULL}, +{117, 369, 0, NULL, NULL}, +{1059, 1266, 0, NULL, NULL}, +{1091, 1267, 0, NULL, NULL} +}; /* compose_tab_10 */ +static int hash_compose_tab_11_33[4] = +{-1,0,1,-1}; /* hash_compose_tab_11_33 */ +static CompEntry compose_tab_11_33[] = { +{85, 473, 0, NULL, NULL}, +{117, 474, 0, NULL, NULL} +}; /* compose_tab_11_33 */ +static int hash_compose_tab_11[68] = +{2,3,-1,4,-1,5,-1,6,7,-1,8,9,-1,-1,10,11,12,13,-1,-1,-1,-1,14,-1,-1,-1,-1,-1, + 33,15,-1,16,17,18,31,19,-1,20,21,22,23,-1,24,25,-1,-1,26,27,28,29,32,-1,-1, + -1,30,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,-1,1}; /* hash_compose_tab_11 */ +static CompEntry compose_tab_11[] = { +{65, 461, 0, NULL, NULL}, +{67, 268, 0, NULL, NULL}, +{68, 270, 0, NULL, NULL}, +{69, 282, 0, NULL, NULL}, +{71, 486, 0, NULL, NULL}, +{73, 463, 0, NULL, NULL}, +{75, 488, 0, NULL, NULL}, +{76, 317, 0, NULL, NULL}, +{78, 327, 0, NULL, NULL}, +{79, 465, 0, NULL, NULL}, +{82, 344, 0, NULL, NULL}, +{83, 352, 0, NULL, NULL}, +{84, 356, 0, NULL, NULL}, +{85, 467, 0, NULL, NULL}, +{90, 381, 0, NULL, NULL}, +{97, 462, 0, NULL, NULL}, +{99, 269, 0, NULL, NULL}, +{100, 271, 0, NULL, NULL}, +{101, 283, 0, NULL, NULL}, +{103, 487, 0, NULL, NULL}, +{105, 464, 0, NULL, NULL}, +{106, 496, 0, NULL, NULL}, +{107, 489, 0, NULL, NULL}, +{108, 318, 0, NULL, NULL}, +{110, 328, 0, NULL, NULL}, +{111, 466, 0, NULL, NULL}, +{114, 345, 0, NULL, NULL}, +{115, 353, 0, NULL, NULL}, +{116, 357, 0, NULL, NULL}, +{117, 468, 0, NULL, NULL}, +{122, 382, 0, NULL, NULL}, +{439, 494, 0, NULL, NULL}, +{658, 495, 0, NULL, NULL}, +{776, 0, 2, compose_tab_11_33, hash_compose_tab_11_33} +}; /* compose_tab_11 */ +static int hash_compose_tab_12_1[4] = +{-1,0,1,-1}; /* hash_compose_tab_12_1 */ +static CompEntry compose_tab_12_1[] = { +{953, 912, 0, NULL, NULL}, +{965, 944, 0, NULL, NULL} +}; /* compose_tab_12_1 */ +static int hash_compose_tab_12[34] = +{11,4,12,5,-1,-1,-1,13,-1,6,-1,-1,-1,14,-1,7,-1,15,-1,8,-1,-1,-1,-1,-1,-1,16, + 9,1,2,-1,10,0,3}; /* hash_compose_tab_12 */ +static CompEntry compose_tab_12[] = { +{168, 901, 0, NULL, NULL}, +{776, 0, 2, compose_tab_12_1, hash_compose_tab_12_1}, +{913, 902, 0, NULL, NULL}, +{917, 904, 0, NULL, NULL}, +{919, 905, 0, NULL, NULL}, +{921, 906, 0, NULL, NULL}, +{927, 908, 0, NULL, NULL}, +{933, 910, 0, NULL, NULL}, +{937, 911, 0, NULL, NULL}, +{945, 940, 0, NULL, NULL}, +{949, 941, 0, NULL, NULL}, +{951, 942, 0, NULL, NULL}, +{953, 943, 0, NULL, NULL}, +{959, 972, 0, NULL, NULL}, +{965, 973, 0, NULL, NULL}, +{969, 974, 0, NULL, NULL}, +{978, 979, 0, NULL, NULL} +}; /* compose_tab_12 */ +static int hash_compose_tab_13[28] = +{-1,5,10,-1,-1,11,-1,-1,-1,0,-1,-1,-1,1,6,-1,-1,2,7,-1,12,8,13,3,-1,-1,4,9}; /* hash_compose_tab_13 */ +static CompEntry compose_tab_13[] = { +{65, 512, 0, NULL, NULL}, +{69, 516, 0, NULL, NULL}, +{73, 520, 0, NULL, NULL}, +{79, 524, 0, NULL, NULL}, +{82, 528, 0, NULL, NULL}, +{85, 532, 0, NULL, NULL}, +{97, 513, 0, NULL, NULL}, +{101, 517, 0, NULL, NULL}, +{105, 521, 0, NULL, NULL}, +{111, 525, 0, NULL, NULL}, +{114, 529, 0, NULL, NULL}, +{117, 533, 0, NULL, NULL}, +{1140, 1142, 0, NULL, NULL}, +{1141, 1143, 0, NULL, NULL} +}; /* compose_tab_13 */ +static int hash_compose_tab_14[24] = +{-1,2,6,-1,-1,7,-1,3,-1,8,4,-1,-1,5,-1,9,-1,0,10,-1,-1,1,11,-1}; /* hash_compose_tab_14 */ +static CompEntry compose_tab_14[] = { +{65, 514, 0, NULL, NULL}, +{69, 518, 0, NULL, NULL}, +{73, 522, 0, NULL, NULL}, +{79, 526, 0, NULL, NULL}, +{82, 530, 0, NULL, NULL}, +{85, 534, 0, NULL, NULL}, +{97, 515, 0, NULL, NULL}, +{101, 519, 0, NULL, NULL}, +{105, 523, 0, NULL, NULL}, +{111, 527, 0, NULL, NULL}, +{114, 531, 0, NULL, NULL}, +{117, 535, 0, NULL, NULL} +}; /* compose_tab_14 */ +static int hash_compose_tab_15_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_15_0 */ +static CompEntry compose_tab_15_0[] = { +{913, 8072, 0, NULL, NULL}, +{919, 8088, 0, NULL, NULL}, +{937, 8104, 0, NULL, NULL}, +{945, 8064, 0, NULL, NULL}, +{951, 8080, 0, NULL, NULL}, +{969, 8096, 0, NULL, NULL} +}; /* compose_tab_15_0 */ +static int hash_compose_tab_15[30] = +{-1,12,-1,-1,-1,13,-1,6,-1,14,-1,-1,-1,1,-1,7,-1,2,-1,3,8,4,9,10,-1,-1,-1,0,5, + 11}; /* hash_compose_tab_15 */ +static CompEntry compose_tab_15[] = { +{837, 0, 6, compose_tab_15_0, hash_compose_tab_15_0}, +{913, 7944, 0, NULL, NULL}, +{917, 7960, 0, NULL, NULL}, +{919, 7976, 0, NULL, NULL}, +{921, 7992, 0, NULL, NULL}, +{927, 8008, 0, NULL, NULL}, +{937, 8040, 0, NULL, NULL}, +{945, 7936, 0, NULL, NULL}, +{949, 7952, 0, NULL, NULL}, +{951, 7968, 0, NULL, NULL}, +{953, 7984, 0, NULL, NULL}, +{959, 8000, 0, NULL, NULL}, +{961, 8164, 0, NULL, NULL}, +{965, 8016, 0, NULL, NULL}, +{969, 8032, 0, NULL, NULL} +}; /* compose_tab_15 */ +static int hash_compose_tab_16_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_16_0 */ +static CompEntry compose_tab_16_0[] = { +{913, 8073, 0, NULL, NULL}, +{919, 8089, 0, NULL, NULL}, +{937, 8105, 0, NULL, NULL}, +{945, 8065, 0, NULL, NULL}, +{951, 8081, 0, NULL, NULL}, +{969, 8097, 0, NULL, NULL} +}; /* compose_tab_16_0 */ +static int hash_compose_tab_16[34] = +{11,3,12,4,-1,-1,-1,13,-1,5,14,6,-1,15,-1,7,-1,16,-1,8,-1,0,-1,-1,-1,-1,-1,9, + -1,1,-1,10,-1,2}; /* hash_compose_tab_16 */ +static CompEntry compose_tab_16[] = { +{837, 0, 6, compose_tab_16_0, hash_compose_tab_16_0}, +{913, 7945, 0, NULL, NULL}, +{917, 7961, 0, NULL, NULL}, +{919, 7977, 0, NULL, NULL}, +{921, 7993, 0, NULL, NULL}, +{927, 8009, 0, NULL, NULL}, +{929, 8172, 0, NULL, NULL}, +{933, 8025, 0, NULL, NULL}, +{937, 8041, 0, NULL, NULL}, +{945, 7937, 0, NULL, NULL}, +{949, 7953, 0, NULL, NULL}, +{951, 7969, 0, NULL, NULL}, +{953, 7985, 0, NULL, NULL}, +{959, 8001, 0, NULL, NULL}, +{961, 8165, 0, NULL, NULL}, +{965, 8017, 0, NULL, NULL}, +{969, 8033, 0, NULL, NULL} +}; /* compose_tab_16 */ +static int hash_compose_tab_17[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_17 */ +static CompEntry compose_tab_17[] = { +{79, 416, 0, NULL, NULL}, +{85, 431, 0, NULL, NULL}, +{111, 417, 0, NULL, NULL}, +{117, 432, 0, NULL, NULL} +}; /* compose_tab_17 */ +static int hash_compose_tab_18_38[8] = +{2,-1,-1,-1,-1,1,3,0}; /* hash_compose_tab_18_38 */ +static CompEntry compose_tab_18_38[] = { +{79, 7906, 0, NULL, NULL}, +{85, 7920, 0, NULL, NULL}, +{111, 7907, 0, NULL, NULL}, +{117, 7921, 0, NULL, NULL} +}; /* compose_tab_18_38 */ +static int hash_compose_tab_18[78] = +{9,10,-1,-1,11,12,13,14,15,16,-1,17,18,-1,-1,38,-1,-1,-1,19,20,-1,21,22,-1,-1, + 23,24,-1,25,26,27,28,29,-1,-1,30,31,32,33,34,35,-1,36,37,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,-1,2,3,-1,-1,4,5,-1,6,7,8}; /* hash_compose_tab_18 */ +static CompEntry compose_tab_18[] = { +{65, 7840, 0, NULL, NULL}, +{66, 7684, 0, NULL, NULL}, +{68, 7692, 0, NULL, NULL}, +{69, 7864, 0, NULL, NULL}, +{72, 7716, 0, NULL, NULL}, +{73, 7882, 0, NULL, NULL}, +{75, 7730, 0, NULL, NULL}, +{76, 7734, 0, NULL, NULL}, +{77, 7746, 0, NULL, NULL}, +{78, 7750, 0, NULL, NULL}, +{79, 7884, 0, NULL, NULL}, +{82, 7770, 0, NULL, NULL}, +{83, 7778, 0, NULL, NULL}, +{84, 7788, 0, NULL, NULL}, +{85, 7908, 0, NULL, NULL}, +{86, 7806, 0, NULL, NULL}, +{87, 7816, 0, NULL, NULL}, +{89, 7924, 0, NULL, NULL}, +{90, 7826, 0, NULL, NULL}, +{97, 7841, 0, NULL, NULL}, +{98, 7685, 0, NULL, NULL}, +{100, 7693, 0, NULL, NULL}, +{101, 7865, 0, NULL, NULL}, +{104, 7717, 0, NULL, NULL}, +{105, 7883, 0, NULL, NULL}, +{107, 7731, 0, NULL, NULL}, +{108, 7735, 0, NULL, NULL}, +{109, 7747, 0, NULL, NULL}, +{110, 7751, 0, NULL, NULL}, +{111, 7885, 0, NULL, NULL}, +{114, 7771, 0, NULL, NULL}, +{115, 7779, 0, NULL, NULL}, +{116, 7789, 0, NULL, NULL}, +{117, 7909, 0, NULL, NULL}, +{118, 7807, 0, NULL, NULL}, +{119, 7817, 0, NULL, NULL}, +{121, 7925, 0, NULL, NULL}, +{122, 7827, 0, NULL, NULL}, +{795, 0, 4, compose_tab_18_38, hash_compose_tab_18_38} +}; /* compose_tab_18 */ +static int hash_compose_tab_19[4] = +{-1,0,1,-1}; /* hash_compose_tab_19 */ +static CompEntry compose_tab_19[] = { +{85, 7794, 0, NULL, NULL}, +{117, 7795, 0, NULL, NULL} +}; /* compose_tab_19 */ +static int hash_compose_tab_20[4] = +{-1,0,1,-1}; /* hash_compose_tab_20 */ +static CompEntry compose_tab_20[] = { +{65, 7680, 0, NULL, NULL}, +{97, 7681, 0, NULL, NULL} +}; /* compose_tab_20 */ +static int hash_compose_tab_21[40] = +{-1,-1,7,8,9,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,10,11,-1,-1,12,13,-1, + -1,0,1,14,15,2,3,16,17,4,5,18,6,19}; /* hash_compose_tab_21 */ +static CompEntry compose_tab_21[] = { +{67, 199, 0, NULL, NULL}, +{68, 7696, 0, NULL, NULL}, +{71, 290, 0, NULL, NULL}, +{72, 7720, 0, NULL, NULL}, +{75, 310, 0, NULL, NULL}, +{76, 315, 0, NULL, NULL}, +{78, 325, 0, NULL, NULL}, +{82, 342, 0, NULL, NULL}, +{83, 350, 0, NULL, NULL}, +{84, 354, 0, NULL, NULL}, +{99, 231, 0, NULL, NULL}, +{100, 7697, 0, NULL, NULL}, +{103, 291, 0, NULL, NULL}, +{104, 7721, 0, NULL, NULL}, +{107, 311, 0, NULL, NULL}, +{108, 316, 0, NULL, NULL}, +{110, 326, 0, NULL, NULL}, +{114, 343, 0, NULL, NULL}, +{115, 351, 0, NULL, NULL}, +{116, 355, 0, NULL, NULL} +}; /* compose_tab_21 */ +static int hash_compose_tab_22[20] = +{-1,6,-1,-1,-1,0,4,7,-1,1,-1,8,-1,2,-1,-1,-1,5,9,3}; /* hash_compose_tab_22 */ +static CompEntry compose_tab_22[] = { +{65, 260, 0, NULL, NULL}, +{69, 280, 0, NULL, NULL}, +{73, 302, 0, NULL, NULL}, +{79, 490, 0, NULL, NULL}, +{85, 370, 0, NULL, NULL}, +{97, 261, 0, NULL, NULL}, +{101, 281, 0, NULL, NULL}, +{105, 303, 0, NULL, NULL}, +{111, 491, 0, NULL, NULL}, +{117, 371, 0, NULL, NULL} +}; /* compose_tab_22 */ +static int hash_compose_tab_23[24] = +{-1,-1,-1,-1,2,6,3,7,-1,-1,-1,-1,4,5,8,9,-1,-1,-1,-1,0,1,10,11}; /* hash_compose_tab_23 */ +static CompEntry compose_tab_23[] = { +{68, 7698, 0, NULL, NULL}, +{69, 7704, 0, NULL, NULL}, +{76, 7740, 0, NULL, NULL}, +{78, 7754, 0, NULL, NULL}, +{84, 7792, 0, NULL, NULL}, +{85, 7798, 0, NULL, NULL}, +{100, 7699, 0, NULL, NULL}, +{101, 7705, 0, NULL, NULL}, +{108, 7741, 0, NULL, NULL}, +{110, 7755, 0, NULL, NULL}, +{116, 7793, 0, NULL, NULL}, +{117, 7799, 0, NULL, NULL} +}; /* compose_tab_23 */ +static int hash_compose_tab_24[4] = +{0,1,-1,-1}; /* hash_compose_tab_24 */ +static CompEntry compose_tab_24[] = { +{72, 7722, 0, NULL, NULL}, +{104, 7723, 0, NULL, NULL} +}; /* compose_tab_24 */ +static int hash_compose_tab_25[12] = +{-1,1,2,-1,-1,3,-1,-1,-1,0,4,5}; /* hash_compose_tab_25 */ +static CompEntry compose_tab_25[] = { +{69, 7706, 0, NULL, NULL}, +{73, 7724, 0, NULL, NULL}, +{85, 7796, 0, NULL, NULL}, +{101, 7707, 0, NULL, NULL}, +{105, 7725, 0, NULL, NULL}, +{117, 7797, 0, NULL, NULL} +}; /* compose_tab_25 */ +static int hash_compose_tab_26[34] = +{1,-1,10,-1,-1,11,12,2,3,13,4,-1,14,-1,5,15,6,-1,-1,-1,16,-1,7,-1,-1,-1,-1,-1, + -1,-1,8,-1,0,9}; /* hash_compose_tab_26 */ +static CompEntry compose_tab_26[] = { +{66, 7686, 0, NULL, NULL}, +{68, 7694, 0, NULL, NULL}, +{75, 7732, 0, NULL, NULL}, +{76, 7738, 0, NULL, NULL}, +{78, 7752, 0, NULL, NULL}, +{82, 7774, 0, NULL, NULL}, +{84, 7790, 0, NULL, NULL}, +{90, 7828, 0, NULL, NULL}, +{98, 7687, 0, NULL, NULL}, +{100, 7695, 0, NULL, NULL}, +{104, 7830, 0, NULL, NULL}, +{107, 7733, 0, NULL, NULL}, +{108, 7739, 0, NULL, NULL}, +{110, 7753, 0, NULL, NULL}, +{114, 7775, 0, NULL, NULL}, +{116, 7791, 0, NULL, NULL}, +{122, 7829, 0, NULL, NULL} +}; /* compose_tab_26 */ +static int hash_compose_tab_27_1[4] = +{-1,0,1,-1}; /* hash_compose_tab_27_1 */ +static CompEntry compose_tab_27_1[] = { +{953, 8151, 0, NULL, NULL}, +{965, 8167, 0, NULL, NULL} +}; /* compose_tab_27_1 */ +static int hash_compose_tab_27_2_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_2_0 */ +static CompEntry compose_tab_27_2_0[] = { +{913, 8078, 0, NULL, NULL}, +{919, 8094, 0, NULL, NULL}, +{937, 8110, 0, NULL, NULL}, +{945, 8070, 0, NULL, NULL}, +{951, 8086, 0, NULL, NULL}, +{969, 8102, 0, NULL, NULL} +}; /* compose_tab_27_2_0 */ +static int hash_compose_tab_27_2[20] = +{-1,3,-1,-1,-1,5,8,-1,-1,9,-1,6,-1,1,7,-1,-1,0,4,2}; /* hash_compose_tab_27_2 */ +static CompEntry compose_tab_27_2[] = { +{837, 0, 6, compose_tab_27_2_0, hash_compose_tab_27_2_0}, +{913, 7950, 0, NULL, NULL}, +{919, 7982, 0, NULL, NULL}, +{921, 7998, 0, NULL, NULL}, +{937, 8046, 0, NULL, NULL}, +{945, 7942, 0, NULL, NULL}, +{951, 7974, 0, NULL, NULL}, +{953, 7990, 0, NULL, NULL}, +{965, 8022, 0, NULL, NULL}, +{969, 8038, 0, NULL, NULL} +}; /* compose_tab_27_2 */ +static int hash_compose_tab_27_3_0[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_27_3_0 */ +static CompEntry compose_tab_27_3_0[] = { +{913, 8079, 0, NULL, NULL}, +{919, 8095, 0, NULL, NULL}, +{937, 8111, 0, NULL, NULL}, +{945, 8071, 0, NULL, NULL}, +{951, 8087, 0, NULL, NULL}, +{969, 8103, 0, NULL, NULL} +}; /* compose_tab_27_3_0 */ +static int hash_compose_tab_27_3[22] = +{-1,0,10,-1,-1,7,-1,8,-1,4,-1,1,-1,5,-1,-1,-1,2,-1,3,9,6}; /* hash_compose_tab_27_3 */ +static CompEntry compose_tab_27_3[] = { +{837, 0, 6, compose_tab_27_3_0, hash_compose_tab_27_3_0}, +{913, 7951, 0, NULL, NULL}, +{919, 7983, 0, NULL, NULL}, +{921, 7999, 0, NULL, NULL}, +{933, 8031, 0, NULL, NULL}, +{937, 8047, 0, NULL, NULL}, +{945, 7943, 0, NULL, NULL}, +{951, 7975, 0, NULL, NULL}, +{953, 7991, 0, NULL, NULL}, +{965, 8023, 0, NULL, NULL}, +{969, 8039, 0, NULL, NULL} +}; /* compose_tab_27_3 */ +static int hash_compose_tab_27_4[6] = +{-1,-1,-1,0,1,2}; /* hash_compose_tab_27_4 */ +static CompEntry compose_tab_27_4[] = { +{945, 8119, 0, NULL, NULL}, +{951, 8135, 0, NULL, NULL}, +{969, 8183, 0, NULL, NULL} +}; /* compose_tab_27_4 */ +static int hash_compose_tab_27[24] = +{0,-1,-1,-1,-1,8,11,-1,1,5,9,-1,-1,-1,-1,6,10,7,-1,2,3,4,-1,-1}; /* hash_compose_tab_27 */ +static CompEntry compose_tab_27[] = { +{168, 8129, 0, NULL, NULL}, +{776, 0, 2, compose_tab_27_1, hash_compose_tab_27_1}, +{787, 0, 10, compose_tab_27_2, hash_compose_tab_27_2}, +{788, 0, 11, compose_tab_27_3, hash_compose_tab_27_3}, +{837, 0, 3, compose_tab_27_4, hash_compose_tab_27_4}, +{945, 8118, 0, NULL, NULL}, +{951, 8134, 0, NULL, NULL}, +{953, 8150, 0, NULL, NULL}, +{965, 8166, 0, NULL, NULL}, +{969, 8182, 0, NULL, NULL}, +{8127, 8143, 0, NULL, NULL}, +{8190, 8159, 0, NULL, NULL} +}; /* compose_tab_27 */ +static int hash_compose_tab_28[12] = +{-1,0,2,4,-1,-1,-1,1,-1,3,5,-1}; /* hash_compose_tab_28 */ +static CompEntry compose_tab_28[] = { +{913, 8124, 0, NULL, NULL}, +{919, 8140, 0, NULL, NULL}, +{937, 8188, 0, NULL, NULL}, +{945, 8115, 0, NULL, NULL}, +{951, 8131, 0, NULL, NULL}, +{969, 8179, 0, NULL, NULL} +}; /* compose_tab_28 */ +static int hash_compose_tab_29[4] = +{0,-1,1,-1}; /* hash_compose_tab_29 */ +static CompEntry compose_tab_29[] = { +{1488, 64302, 0, NULL, NULL}, +{1522, 64287, 0, NULL, NULL} +}; /* compose_tab_29 */ +static int hash_compose_tab_30[2] = +{0,-1}; /* hash_compose_tab_30 */ +static CompEntry compose_tab_30[] = { +{1488, 64303, 0, NULL, NULL} +}; /* compose_tab_30 */ +static int hash_compose_tab_31[2] = +{-1,0}; /* hash_compose_tab_31 */ +static CompEntry compose_tab_31[] = { +{1493, 64331, 0, NULL, NULL} +}; /* compose_tab_31 */ +static int hash_compose_tab_32[44] = +{7,8,9,10,11,-1,12,-1,13,14,-1,15,16,-1,17,18,19,20,21,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,5,6,-1}; /* hash_compose_tab_32 */ +static CompEntry compose_tab_32[] = { +{1488, 64304, 0, NULL, NULL}, +{1489, 64305, 0, NULL, NULL}, +{1490, 64306, 0, NULL, NULL}, +{1491, 64307, 0, NULL, NULL}, +{1492, 64308, 0, NULL, NULL}, +{1493, 64309, 0, NULL, NULL}, +{1494, 64310, 0, NULL, NULL}, +{1496, 64312, 0, NULL, NULL}, +{1497, 64313, 0, NULL, NULL}, +{1498, 64314, 0, NULL, NULL}, +{1499, 64315, 0, NULL, NULL}, +{1500, 64316, 0, NULL, NULL}, +{1502, 64318, 0, NULL, NULL}, +{1504, 64320, 0, NULL, NULL}, +{1505, 64321, 0, NULL, NULL}, +{1507, 64323, 0, NULL, NULL}, +{1508, 64324, 0, NULL, NULL}, +{1510, 64326, 0, NULL, NULL}, +{1511, 64327, 0, NULL, NULL}, +{1512, 64328, 0, NULL, NULL}, +{1513, 64329, 0, NULL, NULL}, +{1514, 64330, 0, NULL, NULL} +}; /* compose_tab_32 */ +static int hash_compose_tab_33[6] = +{-1,0,2,-1,-1,1}; /* hash_compose_tab_33 */ +static CompEntry compose_tab_33[] = { +{1489, 64332, 0, NULL, NULL}, +{1499, 64333, 0, NULL, NULL}, +{1508, 64334, 0, NULL, NULL} +}; /* compose_tab_33 */ +static int hash_compose_tab_34_0[2] = +{-1,0}; /* hash_compose_tab_34_0 */ +static CompEntry compose_tab_34_0[] = { +{1513, 64300, 0, NULL, NULL} +}; /* compose_tab_34_0 */ +static int hash_compose_tab_34[4] = +{0,1,-1,-1}; /* hash_compose_tab_34 */ +static CompEntry compose_tab_34[] = { +{1468, 0, 1, compose_tab_34_0, hash_compose_tab_34_0}, +{1513, 64298, 0, NULL, NULL} +}; /* compose_tab_34 */ +static int hash_compose_tab_35_0[2] = +{-1,0}; /* hash_compose_tab_35_0 */ +static CompEntry compose_tab_35_0[] = { +{1513, 64301, 0, NULL, NULL} +}; /* compose_tab_35_0 */ +static int hash_compose_tab_35[4] = +{0,1,-1,-1}; /* hash_compose_tab_35 */ +static CompEntry compose_tab_35[] = { +{1468, 0, 1, compose_tab_35_0, hash_compose_tab_35_0}, +{1513, 64299, 0, NULL, NULL} +}; /* compose_tab_35 */ +static int hash_compose_tab_36[22] = +{3,10,-1,-1,-1,4,5,-1,-1,-1,-1,-1,6,-1,-1,0,1,2,7,8,9,-1}; /* hash_compose_tab_36 */ +static CompEntry compose_tab_36[] = { +{2325, 2392, 0, NULL, NULL}, +{2326, 2393, 0, NULL, NULL}, +{2327, 2394, 0, NULL, NULL}, +{2332, 2395, 0, NULL, NULL}, +{2337, 2396, 0, NULL, NULL}, +{2338, 2397, 0, NULL, NULL}, +{2344, 2345, 0, NULL, NULL}, +{2347, 2398, 0, NULL, NULL}, +{2351, 2399, 0, NULL, NULL}, +{2352, 2353, 0, NULL, NULL}, +{2355, 2356, 0, NULL, NULL} +}; /* compose_tab_36 */ +static int hash_compose_tab_37[8] = +{-1,0,1,-1,2,-1,-1,3}; /* hash_compose_tab_37 */ +static CompEntry compose_tab_37[] = { +{2465, 2524, 0, NULL, NULL}, +{2466, 2525, 0, NULL, NULL}, +{2476, 2480, 0, NULL, NULL}, +{2479, 2527, 0, NULL, NULL} +}; /* compose_tab_37 */ +static int hash_compose_tab_38[2] = +{-1,0}; /* hash_compose_tab_38 */ +static CompEntry compose_tab_38[] = { +{2503, 2507, 0, NULL, NULL} +}; /* compose_tab_38 */ +static int hash_compose_tab_39[2] = +{-1,0}; /* hash_compose_tab_39 */ +static CompEntry compose_tab_39[] = { +{2503, 2508, 0, NULL, NULL} +}; /* compose_tab_39 */ +static int hash_compose_tab_40[10] = +{-1,-1,0,1,3,4,-1,-1,2,-1}; /* hash_compose_tab_40 */ +static CompEntry compose_tab_40[] = { +{2582, 2649, 0, NULL, NULL}, +{2583, 2650, 0, NULL, NULL}, +{2588, 2651, 0, NULL, NULL}, +{2593, 2652, 0, NULL, NULL}, +{2603, 2654, 0, NULL, NULL} +}; /* compose_tab_40 */ +static int hash_compose_tab_41[6] = +{1,2,-1,-1,-1,0}; /* hash_compose_tab_41 */ +static CompEntry compose_tab_41[] = { +{2849, 2908, 0, NULL, NULL}, +{2850, 2909, 0, NULL, NULL}, +{2863, 2911, 0, NULL, NULL} +}; /* compose_tab_41 */ +static int hash_compose_tab_42[2] = +{-1,0}; /* hash_compose_tab_42 */ +static CompEntry compose_tab_42[] = { +{2887, 2891, 0, NULL, NULL} +}; /* compose_tab_42 */ +static int hash_compose_tab_43[2] = +{-1,0}; /* hash_compose_tab_43 */ +static CompEntry compose_tab_43[] = { +{2887, 2888, 0, NULL, NULL} +}; /* compose_tab_43 */ +static int hash_compose_tab_44[2] = +{-1,0}; /* hash_compose_tab_44 */ +static CompEntry compose_tab_44[] = { +{2887, 2892, 0, NULL, NULL} +}; /* compose_tab_44 */ +static int hash_compose_tab_45[4] = +{-1,-1,0,1}; /* hash_compose_tab_45 */ +static CompEntry compose_tab_45[] = { +{3014, 3018, 0, NULL, NULL}, +{3015, 3019, 0, NULL, NULL} +}; /* compose_tab_45 */ +static int hash_compose_tab_46[4] = +{-1,-1,0,1}; /* hash_compose_tab_46 */ +static CompEntry compose_tab_46[] = { +{2962, 2964, 0, NULL, NULL}, +{3014, 3020, 0, NULL, NULL} +}; /* compose_tab_46 */ +static int hash_compose_tab_47[2] = +{0,-1}; /* hash_compose_tab_47 */ +static CompEntry compose_tab_47[] = { +{3142, 3144, 0, NULL, NULL} +}; /* compose_tab_47 */ +static int hash_compose_tab_48[2] = +{0,-1}; /* hash_compose_tab_48 */ +static CompEntry compose_tab_48[] = { +{3270, 3274, 0, NULL, NULL} +}; /* compose_tab_48 */ +static int hash_compose_tab_49_1[2] = +{0,-1}; /* hash_compose_tab_49_1 */ +static CompEntry compose_tab_49_1[] = { +{3270, 3275, 0, NULL, NULL} +}; /* compose_tab_49_1 */ +static int hash_compose_tab_49[6] = +{2,-1,1,-1,-1,0}; /* hash_compose_tab_49 */ +static CompEntry compose_tab_49[] = { +{3263, 3264, 0, NULL, NULL}, +{3266, 0, 1, compose_tab_49_1, hash_compose_tab_49_1}, +{3270, 3271, 0, NULL, NULL} +}; /* compose_tab_49 */ +static int hash_compose_tab_50[2] = +{0,-1}; /* hash_compose_tab_50 */ +static CompEntry compose_tab_50[] = { +{3270, 3272, 0, NULL, NULL} +}; /* compose_tab_50 */ +static int hash_compose_tab_51[4] = +{-1,-1,0,1}; /* hash_compose_tab_51 */ +static CompEntry compose_tab_51[] = { +{3398, 3402, 0, NULL, NULL}, +{3399, 3403, 0, NULL, NULL} +}; /* compose_tab_51 */ +static int hash_compose_tab_52[2] = +{0,-1}; /* hash_compose_tab_52 */ +static CompEntry compose_tab_52[] = { +{3398, 3404, 0, NULL, NULL} +}; /* compose_tab_52 */ +static int hash_compose_tab_53[2] = +{-1,0}; /* hash_compose_tab_53 */ +static CompEntry compose_tab_53[] = { +{3661, 3635, 0, NULL, NULL} +}; /* compose_tab_53 */ +static int hash_compose_tab_54[2] = +{-1,0}; /* hash_compose_tab_54 */ +static CompEntry compose_tab_54[] = { +{3789, 3763, 0, NULL, NULL} +}; /* compose_tab_54 */ +static int hash_compose_tab_55_2[4] = +{-1,-1,0,1}; /* hash_compose_tab_55_2 */ +static CompEntry compose_tab_55_2[] = { +{4018, 3959, 0, NULL, NULL}, +{4019, 3961, 0, NULL, NULL} +}; /* compose_tab_55_2 */ +static int hash_compose_tab_55[6] = +{0,-1,1,2,-1,-1}; /* hash_compose_tab_55 */ +static CompEntry compose_tab_55[] = { +{3954, 3955, 0, NULL, NULL}, +{3956, 3957, 0, NULL, NULL}, +{3968, 0, 2, compose_tab_55_2, hash_compose_tab_55_2} +}; /* compose_tab_55 */ +static int hash_compose_tab_56[4] = +{-1,-1,0,1}; /* hash_compose_tab_56 */ +static CompEntry compose_tab_56[] = { +{4018, 3958, 0, NULL, NULL}, +{4019, 3960, 0, NULL, NULL} +}; /* compose_tab_56 */ +static int hash_compose_tab_57[4] = +{0,1,-1,-1}; /* hash_compose_tab_57 */ +static CompEntry compose_tab_57[] = { +{3904, 3945, 0, NULL, NULL}, +{3984, 4025, 0, NULL, NULL} +}; /* compose_tab_57 */ +static int hash_compose_tab_58[20] = +{-1,2,7,-1,-1,-1,0,3,5,8,-1,4,9,-1,-1,-1,1,6,-1,-1}; /* hash_compose_tab_58 */ +static CompEntry compose_tab_58[] = { +{3906, 3907, 0, NULL, NULL}, +{3916, 3917, 0, NULL, NULL}, +{3921, 3922, 0, NULL, NULL}, +{3926, 3927, 0, NULL, NULL}, +{3931, 3932, 0, NULL, NULL}, +{3986, 3987, 0, NULL, NULL}, +{3996, 3997, 0, NULL, NULL}, +{4001, 4002, 0, NULL, NULL}, +{4006, 4007, 0, NULL, NULL}, +{4011, 4012, 0, NULL, NULL} +}; /* compose_tab_58 */ +static int hash_compose_tab_59[96] = +{33,12,34,-1,13,35,14,36,15,37,-1,-1,-1,-1,-1,16,38,-1,17,39,-1,18,40,-1,19, + 41,-1,20,42,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,43,44,45, + 46,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,21,47,-1,-1,-1,-1,-1,-1,-1,0,22,-1,-1,-1,1, + 23,2,24,3,25,4,26,5,27,6,28,7,29,8,30,9,31,10,32,11}; /* hash_compose_tab_59 */ +static CompEntry compose_tab_59[] = { +{12358, 12436, 0, NULL, NULL}, +{12363, 12364, 0, NULL, NULL}, +{12365, 12366, 0, NULL, NULL}, +{12367, 12368, 0, NULL, NULL}, +{12369, 12370, 0, NULL, NULL}, +{12371, 12372, 0, NULL, NULL}, +{12373, 12374, 0, NULL, NULL}, +{12375, 12376, 0, NULL, NULL}, +{12377, 12378, 0, NULL, NULL}, +{12379, 12380, 0, NULL, NULL}, +{12381, 12382, 0, NULL, NULL}, +{12383, 12384, 0, NULL, NULL}, +{12385, 12386, 0, NULL, NULL}, +{12388, 12389, 0, NULL, NULL}, +{12390, 12391, 0, NULL, NULL}, +{12392, 12393, 0, NULL, NULL}, +{12399, 12400, 0, NULL, NULL}, +{12402, 12403, 0, NULL, NULL}, +{12405, 12406, 0, NULL, NULL}, +{12408, 12409, 0, NULL, NULL}, +{12411, 12412, 0, NULL, NULL}, +{12445, 12446, 0, NULL, NULL}, +{12454, 12532, 0, NULL, NULL}, +{12459, 12460, 0, NULL, NULL}, +{12461, 12462, 0, NULL, NULL}, +{12463, 12464, 0, NULL, NULL}, +{12465, 12466, 0, NULL, NULL}, +{12467, 12468, 0, NULL, NULL}, +{12469, 12470, 0, NULL, NULL}, +{12471, 12472, 0, NULL, NULL}, +{12473, 12474, 0, NULL, NULL}, +{12475, 12476, 0, NULL, NULL}, +{12477, 12478, 0, NULL, NULL}, +{12479, 12480, 0, NULL, NULL}, +{12481, 12482, 0, NULL, NULL}, +{12484, 12485, 0, NULL, NULL}, +{12486, 12487, 0, NULL, NULL}, +{12488, 12489, 0, NULL, NULL}, +{12495, 12496, 0, NULL, NULL}, +{12498, 12499, 0, NULL, NULL}, +{12501, 12502, 0, NULL, NULL}, +{12504, 12505, 0, NULL, NULL}, +{12507, 12508, 0, NULL, NULL}, +{12527, 12535, 0, NULL, NULL}, +{12528, 12536, 0, NULL, NULL}, +{12529, 12537, 0, NULL, NULL}, +{12530, 12538, 0, NULL, NULL}, +{12541, 12542, 0, NULL, NULL} +}; /* compose_tab_59 */ +static int hash_compose_tab_60[20] = +{-1,7,1,-1,8,2,-1,9,3,-1,-1,4,-1,-1,-1,5,-1,-1,6,0}; /* hash_compose_tab_60 */ +static CompEntry compose_tab_60[] = { +{12399, 12401, 0, NULL, NULL}, +{12402, 12404, 0, NULL, NULL}, +{12405, 12407, 0, NULL, NULL}, +{12408, 12410, 0, NULL, NULL}, +{12411, 12413, 0, NULL, NULL}, +{12495, 12497, 0, NULL, NULL}, +{12498, 12500, 0, NULL, NULL}, +{12501, 12503, 0, NULL, NULL}, +{12504, 12506, 0, NULL, NULL}, +{12507, 12509, 0, NULL, NULL} +}; /* compose_tab_60 */ +static int hash_compose_tab[122] = +{30,31,52,60,32,-1,-1,33,-1,34,35,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,-1,0,1,2,3,4,-1,5,6,7,8,9,10,11,12,36,13,37,14, + 38,15,16,55,40,-1,-1,-1,-1,17,56,-1,-1,-1,-1,-1,41,18,19,20,42,21,22,-1,45, + 39,-1,23,24,-1,25,26,-1,-1,-1,-1,-1,-1,-1,-1,48,-1,43,44,51,53,-1,-1,27,46, + 54,28,-1,-1,47,-1,-1,-1,-1,49,50,-1,-1,57,-1,58,59,29}; /* hash_compose_tab */ +static CompEntry compose_tab[] = { +{768, 0, 39, compose_tab_0, hash_compose_tab_0}, +{769, 0, 70, compose_tab_1, hash_compose_tab_1}, +{770, 0, 27, compose_tab_2, hash_compose_tab_2}, +{771, 0, 19, compose_tab_3, hash_compose_tab_3}, +{772, 0, 28, compose_tab_4, hash_compose_tab_4}, +{774, 0, 30, compose_tab_5, hash_compose_tab_5}, +{775, 0, 40, compose_tab_6, hash_compose_tab_6}, +{776, 0, 50, compose_tab_7, hash_compose_tab_7}, +{777, 0, 15, compose_tab_8, hash_compose_tab_8}, +{778, 0, 6, compose_tab_9, hash_compose_tab_9}, +{779, 0, 6, compose_tab_10, hash_compose_tab_10}, +{780, 0, 34, compose_tab_11, hash_compose_tab_11}, +{781, 0, 17, compose_tab_12, hash_compose_tab_12}, +{783, 0, 14, compose_tab_13, hash_compose_tab_13}, +{785, 0, 12, compose_tab_14, hash_compose_tab_14}, +{787, 0, 15, compose_tab_15, hash_compose_tab_15}, +{788, 0, 17, compose_tab_16, hash_compose_tab_16}, +{795, 0, 4, compose_tab_17, hash_compose_tab_17}, +{803, 0, 39, compose_tab_18, hash_compose_tab_18}, +{804, 0, 2, compose_tab_19, hash_compose_tab_19}, +{805, 0, 2, compose_tab_20, hash_compose_tab_20}, +{807, 0, 20, compose_tab_21, hash_compose_tab_21}, +{808, 0, 10, compose_tab_22, hash_compose_tab_22}, +{813, 0, 12, compose_tab_23, hash_compose_tab_23}, +{814, 0, 2, compose_tab_24, hash_compose_tab_24}, +{816, 0, 6, compose_tab_25, hash_compose_tab_25}, +{817, 0, 17, compose_tab_26, hash_compose_tab_26}, +{834, 0, 12, compose_tab_27, hash_compose_tab_27}, +{837, 0, 6, compose_tab_28, hash_compose_tab_28}, +{1463, 0, 2, compose_tab_29, hash_compose_tab_29}, +{1464, 0, 1, compose_tab_30, hash_compose_tab_30}, +{1465, 0, 1, compose_tab_31, hash_compose_tab_31}, +{1468, 0, 22, compose_tab_32, hash_compose_tab_32}, +{1471, 0, 3, compose_tab_33, hash_compose_tab_33}, +{1473, 0, 2, compose_tab_34, hash_compose_tab_34}, +{1474, 0, 2, compose_tab_35, hash_compose_tab_35}, +{2364, 0, 11, compose_tab_36, hash_compose_tab_36}, +{2492, 0, 4, compose_tab_37, hash_compose_tab_37}, +{2494, 0, 1, compose_tab_38, hash_compose_tab_38}, +{2519, 0, 1, compose_tab_39, hash_compose_tab_39}, +{2620, 0, 5, compose_tab_40, hash_compose_tab_40}, +{2876, 0, 3, compose_tab_41, hash_compose_tab_41}, +{2878, 0, 1, compose_tab_42, hash_compose_tab_42}, +{2902, 0, 1, compose_tab_43, hash_compose_tab_43}, +{2903, 0, 1, compose_tab_44, hash_compose_tab_44}, +{3006, 0, 2, compose_tab_45, hash_compose_tab_45}, +{3031, 0, 2, compose_tab_46, hash_compose_tab_46}, +{3158, 0, 1, compose_tab_47, hash_compose_tab_47}, +{3266, 0, 1, compose_tab_48, hash_compose_tab_48}, +{3285, 0, 3, compose_tab_49, hash_compose_tab_49}, +{3286, 0, 1, compose_tab_50, hash_compose_tab_50}, +{3390, 0, 2, compose_tab_51, hash_compose_tab_51}, +{3415, 0, 1, compose_tab_52, hash_compose_tab_52}, +{3634, 0, 1, compose_tab_53, hash_compose_tab_53}, +{3762, 0, 1, compose_tab_54, hash_compose_tab_54}, +{3953, 0, 3, compose_tab_55, hash_compose_tab_55}, +{3968, 0, 2, compose_tab_56, hash_compose_tab_56}, +{4021, 0, 2, compose_tab_57, hash_compose_tab_57}, +{4023, 0, 10, compose_tab_58, hash_compose_tab_58}, +{12441, 0, 48, compose_tab_59, hash_compose_tab_59}, +{12442, 0, 10, compose_tab_60, hash_compose_tab_60} +}; /* compose_tab */ +#define COMP_CANDIDATE_MAP_OFFSET 24 +static Uint32 comp_candidate_map[] = { + 0x081ABFDFU, + 0x000361B8U, + 0x00000024U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x93800000U, + 0x00000006U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x10000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x50000000U, + 0x00800000U, + 0x00000000U, + 0x00000000U, + 0x10000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x50000000U, + 0x00C00000U, + 0x00000000U, + 0x00000000U, + 0x40000000U, + 0x00800000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00400000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00600004U, + 0x00000000U, + 0x00000000U, + 0x40000000U, + 0x00800000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00040000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00040000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00000000U, + 0x00020000U, + 0x00000001U, + 0x00A00000U +}; diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index cd63401581..e7fd144ec3 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -47,7 +47,7 @@ #define SEQ_TRACE 1 #define CONTEXT_REDS 2000 /* Swap process out after this number */ -#define MAX_ARG 256 /* Max number of arguments allowed */ +#define MAX_ARG 255 /* Max number of arguments allowed */ #define MAX_REG 1024 /* Max number of x(N) registers used */ /* Scheduler stores data for temporary heaps if @@ -120,14 +120,15 @@ * Allocate heap memory, first on the ordinary heap; * failing that, in a heap fragment. */ -#define HAlloc(p, sz) \ +#define HAllocX(p, sz, xtra) \ (ASSERT_EXPR((sz) >= 0), \ ErtsHAllocLockCheck(p), \ (IS_FORCE_HEAP_FRAGS || (((HEAP_LIMIT(p) - HEAP_TOP(p)) < (sz))) \ - ? erts_heap_alloc((p),(sz)) \ + ? erts_heap_alloc((p),(sz),(xtra)) \ : (INIT_HEAP_MEM(p,sz), \ HEAP_TOP(p) = HEAP_TOP(p) + (sz), HEAP_TOP(p) - (sz)))) +#define HAlloc(P, SZ) HAllocX(P,SZ,0) #define HRelease(p, endp, ptr) \ if ((ptr) == (endp)) { \ @@ -199,6 +200,7 @@ extern int BIN_VH_MIN_SIZE; /* minimum virtual (bin) heap */ extern int erts_atom_table_size;/* Atom table size */ #define ORIG_CREATION 0 +#define INTERNAL_CREATION 255 /* macros for extracting bytes from uint16's */ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index d7c8aa84e9..1a102f7187 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -49,10 +49,8 @@ #define in_area(ptr,start,nbytes) ((Uint)((char*)(ptr) - (char*)(start)) < (nbytes)) #define MAX_STRING_LEN 0xffff -#define dec_set_creation(nodename,creat) \ - (((nodename) == erts_this_node->sysname && (creat) == ORIG_CREATION) \ - ? erts_this_node->creation \ - : (creat)) + +#define is_valid_creation(Cre) ((unsigned)(Cre) < MAX_CREATION || (Cre) == INTERNAL_CREATION) #undef ERTS_DEBUG_USE_DIST_SEP #ifdef DEBUG @@ -83,14 +81,14 @@ * */ -static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32); +static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap); static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); -static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins); +static Sint decoded_size(byte *ep, byte* endp, int only_heap_bins, int internal_tags); static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); @@ -461,6 +459,12 @@ Uint erts_encode_ext_size(Eterm term) + 1 /* VERSION_MAGIC */; } +Uint erts_encode_ext_size_ets(Eterm term) +{ + return encode_size_struct2(NULL, term, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS); +} + + void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap *acmp) { byte *ep = *ext; @@ -468,7 +472,7 @@ void erts_encode_dist_ext(Eterm term, byte **ext, Uint32 flags, ErtsAtomCacheMap if (!(flags & DFLAG_DIST_HDR_ATOM_CACHE)) #endif *ep++ = VERSION_MAGIC; - ep = enc_term(acmp, term, ep, flags); + ep = enc_term(acmp, term, ep, flags, NULL); if (!ep) erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_dist_ext(): Internal data structure error\n", @@ -480,7 +484,7 @@ void erts_encode_ext(Eterm term, byte **ext) { byte *ep = *ext; *ep++ = VERSION_MAGIC; - ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS); + ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL); if (!ep) erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_ext(): Internal data structure error\n", @@ -488,6 +492,12 @@ void erts_encode_ext(Eterm term, byte **ext) *ext = ep; } +byte* erts_encode_ext_ets(Eterm term, byte *ep, struct erl_off_heap_header** off_heap) +{ + return enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS|DFLAGS_INTERNAL_TAGS, + off_heap); +} + ErtsDistExternal * erts_make_dist_ext_copy(ErtsDistExternal *edep, Uint xsize) { @@ -813,7 +823,7 @@ erts_decode_dist_ext_size(ErtsDistExternal *edep, int no_refc_bins) goto fail; ep = edep->extp+1; } - res = decoded_size(ep, edep->ext_endp, no_refc_bins); + res = decoded_size(ep, edep->ext_endp, no_refc_bins, 0); if (res >= 0) return res; fail: @@ -825,9 +835,17 @@ Sint erts_decode_ext_size(byte *ext, Uint size, int no_refc_bins) { if (size == 0 || *ext != VERSION_MAGIC) return -1; - return decoded_size(ext+1, ext+size, no_refc_bins); + return decoded_size(ext+1, ext+size, no_refc_bins, 0); } +Sint erts_decode_ext_size_ets(byte *ext, Uint size) +{ + Sint sz = decoded_size(ext, ext+size, 0, 1); + ASSERT(sz >= 0); + return sz; +} + + /* ** hpp is set to either a &p->htop or ** a pointer to a memory pointer (form message buffers) @@ -887,7 +905,13 @@ Eterm erts_decode_ext(Eterm **hpp, ErlOffHeap *off_heap, byte **ext) return obj; } - +Eterm erts_decode_ext_ets(Eterm **hpp, ErlOffHeap *off_heap, byte *ext) +{ + Eterm obj; + ext = dec_term(NULL, hpp, ext, off_heap, &obj); + ASSERT(ext); + return obj; +} /**********************************************************************/ @@ -964,6 +988,7 @@ term_to_binary_1(Process* p, Eterm Term) return erts_term_to_binary(p, Term, 0, TERM_TO_BINARY_DFLAGS); } + Eterm term_to_binary_2(Process* p, Eterm Term, Eterm Flags) { @@ -1075,7 +1100,7 @@ binary2term_prepare(ErtsBinary2TermState *state, byte *data, Sint data_size) goto error; size = (Sint) dest_len; } - res = decoded_size(state->extp, state->extp + size, 0); + res = decoded_size(state->extp, state->extp + size, 0, 0); if (res < 0) goto error; return res; @@ -1183,7 +1208,8 @@ BIF_RETTYPE binary_to_term_2(BIF_ALIST_2) opt = CAR(list_val(opts)); if (opt == am_safe) { fakedep.flags |= ERTS_DIST_EXT_BTT_SAFE; - } else { + } + else { goto error; } opts = CDR(list_val(opts)); @@ -1238,7 +1264,7 @@ external_size_1(Process* p, Eterm Term) Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { - int size; + Uint size; Eterm bin; size_t real_size; byte* endp; @@ -1255,7 +1281,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) bytes = erts_alloc(ERTS_ALC_T_TMP, size); } - if ((endp = enc_term(NULL, Term, bytes, flags)) + if ((endp = enc_term(NULL, Term, bytes, flags, NULL)) == NULL) { erl_exit(1, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); @@ -1300,7 +1326,7 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) bin = new_binary(p, (byte *)NULL, size); bytes = binary_bytes(bin); bytes[0] = VERSION_MAGIC; - if ((endp = enc_term(NULL, Term, bytes+1, flags)) + if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL)) == NULL) { erl_exit(1, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); @@ -1330,6 +1356,21 @@ enc_atom(ErtsAtomCacheMap *acmp, Eterm atom, byte *ep, Uint32 dflags) ASSERT(is_atom(atom)); + if (dflags & DFLAGS_INTERNAL_TAGS) { + Uint aval = atom_val(atom); + ASSERT(aval < (1<<24)); + if (aval >= (1 << 16)) { + *ep++ = ATOM_INTERNAL_REF3; + put_int24(aval, ep); + ep += 3; + } + else { + *ep++ = ATOM_INTERNAL_REF2; + put_int16(aval, ep); + ep += 2; + } + return ep; + } /* * term_to_binary/1,2 and the initial distribution message * don't use the cache. @@ -1379,7 +1420,8 @@ enc_pid(ErtsAtomCacheMap *acmp, Eterm pid, byte* ep, Uint32 dflags) ep += 4; put_int32(os, ep); ep += 4; - *ep++ = pid_creation(pid); + *ep++ = (is_internal_pid(pid) && (dflags & DFLAGS_INTERNAL_TAGS)) ? + INTERNAL_CREATION : pid_creation(pid); return ep; } @@ -1418,6 +1460,23 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) } ep += len; break; + case ATOM_INTERNAL_REF2: + n = get_int16(ep); + ep += 2; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + case ATOM_INTERNAL_REF3: + n = get_int24(ep); + ep += 3; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + default: error: *objp = NIL; /* Don't leave a hole in the heap */ @@ -1426,6 +1485,19 @@ dec_atom(ErtsDistExternal *edep, byte* ep, Eterm* objp) return ep; } +static ERTS_INLINE ErlNode* dec_get_node(Eterm sysname, Uint creation) +{ + switch (creation) { + case INTERNAL_CREATION: + return erts_this_node; + case ORIG_CREATION: + if (sysname == erts_this_node->sysname) { + creation = erts_this_node->creation; + } + } + return erts_find_or_insert_node(sysname,creation); +} + static byte* dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Eterm* objp) { @@ -1449,18 +1521,20 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete ep += 4; if (ser > ERTS_MAX_PID_SERIAL) return NULL; - if ((cre = get_int8(ep)) >= MAX_CREATION) - return NULL; + cre = get_int8(ep); ep += 1; + if (!is_valid_creation(cre)) { + return NULL; + } + data = make_pid_data(ser, num); + /* * We are careful to create the node entry only after all * validity tests are done. */ - cre = dec_set_creation(sysname,cre); - node = erts_find_or_insert_node(sysname,cre); + node = dec_get_node(sysname, cre); - data = make_pid_data(ser, num); if(node == erts_this_node) { *objp = make_internal_pid(data); } else { @@ -1485,7 +1559,8 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3) static byte* -enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) +enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, + struct erl_off_heap_header** off_heap) { DECLARE_WSTACK(s); Uint n; @@ -1637,12 +1712,14 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) Uint32 *ref_num; ASSERT(dflags & DFLAG_EXTENDED_REFERENCES); + *ep++ = NEW_REFERENCE_EXT; i = ref_no_of_numbers(obj); put_int16(i, ep); ep += 2; ep = enc_atom(acmp,ref_node_name(obj),ep,dflags); - *ep++ = ref_creation(obj); + *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_ref(obj)) ? + INTERNAL_CREATION : ref_creation(obj); ref_num = ref_numbers(obj); for (j = 0; j < i; j++) { put_int32(ref_num[j], ep); @@ -1658,7 +1735,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) j = port_number(obj); put_int32(j, ep); ep += 4; - *ep++ = port_creation(obj); + *ep++ = ((dflags & DFLAGS_INTERNAL_TAGS) && is_internal_port(obj)) ? + INTERNAL_CREATION : port_creation(obj); break; case LIST_DEF: @@ -1738,6 +1816,41 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) byte* bytes; ERTS_GET_BINARY_BYTES(obj, bytes, bitoffs, bitsize); + if (dflags & DFLAGS_INTERNAL_TAGS) { + ProcBin* pb = (ProcBin*) binary_val(obj); + Uint bytesize = pb->size; + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sub = (ErlSubBin*)pb; + pb = (ProcBin*) binary_val(sub->orig); + ASSERT(bytesize == sub->size); + bytesize += (bitoffs + bitsize + 7) / 8; + } + if (pb->thing_word == HEADER_PROC_BIN + && heap_bin_size(bytesize) > PROC_BIN_SIZE) { + ProcBin tmp; + if (bitoffs || bitsize) { + *ep++ = BIT_BINARY_INTERNAL_REF; + *ep++ = bitoffs; + *ep++ = bitsize; + } + else { + *ep++ = BINARY_INTERNAL_REF; + } + if (pb->flags) { + erts_emasculate_writable_binary(pb); + } + erts_refc_inc(&pb->val->refc, 2); + + sys_memcpy(&tmp, pb, sizeof(ProcBin)); + tmp.next = *off_heap; + tmp.bytes = bytes; + tmp.size = bytesize; + sys_memcpy(ep, &tmp, sizeof(ProcBin)); + *off_heap = (struct erl_off_heap_header*) ep; + ep += sizeof(ProcBin); + break; + } + } if (bitsize == 0) { /* Plain old byte-sized binary. */ *ep++ = BINARY_EXT; @@ -1773,8 +1886,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) *ep++ = SMALL_INTEGER_EXT; *ep++ = bitsize; } - break; } + break; case EXPORT_DEF: { Export* exp = *((Export **) (export_val(obj) + 1)); @@ -1782,7 +1895,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) *ep++ = EXPORT_EXT; ep = enc_atom(acmp, exp->code[0], ep, dflags); ep = enc_atom(acmp, exp->code[1], ep, dflags); - ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags); + ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap); } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -1818,8 +1931,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) put_int32(funp->num_free, ep); ep += 4; ep = enc_atom(acmp, funp->fe->module, ep, dflags); - ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags); - ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags); + ep = enc_term(acmp, make_small(funp->fe->old_index), ep, dflags, off_heap); + ep = enc_term(acmp, make_small(funp->fe->old_uniq), ep, dflags, off_heap); ep = enc_pid(acmp, funp->creator, ep, dflags); fun_env: @@ -1872,7 +1985,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags) return ep; } -static Uint +static +Uint is_external_string(Eterm list, int* p_is_string) { Uint len = 0; @@ -2162,13 +2276,13 @@ dec_term_atom_common: goto error; } ep += 4; - if ((cre = get_int8(ep)) >= MAX_CREATION) { + cre = get_int8(ep); + ep++; + if (!is_valid_creation(cre)) { goto error; } - ep++; - cre = dec_set_creation(sysname,cre); - node = erts_find_or_insert_node(sysname, cre); + node = dec_get_node(sysname, cre); if(node == erts_this_node) { *objp = make_internal_port(num); } @@ -2205,9 +2319,11 @@ dec_term_atom_common: goto error; ep += 4; - if ((cre = get_int8(ep)) >= MAX_CREATION) - goto error; + cre = get_int8(ep); ep += 1; + if (!is_valid_creation(cre)) { + goto error; + } goto ref_ext_common; case NEW_REFERENCE_EXT: @@ -2220,10 +2336,11 @@ dec_term_atom_common: if ((ep = dec_atom(edep, ep, &sysname)) == NULL) goto error; - if ((cre = get_int8(ep)) >= MAX_CREATION) - goto error; + cre = get_int8(ep); ep += 1; - + if (!is_valid_creation(cre)) { + goto error; + } r0 = get_int32(ep); ep += 4; if (r0 >= MAX_REFERENCE) @@ -2231,8 +2348,7 @@ dec_term_atom_common: ref_ext_common: - cre = dec_set_creation(sysname, cre); - node = erts_find_or_insert_node(sysname, cre); + node = dec_get_node(sysname, cre); if(node == erts_this_node) { RefThing *rtp = (RefThing *) hp; ref_num = (Uint32 *) (hp + REF_THING_HEAD_SIZE); @@ -2560,6 +2676,66 @@ dec_term_atom_common: } break; } + case ATOM_INTERNAL_REF2: + n = get_int16(ep); + ep += 2; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + case ATOM_INTERNAL_REF3: + n = get_int24(ep); + ep += 3; + if (n >= atom_table_size()) { + goto error; + } + *objp = make_atom(n); + break; + + case BINARY_INTERNAL_REF: + { + ProcBin* pb = (ProcBin*) hp; + sys_memcpy(pb, ep, sizeof(ProcBin)); + ep += sizeof(ProcBin); + + erts_refc_inc(&pb->val->refc, 1); + hp += PROC_BIN_SIZE; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; + pb->flags = 0; + *objp = make_binary(pb); + break; + } + case BIT_BINARY_INTERNAL_REF: + { + Sint bitoffs = *ep++; + Sint bitsize = *ep++; + ProcBin* pb = (ProcBin*) hp; + ErlSubBin* sub; + sys_memcpy(pb, ep, sizeof(ProcBin)); + ep += sizeof(ProcBin); + + erts_refc_inc(&pb->val->refc, 1); + hp += PROC_BIN_SIZE; + pb->next = off_heap->first; + off_heap->first = (struct erl_off_heap_header*)pb; + pb->flags = 0; + + sub = (ErlSubBin*)hp; + sub->thing_word = HEADER_SUB_BIN; + sub->size = pb->size - (bitoffs + bitsize + 7)/8; + sub->offs = 0; + sub->bitoffs = bitoffs; + sub->bitsize = bitsize; + sub->is_writable = 0; + sub->orig = make_binary(pb); + + hp += ERL_SUB_BIN_SIZE; + *objp = make_binary(sub); + break; + } + default: error: /* UNDO: @@ -2642,20 +2818,29 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) case NIL_DEF: result++; break; - case ATOM_DEF: { - int alen = atom_tab(atom_val(obj))->len; - if ((MAX_ATOM_LENGTH <= 255 || alen <= 255) - && (dflags & DFLAG_SMALL_ATOM_TAGS)) { - /* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */ - result += 1 + 1 + alen; + case ATOM_DEF: + if (dflags & DFLAGS_INTERNAL_TAGS) { + if (atom_val(obj) >= (1<<16)) { + result += 1 + 3; + } + else { + result += 1 + 2; + } } else { - /* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */ - result += 1 + 2 + alen; + int alen = atom_tab(atom_val(obj))->len; + if ((MAX_ATOM_LENGTH <= 255 || alen <= 255) + && (dflags & DFLAG_SMALL_ATOM_TAGS)) { + /* Make sure a SMALL_ATOM_EXT fits: SMALL_ATOM_EXT l t1 t2... */ + result += 1 + 1 + alen; + } + else { + /* Make sure an ATOM_EXT fits: ATOM_EXT l1 l0 t1 t2... */ + result += 1 + 2 + alen; + } + insert_acache_map(acmp, obj); } - insert_acache_map(acmp, obj); break; - } case SMALL_DEF: { Sint val = signed_val(obj); @@ -2734,8 +2919,25 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } break; case BINARY_DEF: + if (dflags & DFLAGS_INTERNAL_TAGS) { + ProcBin* pb = (ProcBin*) binary_val(obj); + Uint sub_extra = 0; + Uint tot_bytes = pb->size; + if (pb->thing_word == HEADER_SUB_BIN) { + ErlSubBin* sub = (ErlSubBin*) pb; + pb = (ProcBin*) binary_val(sub->orig); + sub_extra = 2; /* bitoffs and bitsize */ + tot_bytes += (sub->bitoffs + sub->bitsize+ 7) / 8; + } + if (pb->thing_word == HEADER_PROC_BIN + && heap_bin_size(tot_bytes) > PROC_BIN_SIZE) { + + result += 1 + sub_extra + sizeof(ProcBin); + break; + } + } result += 1 + 4 + binary_size(obj) + - 5; /* For unaligned binary */ + 5; /* For unaligned binary */ break; case FUN_DEF: { @@ -2807,7 +3009,7 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } static Sint -decoded_size(byte *ep, byte* endp, int no_refc_bins) +decoded_size(byte *ep, byte* endp, int no_refc_bins, int internal_tags) { int heap_size = 0; int terms; @@ -3017,6 +3219,29 @@ decoded_size(byte *ep, byte* endp, int no_refc_bins) heap_size += ERL_FUN_SIZE + num_free; break; } + case ATOM_INTERNAL_REF2: + SKIP(2+atom_extra_skip); + atom_extra_skip = 0; + break; + case ATOM_INTERNAL_REF3: + SKIP(3+atom_extra_skip); + atom_extra_skip = 0; + break; + + case BINARY_INTERNAL_REF: + if (!internal_tags) { + return -1; + } + SKIP(sizeof(ProcBin)); + heap_size += PROC_BIN_SIZE; + break; + case BIT_BINARY_INTERNAL_REF: + if (!internal_tags) { + return -1; + } + SKIP(2+sizeof(ProcBin)); + heap_size += PROC_BIN_SIZE + ERL_SUB_BIN_SIZE; + break; default: return -1; } diff --git a/erts/emulator/beam/external.h b/erts/emulator/beam/external.h index cee48bbeb0..d8287b96a4 100644 --- a/erts/emulator/beam/external.h +++ b/erts/emulator/beam/external.h @@ -54,6 +54,10 @@ #define DIST_HEADER 'D' #define ATOM_CACHE_REF 'R' +#define ATOM_INTERNAL_REF2 'I' +#define ATOM_INTERNAL_REF3 'K' +#define BINARY_INTERNAL_REF 'J' +#define BIT_BINARY_INTERNAL_REF 'L' #define COMPRESSED 'P' #if 0 @@ -156,7 +160,9 @@ Uint erts_encode_dist_ext_size(Eterm, Uint32, ErtsAtomCacheMap *); void erts_encode_dist_ext(Eterm, byte **, Uint32, ErtsAtomCacheMap *); Uint erts_encode_ext_size(Eterm); +Uint erts_encode_ext_size_ets(Eterm); void erts_encode_ext(Eterm, byte **); +byte* erts_encode_ext_ets(Eterm, byte *, struct erl_off_heap_header** ext_off_heap); #ifdef ERTS_WANT_EXTERNAL_TAGS ERTS_GLB_INLINE void erts_peek_dist_header(ErtsDistHeaderPeek *, byte *, Uint); @@ -172,7 +178,9 @@ Sint erts_decode_dist_ext_size(ErtsDistExternal *, int); Eterm erts_decode_dist_ext(Eterm **, ErlOffHeap *, ErtsDistExternal *); Sint erts_decode_ext_size(byte*, Uint, int); +Sint erts_decode_ext_size_ets(byte*, Uint); Eterm erts_decode_ext(Eterm **, ErlOffHeap *, byte**); +Eterm erts_decode_ext_ets(Eterm **, ErlOffHeap *, byte*); Eterm erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index ecd3c8f68a..432bdd705b 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -522,6 +522,7 @@ union erl_off_heap_ptr { struct erl_fun_thing* fun; struct external_thing_* ext; Eterm* ep; + void* voidp; }; /* arrays that get malloced at startup */ @@ -543,7 +544,7 @@ ERTS_GLB_INLINE void erts_may_save_closed_port(Port *prt) if (prt->snapshot != erts_smp_atomic_read(&erts_ports_snapshot)) { /* Dead ports are added from the end of the snapshot buffer */ Eterm* tombstone = (Eterm*) erts_smp_atomic_addtest(&erts_dead_ports_ptr, - -(long)sizeof(Eterm)); + -(erts_aint_t)sizeof(Eterm)); ASSERT(tombstone+1 != NULL); ASSERT(prt->snapshot == (Uint32) erts_smp_atomic_read(&erts_ports_snapshot) - 1); *tombstone = prt->id; @@ -562,7 +563,7 @@ extern Uint display_items; /* no of items to display in traces etc */ extern Uint display_loads; /* print info about loaded modules */ extern int erts_backtrace_depth; -extern erts_smp_atomic_t erts_max_gen_gcs; +extern erts_smp_atomic32_t erts_max_gen_gcs; extern int erts_disable_tolerant_timeofday; @@ -833,7 +834,7 @@ do { \ void erts_emasculate_writable_binary(ProcBin* pb); Eterm erts_new_heap_binary(Process *p, byte *buf, int len, byte** datap); Eterm erts_new_mso_binary(Process*, byte*, int); -Eterm new_binary(Process*, byte*, int); +Eterm new_binary(Process*, byte*, Uint); Eterm erts_realloc_binary(Eterm bin, size_t size); /* erl_bif_info.c */ @@ -889,9 +890,31 @@ void erl_error(char*, va_list); /* copy.c */ void init_copy(void); Eterm copy_object(Eterm, Process*); + +#if HALFWORD_HEAP +Uint size_object_rel(Eterm, Eterm*); +# define size_object(A) size_object_rel(A,NULL) + +Eterm copy_struct_rel(Eterm, Uint, Eterm**, ErlOffHeap*, Eterm* src_base, Eterm* dst_base); +# define copy_struct(OBJ,SZ,HPP,OH) copy_struct_rel(OBJ,SZ,HPP,OH, NULL,NULL) + +Eterm copy_shallow_rel(Eterm*, Uint, Eterm**, ErlOffHeap*, Eterm* src_base); +# define copy_shallow(A,B,C,D) copy_shallow_rel(A,B,C,D,NULL) + +#else /* !HALFWORD_HEAP */ + Uint size_object(Eterm); +# define size_object_rel(A,B) size_object(A) + Eterm copy_struct(Eterm, Uint, Eterm**, ErlOffHeap*); +# define copy_struct_rel(OBJ,SZ,HPP,OH, SB,DB) copy_struct(OBJ,SZ,HPP,OH) + Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); +# define copy_shallow_rel(A,B,C,D, BASE) copy_shallow(A,B,C,D) + +#endif + + void move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, Eterm* refs, unsigned nrefs); @@ -1205,7 +1228,7 @@ ERTS_GLB_INLINE void erts_smp_port_unlock(Port *prt) { #ifdef ERTS_SMP - long refc; + erts_aint_t refc; erts_smp_mtx_unlock(prt->lock); refc = erts_smp_atomic_dectest(&prt->refc); ASSERT(refc >= 0); @@ -1424,84 +1447,6 @@ void erl_drv_thr_init(void); /* time.c */ -ERTS_GLB_INLINE long do_time_read_and_reset(void); -#ifdef ERTS_TIMER_THREAD -ERTS_GLB_INLINE int next_time(void); -ERTS_GLB_INLINE void bump_timer(long); -#else -int next_time(void); -void bump_timer(long); -extern erts_smp_atomic_t do_time; /* set at clock interrupt */ -ERTS_GLB_INLINE void do_time_add(long); -#endif - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -#ifdef ERTS_TIMER_THREAD -ERTS_GLB_INLINE long do_time_read_and_reset(void) { return 0; } -ERTS_GLB_INLINE int next_time(void) { return -1; } -ERTS_GLB_INLINE void bump_timer(long ignore) { } -#else -ERTS_GLB_INLINE long do_time_read_and_reset(void) -{ - return erts_smp_atomic_xchg(&do_time, 0L); -} -ERTS_GLB_INLINE void do_time_add(long elapsed) -{ - erts_smp_atomic_add(&do_time, elapsed); -} -#endif - -#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ - -void init_time(void); -void erl_set_timer(ErlTimer*, ErlTimeoutProc, ErlCancelProc, void*, Uint); -void erl_cancel_timer(ErlTimer*); -Uint time_left(ErlTimer *); - -Uint erts_timer_wheel_memory_size(void); - -#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME)) -# ifndef HAVE_ERTS_NOW_CPU -# define HAVE_ERTS_NOW_CPU -# ifdef HAVE_GETHRVTIME -# define erts_start_now_cpu() sys_start_hrvtime() -# define erts_stop_now_cpu() sys_stop_hrvtime() -# endif -# endif -void erts_get_now_cpu(Uint* megasec, Uint* sec, Uint* microsec); -#endif - -void erts_get_timeval(SysTimeval *tv); -long erts_get_time(void); - -extern SysTimeval erts_first_emu_time; - -void erts_get_emu_time(SysTimeval *); - -ERTS_GLB_INLINE int erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p); - -#if ERTS_GLB_INLINE_INCL_FUNC_DEF - -ERTS_GLB_INLINE int -erts_cmp_timeval(SysTimeval *t1p, SysTimeval *t2p) -{ - if (t1p->tv_sec == t2p->tv_sec) { - if (t1p->tv_usec < t2p->tv_usec) - return -1; - else if (t1p->tv_usec > t2p->tv_usec) - return 1; - return 0; - } - return t1p->tv_sec < t2p->tv_sec ? -1 : 1; -} - -#endif - -#ifdef DEBUG -void p_slpq(void); -#endif - /* utils.c */ /* @@ -1560,16 +1505,30 @@ void erts_init_utils_mem(void); erts_dsprintf_buf_t *erts_create_tmp_dsbuf(Uint); void erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *); +#if HALFWORD_HEAP +int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base); +# define eq(A,B) eq_rel(A,NULL,B,NULL) +#else int eq(Eterm, Eterm); +# define eq_rel(A,A_BASE,B,B_BASE) eq(A,B) +#endif + #define EQ(x,y) (((x) == (y)) || (is_not_both_immed((x),(y)) && eq((x),(y)))) +#if HALFWORD_HEAP +Sint cmp_rel(Eterm, Eterm*, Eterm, Eterm*); +#define CMP(A,B) cmp_rel(A,NULL,B,NULL) +#else Sint cmp(Eterm, Eterm); -#define cmp_lt(a,b) (cmp((a),(b)) < 0) -#define cmp_le(a,b) (cmp((a),(b)) <= 0) -#define cmp_eq(a,b) (cmp((a),(b)) == 0) -#define cmp_ne(a,b) (cmp((a),(b)) != 0) -#define cmp_ge(a,b) (cmp((a),(b)) >= 0) -#define cmp_gt(a,b) (cmp((a),(b)) > 0) +#define cmp_rel(A,A_BASE,B,B_BASE) cmp(A,B) +#define CMP(A,B) cmp(A,B) +#endif +#define cmp_lt(a,b) (CMP((a),(b)) < 0) +#define cmp_le(a,b) (CMP((a),(b)) <= 0) +#define cmp_eq(a,b) (CMP((a),(b)) == 0) +#define cmp_ne(a,b) (CMP((a),(b)) != 0) +#define cmp_ge(a,b) (CMP((a),(b)) >= 0) +#define cmp_gt(a,b) (CMP((a),(b)) > 0) #define CMP_LT(a,b) ((a) != (b) && cmp_lt((a),(b))) #define CMP_GE(a,b) ((a) == (b) || cmp_ge((a),(b))) @@ -1595,6 +1554,19 @@ Sint erts_binary_set_loop_limit(Sint limit); /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); + +void erts_native_filename_put(Eterm ioterm, int encoding, byte *p) ; +Sint erts_native_filename_need(Eterm ioterm, int encoding); +void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars); +int erts_analyze_utf8(byte *source, Uint size, + byte **err_pos, Uint *num_chars, int *left); +char *erts_convert_filename_to_native(Eterm name, ErtsAlcType_t alloc_type, int allow_empty); + +#define ERTS_UTF8_OK 0 +#define ERTS_UTF8_INCOMPLETE 1 +#define ERTS_UTF8_ERROR 2 +#define ERTS_UTF8_ANALYZE_MORE 3 + /* erl_trace.c */ void erts_init_trace(void); void erts_trace_check_exiting(Eterm exiting); @@ -1728,11 +1700,7 @@ Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(int to, void *arg, Process *c_p); -/* - * Interface to erl_init - */ -void erl_init(void); - +int erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* reg); #define seq_trace_output(token, msg, type, receiver, process) \ seq_trace_output_generic((token), (msg), (type), (receiver), (process), NIL) #define seq_trace_output_exit(token, msg, type, receiver, exitfrom) \ @@ -1790,8 +1758,15 @@ do { \ extern Binary *erts_match_set_compile(Process *p, Eterm matchexpr); Eterm erts_match_set_lint(Process *p, Eterm matchexpr); extern void erts_match_set_release_result(Process* p); + +enum erts_pam_run_flags { + ERTS_PAM_TMP_RESULT=0, + ERTS_PAM_COPY_RESULT=1, + ERTS_PAM_CONTIGUOUS_TUPLE=2 +}; extern Eterm erts_match_set_run(Process *p, Binary *mpsp, Eterm *args, int num_args, + enum erts_pam_run_flags in_flags, Uint32 *return_flags); extern Eterm erts_match_set_get_source(Binary *mpsp); extern void erts_match_prog_foreach_offheap(Binary *b, diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 79022d5dd7..f21a96c754 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -428,7 +428,7 @@ setup_port(Port* prt, Eterm pid, erts_driver_t *driver, old_name = prt->name; prt->name = new_name; #ifdef ERTS_SMP - erts_smp_atomic_set(&prt->run_queue, (long) runq); + erts_smp_atomic_set(&prt->run_queue, (erts_aint_t) runq); #endif ASSERT(!prt->drv_ptr); prt->drv_ptr = driver; @@ -670,7 +670,7 @@ erts_open_driver(erts_driver_t* driver, /* Pointer to driver. */ #ifdef ERTS_SMP erts_cancel_smp_ptimer(port->ptimer); #else - erl_cancel_timer(&(port->tm)); + erts_cancel_timer(&(port->tm)); #endif stopq(port); kill_port(port); @@ -1297,7 +1297,7 @@ void init_io(void) erts_port[i].port_data_lock = NULL; } - erts_smp_atomic_init(&erts_ports_snapshot, (long) 0); + erts_smp_atomic_init(&erts_ports_snapshot, (erts_aint_t) 0); last_port_num = 0; erts_smp_spinlock_init(&get_free_port_lck, "get_free_port"); @@ -1839,7 +1839,7 @@ terminate_port(Port *prt) #ifdef ERTS_SMP erts_cancel_smp_ptimer(prt->ptimer); #else - erl_cancel_timer(&prt->tm); + erts_cancel_timer(&prt->tm); #endif drv = prt->drv_ptr; @@ -2802,17 +2802,25 @@ driver_deliver_term(ErlDrvPort port, break; case ERL_DRV_INT: /* signed int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); +#if HALFWORD_HEAP + erts_bld_sint64(NULL, &need, (Sint64)ptr[0]); +#else /* check for bignum */ if (!IS_SSMALL((Sint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ +#endif ptr++; depth++; break; case ERL_DRV_UINT: /* unsigned int argument */ ERTS_DDT_CHK_ENOUGH_ARGS(1); +#if HALFWORD_HEAP + erts_bld_uint64(NULL, &need, (Uint64)ptr[0]); +#else /* check for bignum */ if (!IS_USMALL(0, (Uint)ptr[0])) need += BIG_UINT_HEAP_SIZE; /* use small_to_big */ +#endif ptr++; depth++; break; @@ -2979,22 +2987,30 @@ driver_deliver_term(ErlDrvPort port, break; case ERL_DRV_INT: /* signed int argument */ +#if HALFWORD_HEAP + mess = erts_bld_sint64(&hp, NULL, (Sint64)ptr[0]); +#else if (IS_SSMALL((Sint)ptr[0])) mess = make_small((Sint)ptr[0]); else { mess = small_to_big((Sint)ptr[0], hp); hp += BIG_UINT_HEAP_SIZE; } +#endif ptr++; break; case ERL_DRV_UINT: /* unsigned int argument */ +#if HALFWORD_HEAP + mess = erts_bld_uint64(&hp, NULL, (Uint64)ptr[0]); +#else if (IS_USMALL(0, (Uint)ptr[0])) mess = make_small((Uint)ptr[0]); else { mess = uint_to_big((Uint)ptr[0], hp); hp += BIG_UINT_HEAP_SIZE; } +#endif ptr++; break; @@ -3236,7 +3252,7 @@ int driver_output_binary(ErlDrvPort ix, char* hbuf, int hlen, return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len)); + erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { return erts_net_message(prt, prt->dist_entry, @@ -3271,7 +3287,7 @@ int driver_output2(ErlDrvPort ix, char* hbuf, int hlen, char* buf, int len) return 0; prt->bytes_in += (hlen + len); - erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + len)); + erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + len)); if (prt->status & ERTS_PORT_SFLG_DISTRIBUTION) { if (len == 0) return erts_net_message(prt, @@ -3348,7 +3364,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, int hlen, ErlIOVec* vec, int skip) /* XXX handle distribution !!! */ prt->bytes_in += (hlen + size); - erts_smp_atomic_add(&erts_bytes_in, (long) (hlen + size)); + erts_smp_atomic_add(&erts_bytes_in, (erts_aint_t) (hlen + size)); deliver_vec_message(prt, prt->connected, hbuf, hlen, binv, iov, n, size); return 0; } @@ -3392,25 +3408,25 @@ int len; * reference count on driver binaries... */ -long +ErlDrvSInt driver_binary_get_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return erts_refc_read(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_read(&bp->refc, 1); } -long +ErlDrvSInt driver_binary_inc_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return erts_refc_inctest(&bp->refc, 2); + return (ErlDrvSInt) erts_refc_inctest(&bp->refc, 2); } -long +ErlDrvSInt driver_binary_dec_refc(ErlDrvBinary *dbp) { Binary* bp = ErlDrvBinary2Binary(dbp); - return erts_refc_dectest(&bp->refc, 1); + return (ErlDrvSInt) erts_refc_dectest(&bp->refc, 1); } @@ -3525,12 +3541,12 @@ pdl_init_refc(ErlDrvPDL pdl) erts_atomic_init(&pdl->refc, 1); } -static ERTS_INLINE long +static ERTS_INLINE ErlDrvSInt pdl_read_refc(ErlDrvPDL pdl) { - long refc = erts_atomic_read(&pdl->refc); + erts_aint_t refc = erts_atomic_read(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); - return refc; + return (ErlDrvSInt) refc; } static ERTS_INLINE void @@ -3540,12 +3556,12 @@ pdl_inc_refc(ErlDrvPDL pdl) ERTS_LC_ASSERT(driver_pdl_get_refc(pdl) > 1); } -static ERTS_INLINE long +static ERTS_INLINE ErlDrvSInt pdl_inctest_refc(ErlDrvPDL pdl) { - long refc = erts_atomic_inctest(&pdl->refc); + erts_aint_t refc = erts_atomic_inctest(&pdl->refc); ERTS_LC_ASSERT(refc > 1); - return refc; + return (ErlDrvSInt) refc; } #if 0 /* unused */ @@ -3557,12 +3573,12 @@ pdl_dec_refc(ErlDrvPDL pdl) } #endif -static ERTS_INLINE long +static ERTS_INLINE ErlDrvSInt pdl_dectest_refc(ErlDrvPDL pdl) { - long refc = erts_atomic_dectest(&pdl->refc); + erts_aint_t refc = erts_atomic_dectest(&pdl->refc); ERTS_LC_ASSERT(refc >= 0); - return refc; + return (ErlDrvSInt) refc; } static ERTS_INLINE void pdl_destroy(ErlDrvPDL pdl) @@ -3633,7 +3649,7 @@ driver_pdl_lock(ErlDrvPDL pdl) void driver_pdl_unlock(ErlDrvPDL pdl) { - long refc; + ErlDrvSInt refc; #ifdef HARDDEBUG erts_fprintf(stderr, "driver_pdl_unlock(0x%08X)\r\n",(unsigned) pdl); #endif @@ -3643,28 +3659,30 @@ driver_pdl_unlock(ErlDrvPDL pdl) pdl_destroy(pdl); } -long +ErlDrvSInt driver_pdl_get_refc(ErlDrvPDL pdl) { return pdl_read_refc(pdl); } -long +ErlDrvSInt driver_pdl_inc_refc(ErlDrvPDL pdl) { - long refc = pdl_inctest_refc(pdl); + ErlDrvSInt refc = pdl_inctest_refc(pdl); #ifdef HARDDEBUG - erts_fprintf(stderr, "driver_pdl_inc_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc); + erts_fprintf(stderr, "driver_pdl_inc_refc(%p) -> %bpd\r\n", + pdl, refc); #endif return refc; } -long +ErlDrvSInt driver_pdl_dec_refc(ErlDrvPDL pdl) { - long refc = pdl_dectest_refc(pdl); + ErlDrvSInt refc = pdl_dectest_refc(pdl); #ifdef HARDDEBUG - erts_fprintf(stderr, "driver_pdl_dec_refc(0x%08X) -> %ld\r\n",(unsigned) pdl, refc); + erts_fprintf(stderr, "driver_pdl_dec_refc(%p) -> %bpd\r\n", + pdl, refc); #endif if (!refc) pdl_destroy(pdl); @@ -4050,7 +4068,7 @@ drv_cancel_timer(Port *prt) #ifdef ERTS_SMP erts_cancel_smp_ptimer(prt->ptimer); #else - erl_cancel_timer(&prt->tm); + erts_cancel_timer(&prt->tm); #endif if (erts_port_task_is_scheduled(&prt->timeout_task)) erts_port_task_abort(prt->id, &prt->timeout_task); @@ -4074,7 +4092,7 @@ int driver_set_timer(ErlDrvPort ix, UWord t) (ErlTimeoutProc) schedule_port_timeout, t); #else - erl_set_timer(&prt->tm, + erts_set_timer(&prt->tm, (ErlTimeoutProc) schedule_port_timeout, NULL, prt, @@ -4105,9 +4123,9 @@ driver_read_timer(ErlDrvPort ix, unsigned long* t) return -1; ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); #ifdef ERTS_SMP - *t = prt->ptimer ? time_left(&prt->ptimer->timer.tm) : 0; + *t = prt->ptimer ? erts_time_left(&prt->ptimer->timer.tm) : 0; #else - *t = time_left(&prt->tm); + *t = erts_time_left(&prt->tm); #endif return 0; } diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index a2439d5582..6caa1e0b2d 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -101,16 +101,16 @@ return %macro: test_heap TestHeap -pack allocate t t -allocate_heap I I I +allocate_heap t I t deallocate I init y allocate_zero t t -allocate_heap_zero I I I +allocate_heap_zero t I t trim N Remaining => i_trim N i_trim I -test_heap I I +test_heap I t allocate_heap S u==0 R => allocate S R allocate_heap_zero S u==0 R => allocate_zero S R @@ -124,7 +124,7 @@ init Y1 | init Y2 => init2 Y1 Y2 # Selecting values -select_val S=q Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest) +select_val S=aiq Fail=f Size=u Rest=* => const_select_val(S, Fail, Size, Rest) select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \ gen_jump_tab(S, Fail, Size, Rest) @@ -132,34 +132,59 @@ select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \ is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | use_jump_tab(Size, Rest) => \ gen_jump_tab(S, Fail, Size, Rest) +is_integer TypeFail=f S | select_val S=s Fail=f Size=u Rest=* | \ + mixed_types(Size, Rest) => \ + gen_split_values(S, TypeFail, Fail, Size, Rest) + select_val S=s Fail=f Size=u Rest=* | mixed_types(Size, Rest) => \ - gen_split_values(S, Fail, Size, Rest) + gen_split_values(S, Fail, Fail, Size, Rest) -is_integer Fail=f S | select_val S=s Fail=f Size=u Rest=* | \ +is_integer Fail=f S | select_val S=d Fail=f Size=u Rest=* | \ fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest) -is_atom Fail=f S | select_val S=s Fail=f Size=u Rest=* | \ +is_atom Fail=f S | select_val S=d Fail=f Size=u Rest=* | \ fixed_size_values(Size, Rest) => gen_select_val(S, Fail, Size, Rest) -select_val S=s Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \ - gen_select_val(S, Fail, Size, Rest) +select_val S=s Fail=f Size=u Rest=* | floats_or_bignums(Size, Rest) => \ + gen_select_literals(S, Fail, Size, Rest) -select_val S=s Fail=f Size=u Rest=* | all_values_are_big(Size, Rest) => \ - gen_select_big(S, Fail, Size, Rest) +select_val S=d Fail=f Size=u Rest=* | fixed_size_values(Size, Rest) => \ + gen_select_val(S, Fail, Size, Rest) -is_tuple Fail=f S | select_tuple_arity S=s Fail=f Size=u Rest=* => \ +is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -select_tuple_arity S=s Fail=f Size=u Rest=* => \ +select_tuple_arity S=d Fail=f Size=u Rest=* => \ gen_select_tuple_arity(S, Fail, Size, Rest) -i_select_val s f I -i_select_tuple_arity s f I -i_select_big s f -i_select_float s f I +i_select_val r f I +i_select_val x f I +i_select_val y f I + +i_select_val2 r f c f c f +i_select_val2 x f c f c f +i_select_val2 y f c f c f + +i_select_tuple_arity2 r f A f A f +i_select_tuple_arity2 x f A f A f +i_select_tuple_arity2 y f A f A f + +i_select_tuple_arity r f I +i_select_tuple_arity x f I +i_select_tuple_arity y f I + +i_jump_on_val_zero r f I +i_jump_on_val_zero x f I +i_jump_on_val_zero y f I + +i_jump_on_val r f I I +i_jump_on_val x f I I +i_jump_on_val y f I I -i_jump_on_val_zero s f I -i_jump_on_val s f I I +jump Target | label Lbl | same_label(Target, Lbl) => label Lbl + +is_ne_exact L1 S1 S2 | jump Fail | label L2 | same_label(L1, L2) => \ + is_eq_exact Fail S1 S2 | label L2 %macro: get_list GetList -pack get_list x x x @@ -234,11 +259,17 @@ is_number Fail Literal=q => move Literal x | is_number Fail x jump f -case_end Literal=q => move Literal x | case_end x -badmatch Literal=q => move Literal x | badmatch x +case_end Literal=cq => move Literal x | case_end x +badmatch Literal=cq => move Literal x | badmatch x + +case_end r +case_end x +case_end y + +badmatch r +badmatch x +badmatch y -case_end s -badmatch s if_end raise s s @@ -248,12 +279,33 @@ system_limit j move R R => +move C=cxy r | jump Lbl => move_jump Lbl C + +%macro: move_jump MoveJump -nonext +move_jump f n +move_jump f c +move_jump f x +move_jump f y + move X1=x Y1=y | move X2=x Y2=y => move2 X1 Y1 X2 Y2 move Y1=y X1=x | move Y2=y X2=x => move2 Y1 X1 Y2 X2 +move X1=x X2=x | move X3=x X4=x => move2 X1 X2 X3 X4 + +move C=aiq X=x==1 => move_x1 C +move C=aiq X=x==2 => move_x2 C + +move_x1 c +move_x2 c %macro: move2 Move2 -pack move2 x y x y move2 y x y x +move2 x x x x + +# The compiler almost never generates a "move Literal y(Y)" instruction, +# so let's cheat if we encounter one. +move S=n D=y => init D +move S=c D=y => move S x | move x D %macro:move Move -pack -gen_dest move x x @@ -265,15 +317,10 @@ move r x move r y move c r move c x -move c y move n x move n r move y y -%cold -move s d -%hot - # Receive operations. loop_rec Fail Src | smp_mark_target_label(Fail) => i_loop_rec Fail Src @@ -306,55 +353,78 @@ i_wait_error_locked send # -# Comparisions. +# Optimized comparisons with one immediate/literal operand. +# + +is_eq_exact Lbl R=rxy C=ian => i_is_eq_exact_immed Lbl R C +is_eq_exact Lbl R=rxy C=q => i_is_eq_exact_literal R Lbl C + +is_ne_exact Lbl R=rxy C=ian => i_is_ne_exact_immed Lbl R C +is_ne_exact Lbl R=rxy C=q => i_is_ne_exact_literal R Lbl C + +%macro: i_is_eq_exact_immed EqualImmed -fail_action +i_is_eq_exact_immed f r c +i_is_eq_exact_immed f x c +i_is_eq_exact_immed f y c + +i_is_eq_exact_literal r f c +i_is_eq_exact_literal x f c +i_is_eq_exact_literal y f c + +%macro: i_is_ne_exact_immed NotEqualImmed -fail_action +i_is_ne_exact_immed f r c +i_is_ne_exact_immed f x c +i_is_ne_exact_immed f y c + +i_is_ne_exact_literal r f c +i_is_ne_exact_literal x f c +i_is_ne_exact_literal y f c + +# +# All other comparisons. # -is_eq_exact Lbl=f R=rxy C=ian => i_is_eq_immed Lbl R C -is_eq Lbl=f R=rxy C=an => i_is_eq_immed Lbl R C +is_eq_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl +is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl is_ge Lbl S1 S2 => i_fetch S1 S2 | i_is_ge Lbl is_lt Lbl S1 S2 => i_fetch S1 S2 | i_is_lt Lbl is_eq Lbl S1 S2 => i_fetch S1 S2 | i_is_eq Lbl is_ne Lbl S1 S2 => i_fetch S1 S2 | i_is_ne Lbl -is_eq_exact Lbl=f S1 S2 => i_fetch S1 S2 | i_is_eq_exact Lbl -is_ne_exact Lbl S1 S2 => i_fetch S1 S2 | i_is_ne_exact Lbl - +i_is_eq_exact f +i_is_ne_exact f i_is_lt f i_is_ge f i_is_eq f i_is_ne f -i_is_eq_exact f -i_is_ne_exact f - -%macro: i_is_eq_immed EqualImmed -fail_action -i_is_eq_immed f r c -i_is_eq_immed f x c -i_is_eq_immed f y c # # Putting things. # -put_tuple Arity Dst | put V => i_put_tuple Arity V Dst +put_tuple Arity Dst => i_put_tuple Dst u -%macro: i_put_tuple PutTuple -pack -i_put_tuple A x x -i_put_tuple A y x -i_put_tuple A r x -i_put_tuple A n x -i_put_tuple A c x -i_put_tuple A x y -i_put_tuple A x r -i_put_tuple A y r -i_put_tuple A n r -i_put_tuple A c r +i_put_tuple Dst Arity Puts=* | put S1 | put S2 | \ + put S3 | put S4 | put S5 => \ + tuple_append_put5(Arity, Dst, Puts, S1, S2, S3, S4, S5) -%cold -i_put_tuple A r y -i_put_tuple A y y -i_put_tuple A c y -%hot +i_put_tuple Dst Arity Puts=* | put S => \ + tuple_append_put(Arity, Dst, Puts, S) + +i_put_tuple/2 + +%macro:i_put_tuple PutTuple -pack -goto:do_put_tuple +i_put_tuple r I +i_put_tuple x I +i_put_tuple y I + +# +# The instruction "put_list Const [] Dst" will not be generated by +# the current BEAM compiler. But until R15A, play it safe by handling +# that instruction with the following transformation. +# +put_list Const=c n Dst => move Const x | put_list x n Dst %macro:put_list PutList -pack -gen_dest @@ -362,10 +432,8 @@ put_list x n x put_list y n x put_list x x x put_list y x x -put_list c n x put_list x x r put_list y r r -put_list c n r put_list y y x put_list x y x @@ -376,6 +444,13 @@ put_list y y r put_list y r x put_list r n x +put_list x r x +put_list x y r +put_list y x r +put_list y x x + +put_list x r r + # put_list SrcReg Constant Dst put_list r c r put_list r c x @@ -403,17 +478,9 @@ put_list c y x put_list c y y %cold -put_list x r r put_list s s d %hot -%macro: put Put -put x -put r -put y -put c -put n - %macro: i_fetch FetchArgs -pack i_fetch c c i_fetch c r @@ -464,19 +531,20 @@ move_return n r move S r | deallocate D | return => move_deallocate_return S r D -%macro: move_deallocate_return MoveDeallocateReturn -nonext -move_deallocate_return x r P -move_deallocate_return y r P -move_deallocate_return c r P -move_deallocate_return n r P +%macro: move_deallocate_return MoveDeallocateReturn -pack -nonext +move_deallocate_return x r Q +move_deallocate_return y r Q +move_deallocate_return c r Q +move_deallocate_return n r Q deallocate D | return => deallocate_return D %macro: deallocate_return DeallocateReturn -nonext -deallocate_return P +deallocate_return Q test_heap Need u==1 | put_list Y=y r r => test_heap_1_put_list Need Y +%macro: test_heap_1_put_list TestHeapPutList -pack test_heap_1_put_list I y # Test tuple & arity (head) @@ -576,14 +644,14 @@ is_list f y is_nonempty_list Fail=f S=rx | allocate Need Rs => is_nonempty_list_allocate Fail S Need Rs -%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -is_nonempty_list_allocate f x I I -is_nonempty_list_allocate f r I I +%macro:is_nonempty_list_allocate IsNonemptyListAllocate -fail_action -pack +is_nonempty_list_allocate f x I t +is_nonempty_list_allocate f r I t is_nonempty_list F=f r | test_heap I1 I2 => is_non_empty_list_test_heap F r I1 I2 -%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -is_non_empty_list_test_heap f r I I +%macro: is_non_empty_list_test_heap IsNonemptyListTestHeap -fail_action -pack +is_non_empty_list_test_heap f r I t %macro: is_nonempty_list IsNonemptyList -fail_action is_nonempty_list f x @@ -912,8 +980,13 @@ node x node y %hot -i_fast_element j I s d -i_element j s s d +i_fast_element r j I d +i_fast_element x j I d +i_fast_element y j I d + +i_element r j s d +i_element x j s d +i_element y j s d bif1 f b s d bif1_body b s d @@ -940,11 +1013,11 @@ move S r | call_last Ar P=f D => move_call_last S r P D i_move_call_last f P c r -%macro:move_call_last MoveCallLast -arg_f -nonext +%macro:move_call_last MoveCallLast -arg_f -nonext -pack move_call_last/4 -move_call_last x r f P -move_call_last y r f P +move_call_last x r f Q +move_call_last y r f Q move S=c r | call_only Ar P=f => i_move_call_only P S r move S=x r | call_only Ar P=f => move_call_only S r P @@ -993,7 +1066,7 @@ is_function f y is_function f r is_function Fail=f c => jump Fail -func_info M=a F=a A=u | label L => gen_func_info(M, F, A, L) +func_info M F A => i_func_info u M F A # ================================================================ # New bit syntax matching (R11B). @@ -1307,6 +1380,8 @@ fconv Arg=iqan Dst=l => move Arg x | fconv x Dst fmove q l fmove d l +fmove l d + fconv d l i_fadd l l l @@ -1322,12 +1397,6 @@ fcheckerror p => i_fcheckerror i_fcheckerror fclearerror -fmove FR=l Dst=d | new_float_allocation() => fmove_new FR Dst - -# The new instruction for moving a float out of a floating point register. -# (No allocation.) -fmove_new l d - # # New apply instructions in R10B. # @@ -1336,7 +1405,21 @@ apply I apply_last I P # -# New GCing arithmetic instructions. +# Optimize addition and subtraction of small literals using +# the i_increment/4 instruction (in bodies, not in guards). +# + +gc_bif2 p Live u$bif:erlang:splus/2 Int=i Reg=d Dst => \ + gen_increment(Reg, Int, Live, Dst) +gc_bif2 p Live u$bif:erlang:splus/2 Reg=d Int=i Dst => \ + gen_increment(Reg, Int, Live, Dst) + +gc_bif2 p Live u$bif:erlang:sminus/2 Reg=d Int=i Dst | \ + negation_is_small(Int) => \ + gen_increment_from_minus(Reg, Int, Live, Dst) + +# +# GCing arithmetic instructions. # gc_bif2 Fail I u$bif:erlang:splus/2 S1 S2 Dst=d => i_fetch S1 S2 | i_plus Fail I Dst @@ -1359,6 +1442,10 @@ gc_bif1 Fail I u$bif:erlang:bnot/1 Src Dst=d => i_int_bnot Fail Src I Dst gc_bif1 Fail I u$bif:erlang:sminus/1 Src Dst=d => i_fetch i Src | i_minus Fail I Dst gc_bif1 Fail I u$bif:erlang:splus/1 Src Dst=d => i_fetch i Src | i_plus Fail I Dst +i_increment r I I d +i_increment x I I d +i_increment y I I d + i_plus j I d i_minus j I d i_times j I d diff --git a/erts/emulator/beam/packet_parser.c b/erts/emulator/beam/packet_parser.c index 5bcd567b5f..a66d60aa22 100644 --- a/erts/emulator/beam/packet_parser.c +++ b/erts/emulator/beam/packet_parser.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2009. All Rights Reserved. + * Copyright Ericsson AB 2008-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -47,11 +47,6 @@ (((unsigned char*) (s))[1] << 8) | \ (((unsigned char*) (s))[0])) -#define put_int24(s, x) ((((unsigned char*)(s))[0] = ((x) >> 16) & 0xff), \ - (((unsigned char*)(s))[1] = ((x) >> 8) & 0xff), \ - (((unsigned char*)(s))[2] = (x) & 0xff)) - - #if !defined(__WIN32__) && !defined(HAVE_STRNCASECMP) #define STRNCASECMP my_strncasecmp @@ -833,7 +828,7 @@ int packet_parse_ssl(const char* buf, int len, char prefix[4]; /* <<1:8,Length:24,Data/binary>> */ prefix[0] = 1; - put_int24(&prefix[1],len-3); + put_int24(len-3,&prefix[1]); return pcb->ssl_tls(arg, 22, major, minor, buf+3, len-3, prefix, sizeof(prefix)); } else { diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 0031568af6..e64c43de6e 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -39,13 +39,6 @@ #define ENABLE_CHILD_WAITER_THREAD 1 #endif -/* The ERTS_TIMER_TREAD #define must be visible to the - erl_${OS}_sys.h #include files: it controls whether - certain optional facilities should be defined or not. */ -#if defined(ERTS_SMP) && 0 -#define ERTS_TIMER_THREAD -#endif - #if defined (__WIN32__) # include "erl_win_sys.h" #elif defined (VXWORKS) @@ -232,14 +225,14 @@ int real_printf(const char *fmt, ...); #else #error Neither 32 nor 64 bit architecture #endif -#ifdef ARCH_64 -# ifdef HALFWORD_HEAP_EMULATOR +#if defined(ARCH_64) && defined(HALFWORD_HEAP_EMULATOR) # define HALFWORD_HEAP 1 # define HALFWORD_ASSERT 0 -# else +# define ASSERT_HALFWORD(COND) ASSERT(COND) +#else # define HALFWORD_HEAP 0 # define HALFWORD_ASSERT 0 -# endif +# define ASSERT_HALFWORD(COND) #endif #if SIZEOF_VOID_P != SIZEOF_SIZE_T @@ -338,12 +331,16 @@ typedef unsigned char byte; (((size_t) 8) - (((size_t) (X)) & ((size_t) 7))) #include "erl_lock_check.h" + +/* needed by erl_smp.h */ +int erts_send_warning_to_logger_str_nogl(char *); + #include "erl_smp.h" #ifdef ERTS_WANT_BREAK_HANDLING # ifdef ERTS_SMP -extern erts_smp_atomic_t erts_break_requested; -# define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic_read(&erts_break_requested)) +extern erts_smp_atomic32_t erts_break_requested; +# define ERTS_BREAK_REQUESTED ((int) erts_smp_atomic32_read(&erts_break_requested)) # else extern volatile int erts_break_requested; # define ERTS_BREAK_REQUESTED erts_break_requested @@ -356,8 +353,8 @@ void erts_do_break_handling(void); # define ERTS_GOT_SIGUSR1 0 # else # ifdef ERTS_SMP -extern erts_smp_atomic_t erts_got_sigusr1; -# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic_read(&erts_got_sigusr1)) +extern erts_smp_atomic32_t erts_got_sigusr1; +# define ERTS_GOT_SIGUSR1 ((int) erts_smp_atomic32_read(&erts_got_sigusr1)) # else extern volatile int erts_got_sigusr1; # define ERTS_GOT_SIGUSR1 erts_got_sigusr1 @@ -466,8 +463,6 @@ static const int zero_value = 0, one_value = 1; # endif /* !__WIN32__ */ #endif /* WANT_NONBLOCKING */ -extern erts_cpu_info_t *erts_cpuinfo; /* erl_init.c */ - __decl_noreturn void __noreturn erl_exit(int n, char*, ...); /* Some special erl_exit() codes: */ @@ -526,7 +521,8 @@ int erts_send_info_to_logger_nogl(erts_dsprintf_buf_t *); int erts_send_warning_to_logger_nogl(erts_dsprintf_buf_t *); int erts_send_error_to_logger_nogl(erts_dsprintf_buf_t *); int erts_send_info_to_logger_str_nogl(char *); -int erts_send_warning_to_logger_str_nogl(char *); +/* needed by erl_smp.h (declared above) + int erts_send_warning_to_logger_str_nogl(char *); */ int erts_send_error_to_logger_str_nogl(char *); typedef struct preload { @@ -542,8 +538,8 @@ typedef struct preload { */ typedef struct _SysDriverOpts { - int ifd; /* Input file descriptor (fd driver). */ - int ofd; /* Outputfile descriptor (fd driver). */ + Uint ifd; /* Input file descriptor (fd driver). */ + Uint ofd; /* Outputfile descriptor (fd driver). */ int packet_bytes; /* Number of bytes in packet header. */ int read_write; /* Read and write bits. */ int use_stdio; /* Use standard I/O: TRUE or FALSE. */ @@ -564,11 +560,7 @@ extern char *erts_default_arg0; extern char os_type[]; extern int sys_init_time(void); -#if defined(ERTS_TIMER_THREAD) -#define erts_deliver_time() -#else extern void erts_deliver_time(void); -#endif extern void erts_time_remaining(SysTimeval *); extern int erts_init_time_sup(void); extern void erts_sys_init_float(void); @@ -730,11 +722,11 @@ typedef enum { } erts_activity_error_t; typedef struct { - erts_smp_atomic_t do_block; + erts_smp_atomic32_t do_block; struct { - erts_smp_atomic_t wait; - erts_smp_atomic_t gc; - erts_smp_atomic_t io; + erts_smp_atomic32_t wait; + erts_smp_atomic32_t gc; + erts_smp_atomic32_t io; } in_activity; } erts_system_block_state_t; @@ -885,7 +877,7 @@ ERTS_GLB_INLINE int erts_smp_pending_system_block(void) { #ifdef ERTS_SMP - return erts_smp_atomic_read(&erts_system_block_state.do_block); + return (int) erts_smp_atomic32_read(&erts_system_block_state.do_block); #else return 0; #endif @@ -921,7 +913,7 @@ erts_smp_set_activity(erts_activity_t old_activity, case ERTS_ACTIVITY_UNDEFINED: break; case ERTS_ACTIVITY_WAIT: - erts_smp_atomic_dec(&erts_system_block_state.in_activity.wait); + erts_smp_atomic32_dec(&erts_system_block_state.in_activity.wait); if (locked) { /* You are not allowed to leave activity waiting * without supplying the possibility to block @@ -932,10 +924,10 @@ erts_smp_set_activity(erts_activity_t old_activity, } break; case ERTS_ACTIVITY_GC: - erts_smp_atomic_dec(&erts_system_block_state.in_activity.gc); + erts_smp_atomic32_dec(&erts_system_block_state.in_activity.gc); break; case ERTS_ACTIVITY_IO: - erts_smp_atomic_dec(&erts_system_block_state.in_activity.io); + erts_smp_atomic32_dec(&erts_system_block_state.in_activity.io); break; default: erts_set_activity_error(ERTS_ACT_ERR_LEAVE_UNKNOWN_ACTIVITY, @@ -951,13 +943,13 @@ erts_smp_set_activity(erts_activity_t old_activity, case ERTS_ACTIVITY_UNDEFINED: break; case ERTS_ACTIVITY_WAIT: - erts_smp_atomic_inc(&erts_system_block_state.in_activity.wait); + erts_smp_atomic32_inc(&erts_system_block_state.in_activity.wait); break; case ERTS_ACTIVITY_GC: - erts_smp_atomic_inc(&erts_system_block_state.in_activity.gc); + erts_smp_atomic32_inc(&erts_system_block_state.in_activity.gc); break; case ERTS_ACTIVITY_IO: - erts_smp_atomic_inc(&erts_system_block_state.in_activity.io); + erts_smp_atomic32_inc(&erts_system_block_state.in_activity.io); break; default: erts_set_activity_error(ERTS_ACT_ERR_ENTER_UNKNOWN_ACTIVITY, @@ -992,27 +984,31 @@ erts_smp_set_activity(erts_activity_t old_activity, typedef erts_smp_atomic_t erts_refc_t; -ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, long val); -ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE long erts_refc_inctest(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE long erts_refc_dectest(erts_refc_t *refcp, long min_val); -ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, long diff, long min_val); -ERTS_GLB_INLINE long erts_refc_read(erts_refc_t *refcp, long min_val); +ERTS_GLB_INLINE void erts_refc_init(erts_refc_t *refcp, erts_aint_t val); +ERTS_GLB_INLINE void erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_inctest(erts_refc_t *refcp, + erts_aint_t min_val); +ERTS_GLB_INLINE void erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_dectest(erts_refc_t *refcp, + erts_aint_t min_val); +ERTS_GLB_INLINE void erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, + erts_aint_t min_val); +ERTS_GLB_INLINE erts_aint_t erts_refc_read(erts_refc_t *refcp, + erts_aint_t min_val); #if ERTS_GLB_INLINE_INCL_FUNC_DEF ERTS_GLB_INLINE void -erts_refc_init(erts_refc_t *refcp, long val) +erts_refc_init(erts_refc_t *refcp, erts_aint_t val) { erts_smp_atomic_init((erts_smp_atomic_t *) refcp, val); } ERTS_GLB_INLINE void -erts_refc_inc(erts_refc_t *refcp, long min_val) +erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", @@ -1022,10 +1018,10 @@ erts_refc_inc(erts_refc_t *refcp, long min_val) #endif } -ERTS_GLB_INLINE long -erts_refc_inctest(erts_refc_t *refcp, long min_val) +ERTS_GLB_INLINE erts_aint_t +erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) { - long val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_inctest((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1036,10 +1032,10 @@ erts_refc_inctest(erts_refc_t *refcp, long min_val) } ERTS_GLB_INLINE void -erts_refc_dec(erts_refc_t *refcp, long min_val) +erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", @@ -1049,10 +1045,10 @@ erts_refc_dec(erts_refc_t *refcp, long min_val) #endif } -ERTS_GLB_INLINE long -erts_refc_dectest(erts_refc_t *refcp, long min_val) +ERTS_GLB_INLINE erts_aint_t +erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) { - long val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_dectest((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1063,10 +1059,10 @@ erts_refc_dectest(erts_refc_t *refcp, long min_val) } ERTS_GLB_INLINE void -erts_refc_add(erts_refc_t *refcp, long diff, long min_val) +erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) { #ifdef ERTS_REFC_DEBUG - long val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff); + erts_aint_t val = erts_smp_atomic_addtest((erts_smp_atomic_t *) refcp, diff); if (val < min_val) erl_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", @@ -1076,10 +1072,10 @@ erts_refc_add(erts_refc_t *refcp, long diff, long min_val) #endif } -ERTS_GLB_INLINE long -erts_refc_read(erts_refc_t *refcp, long min_val) +ERTS_GLB_INLINE erts_aint_t +erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) { - long val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp); + erts_aint_t val = erts_smp_atomic_read((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) erl_exit(ERTS_ABORT_EXIT, @@ -1168,6 +1164,15 @@ void* sys_calloc2(Uint, Uint); ((char*)(s))[3] = (char)(i) & 0xff;} \ while (0) +#define get_int24(s) ((((unsigned char*) (s))[0] << 16) | \ + (((unsigned char*) (s))[1] << 8) | \ + (((unsigned char*) (s))[2])) + +#define put_int24(i, s) do {((char*)(s))[0] = (char)((i) >> 16) & 0xff; \ + ((char*)(s))[1] = (char)((i) >> 8) & 0xff; \ + ((char*)(s))[2] = (char)(i) & 0xff;} \ + while (0) + #define get_int16(s) ((((unsigned char*) (s))[0] << 8) | \ (((unsigned char*) (s))[1])) @@ -1181,6 +1186,7 @@ void* sys_calloc2(Uint, Uint); #define put_int8(i, s) do {((unsigned char*)(s))[0] = (i) & 0xff;} while (0) + /* * Use DEBUGF as you would use printf, but use double parentheses: * @@ -1245,6 +1251,22 @@ char* win32_errorstr(int); #endif +/************************************************************************ + * Find out the native filename encoding of the process (look at locale of + * Unix processes and just do UTF16 on windows + ************************************************************************/ +#define ERL_FILENAME_UNKNOWN 0 +#define ERL_FILENAME_LATIN1 1 +#define ERL_FILENAME_UTF8 2 +#define ERL_FILENAME_UTF8_MAC 3 +#define ERL_FILENAME_WIN_WCHAR 4 + +int erts_get_native_filename_encoding(void); +/* The set function is only to be used by erl_init! */ +void erts_set_user_requested_filename_encoding(int encoding); +int erts_get_user_requested_filename_encoding(void); + +void erts_init_sys_common_misc(void); #endif diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 53d39aef0e..a00faff912 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -99,80 +99,37 @@ static erts_smp_mtx_t tiw_lock; static ErlTimer** tiw; /* the timing wheel, allocated in init_time() */ static Uint tiw_pos; /* current position in wheel */ static Uint tiw_nto; /* number of timeouts in wheel */ +static Uint tiw_min; +static ErlTimer *tiw_min_ptr; /* END tiw_lock protected variables */ /* Actual interval time chosen by sys_init_time() */ static int itime; /* Constant after init */ -#if defined(ERTS_TIMER_THREAD) -static SysTimeval time_start; /* start of current time interval */ -static long ticks_end; /* time_start+ticks_end == time_wakeup */ -static long ticks_latest; /* delta from time_start at latest time update*/ - -static ERTS_INLINE long time_gettimeofday(SysTimeval *now) -{ - long elapsed; - - erts_get_timeval(now); - now->tv_usec = 1000 * (now->tv_usec / 1000); /* ms resolution */ - elapsed = (1000 * (now->tv_sec - time_start.tv_sec) + - (now->tv_usec - time_start.tv_usec) / 1000); - // elapsed /= CLOCK_RESOLUTION; - return elapsed; -} - -static long do_time_update(void) -{ - SysTimeval now; - long elapsed; - - elapsed = time_gettimeofday(&now); - ticks_latest = elapsed; - return elapsed; -} - -static ERTS_INLINE long do_time_read(void) -{ - return ticks_latest; -} - -static long do_time_reset(void) -{ - SysTimeval now; - long elapsed; - - elapsed = time_gettimeofday(&now); - time_start = now; - ticks_end = LONG_MAX; - ticks_latest = 0; - return elapsed; -} - -static ERTS_INLINE void do_time_init(void) -{ - (void)do_time_reset(); -} - -#else erts_smp_atomic_t do_time; /* set at clock interrupt */ -static ERTS_INLINE long do_time_read(void) { return erts_smp_atomic_read(&do_time); } -static ERTS_INLINE long do_time_update(void) { return do_time_read(); } +static ERTS_INLINE erts_aint_t do_time_read(void) { return erts_smp_atomic_read(&do_time); } +static ERTS_INLINE erts_aint_t do_time_update(void) { return do_time_read(); } static ERTS_INLINE void do_time_init(void) { erts_smp_atomic_init(&do_time, 0L); } -#endif /* get the time (in units of itime) to the next timeout, or -1 if there are no timeouts */ -static int next_time_internal(void) /* PRE: tiw_lock taken by caller */ +static erts_aint_t next_time_internal(void) /* PRE: tiw_lock taken by caller */ { int i, tm, nto; unsigned int min; ErlTimer* p; - long dt; + erts_aint_t dt; if (tiw_nto == 0) return -1; /* no timeouts in wheel */ + + if (tiw_min_ptr) { + min = tiw_min; + dt = do_time_read(); + return ((min >= dt) ? (min - dt) : 0); + } /* start going through wheel to find next timeout */ tm = nto = 0; @@ -185,11 +142,17 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */ if (p->count == 0) { /* found next timeout */ dt = do_time_read(); + /* p->count is zero */ + tiw_min_ptr = p; + tiw_min = tm; return ((tm >= dt) ? (tm - dt) : 0); } else { /* keep shortest time in 'min' */ - if (tm + p->count*TIW_SIZE < min) + if (tm + p->count*TIW_SIZE < min) { min = tm + p->count*TIW_SIZE; + tiw_min_ptr = p; + tiw_min = min; + } } p = p->next; } @@ -202,11 +165,35 @@ static int next_time_internal(void) /* PRE: tiw_lock taken by caller */ return ((min >= dt) ? (min - dt) : 0); } -#if !defined(ERTS_TIMER_THREAD) +static void remove_timer(ErlTimer *p) { + /* first */ + if (!p->prev) { + tiw[p->slot] = p->next; + if(p->next) + p->next->prev = NULL; + } else { + p->prev->next = p->next; + } + + /* last */ + if (!p->next) { + if (p->prev) + p->prev->next = NULL; + } else { + p->next->prev = p->prev; + } + + p->next = NULL; + p->prev = NULL; + /* Make sure cancel callback isn't called */ + p->active = 0; + tiw_nto--; +} + /* Private export to erl_time_sup.c */ -int next_time(void) +erts_aint_t erts_next_time(void) { - int ret; + erts_aint_t ret; erts_smp_mtx_lock(&tiw_lock); (void)do_time_update(); @@ -214,14 +201,13 @@ int next_time(void) erts_smp_mtx_unlock(&tiw_lock); return ret; } -#endif -static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-locked */ +static ERTS_INLINE void bump_timer_internal(erts_aint_t dt) /* PRE: tiw_lock is write-locked */ { Uint keep_pos; Uint count; ErlTimer *p, **prev, *timeout_head, **timeout_tail; - Uint dtime = (unsigned long)dt; + Uint dtime = (Uint) dt; /* no need to bump the position if there aren't any timeouts */ if (tiw_nto == 0) { @@ -242,12 +228,16 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l if (tiw_pos == keep_pos) count--; prev = &tiw[tiw_pos]; while ((p = *prev) != NULL) { + ASSERT( p != p->next); if (p->count < count) { /* we have a timeout */ - *prev = p->next; /* Remove from list */ - tiw_nto--; - p->next = NULL; - p->active = 0; /* Make sure cancel callback - isn't called */ + /* remove min time */ + if (tiw_min_ptr == p) { + tiw_min_ptr = NULL; + tiw_min = 0; + } + + /* Remove from list */ + remove_timer(p); *timeout_tail = p; /* Insert in timeout queue */ timeout_tail = &p->next; } @@ -261,6 +251,8 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l dtime--; } tiw_pos = keep_pos; + if (tiw_min_ptr) + tiw_min -= dt; erts_smp_mtx_unlock(&tiw_lock); @@ -275,24 +267,17 @@ static ERTS_INLINE void bump_timer_internal(long dt) /* PRE: tiw_lock is write-l * callback is called. */ p->next = NULL; + p->prev = NULL; p->slot = 0; (*p->timeout)(p->arg); } } -#if defined(ERTS_TIMER_THREAD) -static void timer_thread_bump_timer(void) -{ - erts_smp_mtx_lock(&tiw_lock); - bump_timer_internal(do_time_reset()); -} -#else -void bump_timer(long dt) /* dt is value from do_time */ +void erts_bump_timer(erts_aint_t dt) /* dt is value from do_time */ { erts_smp_mtx_lock(&tiw_lock); bump_timer_internal(dt); } -#endif Uint erts_timer_wheel_memory_size(void) @@ -300,82 +285,10 @@ erts_timer_wheel_memory_size(void) return (Uint) TIW_SIZE * sizeof(ErlTimer*); } -#if defined(ERTS_TIMER_THREAD) -static struct erts_iwait *timer_thread_iwait; - -static int timer_thread_setup_delay(SysTimeval *rem_time) -{ - long elapsed; - int ticks; - - erts_smp_mtx_lock(&tiw_lock); - elapsed = do_time_update(); - ticks = next_time_internal(); - if (ticks == -1) /* timer queue empty */ - ticks = 100*1000*1000; - if (elapsed > ticks) - elapsed = ticks; - ticks -= elapsed; - //ticks *= CLOCK_RESOLUTION; - rem_time->tv_sec = ticks / 1000; - rem_time->tv_usec = 1000 * (ticks % 1000); - ticks_end = ticks; - erts_smp_mtx_unlock(&tiw_lock); - return ticks; -} - -static void *timer_thread_start(void *ignore) -{ - SysTimeval delay; - -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_set_thread_name("timer"); -#endif - erts_register_blockable_thread(); - - for(;;) { - if (timer_thread_setup_delay(&delay)) { - erts_smp_activity_begin(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - ASSERT_NO_LOCKED_LOCKS; - erts_iwait_wait(timer_thread_iwait, &delay); - ASSERT_NO_LOCKED_LOCKS; - erts_smp_activity_end(ERTS_ACTIVITY_WAIT, NULL, NULL, NULL); - } - else - erts_smp_chk_system_block(NULL, NULL, NULL); - timer_thread_bump_timer(); - ASSERT_NO_LOCKED_LOCKS; - } - /*NOTREACHED*/ - return NULL; -} - -static ERTS_INLINE void timer_thread_post_insert(Uint ticks) -{ - if ((Sint)ticks < ticks_end) - erts_iwait_interrupt(timer_thread_iwait); -} - -static void timer_thread_init(void) -{ - erts_thr_opts_t opts = ERTS_THR_OPTS_DEFAULT_INITER; - erts_tid_t tid; - - opts->detached = 1; - - timer_thread_iwait = erts_iwait_init(); - erts_thr_create(&tid, timer_thread_start, NULL, &opts); -} - -#else -static ERTS_INLINE void timer_thread_post_insert(Uint ticks) { } -static ERTS_INLINE void timer_thread_init(void) { } -#endif - /* this routine links the time cells into a free list at the start and sets the time queue as empty */ void -init_time(void) +erts_init_time(void) { int i; @@ -391,10 +304,13 @@ init_time(void) tiw[i] = NULL; do_time_init(); tiw_pos = tiw_nto = 0; - - timer_thread_init(); + tiw_min_ptr = NULL; + tiw_min = 0; } + + + /* ** Insert a process into the time queue, with a timeout 't' */ @@ -424,16 +340,31 @@ insert_timer(ErlTimer* p, Uint t) /* insert at head of list at slot */ p->next = tiw[tm]; + p->prev = NULL; + if (p->next != NULL) + p->next->prev = p; tiw[tm] = p; - tiw_nto++; - timer_thread_post_insert(ticks); + + /* insert min time */ + if ((tiw_nto == 0) || ((tiw_min_ptr != NULL) && (ticks < tiw_min))) { + tiw_min = ticks; + tiw_min_ptr = p; + } + if ((tiw_min_ptr == p) && (ticks > tiw_min)) { + /* some other timer might be 'min' now */ + tiw_min = 0; + tiw_min_ptr = NULL; + } + + tiw_nto++; } void -erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, +erts_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, void* arg, Uint t) { + erts_deliver_time(); erts_smp_mtx_lock(&tiw_lock); if (p->active) { /* XXX assert ? */ @@ -446,42 +377,34 @@ erl_set_timer(ErlTimer* p, ErlTimeoutProc timeout, ErlCancelProc cancel, p->active = 1; insert_timer(p, t); erts_smp_mtx_unlock(&tiw_lock); -#if defined(ERTS_SMP) && !defined(ERTS_TIMER_THREAD) +#if defined(ERTS_SMP) if (t <= (Uint) LONG_MAX) erts_sys_schedule_interrupt_timed(1, (long) t); #endif } void -erl_cancel_timer(ErlTimer* p) +erts_cancel_timer(ErlTimer* p) { - ErlTimer *tp; - ErlTimer **prev; - erts_smp_mtx_lock(&tiw_lock); if (!p->active) { /* allow repeated cancel (drivers) */ erts_smp_mtx_unlock(&tiw_lock); return; } - /* find p in linked list at slot p->slot and remove it */ - prev = &tiw[p->slot]; - while ((tp = *prev) != NULL) { - if (tp == p) { - *prev = p->next; /* Remove from list */ - tiw_nto--; - p->next = NULL; - p->slot = p->count = 0; - p->active = 0; - if (p->cancel != NULL) { - erts_smp_mtx_unlock(&tiw_lock); - (*p->cancel)(p->arg); - } else { - erts_smp_mtx_unlock(&tiw_lock); - } - return; - } else { - prev = &tp->next; - } + + /* is it the 'min' timer, remove min */ + if (p == tiw_min_ptr) { + tiw_min_ptr = NULL; + tiw_min = 0; + } + + remove_timer(p); + p->slot = p->count = 0; + + if (p->cancel != NULL) { + erts_smp_mtx_unlock(&tiw_lock); + (*p->cancel)(p->arg); + return; } erts_smp_mtx_unlock(&tiw_lock); } @@ -493,10 +416,10 @@ erl_cancel_timer(ErlTimer* p) immediately if it hadn't been cancelled). */ Uint -time_left(ErlTimer *p) +erts_time_left(ErlTimer *p) { Uint left; - long dt; + erts_aint_t dt; erts_smp_mtx_lock(&tiw_lock); @@ -517,12 +440,11 @@ time_left(ErlTimer *p) erts_smp_mtx_unlock(&tiw_lock); - return left * itime; + return (Uint) left * itime; } #ifdef DEBUG - -void p_slpq() +void erts_p_slpq() { int i; ErlTimer* p; @@ -551,5 +473,4 @@ void p_slpq() erts_smp_mtx_unlock(&tiw_lock); } - #endif /* DEBUG */ diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index ab5e8b5d4a..f531d1430b 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -59,13 +59,6 @@ /* profile_scheduler mini message queue */ -#ifdef ERTS_TIMER_THREAD -/* A timer thread is not welcomed with this lock violation work around. - * - Bj�rn-Egil - */ -#error Timer thread may not be enabled due to lock violation. -#endif - typedef struct { Uint scheduler_id; Uint no_schedulers; @@ -98,7 +91,7 @@ dispatch_profile_msg_q(profile_sched_msg_q *psmq) Eterm* -erts_heap_alloc(Process* p, Uint need) +erts_heap_alloc(Process* p, Uint need, Uint xtra) { ErlHeapFragment* bp; Eterm* htop; @@ -124,7 +117,7 @@ erts_heap_alloc(Process* p, Uint need) p->space_verified_from = NULL; #endif /* FORCE_HEAP_FRAGS */ - n = need; + n = need + xtra; bp = MBUF(p); if (bp != NULL && need <= (bp->alloc_size - bp->used_size)) { Eterm* ret = bp->mem + bp->used_size; @@ -160,7 +153,7 @@ erts_heap_alloc(Process* p, Uint need) bp->next = MBUF(p); MBUF(p) = bp; bp->alloc_size = n; - bp->used_size = n; + bp->used_size = need; MBUF_SIZE(p) += n; bp->off_heap.first = NULL; bp->off_heap.overhead = 0; @@ -1901,34 +1894,36 @@ erts_destroy_tmp_dsbuf(erts_dsprintf_buf_t *dsbufp) erts_free(ERTS_ALC_T_TMP_DSBUF, (void *) dsbufp); } - /* eq and cmp are written as separate functions a eq is a little faster */ /* * Test for equality of two terms. * Returns 0 if not equal, or a non-zero value otherwise. */ - +#if HALFWORD_HEAP +int eq_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) +#else int eq(Eterm a, Eterm b) +#endif { DECLARE_WSTACK(stack); Sint sz; Eterm* aa; - Eterm* bb; + Eterm* bb; tailrecur: - if (a == b) goto pop_next; + if (is_same(a, a_base, b, b_base)) goto pop_next; tailrecur_ne: switch (primary_tag(a)) { case TAG_PRIMARY_LIST: if (is_list(b)) { - Eterm* aval = list_val(a); - Eterm* bval = list_val(b); + Eterm* aval = list_val_rel(a, a_base); + Eterm* bval = list_val_rel(b, b_base); while (1) { Eterm atmp = CAR(aval); Eterm btmp = CAR(bval); - if (atmp != btmp) { + if (!is_same(atmp,a_base,btmp,b_base)) { WSTACK_PUSH2(stack,(UWord) CDR(bval),(UWord) CDR(aval)); a = atmp; b = btmp; @@ -1936,7 +1931,7 @@ tailrecur_ne: } atmp = CDR(aval); btmp = CDR(bval); - if (atmp == btmp) { + if (is_same(atmp,a_base,btmp,b_base)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -1944,22 +1939,22 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aval = list_val(atmp); - bval = list_val(btmp); + aval = list_val_rel(atmp, a_base); + bval = list_val_rel(btmp, b_base); } } break; /* not equal */ case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(a); + Eterm hdr = *boxed_val_rel(a,a_base); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: { - aa = tuple_val(a); - if (!is_boxed(b) || *boxed_val(b) != *aa) + aa = tuple_val_rel(a, a_base); + if (!is_boxed(b) || *boxed_val_rel(b,b_base) != *aa) goto not_equal; - bb = tuple_val(b); + bb = tuple_val_rel(b,b_base); if ((sz = arityval(*aa)) == 0) goto pop_next; ++aa; ++bb; @@ -1978,16 +1973,16 @@ tailrecur_ne: Uint a_bitoffs; Uint b_bitoffs; - if (is_not_binary(b)) { + if (!is_binary_rel(b,b_base)) { goto not_equal; } - a_size = binary_size(a); - b_size = binary_size(b); + a_size = binary_size_rel(a,a_base); + b_size = binary_size_rel(b,b_base); if (a_size != b_size) { goto not_equal; } - ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); - ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); + ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); + ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { if (sys_memcmp(a_ptr, b_ptr, a_size) == 0) goto pop_next; } else if (a_bitsize == b_bitsize) { @@ -1998,9 +1993,9 @@ tailrecur_ne: } case EXPORT_SUBTAG: { - if (is_export(b)) { - Export* a_exp = *((Export **) (export_val(a) + 1)); - Export* b_exp = *((Export **) (export_val(b) + 1)); + if (is_export_rel(b,b_base)) { + Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); + Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); if (a_exp == b_exp) goto pop_next; } break; /* not equal */ @@ -2010,10 +2005,10 @@ tailrecur_ne: ErlFunThing* f1; ErlFunThing* f2; - if (is_not_fun(b)) + if (!is_fun_rel(b,b_base)) goto not_equal; - f1 = (ErlFunThing *) fun_val(a); - f2 = (ErlFunThing *) fun_val(b); + f1 = (ErlFunThing *) fun_val_rel(a,a_base); + f2 = (ErlFunThing *) fun_val_rel(b,b_base); if (f1->fe->module != f2->fe->module || f1->fe->old_index != f2->fe->old_index || f1->fe->old_uniq != f2->fe->old_uniq || @@ -2031,15 +2026,15 @@ tailrecur_ne: ExternalThing *ap; ExternalThing *bp; - if(is_not_external(b)) + if(!is_external_rel(b,b_base)) goto not_equal; - ap = external_thing_ptr(a); - bp = external_thing_ptr(b); + ap = external_thing_ptr_rel(a,a_base); + bp = external_thing_ptr_rel(b,b_base); if(ap->header == bp->header && ap->node == bp->node) { - ASSERT(1 == external_data_words(a)); - ASSERT(1 == external_data_words(b)); + ASSERT(1 == external_data_words_rel(a,a_base)); + ASSERT(1 == external_data_words_rel(b,b_base)); if (ap->data.ui[0] == bp->data.ui[0]) goto pop_next; } @@ -2057,27 +2052,36 @@ tailrecur_ne: Uint alen; Uint blen; Uint i; + ExternalThing* athing; + ExternalThing* bthing; - if(is_not_external_ref(b)) + if(!is_external_ref_rel(b,b_base)) goto not_equal; - if(external_node(a) != external_node(b)) + athing = external_thing_ptr_rel(a,a_base); + bthing = external_thing_ptr_rel(b,b_base); + + if(athing->node != bthing->node) goto not_equal; - anum = external_ref_numbers(a); - bnum = external_ref_numbers(b); - alen = external_ref_no_of_numbers(a); - blen = external_ref_no_of_numbers(b); + anum = external_thing_ref_numbers(athing); + bnum = external_thing_ref_numbers(bthing); + alen = external_thing_ref_no_of_numbers(athing); + blen = external_thing_ref_no_of_numbers(bthing); goto ref_common; case REF_SUBTAG: - - if (is_not_internal_ref(b)) + if (!is_internal_ref_rel(b,b_base)) goto not_equal; - alen = internal_ref_no_of_numbers(a); - blen = internal_ref_no_of_numbers(b); - anum = internal_ref_numbers(a); - bnum = internal_ref_numbers(b); + + { + RefThing* athing = ref_thing_ptr_rel(a,a_base); + RefThing* bthing = ref_thing_ptr_rel(b,b_base); + alen = internal_thing_ref_no_of_numbers(athing); + blen = internal_thing_ref_no_of_numbers(bthing); + anum = internal_thing_ref_numbers(athing); + bnum = internal_thing_ref_numbers(bthing); + } ref_common: ASSERT(alen > 0 && blen > 0); @@ -2122,10 +2126,10 @@ tailrecur_ne: { int i; - if (is_not_big(b)) + if (!is_big_rel(b,b_base)) goto not_equal; - aa = big_val(a); /* get pointer to thing */ - bb = big_val(b); + aa = big_val_rel(a,a_base); + bb = big_val_rel(b,b_base); if (*aa != *bb) goto not_equal; i = BIG_ARITY(aa); @@ -2140,9 +2144,9 @@ tailrecur_ne: FloatDef af; FloatDef bf; - if (is_float(b)) { - GET_DOUBLE(a, af); - GET_DOUBLE(b, bf); + if (is_float_rel(b,b_base)) { + GET_DOUBLE_REL(a, af, a_base); + GET_DOUBLE_REL(b, bf, b_base); if (af.fd == bf.fd) goto pop_next; } break; /* not equal */ @@ -2161,7 +2165,7 @@ term_array: /* arrays in 'aa' and 'bb', length in 'sz' */ Eterm* bp = bb; Sint i = sz; for (;;) { - if (*ap != *bp) break; + if (!is_same(*ap,a_base,*bp,b_base)) break; if (--i == 0) goto pop_next; ++ap; ++bp; @@ -2250,7 +2254,11 @@ static int cmp_atoms(Eterm a, Eterm b) bb->name+3, bb->len-3); } +#if HALFWORD_HEAP +Sint cmp_rel(Eterm a, Eterm* a_base, Eterm b, Eterm* b_base) +#else Sint cmp(Eterm a, Eterm b) +#endif { DECLARE_WSTACK(stack); Eterm* aa; @@ -2284,7 +2292,7 @@ Sint cmp(Eterm a, Eterm b) tailrecur: - if (a == b) { /* Equal values or pointers. */ + if (is_same(a,a_base,b,b_base)) { /* Equal values or pointers. */ goto pop_next; } tailrecur_ne: @@ -2310,9 +2318,9 @@ tailrecur_ne: if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port(b)) { - bnode = external_port_node(b); - bdata = external_port_data(b); + } else if (is_external_port_rel(b,b_base)) { + bnode = external_port_node_rel(b,b_base); + bdata = external_port_data_rel(b,b_base); } else { a_tag = PORT_DEF; goto mixed_types; @@ -2328,9 +2336,9 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid(b)) { - bnode = external_pid_node(b); - bdata = external_pid_data(b); + } else if (is_external_pid_rel(b,b_base)) { + bnode = external_pid_node_rel(b,b_base); + bdata = external_pid_data_rel(b,b_base); } else { a_tag = PID_DEF; goto mixed_types; @@ -2363,12 +2371,12 @@ tailrecur_ne: a_tag = LIST_DEF; goto mixed_types; } - aa = list_val(a); - bb = list_val(b); + aa = list_val_rel(a,a_base); + bb = list_val_rel(b,b_base); while (1) { Eterm atmp = CAR(aa); Eterm btmp = CAR(bb); - if (atmp != btmp) { + if (!is_same(atmp,a_base,btmp,b_base)) { WSTACK_PUSH2(stack,(UWord) CDR(bb),(UWord) CDR(aa)); a = atmp; b = btmp; @@ -2376,7 +2384,7 @@ tailrecur_ne: } atmp = CDR(aa); btmp = CDR(bb); - if (atmp == btmp) { + if (is_same(atmp,a_base,btmp,b_base)) { goto pop_next; } if (is_not_list(atmp) || is_not_list(btmp)) { @@ -2384,20 +2392,20 @@ tailrecur_ne: b = btmp; goto tailrecur_ne; } - aa = list_val(atmp); - bb = list_val(btmp); + aa = list_val_rel(atmp,a_base); + bb = list_val_rel(btmp,b_base); } case TAG_PRIMARY_BOXED: { - Eterm ahdr = *boxed_val(a); + Eterm ahdr = *boxed_val_rel(a,a_base); switch ((ahdr & _TAG_HEADER_MASK) >> _TAG_PRIMARY_SIZE) { case (_TAG_HEADER_ARITYVAL >> _TAG_PRIMARY_SIZE): - if (is_not_tuple(b)) { + if (!is_tuple_rel(b,b_base)) { a_tag = TUPLE_DEF; goto mixed_types; } - aa = tuple_val(a); - bb = tuple_val(b); + aa = tuple_val_rel(a,a_base); + bb = tuple_val_rel(b,b_base); /* compare the arities */ i = arityval(ahdr); /* get the arity*/ if (i != arityval(*bb)) { @@ -2411,31 +2419,31 @@ tailrecur_ne: goto term_array; case (_TAG_HEADER_FLOAT >> _TAG_PRIMARY_SIZE): - if (is_not_float(b)) { + if (!is_float_rel(b,b_base)) { a_tag = FLOAT_DEF; goto mixed_types; } else { FloatDef af; FloatDef bf; - GET_DOUBLE(a, af); - GET_DOUBLE(b, bf); + GET_DOUBLE_REL(a, af, a_base); + GET_DOUBLE_REL(b, bf, b_base); ON_CMP_GOTO(float_comp(af.fd, bf.fd)); } case (_TAG_HEADER_POS_BIG >> _TAG_PRIMARY_SIZE): case (_TAG_HEADER_NEG_BIG >> _TAG_PRIMARY_SIZE): - if (is_not_big(b)) { + if (!is_big_rel(b,b_base)) { a_tag = BIG_DEF; goto mixed_types; } - ON_CMP_GOTO(big_comp(a, b)); + ON_CMP_GOTO(big_comp(rterm2wterm(a,a_base), rterm2wterm(b,b_base))); case (_TAG_HEADER_EXPORT >> _TAG_PRIMARY_SIZE): - if (is_not_export(b)) { + if (!is_export_rel(b,b_base)) { a_tag = EXPORT_DEF; goto mixed_types; } else { - Export* a_exp = *((Export **) (export_val(a) + 1)); - Export* b_exp = *((Export **) (export_val(b) + 1)); + Export* a_exp = *((Export **) (export_val_rel(a,a_base) + 1)); + Export* b_exp = *((Export **) (export_val_rel(b,b_base) + 1)); if ((j = cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { RETURN_NEQ(j); @@ -2447,12 +2455,12 @@ tailrecur_ne: } break; case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): - if (is_not_fun(b)) { + if (!is_fun_rel(b,b_base)) { a_tag = FUN_DEF; goto mixed_types; } else { - ErlFunThing* f1 = (ErlFunThing *) fun_val(a); - ErlFunThing* f2 = (ErlFunThing *) fun_val(b); + ErlFunThing* f1 = (ErlFunThing *) fun_val_rel(a,a_base); + ErlFunThing* f2 = (ErlFunThing *) fun_val_rel(b,b_base); Sint diff; diff = cmpbytes(atom_tab(atom_val(f1->fe->module))->name, @@ -2484,51 +2492,57 @@ tailrecur_ne: if (is_internal_pid(b)) { bnode = erts_this_node; bdata = internal_pid_data(b); - } else if (is_external_pid(b)) { - bnode = external_pid_node(b); - bdata = external_pid_data(b); + } else if (is_external_pid_rel(b,b_base)) { + bnode = external_pid_node_rel(b,b_base); + bdata = external_pid_data_rel(b,b_base); } else { a_tag = EXTERNAL_PID_DEF; goto mixed_types; } - anode = external_pid_node(a); - adata = external_pid_data(a); + anode = external_pid_node_rel(a,a_base); + adata = external_pid_data_rel(a,a_base); goto pid_common; case (_TAG_HEADER_EXTERNAL_PORT >> _TAG_PRIMARY_SIZE): if (is_internal_port(b)) { bnode = erts_this_node; bdata = internal_port_data(b); - } else if (is_external_port(b)) { - bnode = external_port_node(b); - bdata = external_port_data(b); + } else if (is_external_port_rel(b,b_base)) { + bnode = external_port_node_rel(b,b_base); + bdata = external_port_data_rel(b,b_base); } else { a_tag = EXTERNAL_PORT_DEF; goto mixed_types; } - anode = external_port_node(a); - adata = external_port_data(a); + anode = external_port_node_rel(a,a_base); + adata = external_port_data_rel(a,a_base); goto port_common; case (_TAG_HEADER_REF >> _TAG_PRIMARY_SIZE): /* * Note! When comparing refs we need to compare ref numbers * (32-bit words), *not* ref data words. */ + - if (is_internal_ref(b)) { + if (is_internal_ref_rel(b,b_base)) { + RefThing* bthing = ref_thing_ptr_rel(b,b_base); bnode = erts_this_node; - bnum = internal_ref_numbers(b); - blen = internal_ref_no_of_numbers(b); - } else if(is_external_ref(b)) { - bnode = external_ref_node(b); - bnum = external_ref_numbers(b); - blen = external_ref_no_of_numbers(b); + bnum = internal_thing_ref_numbers(bthing); + blen = internal_thing_ref_no_of_numbers(bthing); + } else if(is_external_ref_rel(b,b_base)) { + ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + bnode = bthing->node; + bnum = external_thing_ref_numbers(bthing); + blen = external_thing_ref_no_of_numbers(bthing); } else { a_tag = REF_DEF; goto mixed_types; } - anode = erts_this_node; - anum = internal_ref_numbers(a); - alen = internal_ref_no_of_numbers(a); + { + RefThing* athing = ref_thing_ptr_rel(a,a_base); + anode = erts_this_node; + anum = internal_thing_ref_numbers(athing); + alen = internal_thing_ref_no_of_numbers(athing); + } ref_common: CMP_NODES(anode, bnode); @@ -2557,31 +2571,36 @@ tailrecur_ne: RETURN_NEQ((Sint32) (anum[i] - bnum[i])); goto pop_next; case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): - if (is_internal_ref(b)) { + if (is_internal_ref_rel(b,b_base)) { + RefThing* bthing = ref_thing_ptr_rel(b,b_base); bnode = erts_this_node; - bnum = internal_ref_numbers(b); - blen = internal_ref_no_of_numbers(b); - } else if (is_external_ref(b)) { - bnode = external_ref_node(b); - bnum = external_ref_numbers(b); - blen = external_ref_no_of_numbers(b); + bnum = internal_thing_ref_numbers(bthing); + blen = internal_thing_ref_no_of_numbers(bthing); + } else if (is_external_ref_rel(b,b_base)) { + ExternalThing* bthing = external_thing_ptr_rel(b,b_base); + bnode = bthing->node; + bnum = external_thing_ref_numbers(bthing); + blen = external_thing_ref_no_of_numbers(bthing); } else { a_tag = EXTERNAL_REF_DEF; goto mixed_types; } - anode = external_ref_node(a); - anum = external_ref_numbers(a); - alen = external_ref_no_of_numbers(a); + { + ExternalThing* athing = external_thing_ptr_rel(a,a_base); + anode = athing->node; + anum = external_thing_ref_numbers(athing); + alen = external_thing_ref_no_of_numbers(athing); + } goto ref_common; default: /* Must be a binary */ - ASSERT(is_binary(a)); - if (is_not_binary(b)) { + ASSERT(is_binary_rel(a,a_base)); + if (!is_binary_rel(b,b_base)) { a_tag = BINARY_DEF; goto mixed_types; } else { - Uint a_size = binary_size(a); - Uint b_size = binary_size(b); + Uint a_size = binary_size_rel(a,a_base); + Uint b_size = binary_size_rel(b,b_base); Uint a_bitsize; Uint b_bitsize; Uint a_bitoffs; @@ -2590,8 +2609,8 @@ tailrecur_ne: int cmp; byte* a_ptr; byte* b_ptr; - ERTS_GET_BINARY_BYTES(a, a_ptr, a_bitoffs, a_bitsize); - ERTS_GET_BINARY_BYTES(b, b_ptr, b_bitoffs, b_bitsize); + ERTS_GET_BINARY_BYTES_REL(a, a_ptr, a_bitoffs, a_bitsize, a_base); + ERTS_GET_BINARY_BYTES_REL(b, b_ptr, b_bitoffs, b_bitsize, b_base); if ((a_bitsize | b_bitsize | a_bitoffs | b_bitoffs) == 0) { min_size = (a_size < b_size) ? a_size : b_size; if ((cmp = sys_memcmp(a_ptr, b_ptr, min_size)) != 0) { @@ -2618,7 +2637,6 @@ tailrecur_ne: */ mixed_types: - b_tag = tag_val_def(b); { FloatDef f1, f2; @@ -2628,39 +2646,47 @@ tailrecur_ne: #else Eterm *big_buf = erts_get_scheduler_data()->cmp_tmp_heap; #endif +#if HALFWORD_HEAP + Wterm aw = is_immed(a) ? a : rterm2wterm(a,a_base); + Wterm bw = is_immed(b) ? b : rterm2wterm(b,b_base); +#else + Eterm aw = a; + Eterm bw = b; +#endif + b_tag = tag_val_def(bw); switch(_NUMBER_CODE(a_tag, b_tag)) { case SMALL_BIG: big = small_to_big(signed_val(a), big_buf); - j = big_comp(big, b); + j = big_comp(big, bw); break; case SMALL_FLOAT: f1.fd = signed_val(a); - GET_DOUBLE(b, f2); + GET_DOUBLE(bw, f2); j = float_comp(f1.fd, f2.fd); break; case BIG_SMALL: big = small_to_big(signed_val(b), big_buf); - j = big_comp(a, big); + j = big_comp(aw, big); break; case BIG_FLOAT: - if (big_to_double(a, &f1.fd) < 0) { + if (big_to_double(aw, &f1.fd) < 0) { j = big_sign(a) ? -1 : 1; } else { - GET_DOUBLE(b, f2); + GET_DOUBLE(bw, f2); j = float_comp(f1.fd, f2.fd); } break; case FLOAT_SMALL: - GET_DOUBLE(a, f1); + GET_DOUBLE(aw, f1); f2.fd = signed_val(b); j = float_comp(f1.fd, f2.fd); break; case FLOAT_BIG: - if (big_to_double(b, &f2.fd) < 0) { + if (big_to_double(bw, &f2.fd) < 0) { j = big_sign(b) ? 1 : -1; } else { - GET_DOUBLE(a, f1); + GET_DOUBLE(aw, f1); j = float_comp(f1.fd, f2.fd); } break; @@ -3183,7 +3209,7 @@ erts_create_smp_ptimer(ErtsSmpPTimer **timer_ref, *timer_ref = res; - erl_set_timer(&res->timer.tm, + erts_set_timer(&res->timer.tm, (ErlTimeoutProc) ptimer_timeout, (ErlCancelProc) ptimer_cancelled, (void*) res, @@ -3197,7 +3223,7 @@ erts_cancel_smp_ptimer(ErtsSmpPTimer *ptimer) ASSERT(*ptimer->timer.timer_ref == ptimer); *ptimer->timer.timer_ref = NULL; ptimer->timer.flags |= ERTS_PTMR_FLG_CANCELLED; - erl_cancel_timer(&ptimer->timer.tm); + erts_cancel_timer(&ptimer->timer.tm); } } @@ -3637,19 +3663,19 @@ erts_set_activity_error(erts_activity_error_t error, char *file, int line) } -static ERTS_INLINE int +static ERTS_INLINE erts_aint32_t threads_not_under_control(void) { - int res = system_block_state.threads_to_block; + erts_aint32_t res = system_block_state.threads_to_block; /* Waiting is always an allowed activity... */ - res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.wait); + res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.wait); if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_GC) - res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.gc); + res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.gc); if (system_block_state.allowed_activities & ERTS_BS_FLG_ALLOW_IO) - res -= erts_smp_atomic_read(&erts_system_block_state.in_activity.io); + res -= erts_smp_atomic32_read(&erts_system_block_state.in_activity.io); if (res < 0) { ASSERT(0); @@ -3709,7 +3735,7 @@ erts_block_system(Uint32 allowed_activities) } else { - erts_smp_atomic_inc(&erts_system_block_state.do_block); + erts_smp_atomic32_inc(&erts_system_block_state.do_block); /* Someone else might be waiting for us to block... */ if (do_block) { @@ -3761,11 +3787,11 @@ erts_emergency_block_system(long timeout, Uint32 allowed_activities) another_blocker = erts_smp_pending_system_block(); system_block_state.emergency = 1; - erts_smp_atomic_inc(&erts_system_block_state.do_block); + erts_smp_atomic32_inc(&erts_system_block_state.do_block); if (another_blocker) { if (is_blocker()) { - erts_smp_atomic_dec(&erts_system_block_state.do_block); + erts_smp_atomic32_dec(&erts_system_block_state.do_block); res = 0; goto done; } @@ -3822,7 +3848,7 @@ erts_release_system(void) if (system_block_state.recursive_block) system_block_state.recursive_block--; else { - do_block = erts_smp_atomic_dectest(&erts_system_block_state.do_block); + do_block = erts_smp_atomic32_dectest(&erts_system_block_state.do_block); system_block_state.have_blocker = 0; if (is_blockable_thread()) system_block_state.threads_to_block++; @@ -3957,10 +3983,10 @@ erts_system_block_init(void) /* Global state... */ - erts_smp_atomic_init(&erts_system_block_state.do_block, 0L); - erts_smp_atomic_init(&erts_system_block_state.in_activity.wait, 0L); - erts_smp_atomic_init(&erts_system_block_state.in_activity.gc, 0L); - erts_smp_atomic_init(&erts_system_block_state.in_activity.io, 0L); + erts_smp_atomic32_init(&erts_system_block_state.do_block, 0); + erts_smp_atomic32_init(&erts_system_block_state.in_activity.wait, 0); + erts_smp_atomic32_init(&erts_system_block_state.in_activity.gc, 0); + erts_smp_atomic32_init(&erts_system_block_state.in_activity.io, 0); /* Make sure blockable threads unregister when exiting... */ erts_smp_install_exit_handler(erts_unregister_blockable_thread); diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index c450f10f48..4e9b5005c1 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -67,6 +67,8 @@ #define FILE_RESP_LDATA 6 #define FILE_RESP_N2DATA 7 #define FILE_RESP_EOF 8 +#define FILE_RESP_FNAME 9 +#define FILE_RESP_ALL_DATA 10 /* Options */ @@ -109,11 +111,11 @@ void erl_exit(int n, char *fmt, ...); static ErlDrvSysInfo sys_info; -/*#define TRACE 1*/ +/* #define TRACE 1 */ #ifdef TRACE -# define TRACE_C(c) (putchar(c)) -# define TRACE_S(s) (fputs((s), stdout)) -# define TRACE_F(args) (printf args) +# define TRACE_C(c) do { putchar(c); fflush(stdout); } while (0) +# define TRACE_S(s) do { fputs((s), stdout); fflush(stdout); } while (0) +# define TRACE_F(args) do { printf args ;fflush(stdout); } while (0) #else # define TRACE_C(c) ((void)(0)) # define TRACE_S(s) ((void)(0)) @@ -137,24 +139,54 @@ static ErlDrvSysInfo sys_info; #define MUTEX_UNLOCK(m) #endif - - #if 0 /* Experimental, for forcing all file operations to use the same thread. */ -static unsigned file_fixed_key = 1; -#define KEY(desc) (&file_fixed_key) + static unsigned file_fixed_key = 1; +# define KEY(desc) (&file_fixed_key) #else -#define KEY(desc) (&(desc)->key) +# define KEY(desc) (&(desc)->key) #endif +#ifdef FILENAMES_16BIT +# define FILENAME_BYTELEN(Str) filename_len_16bit(Str) +# define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) +# define FILENAME_CHARSIZE 2 + + static int filename_len_16bit(char *str) + { + char *p = str; + while(*p != '\0' || p[1] != '\0') { + p += 2; + } + return (p - str); + } + + static void filename_cpy_16bit(char *to, char *from) + { + while(*from != '\0' || from[1] != '\0') { + *to++ = *from++; + *to++ = *from++; + } + *to++ = *from++; + *to++ = *from++; + } + +#else +# define FILENAME_BYTELEN(Str) strlen(Str) +# define FILENAME_COPY(To,From) strcpy(To,From) +# define FILENAME_CHARSIZE 1 +#endif -#if MAXPATHLEN >= BUFSIZ -#define RESBUFSIZE MAXPATHLEN+1 +#if (MAXPATHLEN+1)*FILENAME_CHARSIZE+1 > BUFSIZ +# define RESBUFSIZE ((MAXPATHLEN+1)*FILENAME_CHARSIZE+1) #else -#define RESBUFSIZE BUFSIZ +# define RESBUFSIZE BUFSIZ #endif + + + #define GET_TIME(i, b) \ (i).year = get_int32((b) + 0 * 4); \ (i).month = get_int32((b) + 1 * 4); \ @@ -286,9 +318,9 @@ struct t_preadv { }; #define READDIR_BUFSIZE (8*1024) -#if READDIR_BUFSIZE < (2*MAXPATHLEN) -#undef READDIR_BUFSIZE -#define READDIR_BUFSIZE (2*MAXPATHLEN) +#if READDIR_BUFSIZE < (FILENAME_CHARSIZE*2*(MAXPATHLEN+1)) +# undef READDIR_BUFSIZE +# define READDIR_BUFSIZE (FILENAME_CHARSIZE*2*(MAXPATHLEN+1)) #endif struct t_readdir_buf { @@ -353,7 +385,6 @@ struct t_data ErlDrvBinary *binp; int size; int offset; - char name[1]; } read_file; struct { struct t_readdir_buf *first_buf; @@ -369,6 +400,7 @@ struct t_data }; + #define EF_ALLOC(S) driver_alloc((S)) #define EF_REALLOC(P, S) driver_realloc((P), (S)) #define EF_SAFE_ALLOC(S) ef_safe_alloc((S)) @@ -1084,7 +1116,7 @@ static void invoke_read_file(void *data) Sint64 size; if (! (d->result_ok = - efile_openfile(&d->errInfo, d->c.read_file.name, + efile_openfile(&d->errInfo, d->b, EFILE_MODE_READ, &fd, &size))) { goto done; } @@ -1288,7 +1320,7 @@ static void invoke_writev(void *data) { p < size && iovcnt < iovlen; p += iov0[iovcnt++].iov_len) ; - iov = EF_ALLOC(sizeof(SysIOVec)*iovcnt); + iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovcnt); memcpy(iov,iov0,iovcnt*sizeof(SysIOVec)); MUTEX_UNLOCK(d->c.writev.q_mtx); /* Let go of lock until we deque from original vector */ @@ -1368,7 +1400,7 @@ static void invoke_readlink(void *data) d->result_ok = efile_readlink(&d->errInfo, d->b, resbuf+1, RESBUFSIZE-1); if (d->result_ok != 0) - strcpy((char *) d->b + 1, resbuf+1); + FILENAME_COPY((char *) d->b + 1, resbuf+1); } static void invoke_altname(void *data) @@ -1380,7 +1412,7 @@ static void invoke_altname(void *data) d->result_ok = efile_altname(&d->errInfo, d->b, resbuf+1, RESBUFSIZE-1); if (d->result_ok != 0) - strcpy((char *) d->b + 1, resbuf+1); + FILENAME_COPY((char *) d->b + 1, resbuf+1); } static void invoke_pwritev(void *data) { @@ -1405,7 +1437,7 @@ static void invoke_pwritev(void *data) { /* Lock the queue just for a while, we don't want it locked during write */ MUTEX_LOCK(c->q_mtx); iov0 = driver_peekq(c->port, &iovlen); - iov = EF_ALLOC(sizeof(SysIOVec)*iovlen); + iov = EF_SAFE_ALLOC(sizeof(SysIOVec)*iovlen); memcpy(iov,iov0,sizeof(SysIOVec)*iovlen); MUTEX_UNLOCK(c->q_mtx); @@ -1499,7 +1531,7 @@ static void invoke_link(void *data) char *new_name; d->again = 0; - new_name = name+strlen(name)+1; + new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; d->result_ok = efile_link(&d->errInfo, name, new_name); } @@ -1510,7 +1542,7 @@ static void invoke_symlink(void *data) char *new_name; d->again = 0; - new_name = name+strlen(name)+1; + new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; d->result_ok = efile_symlink(&d->errInfo, name, new_name); } @@ -1521,7 +1553,7 @@ static void invoke_rename(void *data) char *new_name; d->again = 0; - new_name = name+strlen(name)+1; + new_name = name+FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; d->result_ok = efile_rename(&d->errInfo, name, new_name); } @@ -1569,13 +1601,15 @@ static void invoke_readdir(void *data) int s; char *p = NULL; int buf_sz = 0; + size_t tmp_bs; d->again = 0; d->errInfo.posix_errno = 0; while (1) { char *str; - if (buf_sz < (4 /* sz */ + 1 /* cmd */ + MAXPATHLEN + 1 /* '\0' */)) { + if (buf_sz < (4 /* sz */ + 1 /* cmd */ + + FILENAME_CHARSIZE*(MAXPATHLEN + 1))) { struct t_readdir_buf *b; if (p) { put_int32(0, p); /* EOB */ @@ -1591,18 +1625,18 @@ static void invoke_readdir(void *data) buf_sz = READDIR_BUFSIZE - 4/* EOB */; } - p[4] = FILE_RESP_OK; + p[4] = FILE_RESP_FNAME; buf_sz -= 4 + 1; str = p + 4 + 1; ASSERT(buf_sz >= MAXPATHLEN + 1); - s = efile_readdir(&d->errInfo, d->b, &d->dir_handle, str, buf_sz); + tmp_bs = buf_sz; + s = efile_readdir(&d->errInfo, d->b, &d->dir_handle, str, &tmp_bs); if (s) { - int str_sz = strlen(str); - int sz = str_sz + 1; - put_int32(sz, p); - p += 4 + sz; - buf_sz -= str_sz; + put_int32(tmp_bs + 1 /* 1 byte for opcode */, p); + p += 4 + tmp_bs + 1; + ASSERT(p == (str + tmp_bs)); + buf_sz -= tmp_bs; } else { put_int32(1, p); @@ -1911,7 +1945,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) if (!d->result_ok) reply_error(desc, &d->errInfo); else { - header[0] = FILE_RESP_OK; + header[0] = FILE_RESP_ALL_DATA; TRACE_C('R'); driver_output_binary(desc->port, header, 1, d->c.read_file.binp, @@ -1968,10 +2002,10 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) if (!d->result_ok) reply_error(desc, &d->errInfo); else { - resbuf[0] = FILE_RESP_OK; - length = 1+strlen((char*) resbuf+1); + resbuf[0] = FILE_RESP_FNAME; + length = 1+FILENAME_BYTELEN((char*) resbuf+1); TRACE_C('R'); - driver_output2(desc->port, resbuf, length, NULL, 0); + driver_output2(desc->port, resbuf, 1, resbuf+1, length-1); } free_data(data); break; @@ -2031,13 +2065,18 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) int sz = get_int32(p); while (sz) { /* 0 == EOB */ p += 4; - driver_output2(desc->port, p, sz, NULL, 0); + if (sz - 1 > 0) { + driver_output2(desc->port, p, 1, p+1, sz-1); + } else { + driver_output2(desc->port, p, 1, NULL, 0); + } p += sz; sz = get_int32(p); } b1 = b1->next; EF_FREE(b2); } + d->c.read_dir.first_buf = NULL; d->c.read_dir.last_buf = NULL; } @@ -2113,9 +2152,9 @@ file_output(ErlDrvData e, char* buf, int count) case FILE_MKDIR: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_mkdir; d->free = free_data; @@ -2124,9 +2163,9 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_RMDIR: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_rmdir; d->free = free_data; @@ -2135,9 +2174,9 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_DELETE: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_delete_file; d->free = free_data; @@ -2147,14 +2186,14 @@ file_output(ErlDrvData e, char* buf, int count) case FILE_RENAME: { char* new_name; - - new_name = name+strlen(name)+1; + int namelen = FILENAME_BYTELEN(name)+FILENAME_CHARSIZE; + new_name = name+namelen; d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + strlen(name) + 1 - + strlen(new_name) + 1); + + namelen + + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE); - strcpy(d->b, name); - strcpy(d->b + strlen(name) + 1, new_name); + FILENAME_COPY(d->b, name); + FILENAME_COPY(d->b + namelen, new_name); d->flags = desc->flags; d->fd = fd; d->command = command; @@ -2165,9 +2204,9 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_CHDIR: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_chdir; d->free = free_data; @@ -2190,9 +2229,10 @@ file_output(ErlDrvData e, char* buf, int count) #ifdef USE_THREADS if (sys_info.async_threads > 0) { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + + FILENAME_CHARSIZE); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->dir_handle = NULL; d->command = command; d->invoke = invoke_readdir; @@ -2205,17 +2245,19 @@ file_output(ErlDrvData e, char* buf, int count) else #endif { + size_t resbufsize; char resbuf[RESBUFSIZE+1]; EFILE_DIR_HANDLE dir_handle; /* Handle to open directory. */ errInfo.posix_errno = 0; dir_handle = NULL; - resbuf[0] = FILE_RESP_OK; + resbuf[0] = FILE_RESP_FNAME; + resbufsize = RESBUFSIZE; while (efile_readdir(&errInfo, name, &dir_handle, - resbuf+1, RESBUFSIZE)) { - int length = 1 + strlen(resbuf+1); - driver_output2(desc->port, resbuf, length, NULL, 0); + resbuf+1, &resbufsize)) { + driver_output2(desc->port, resbuf, 1, resbuf+1, resbufsize); + resbufsize = RESBUFSIZE; } if (errInfo.posix_errno != 0) { reply_error(desc, &errInfo); @@ -2227,11 +2269,12 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_OPEN: { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(buf+4) + 1); + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(buf+4) + + FILENAME_CHARSIZE); d->flags = get_int32((uchar*)buf); name = buf+4; - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_open; d->free = free_data; @@ -2240,44 +2283,45 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_FDATASYNC: - { + { d = EF_SAFE_ALLOC(sizeof(struct t_data)); - + d->fd = fd; d->command = command; d->invoke = invoke_fdatasync; d->free = free_data; d->level = 2; goto done; - } + } case FILE_FSYNC: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data)); - - d->fd = fd; - d->command = command; - d->invoke = invoke_fsync; - d->free = free_data; - d->level = 2; - goto done; - } + { + d = EF_SAFE_ALLOC(sizeof(struct t_data)); + + d->fd = fd; + d->command = command; + d->invoke = invoke_fsync; + d->free = free_data; + d->level = 2; + goto done; + } case FILE_FSTAT: case FILE_LSTAT: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + strlen(name) + 1); + { + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + FILENAME_BYTELEN(name) + + FILENAME_CHARSIZE); + + FILENAME_COPY(d->b, name); + d->fd = fd; + d->command = command; + d->invoke = invoke_flstat; + d->free = free_data; + d->level = 2; + goto done; + } - strcpy(d->b, name); - d->fd = fd; - d->command = command; - d->invoke = invoke_flstat; - d->free = free_data; - d->level = 2; - goto done; - } - case FILE_TRUNCATE: { d = EF_SAFE_ALLOC(sizeof(struct t_data)); @@ -2294,7 +2338,7 @@ file_output(ErlDrvData e, char* buf, int count) case FILE_WRITE_INFO: { d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + strlen(buf+21*4) + 1); + + FILENAME_BYTELEN(buf+21*4) + FILENAME_CHARSIZE); d->info.mode = get_int32(buf + 0 * 4); d->info.uid = get_int32(buf + 1 * 4); @@ -2302,7 +2346,7 @@ file_output(ErlDrvData e, char* buf, int count) GET_TIME(d->info.accessTime, buf + 3 * 4); GET_TIME(d->info.modifyTime, buf + 9 * 4); GET_TIME(d->info.cTime, buf + 15 * 4); - strcpy(d->b, buf+21*4); + FILENAME_COPY(d->b, buf+21*4); d->command = command; d->invoke = invoke_write_info; d->free = free_data; @@ -2314,7 +2358,7 @@ file_output(ErlDrvData e, char* buf, int count) { d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); - strcpy(d->b, name); + FILENAME_COPY(d->b, name); d->command = command; d->invoke = invoke_readlink; d->free = free_data; @@ -2323,28 +2367,29 @@ file_output(ErlDrvData e, char* buf, int count) } case FILE_ALTNAME: - { - d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); - strcpy(d->b, name); - d->command = command; - d->invoke = invoke_altname; - d->free = free_data; - d->level = 2; - goto done; - } + { + d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 + RESBUFSIZE + 1); + FILENAME_COPY(d->b, name); + d->command = command; + d->invoke = invoke_altname; + d->free = free_data; + d->level = 2; + goto done; + } case FILE_LINK: { char* new_name; + int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; - new_name = name+strlen(name)+1; + new_name = name+namelen; d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + strlen(name) + 1 - + strlen(new_name) + 1); + + namelen + + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE); - strcpy(d->b, name); - strcpy(d->b + strlen(name) + 1, new_name); + FILENAME_COPY(d->b, name); + FILENAME_COPY(d->b + namelen, new_name); d->flags = desc->flags; d->fd = fd; d->command = command; @@ -2357,14 +2402,15 @@ file_output(ErlDrvData e, char* buf, int count) case FILE_SYMLINK: { char* new_name; + int namelen = FILENAME_BYTELEN(name) + FILENAME_CHARSIZE; - new_name = name+strlen(name)+1; + new_name = name+namelen; d = EF_SAFE_ALLOC(sizeof(struct t_data) - 1 - + strlen(name) + 1 - + strlen(new_name) + 1); + + namelen + + FILENAME_BYTELEN(new_name) + FILENAME_CHARSIZE); - strcpy(d->b, name); - strcpy(d->b + strlen(name) + 1, new_name); + FILENAME_COPY(d->b, name); + FILENAME_COPY(d->b + namelen, new_name); d->flags = desc->flags; d->fd = fd; d->command = command; @@ -3004,6 +3050,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { case FILE_READ_FILE: { struct t_data *d; + char *filename; if (ev->size < 1+1) { /* Buffer contains empty name */ reply_posix_error(desc, ENOENT); @@ -3014,7 +3061,8 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { reply_posix_error(desc, EINVAL); goto done; } - d = EF_ALLOC(sizeof(struct t_data) + ev->size); + filename = EV_CHAR_P(ev, p, q); + d = EF_ALLOC(sizeof(struct t_data) -1 + FILENAME_BYTELEN(filename) + FILENAME_CHARSIZE); if (! d) { reply_posix_error(desc, ENOMEM); goto done; @@ -3022,8 +3070,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { d->command = command; d->reply = !0; /* Copy name */ - memcpy(d->c.read_file.name, EV_CHAR_P(ev, p, q), ev->size-1); - d->c.read_file.name[ev->size-1] = '\0'; + FILENAME_COPY(d->b, filename); d->c.read_file.binp = NULL; d->invoke = invoke_read_file; d->free = free_read_file; diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index ac95c1f949..3097ded3f1 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -59,6 +59,14 @@ #define FA_WRITE 1 #define FA_READ 2 +/* Some OS'es (i.e. Windows) has filenames in wide charaqcters. That requires special handling */ +/* Note that we do *not* honor alignment in the communication to the OS specific driver, */ +/* which is not a problem on x86, but might be on other platforms. The OS specific efile */ +/* implementation is expected to align if needed */ +#ifdef __WIN32__ +#define FILENAMES_16BIT 1 +#endif + /* * An handle to an open directory. To be cast to the correct type * in the system-dependent directory functions. @@ -123,7 +131,7 @@ int efile_getdcwd(Efile_error* errInfo, int drive, char* buffer, size_t size); int efile_readdir(Efile_error* errInfo, char* name, EFILE_DIR_HANDLE* dir_handle, - char* buffer, size_t size); + char* buffer, size_t *size); int efile_openfile(Efile_error* errInfo, char* name, int flags, int* pfd, Sint64* pSize); void efile_closefile(int fd); diff --git a/erts/emulator/drivers/common/gzio.c b/erts/emulator/drivers/common/gzio.c index 801bc61d4d..5531a275ea 100644 --- a/erts/emulator/drivers/common/gzio.c +++ b/erts/emulator/drivers/common/gzio.c @@ -28,6 +28,7 @@ #ifdef __WIN32__ #define HAVE_CONFLICTING_FREAD_DECLARATION +#define FILENAMES_16BIT 1 #endif #ifdef STDC @@ -102,6 +103,40 @@ local uLong getLong OF((gz_stream *s)); # define ERTS_GZREAD(File, Buf, Count) fread((Buf), 1, (Count), (File)) #endif +/* + * Ripped from efile_drv.c + */ + +#ifdef FILENAMES_16BIT +# define FILENAME_BYTELEN(Str) filename_len_16bit(Str) +# define FILENAME_COPY(To,From) filename_cpy_16bit((To),(From)) +# define FILENAME_CHARSIZE 2 + + static int filename_len_16bit(const char *str) + { + const char *p = str; + while(*p != '\0' || p[1] != '\0') { + p += 2; + } + return (p - str); + } + + static void filename_cpy_16bit(char *to, const char *from) + { + while(*from != '\0' || from[1] != '\0') { + *to++ = *from++; + *to++ = *from++; + } + *to++ = *from++; + *to++ = *from++; + } + +#else +# define FILENAME_BYTELEN(Str) strlen(Str) +# define FILENAME_COPY(To,From) strcpy(To,From) +# define FILENAME_CHARSIZE 1 +#endif + /* =========================================================================== Opens a gzip (.gz) file for reading or writing. The mode parameter is as in fopen ("rb" or "wb"). The file is given either by file descriptor @@ -144,11 +179,11 @@ local gzFile gz_open (path, mode) s->position = 0; s->destroy = destroy; - s->path = (char*)ALLOC(strlen(path)+1); + s->path = (char*)ALLOC(FILENAME_BYTELEN(path)+FILENAME_CHARSIZE); if (s->path == NULL) { return s->destroy(s), (gzFile)Z_NULL; } - strcpy(s->path, path); /* do this early for debugging */ + FILENAME_COPY(s->path, path); /* do this early for debugging */ s->mode = '\0'; do { @@ -194,7 +229,22 @@ local gzFile gz_open (path, mode) s->stream.avail_out = Z_BUFSIZE; errno = 0; -#ifdef UNIX +#if defined(FILENAMES_16BIT) + { + char wfmode[160]; + int i=0,j; + for(j=0;fmode[j] != '\0';++j) { + wfmode[i++]=fmode[j]; + wfmode[i++]='\0'; + } + wfmode[i++] = '\0'; + wfmode[i++] = '\0'; + s->file = F_OPEN(path, wfmode); + if (s->file == NULL) { + return s->destroy(s), (gzFile)Z_NULL; + } + } +#elif defined(UNIX) if (s->mode == 'r') { s->file = open(path, O_RDONLY); } else { diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 3de48194fb..59f4cfb9b4 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * Copyright Ericsson AB 1997-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -54,6 +54,9 @@ #ifdef HAVE_IFADDRS_H #include <ifaddrs.h> #endif +#ifdef HAVE_NETPACKET_PACKET_H +#include <netpacket/packet.h> +#endif /* All platforms fail on malloc errors. */ #define FATAL_MALLOC @@ -85,8 +88,21 @@ #include <winsock2.h> #endif #include <windows.h> +#include <Ws2tcpip.h> /* NEED VC 6.0 or higher */ + +/* Visual studio 2008+: NTDDI_VERSION needs to be set for iphlpapi.h + to define the right structures. It needs to be set to WINXP (or LONGHORN) + for IPV6 to work and it's set lower by default, so we need to change it. */ +#ifdef HAVE_SDKDDKVER_H +# include <sdkddkver.h> +# ifdef NTDDI_VERSION +# undef NTDDI_VERSION +# endif +# define NTDDI_VERSION NTDDI_WINXP +#endif + +#include <iphlpapi.h> -#include <Ws2tcpip.h> /* NEED VC 6.0 !!! */ #undef WANT_NONBLOCKING #include "sys.h" @@ -467,6 +483,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define INET_REQ_IFGET 22 #define INET_REQ_IFSET 23 #define INET_REQ_SUBSCRIBE 24 +#define INET_REQ_GETIFADDRS 25 /* TCP requests */ #define TCP_REQ_ACCEPT 40 #define TCP_REQ_LISTEN 41 @@ -632,15 +649,12 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define IS_BUSY(d) \ (((d)->state & INET_F_BUSY) == INET_F_BUSY) +#define INET_MAX_OPT_BUFFER (64*1024) + #define INET_DEF_BUFFER 1460 /* default buffer size */ #define INET_MIN_BUFFER 1 /* internal min buffer */ -#define INET_MAX_BUFFER (1024*64) /* internal max buffer */ -/* Note: INET_HIGH_WATERMARK MUST be less than 2*INET_MAX_BUFFER */ #define INET_HIGH_WATERMARK (1024*8) /* 8k pending high => busy */ -/* Note: INET_LOW_WATERMARK MUST be less than INET_MAX_BUFFER and -** less than INET_HIGH_WATERMARK -*/ #define INET_LOW_WATERMARK (1024*4) /* 4k pending => allow more */ #define INET_INFINITY 0xffffffff /* infinity value */ @@ -1251,139 +1265,136 @@ static int load_ip_and_port LOAD_ATOM((spec), (i), (flag) ? am_true : am_false); #endif /* HAVE_SCTP */ +/* Assume a cache line size of 64 bytes */ +#define INET_DRV_CACHE_LINE_SIZE ((ErlDrvUInt) 64) +#define INET_DRV_CACHE_LINE_MASK (INET_DRV_CACHE_LINE_SIZE - 1) + /* ** Binary Buffer Managment ** We keep a stack of usable buffers */ -#define BUFFER_STACK_SIZE 16 - -static erts_smp_spinlock_t inet_buffer_stack_lock; -static ErlDrvBinary* buffer_stack[BUFFER_STACK_SIZE]; -static int buffer_stack_pos = 0; +#define BUFFER_STACK_SIZE 14 +#define BUFFER_STACK_MAX_MEM_SIZE (1024*1024) +ErlDrvTSDKey buffer_stack_key; -/* - * XXX - * The erts_smp_spin_* functions should not be used by drivers (but this - * driver is special). Replace when driver locking api has been implemented. - * /rickard - */ -#define BUFSTK_LOCK erts_smp_spin_lock(&inet_buffer_stack_lock); -#define BUFSTK_UNLOCK erts_smp_spin_unlock(&inet_buffer_stack_lock); - -#ifdef DEBUG -static int tot_buf_allocated = 0; /* memory in use for i_buf */ -static int tot_buf_stacked = 0; /* memory on stack */ -static int max_buf_allocated = 0; /* max allocated */ - -#define COUNT_BUF_ALLOC(sz) do { \ - BUFSTK_LOCK; \ - tot_buf_allocated += (sz); \ - if (tot_buf_allocated > max_buf_allocated) \ - max_buf_allocated = tot_buf_allocated; \ - BUFSTK_UNLOCK; \ -} while(0) - -#define COUNT_BUF_FREE(sz) do { \ - BUFSTK_LOCK; \ - tot_buf_allocated -= (sz); \ - BUFSTK_UNLOCK; \ - } while(0) - -#define COUNT_BUF_STACK(sz) do { \ - BUFSTK_LOCK; \ - tot_buf_stacked += (sz); \ - BUFSTK_UNLOCK; \ - } while(0) +typedef struct { + int mem_size; + int pos; + ErlDrvBinary* stk[BUFFER_STACK_SIZE]; +} InetDrvBufStkBase; -#else +typedef struct { + InetDrvBufStkBase buf; + char align[(((sizeof(InetDrvBufStkBase) - 1) / INET_DRV_CACHE_LINE_SIZE) + 1) + * INET_DRV_CACHE_LINE_SIZE]; +} InetDrvBufStk; + +static InetDrvBufStk *get_bufstk(void) +{ + InetDrvBufStk *bs = erl_drv_tsd_get(buffer_stack_key); + if (bs) + return bs; + bs = driver_alloc(sizeof(InetDrvBufStk) + + INET_DRV_CACHE_LINE_SIZE - 1); + if (!bs) + return NULL; + if ((((ErlDrvUInt) bs) & INET_DRV_CACHE_LINE_MASK) != 0) + bs = ((InetDrvBufStk *) + ((((ErlDrvUInt) bs) & ~INET_DRV_CACHE_LINE_MASK) + + INET_DRV_CACHE_LINE_SIZE)); + erl_drv_tsd_set(buffer_stack_key, bs); + bs->buf.pos = 0; + bs->buf.mem_size = 0; -#define COUNT_BUF_ALLOC(sz) -#define COUNT_BUF_FREE(sz) -#define COUNT_BUF_STACK(sz) + ASSERT(bs == erl_drv_tsd_get(buffer_stack_key)); -#endif + return bs; +} static ErlDrvBinary* alloc_buffer(long minsz) { - ErlDrvBinary* buf = NULL; + InetDrvBufStk *bs = get_bufstk(); + + DEBUGF(("alloc_buffer: %ld\r\n", minsz)); + + if (bs && bs->buf.pos > 0) { + long size; + ErlDrvBinary* buf = bs->buf.stk[--bs->buf.pos]; + size = buf->orig_size; + bs->buf.mem_size -= size; + ASSERT(0 <= bs->buf.mem_size + && bs->buf.mem_size <= BUFFER_STACK_MAX_MEM_SIZE); + if (size >= minsz) + return buf; - BUFSTK_LOCK; + driver_free_binary(buf); + } - DEBUGF(("alloc_buffer: sz = %ld, tot = %d, max = %d\r\n", - minsz, tot_buf_allocated, max_buf_allocated)); + ASSERT(!bs || bs->buf.pos != 0 || bs->buf.mem_size == 0); - if (buffer_stack_pos > 0) { - int origsz; + return driver_alloc_binary(minsz); +} - buf = buffer_stack[--buffer_stack_pos]; - origsz = buf->orig_size; - BUFSTK_UNLOCK; - COUNT_BUF_STACK(-origsz); - if (origsz < minsz) { - if ((buf = driver_realloc_binary(buf, minsz)) == NULL) - return NULL; - COUNT_BUF_ALLOC(buf->orig_size - origsz); +/*#define CHECK_DOUBLE_RELEASE 1*/ +#ifdef CHECK_DOUBLE_RELEASE +static void +check_double_release(InetDrvBufStk *bs, ErlDrvBinary* buf) +{ +#ifdef __GNUC__ +#warning CHECK_DOUBLE_RELEASE is enabled, this is a custom build emulator +#endif + int i; + for (i = 0; i < bs->buf.pos; ++i) { + if (bs->buf.stk[i] == buf) { + erl_exit(ERTS_ABORT_EXIT, + "Multiple buffer release in inet_drv, this " + "is a bug, save the core and send it to " + "[email protected]!"); } } - else { - BUFSTK_UNLOCK; - if ((buf = driver_alloc_binary(minsz)) == NULL) - return NULL; - COUNT_BUF_ALLOC(buf->orig_size); - } - return buf; } +#endif -/* -** Max buffer memory "cached" BUFFER_STACK_SIZE * INET_MAX_BUFFER -** (16 * 64k ~ 1M) -*/ -/*#define CHECK_DOUBLE_RELEASE 1*/ static void release_buffer(ErlDrvBinary* buf) { + InetDrvBufStk *bs; + long size; + DEBUGF(("release_buffer: %ld\r\n", (buf==NULL) ? 0 : buf->orig_size)); - if (buf == NULL) + + if (!buf) return; - BUFSTK_LOCK; - if ((buf->orig_size > INET_MAX_BUFFER) || - (buffer_stack_pos >= BUFFER_STACK_SIZE)) { - BUFSTK_UNLOCK; - COUNT_BUF_FREE(buf->orig_size); + + size = buf->orig_size; + + if (size > BUFFER_STACK_MAX_MEM_SIZE) + goto free_binary; + + bs = get_bufstk(); + if (!bs + || (bs->buf.mem_size + size > BUFFER_STACK_MAX_MEM_SIZE) + || (bs->buf.pos >= BUFFER_STACK_SIZE)) { + free_binary: driver_free_binary(buf); } else { #ifdef CHECK_DOUBLE_RELEASE -#ifdef __GNUC__ -#warning CHECK_DOUBLE_RELEASE is enabled, this is a custom build emulator -#endif - int i; - for (i = 0; i < buffer_stack_pos; ++i) { - if (buffer_stack[i] == buf) { - erl_exit(1,"Multiple buffer release in inet_drv, this is a " - "bug, save the core and send it to " - "[email protected]!"); - } - } + check_double_release(bs, buf); #endif - buffer_stack[buffer_stack_pos++] = buf; - BUFSTK_UNLOCK; - COUNT_BUF_STACK(buf->orig_size); + ASSERT(bs->buf.pos != 0 || bs->buf.mem_size == 0); + + bs->buf.mem_size += size; + bs->buf.stk[bs->buf.pos++] = buf; + + ASSERT(0 <= bs->buf.mem_size + && bs->buf.mem_size <= BUFFER_STACK_MAX_MEM_SIZE); } } static ErlDrvBinary* realloc_buffer(ErlDrvBinary* buf, long newsz) { - ErlDrvBinary* bin; -#ifdef DEBUG - long orig_size = buf->orig_size; -#endif - - if ((bin = driver_realloc_binary(buf,newsz)) != NULL) { - COUNT_BUF_ALLOC(newsz - orig_size); - ; - } - return bin; + return driver_realloc_binary(buf, newsz); } /* use a TRICK, access the refc field to see if any one else has @@ -1397,10 +1408,8 @@ static void free_buffer(ErlDrvBinary* buf) if (buf != NULL) { if (driver_binary_get_refc(buf) == 1) release_buffer(buf); - else { - COUNT_BUF_FREE(buf->orig_size); + else driver_free_binary(buf); - } } } @@ -3404,20 +3413,14 @@ static int inet_init() if (!sock_init()) goto error; - buffer_stack_pos = 0; - - erts_smp_spinlock_init(&inet_buffer_stack_lock, "inet_buffer_stack_lock"); + if (0 != erl_drv_tsd_key_create("inet_buffer_stack_key", &buffer_stack_key)) + goto error; ASSERT(sizeof(struct in_addr) == 4); # if defined(HAVE_IN6) && defined(AF_INET6) ASSERT(sizeof(struct in6_addr) == 16); # endif -#ifdef DEBUG - tot_buf_allocated = 0; - max_buf_allocated = 0; - tot_buf_stacked = 0; -#endif INIT_ATOM(ok); INIT_ATOM(tcp); INIT_ATOM(udp); @@ -3824,39 +3827,81 @@ do { if ((end)-(ptr) < (n)) goto error; } while(0) static char* sockaddr_to_buf(struct sockaddr* addr, char* ptr, char* end) { if (addr->sa_family == AF_INET || addr->sa_family == 0) { - struct in_addr a; - buf_check(ptr,end,sizeof(struct in_addr)); - a = ((struct sockaddr_in*) addr)->sin_addr; - sys_memcpy(ptr, (char*)&a, sizeof(struct in_addr)); - return ptr + sizeof(struct in_addr); + struct in_addr *p = &(((struct sockaddr_in*) addr)->sin_addr); + buf_check(ptr, end, 1 + sizeof(struct in_addr)); + *ptr = INET_AF_INET; + sys_memcpy(ptr+1, (char*)p, sizeof(struct in_addr)); + return ptr + 1 + sizeof(struct in_addr); } #if defined(HAVE_IN6) && defined(AF_INET6) else if (addr->sa_family == AF_INET6) { - struct in6_addr a; - buf_check(ptr,end,sizeof(struct in6_addr)); - a = ((struct sockaddr_in6*) addr)->sin6_addr; - sys_memcpy(ptr, (char*)&a, sizeof(struct in6_addr)); - return ptr + sizeof(struct in6_addr); + struct in6_addr *p = &(((struct sockaddr_in6*) addr)->sin6_addr); + buf_check(ptr, end, 1 + sizeof(struct in6_addr)); + *ptr = INET_AF_INET6; + sys_memcpy(ptr+1, (char*)p, sizeof(struct in6_addr)); + return ptr + 1 + sizeof(struct in6_addr); + } +#endif +#if defined(AF_LINK) + else if (addr->sa_family == AF_LINK) { + struct sockaddr_dl *sdl_p = (struct sockaddr_dl*) addr; + buf_check(ptr, end, 2 + sdl_p->sdl_alen); + put_int16(sdl_p->sdl_alen, ptr); ptr += 2; + sys_memcpy(ptr, sdl_p->sdl_data + sdl_p->sdl_nlen, sdl_p->sdl_alen); + return ptr + sdl_p->sdl_alen; + } +#endif +#if defined(AF_PACKET) && defined(HAVE_NETPACKET_PACKET_H) + else if(addr->sa_family == AF_PACKET) { + struct sockaddr_ll *sll_p = (struct sockaddr_ll*) addr; + buf_check(ptr, end, 2 + sll_p->sll_halen); + put_int16(sll_p->sll_halen, ptr); ptr += 2; + sys_memcpy(ptr, sll_p->sll_addr, sll_p->sll_halen); + return ptr + sll_p->sll_halen; } #endif + return ptr; error: return NULL; - } static char* buf_to_sockaddr(char* ptr, char* end, struct sockaddr* addr) { - buf_check(ptr,end,sizeof(struct in_addr)); - sys_memcpy((char*) &((struct sockaddr_in*)addr)->sin_addr, ptr, - sizeof(struct in_addr)); - addr->sa_family = AF_INET; - return ptr + sizeof(struct in_addr); - + buf_check(ptr,end,1); + switch (*ptr++) { + case INET_AF_INET: { + struct in_addr *p = &((struct sockaddr_in*)addr)->sin_addr; + buf_check(ptr,end,sizeof(struct in_addr)); + sys_memcpy((char*) p, ptr, sizeof(struct in_addr)); + addr->sa_family = AF_INET; + return ptr + sizeof(struct in_addr); + } + case INET_AF_INET6: { + struct in6_addr *p = &((struct sockaddr_in6*)addr)->sin6_addr; + buf_check(ptr,end,sizeof(struct in6_addr)); + sys_memcpy((char*) p, ptr, sizeof(struct in6_addr)); + addr->sa_family = AF_INET6; + return ptr + sizeof(struct in6_addr); + } + } error: return NULL; } +#if defined (IFF_POINTOPOINT) +#define IFGET_FLAGS(cflags) IFGET_FLAGS_P2P(cflags, IFF_POINTOPOINT) +#elif defined IFF_POINTTOPOINT +#define IFGET_FLAGS(cflags) IFGET_FLAGS_P2P(cflags, IFF_POINTTOPOINT) +#endif + +#define IFGET_FLAGS_P2P(cflags, iff_ptp) \ + ((((cflags) & IFF_UP) ? INET_IFF_UP : 0) | \ + (((cflags) & IFF_BROADCAST) ? INET_IFF_BROADCAST : 0) | \ + (((cflags) & IFF_LOOPBACK) ? INET_IFF_LOOPBACK : 0) | \ + (((cflags) & iff_ptp) ? INET_IFF_POINTTOPOINT : 0) | \ + (((cflags) & IFF_UP) ? INET_IFF_RUNNING : 0) | /* emulate running ? */ \ + (((cflags) & IFF_MULTICAST) ? INET_IFF_MULTICAST : 0)) #if defined(__WIN32__) && defined(SIO_GET_INTERFACE_LIST) @@ -3894,7 +3939,6 @@ static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize) return ctl_reply(INET_REP_OK, sbuf, sptr - sbuf, rbuf, rsize); } - /* input is an ip-address in string format i.e A.B.C.D ** scan the INTERFACE_LIST to get the options */ @@ -3980,27 +4024,12 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, break; case INET_IFOPT_FLAGS: { - long eflags = 0; int flags = ifp->iiFlags; /* just enumerate the interfaces (no names) */ - /* translate flags */ - if (flags & IFF_UP) - eflags |= INET_IFF_UP; - if (flags & IFF_BROADCAST) - eflags |= INET_IFF_BROADCAST; - if (flags & IFF_LOOPBACK) - eflags |= INET_IFF_LOOPBACK; - if (flags & IFF_POINTTOPOINT) - eflags |= INET_IFF_POINTTOPOINT; - if (flags & IFF_UP) /* emulate runnign ? */ - eflags |= INET_IFF_RUNNING; - if (flags & IFF_MULTICAST) - eflags |= INET_IFF_MULTICAST; - buf_check(sptr, s_end, 5); *sptr++ = INET_IFOPT_FLAGS; - put_int32(eflags, sptr); + put_int32(IFGET_FLAGS(flags), sptr); sptr += 4; break; } @@ -4021,7 +4050,6 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, return ctl_reply(INET_REP_OK, NULL, 0, rbuf, rsize); } - #elif defined(SIOCGIFCONF) && defined(SIOCSIFFLAGS) /* cygwin has SIOCGIFCONF but not SIOCSIFFLAGS (Nov 2002) */ @@ -4032,69 +4060,77 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, #define SIZEA(p) (sizeof (p)) #endif - -static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize) -{ - struct ifconf ifc; - struct ifreq *ifr; - char *buf; - int buflen, ifc_len, i; - char *sbuf, *sp; - - /* Courtesy of Per Bergqvist and W. Richard Stevens */ - - ifc_len = 0; - buflen = 100 * sizeof(struct ifreq); - buf = ALLOC(buflen); +static int get_ifconf(SOCKET s, struct ifconf *ifcp) { + int ifc_len = 0; + int buflen = 100 * sizeof(struct ifreq); + char *buf = ALLOC(buflen); for (;;) { - ifc.ifc_len = buflen; - ifc.ifc_buf = buf; - if (ioctl(desc->s, SIOCGIFCONF, (char *)&ifc) < 0) { + ifcp->ifc_len = buflen; + ifcp->ifc_buf = buf; + if (ioctl(s, SIOCGIFCONF, (char *)ifcp) < 0) { int res = sock_errno(); if (res != EINVAL || ifc_len) { FREE(buf); - return ctl_error(res, rbuf, rsize); + return -1; } } else { - if (ifc.ifc_len == ifc_len) break; /* buf large enough */ - ifc_len = ifc.ifc_len; + if (ifcp->ifc_len == ifc_len) break; /* buf large enough */ + ifc_len = ifcp->ifc_len; } buflen += 10 * sizeof(struct ifreq); buf = (char *)REALLOC(buf, buflen); } - - sp = sbuf = ALLOC(ifc_len+1); + return 0; +} + +static void free_ifconf(struct ifconf *ifcp) { + FREE(ifcp->ifc_buf); +} + +static int inet_ctl_getiflist(inet_descriptor* desc, char** rbuf, int rsize) +{ + struct ifconf ifc; + struct ifreq *ifrp; + char *sbuf, *sp; + int i; + + /* Courtesy of Per Bergqvist and W. Richard Stevens */ + + if (get_ifconf(desc->s, &ifc) < 0) { + return ctl_error(sock_errno(), rbuf, rsize); + } + + sp = sbuf = ALLOC(ifc.ifc_len+1); *sp++ = INET_REP_OK; i = 0; for (;;) { int n; - - ifr = (struct ifreq *) VOIDP(buf + i); - n = sizeof(ifr->ifr_name) + SIZEA(ifr->ifr_addr); - if (n < sizeof(*ifr)) n = sizeof(*ifr); - if (i+n > ifc_len) break; + + ifrp = (struct ifreq *) VOIDP(ifc.ifc_buf + i); + n = sizeof(ifrp->ifr_name) + SIZEA(ifrp->ifr_addr); + if (n < sizeof(*ifrp)) n = sizeof(*ifrp); + if (i+n > ifc.ifc_len) break; i += n; - - switch (ifr->ifr_addr.sa_family) { + + switch (ifrp->ifr_addr.sa_family) { #if defined(HAVE_IN6) && defined(AF_INET6) case AF_INET6: #endif case AF_INET: - ASSERT(sp+IFNAMSIZ+1 < sbuf+buflen+1) - strncpy(sp, ifr->ifr_name, IFNAMSIZ); + ASSERT(sp+IFNAMSIZ+1 < sbuf+ifc.ifc_len+1) + strncpy(sp, ifrp->ifr_name, IFNAMSIZ); sp[IFNAMSIZ] = '\0'; sp += strlen(sp), ++sp; } - - if (i >= ifc_len) break; + + if (i >= ifc.ifc_len) break; } - FREE(buf); + free_ifconf(&ifc); *rbuf = sbuf; return sp - sbuf; } - /* FIXME: temporary hack */ #ifndef IFHWADDRLEN #define IFHWADDRLEN 6 @@ -4133,37 +4169,52 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, #ifdef SIOCGIFHWADDR if (ioctl(desc->s, SIOCGIFHWADDR, (char *)&ifreq) < 0) break; - buf_check(sptr, s_end, 1+IFHWADDRLEN); + buf_check(sptr, s_end, 1+2+IFHWADDRLEN); *sptr++ = INET_IFOPT_HWADDR; + put_int16(IFHWADDRLEN, sptr); sptr += 2; /* raw memcpy (fix include autoconf later) */ sys_memcpy(sptr, (char*)(&ifreq.ifr_hwaddr.sa_data), IFHWADDRLEN); sptr += IFHWADDRLEN; -#elif defined(HAVE_GETIFADDRS) - struct ifaddrs *ifa, *ifp; - int found = 0; - - if (getifaddrs(&ifa) == -1) - goto error; +#elif defined(SIOCGENADDR) + if (ioctl(desc->s, SIOCGENADDR, (char *)&ifreq) < 0) + break; + buf_check(sptr, s_end, 1+2+sizeof(ifreq.ifr_enaddr)); + *sptr++ = INET_IFOPT_HWADDR; + put_int16(sizeof(ifreq.ifr_enaddr), sptr); sptr += 2; + /* raw memcpy (fix include autoconf later) */ + sys_memcpy(sptr, (char*)(&ifreq.ifr_enaddr), + sizeof(ifreq.ifr_enaddr)); + sptr += sizeof(ifreq.ifr_enaddr); +#elif defined(HAVE_GETIFADDRS) && defined(AF_LINK) + struct ifaddrs *ifa, *ifp; + struct sockaddr_dl *sdlp; + int found = 0; + + if (getifaddrs(&ifa) == -1) + goto error; - for (ifp = ifa; ifp; ifp = ifp->ifa_next) { - if ((ifp->ifa_addr->sa_family == AF_LINK) && - (sys_strcmp(ifp->ifa_name, ifreq.ifr_name) == 0)) { - found = 1; - break; - } - } + for (ifp = ifa; ifp; ifp = ifp->ifa_next) { + if ((ifp->ifa_addr->sa_family == AF_LINK) && + (sys_strcmp(ifp->ifa_name, ifreq.ifr_name) == 0)) { + found = 1; + break; + } + } - if (found == 0) { - freeifaddrs(ifa); - break; - } + if (found == 0) { + freeifaddrs(ifa); + break; + } + sdlp = (struct sockaddr_dl *)ifp->ifa_addr; - buf_check(sptr, s_end, 1+IFHWADDRLEN); - *sptr++ = INET_IFOPT_HWADDR; - sys_memcpy(sptr, ((struct sockaddr_dl *)ifp->ifa_addr)->sdl_data + - ((struct sockaddr_dl *)ifp->ifa_addr)->sdl_nlen, IFHWADDRLEN); - freeifaddrs(ifa); - sptr += IFHWADDRLEN; + buf_check(sptr, s_end, 1+2+sdlp->sdl_alen); + *sptr++ = INET_IFOPT_HWADDR; + put_int16(sdlp->sdl_alen, sptr); sptr += 2; + sys_memcpy(sptr, + sdlp->sdl_data + sdlp->sdl_nlen, + sdlp->sdl_alen); + freeifaddrs(ifa); + sptr += sdlp->sdl_alen; #endif break; } @@ -4240,29 +4291,15 @@ static int inet_ctl_ifget(inet_descriptor* desc, char* buf, int len, case INET_IFOPT_FLAGS: { int flags; - int eflags = 0; if (ioctl(desc->s, SIOCGIFFLAGS, (char*)&ifreq) < 0) flags = 0; else flags = ifreq.ifr_flags; - /* translate flags */ - if (flags & IFF_UP) - eflags |= INET_IFF_UP; - if (flags & IFF_BROADCAST) - eflags |= INET_IFF_BROADCAST; - if (flags & IFF_LOOPBACK) - eflags |= INET_IFF_LOOPBACK; - if (flags & IFF_POINTOPOINT) - eflags |= INET_IFF_POINTTOPOINT; - if (flags & IFF_RUNNING) - eflags |= INET_IFF_RUNNING; - if (flags & IFF_MULTICAST) - eflags |= INET_IFF_MULTICAST; buf_check(sptr, s_end, 5); *sptr++ = INET_IFOPT_FLAGS; - put_int32(eflags, sptr); + put_int32(IFGET_FLAGS(flags), sptr); sptr += 4; break; } @@ -4300,17 +4337,22 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, (void) ioctl(desc->s, SIOCSIFADDR, (char*)&ifreq); break; - case INET_IFOPT_HWADDR: - buf_check(buf, b_end, IFHWADDRLEN); + case INET_IFOPT_HWADDR: { + unsigned int len; + buf_check(buf, b_end, 2); + len = get_int16(buf); buf += 2; + buf_check(buf, b_end, len); #ifdef SIOCSIFHWADDR /* raw memcpy (fix include autoconf later) */ - sys_memcpy((char*)(&ifreq.ifr_hwaddr.sa_data), buf, IFHWADDRLEN); + sys_memset((char*)(&ifreq.ifr_hwaddr.sa_data), + '\0', sizeof(ifreq.ifr_hwaddr.sa_data)); + sys_memcpy((char*)(&ifreq.ifr_hwaddr.sa_data), buf, len); (void) ioctl(desc->s, SIOCSIFHWADDR, (char *)&ifreq); #endif - buf += IFHWADDRLEN; + buf += len; break; - + } case INET_IFOPT_BROADADDR: #ifdef SIOCSIFBRDADDR @@ -4415,6 +4457,557 @@ static int inet_ctl_ifset(inet_descriptor* desc, char* buf, int len, #endif + + +/* Latin-1 to utf8 */ + +static int utf8_len(const char *c, int m) { + int l; + for (l = 0; m; c++, l++, m--) { + if (*c == '\0') break; + if ((*c & 0x7f) != *c) l++; + } + return l; +} + +static void utf8_encode(const char *c, int m, char *p) { + for (; m; c++, m--) { + if (*c == '\0') break; + if ((*c & 0x7f) != *c) { + *p++ = (char) (0xC0 | (0x03 & (*c >> 6))); + *p++ = (char) (0x80 | (0x3F & *c)); + } else { + *p++ = (char) *c; + } + } +} + +#if defined(__WIN32__) + +static void set_netmask_bytes(char *c, int len, int pref_len) { + int i, m; + for (i = 0, m = pref_len >> 3; i < m && i < len; i++) c[i] = '\xFF'; + if (i < len) c[i++] = 0xFF << (8 - (pref_len & 7)); + for (; i < len; i++) c[i] = '\0'; +} + + +int eq_masked_bytes(char *a, char *b, int pref_len) { + int i, m; + for (i = 0, m = pref_len >> 3; i < m; i++) { + if (a[i] != b[i]) return 0; + } + m = pref_len & 7; + if (m) { + m = 0xFF & (0xFF << (8 - m)); + if ((a[i] & m) != (b[i] & m)) return 0; + } + return !0; +} + +static int inet_ctl_getifaddrs(inet_descriptor* desc_p, + char **rbuf_pp, int rsize) +{ + int i; + DWORD ret, n; + IP_INTERFACE_INFO *info_p; + MIB_IPADDRTABLE *ip_addrs_p; + IP_ADAPTER_ADDRESSES *ip_adaddrs_p, *ia_p; + + char *buf_p; + char *buf_alloc_p; + int buf_size =512; +# define BUF_ENSURE(Size) \ + do { \ + int NEED_, GOT_ = buf_p - buf_alloc_p; \ + NEED_ = GOT_ + (Size); \ + if (NEED_ > buf_size) { \ + buf_size = NEED_ + 512; \ + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); \ + buf_p = buf_alloc_p + GOT_; \ + } \ + } while(0) +# define SOCKADDR_TO_BUF(opt, sa) \ + do { \ + if (sa) { \ + char *P_; \ + *buf_p++ = (opt); \ + while (! (P_ = sockaddr_to_buf((sa), buf_p, \ + buf_alloc_p+buf_size))) { \ + int GOT_ = buf_p - buf_alloc_p; \ + buf_size += 512; \ + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); \ + buf_p = buf_alloc_p + GOT_; \ + } \ + if (P_ == buf_p) { \ + buf_p--; \ + } else { \ + buf_p = P_; \ + } \ + } \ + } while (0) + + { + /* Try GetAdaptersAddresses, if it is available */ + unsigned long ip_adaddrs_size = 16 * 1024; + ULONG family = AF_UNSPEC; + ULONG flags = + GAA_FLAG_INCLUDE_PREFIX | GAA_FLAG_SKIP_ANYCAST | + GAA_FLAG_SKIP_DNS_SERVER | GAA_FLAG_SKIP_FRIENDLY_NAME | + GAA_FLAG_SKIP_MULTICAST; + ULONG (WINAPI *fpGetAdaptersAddresses) + (ULONG, ULONG, PVOID, PIP_ADAPTER_ADDRESSES, PULONG); + HMODULE iphlpapi = GetModuleHandle("iphlpapi"); + fpGetAdaptersAddresses = (void *) + (iphlpapi ? + GetProcAddress(iphlpapi, "GetAdaptersAddresses") : + NULL); + if (fpGetAdaptersAddresses) { + ip_adaddrs_p = ALLOC(ip_adaddrs_size); + for (i = 17; i; i--) { + ret = fpGetAdaptersAddresses( + family, flags, NULL, ip_adaddrs_p, &ip_adaddrs_size); + ip_adaddrs_p = REALLOC(ip_adaddrs_p, ip_adaddrs_size); + if (ret == NO_ERROR) break; + if (ret == ERROR_BUFFER_OVERFLOW) continue; + i = 0; + } + if (! i) { + FREE(ip_adaddrs_p); + ip_adaddrs_p = NULL; + } + } else ip_adaddrs_p = NULL; + } + + { + /* Load the IP_INTERFACE_INFO table (only IPv4 interfaces), + * reliable source of interface names on XP + */ + unsigned long info_size = 4 * 1024; + info_p = ALLOC(info_size); + for (i = 17; i; i--) { + ret = GetInterfaceInfo(info_p, &info_size); + info_p = REALLOC(info_p, info_size); + if (ret == NO_ERROR) break; + if (ret == ERROR_INSUFFICIENT_BUFFER) continue; + i = 0; + } + if (! i) { + FREE(info_p); + info_p = NULL; + } + } + + if (! ip_adaddrs_p) { + /* If GetAdaptersAddresses gave nothing we fall back to + * MIB_IPADDRTABLE (only IPv4 interfaces) + */ + unsigned long ip_addrs_size = 16 * sizeof(*ip_addrs_p); + ip_addrs_p = ALLOC(ip_addrs_size); + for (i = 17; i; i--) { + ret = GetIpAddrTable(ip_addrs_p, &ip_addrs_size, FALSE); + ip_addrs_p = REALLOC(ip_addrs_p, ip_addrs_size); + if (ret == NO_ERROR) break; + if (ret == ERROR_INSUFFICIENT_BUFFER) continue; + i = 0; + } + if (! i) { + if (info_p) FREE(info_p); + FREE(ip_addrs_p); + return ctl_reply(INET_REP_OK, NULL, 0, rbuf_pp, rsize); + } + } else ip_addrs_p = NULL; + + buf_p = buf_alloc_p = ALLOC(buf_size); + *buf_p++ = INET_REP_OK; + + /* Iterate over MIB_IPADDRTABLE or IP_ADAPTER_ADDRESSES */ + for (ia_p = NULL, ip_addrs_p ? ((void *)(i = 0)) : (ia_p = ip_adaddrs_p); + ip_addrs_p ? (i < ip_addrs_p->dwNumEntries) : (ia_p != NULL); + ip_addrs_p ? ((void *)(i++)) : (ia_p = ia_p->Next)) { + MIB_IPADDRROW *ipaddrrow_p = NULL; + DWORD flags = INET_IFF_MULTICAST; + DWORD index = 0; + WCHAR *wname_p = NULL; + MIB_IFROW ifrow; + + if (ip_addrs_p) { + ipaddrrow_p = ip_addrs_p->table + i; + index = ipaddrrow_p->dwIndex; + } else { + index = ia_p->IfIndex; + if (ia_p->Flags & IP_ADAPTER_NO_MULTICAST) { + flags &= ~INET_IFF_MULTICAST; + } + } +index: + if (! index) goto done; + sys_memzero(&ifrow, sizeof(ifrow)); + ifrow.dwIndex = index; + if (GetIfEntry(&ifrow) != NO_ERROR) break; + /* Find the interface name - first try MIB_IFROW.wzname */ + if (ifrow.wszName[0] != 0) { + wname_p = ifrow.wszName; + } else { + /* Then try IP_ADAPTER_INDEX_MAP.Name (only IPv4 adapters) */ + int j; + for (j = 0; j < info_p->NumAdapters; j++) { + if (info_p->Adapter[j].Index == (ULONG) ifrow.dwIndex) { + if (info_p->Adapter[j].Name[0] != 0) { + wname_p = info_p->Adapter[j].Name; + } + break; + } + } + } + if (wname_p) { + int len; + /* Convert interface name to UTF-8 */ + len = + WideCharToMultiByte( + CP_UTF8, 0, wname_p, -1, NULL, 0, NULL, NULL); + if (! len) break; + BUF_ENSURE(len); + WideCharToMultiByte( + CP_UTF8, 0, wname_p, -1, buf_p, len, NULL, NULL); + buf_p += len; + } else { + /* Found no name - + * use "MIB_IFROW.dwIndex: MIB_IFROW.bDescr" as name instead */ + int l; + l = utf8_len(ifrow.bDescr, ifrow.dwDescrLen); + BUF_ENSURE(9 + l+1); + buf_p += + erts_sprintf( + buf_p, "%lu: ", (unsigned long) ifrow.dwIndex); + utf8_encode(ifrow.bDescr, ifrow.dwDescrLen, buf_p); + buf_p += l; + *buf_p++ = '\0'; + } + /* Interface flags, often make up broadcast and multicast flags */ + switch (ifrow.dwType) { + case IF_TYPE_ETHERNET_CSMACD: + flags |= INET_IFF_BROADCAST; + break; + case IF_TYPE_SOFTWARE_LOOPBACK: + flags |= INET_IFF_LOOPBACK; + flags &= ~INET_IFF_MULTICAST; + break; + default: + flags &= ~INET_IFF_MULTICAST; + break; + } + if (ifrow.dwAdminStatus) { + flags |= INET_IFF_UP; + switch (ifrow.dwOperStatus) { + case IF_OPER_STATUS_CONNECTING: + flags |= INET_IFF_POINTTOPOINT; + break; + case IF_OPER_STATUS_CONNECTED: + flags |= INET_IFF_RUNNING | INET_IFF_POINTTOPOINT; + break; + case IF_OPER_STATUS_OPERATIONAL: + flags |= INET_IFF_RUNNING; + break; + } + } + BUF_ENSURE(1 + 4); + *buf_p++ = INET_IFOPT_FLAGS; + put_int32(flags, buf_p); buf_p += 4; + if (ipaddrrow_p) { + /* Legacy implementation through GetIpAddrTable */ + struct sockaddr_in sin; + /* IP Address */ + sys_memzero(&sin, sizeof(sin)); + sin.sin_family = AF_INET; + sin.sin_addr.s_addr = ipaddrrow_p->dwAddr; + BUF_ENSURE(1); + /* Netmask */ + SOCKADDR_TO_BUF(INET_IFOPT_ADDR, (struct sockaddr *) &sin); + sin.sin_addr.s_addr = ipaddrrow_p->dwMask; + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, (struct sockaddr *) &sin); + if (flags & INET_IFF_BROADCAST) { + /* Broadcast address - fake it*/ + sin.sin_addr.s_addr = ipaddrrow_p->dwAddr; + sin.sin_addr.s_addr |= ~ipaddrrow_p->dwMask; + BUF_ENSURE(1); + SOCKADDR_TO_BUF( + INET_IFOPT_BROADADDR, (struct sockaddr *) &sin); + } + } else { + IP_ADAPTER_UNICAST_ADDRESS *p; + /* IP Address(es) */ + for (p = ia_p->FirstUnicastAddress; + p; + p = p->Next) + { + IP_ADAPTER_PREFIX *q; + ULONG shortest_length; + struct sockaddr *shortest_p, *sa_p = p->Address.lpSockaddr; + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_ADDR, sa_p); + shortest_p = NULL; + shortest_length = 0; + for (q = ia_p->FirstPrefix; + q; + q = q->Next) { + struct sockaddr *sp_p = q->Address.lpSockaddr; + if (sa_p->sa_family != sp_p->sa_family) continue; + switch (sa_p->sa_family) { + case AF_INET: { + struct sockaddr_in sin; + DWORD sa, sp, mask; + sa = ntohl((DWORD) + ((struct sockaddr_in *) + sa_p)->sin_addr.s_addr); + sp = ntohl((DWORD) + ((struct sockaddr_in *) + sp_p)->sin_addr.s_addr); + mask = 0xFFFFFFFF << (32 - q->PrefixLength); + if ((sa & mask) != (sp & mask)) continue; + if ((! shortest_p) + || q->PrefixLength < shortest_length) { + shortest_p = sp_p; + shortest_length = q->PrefixLength; + } + } break; + case AF_INET6: { + struct sockaddr_in6 sin6; + if (!eq_masked_bytes((char *) + &((struct sockaddr_in6 *) + sa_p)->sin6_addr, + (char *) + &((struct sockaddr_in6 *) + sp_p)->sin6_addr, + q->PrefixLength)) { + continue; + } + if ((! shortest_p) + || q->PrefixLength < shortest_length) { + shortest_p = sp_p; + shortest_length = q->PrefixLength; + } + } break; + } + } + if (! shortest_p) { + /* Found no shortest prefix */ + shortest_p = sa_p; + switch (shortest_p->sa_family) { + case AF_INET: { + /* Fall back to old classfull network addresses */ + DWORD addr = ntohl(((struct sockaddr_in *)shortest_p) + ->sin_addr.s_addr); + if (! (addr & 0x800000)) { + /* Class A */ + shortest_length = 8; + } else if (! (addr & 0x400000)) { + /* Class B */ + shortest_length = 16; + } else if (! (addr & 0x200000)) { + /* Class C */ + shortest_length = 24; + } else { + shortest_length = 32; + } + } break; + case AF_INET6: { + /* Just play it safe */ + shortest_length = 128; + } break; + } + } + switch (shortest_p->sa_family) { + case AF_INET: { + struct sockaddr_in sin; + DWORD mask = 0xFFFFFFFF << (32 - shortest_length); + sys_memzero(&sin, sizeof(sin)); + sin.sin_family = shortest_p->sa_family; + sin.sin_addr.s_addr = htonl(mask); + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, + (struct sockaddr *) &sin); + if (flags & INET_IFF_BROADCAST) { + DWORD sp = + ntohl((DWORD) + ((struct sockaddr_in *)shortest_p) + -> sin_addr.s_addr); + sin.sin_addr.s_addr = htonl(sp | ~mask); + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_BROADADDR, + (struct sockaddr *) &sin); + } + } break; + case AF_INET6: { + struct sockaddr_in6 sin6; + sys_memzero(&sin6, sizeof(sin6)); + sin6.sin6_family = shortest_p->sa_family; + set_netmask_bytes((char *) &sin6.sin6_addr, + 16, + shortest_length); + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, + (struct sockaddr *) &sin6); + } break; + } + } + } + if (ifrow.dwPhysAddrLen) { + /* Hardware Address */ + BUF_ENSURE(1 + 2 + ifrow.dwPhysAddrLen); + *buf_p++ = INET_IFOPT_HWADDR; + put_int16(ifrow.dwPhysAddrLen, buf_p); buf_p += 2; + sys_memcpy(buf_p, ifrow.bPhysAddr, ifrow.dwPhysAddrLen); + buf_p += ifrow.dwPhysAddrLen; + } + +done: + /* That is all for this interface */ + BUF_ENSURE(1); + *buf_p++ = '\0'; + if (ia_p && + ia_p->Ipv6IfIndex && + ia_p->Ipv6IfIndex != index) + { + /* Oops, there was an other interface for IPv6. Possible? XXX */ + index = ia_p->Ipv6IfIndex; + goto index; + } + } + + if (ip_adaddrs_p) FREE(ip_adaddrs_p); + if (info_p) FREE(info_p); + if (ip_addrs_p) FREE(ip_addrs_p); + + buf_size = buf_p - buf_alloc_p; + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); + /* buf_p is now unreliable */ + *rbuf_pp = buf_alloc_p; + return buf_size; +# undef BUF_ENSURE +} + +#elif defined(HAVE_GETIFADDRS) + +static int inet_ctl_getifaddrs(inet_descriptor* desc_p, + char **rbuf_pp, int rsize) +{ + struct ifaddrs *ifa_p, *ifa_free_p; + + int buf_size; + char *buf_p; + char *buf_alloc_p; + + buf_size = 512; + buf_alloc_p = ALLOC(buf_size); + buf_p = buf_alloc_p; +# define BUF_ENSURE(Size) \ + do { \ + int NEED_, GOT_ = buf_p - buf_alloc_p; \ + NEED_ = GOT_ + (Size); \ + if (NEED_ > buf_size) { \ + buf_size = NEED_ + 512; \ + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); \ + buf_p = buf_alloc_p + GOT_; \ + } \ + } while (0) +# define SOCKADDR_TO_BUF(opt, sa) \ + do { \ + if (sa) { \ + char *P_; \ + *buf_p++ = (opt); \ + while (! (P_ = sockaddr_to_buf((sa), buf_p, \ + buf_alloc_p+buf_size))) { \ + int GOT_ = buf_p - buf_alloc_p; \ + buf_size += 512; \ + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); \ + buf_p = buf_alloc_p + GOT_; \ + } \ + if (P_ == buf_p) { \ + buf_p--; \ + } else { \ + buf_p = P_; \ + } \ + } \ + } while (0) + + if (getifaddrs(&ifa_p) < 0) { + return ctl_error(sock_errno(), rbuf_pp, rsize); + } + ifa_free_p = ifa_p; + *buf_p++ = INET_REP_OK; + for (; ifa_p; ifa_p = ifa_p->ifa_next) { + int len = utf8_len(ifa_p->ifa_name, -1); + BUF_ENSURE(len+1 + 1+4 + 1); + utf8_encode(ifa_p->ifa_name, -1, buf_p); + buf_p += len; + *buf_p++ = '\0'; + *buf_p++ = INET_IFOPT_FLAGS; + put_int32(IFGET_FLAGS(ifa_p->ifa_flags), buf_p); buf_p += 4; + if (ifa_p->ifa_addr) { + if (ifa_p->ifa_addr->sa_family == AF_INET +#if defined(AF_INET6) + || ifa_p->ifa_addr->sa_family == AF_INET6 +#endif + ) { + SOCKADDR_TO_BUF(INET_IFOPT_ADDR, ifa_p->ifa_addr); + if (ifa_p->ifa_netmask) { + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_NETMASK, ifa_p->ifa_netmask); + } + if (ifa_p->ifa_dstaddr && + (ifa_p->ifa_flags & IFF_POINTOPOINT)) { + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_DSTADDR, ifa_p->ifa_dstaddr); + } else if (ifa_p->ifa_broadaddr && + (ifa_p->ifa_flags & IFF_BROADCAST)) { + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_BROADADDR, ifa_p->ifa_broadaddr); + } + } +#if defined(AF_LINK) || defined(AF_PACKET) + else if ( +#if defined(AF_LINK) + ifa_p->ifa_addr->sa_family == AF_LINK +#else + 0 +#endif +#if defined(AF_PACKET) + || ifa_p->ifa_addr->sa_family == AF_PACKET +#endif + ) { + char *bp = buf_p; + BUF_ENSURE(1); + SOCKADDR_TO_BUF(INET_IFOPT_HWADDR, ifa_p->ifa_addr); + if (buf_p - bp < 4) buf_p = bp; /* Empty hwaddr */ + } +#endif + } + BUF_ENSURE(1); + *buf_p++ = '\0'; + } + buf_size = buf_p - buf_alloc_p; + buf_alloc_p = REALLOC(buf_alloc_p, buf_size); + /* buf_p is now unreliable */ + freeifaddrs(ifa_free_p); + *rbuf_pp = buf_alloc_p; + return buf_size; +# undef BUF_ENSURE +} + +#else + +static int inet_ctl_getifaddrs(inet_descriptor* desc_p, + char **rbuf_pp, int rsize) +{ + return ctl_error(ENOTSUP, rbuf_pp, rsize); +} + +#endif + + + #ifdef VXWORKS /* ** THIS is a terrible creature, a bug in the TCP part @@ -4457,9 +5050,17 @@ static STATUS wrap_sockopt(STATUS (*function)() /* Yep, no parameter } #endif +/* Per H @ Tail-f: The original code here had problems that possibly + only occur if you abuse it for non-INET sockets, but anyway: + a) If the getsockopt for SO_PRIORITY or IP_TOS failed, the actual + requested setsockopt was never even attempted. + b) If {get,set}sockopt for one of IP_TOS and SO_PRIORITY failed, + but ditto for the other worked and that was actually the requested + option, failure was still reported to erlang. */ + #if defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY) static int setopt_prio_tos_trick - (int fd, int proto, int type, char* arg_ptr, int arg_sz) + (int fd, int proto, int type, char* arg_ptr, int arg_sz, int propagate) { /* The relations between SO_PRIORITY, TOS and other options is not what you (or at least I) would expect...: @@ -4472,6 +5073,8 @@ static int setopt_prio_tos_trick int tmp_ival_prio; int tmp_ival_tos; int res; + int res_prio; + int res_tos; #ifdef HAVE_SOCKLEN_T socklen_t #else @@ -4480,28 +5083,35 @@ static int setopt_prio_tos_trick tmp_arg_sz_prio = sizeof(tmp_ival_prio), tmp_arg_sz_tos = sizeof(tmp_ival_tos); - res = sock_getopt(fd, SOL_SOCKET, SO_PRIORITY, + res_prio = sock_getopt(fd, SOL_SOCKET, SO_PRIORITY, (char *) &tmp_ival_prio, &tmp_arg_sz_prio); - if (res == 0) { - res = sock_getopt(fd, SOL_IP, IP_TOS, + res_tos = sock_getopt(fd, SOL_IP, IP_TOS, (char *) &tmp_ival_tos, &tmp_arg_sz_tos); - if (res == 0) { res = sock_setopt(fd, proto, type, arg_ptr, arg_sz); if (res == 0) { if (type != SO_PRIORITY) { - if (type != IP_TOS) { - res = sock_setopt(fd, + if (type != IP_TOS && res_tos == 0) { + res_tos = sock_setopt(fd, SOL_IP, IP_TOS, (char *) &tmp_ival_tos, tmp_arg_sz_tos); + if (propagate) + res = res_tos; } - if (res == 0) { - res = sock_setopt(fd, + if (res == 0 && res_prio == 0) { + res_prio = sock_setopt(fd, SOL_SOCKET, SO_PRIORITY, (char *) &tmp_ival_prio, tmp_arg_sz_prio); + if (propagate) { + /* Some kernels set a SO_PRIORITY by default that you are not permitted to reset, + silently ignore this error condition */ + if (res_prio != 0 && sock_errno() == EPERM) { + res = 0; + } else { + res = res_prio; } } } @@ -4576,8 +5186,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) case INET_LOPT_BUFFER: DEBUGF(("inet_set_opts(%ld): s=%d, BUFFER=%d\r\n", (long)desc->port, desc->s, ival)); - if (ival > INET_MAX_BUFFER) ival = INET_MAX_BUFFER; - else if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER; + if (ival < INET_MIN_BUFFER) ival = INET_MIN_BUFFER; desc->bufsz = ival; continue; @@ -4642,7 +5251,6 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) if (desc->stype == SOCK_STREAM) { tcp_descriptor* tdesc = (tcp_descriptor*) desc; if (ival < 0) ival = 0; - else if (ival > INET_MAX_BUFFER*2) ival = INET_MAX_BUFFER*2; if (tdesc->low > ival) tdesc->low = ival; tdesc->high = ival; @@ -4653,7 +5261,6 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) if (desc->stype == SOCK_STREAM) { tcp_descriptor* tdesc = (tcp_descriptor*) desc; if (ival < 0) ival = 0; - else if (ival > INET_MAX_BUFFER) ival = INET_MAX_BUFFER; if (tdesc->high < ival) tdesc->high = ival; tdesc->low = ival; @@ -4850,7 +5457,7 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) return -1; } #if defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY) - res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz); + res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz, propagate); #else res = sock_setopt (desc->s, proto, type, arg_ptr, arg_sz); #endif @@ -4999,9 +5606,6 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) case INET_LOPT_BUFFER: desc->bufsz = get_int32(curr); curr += 4; - if (desc->bufsz > INET_MAX_BUFFER) - desc->bufsz = INET_MAX_BUFFER; - else if (desc->bufsz < INET_MIN_BUFFER) desc->bufsz = INET_MIN_BUFFER; res = 0; /* This does not affect the kernel buffer size */ @@ -5242,9 +5846,12 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) char *after; # ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_FLAGS int eflags, cflags, hb_enable, hb_disable, - pmtud_enable, pmtud_disable, + pmtud_enable, pmtud_disable; +# ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY + int sackdelay_enable, sackdelay_disable; # endif +# endif CHKLEN(curr, ASSOC_ID_LEN); arg.pap.spp_assoc_id = GET_ASSOC_ID(curr); curr += ASSOC_ID_LEN; @@ -5293,12 +5900,15 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) if (pmtud_enable) cflags |= SPP_PMTUD_ENABLE; if (pmtud_disable) cflags |= SPP_PMTUD_DISABLE; +# ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY + /* The followings are missing in FreeBSD 7.1 */ sackdelay_enable =eflags& SCTP_FLAG_SACDELAY_ENABLE; sackdelay_disable=eflags& SCTP_FLAG_SACDELAY_DISABLE; if (sackdelay_enable && sackdelay_disable) return -1; if (sackdelay_enable) cflags |= SPP_SACKDELAY_ENABLE; if (sackdelay_disable) cflags |= SPP_SACKDELAY_DISABLE; +# endif arg.pap.spp_flags = cflags; # endif @@ -5377,7 +5987,7 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) return -1; } #if defined(IP_TOS) && defined(SOL_IP) && defined(SO_PRIORITY) - res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz); + res = setopt_prio_tos_trick (desc->s, proto, type, arg_ptr, arg_sz, 1); #else res = sock_setopt (desc->s, proto, type, arg_ptr, arg_sz); #endif @@ -5436,7 +6046,7 @@ static int inet_fill_opts(inet_descriptor* desc, #define PLACE_FOR(Size,Ptr) \ do { \ int need = dest_used + (Size); \ - if (need > INET_MAX_BUFFER) { \ + if (need > INET_MAX_OPT_BUFFER) { \ RETURN_ERROR(); \ } \ if (need > dest_allocated) { \ @@ -5660,7 +6270,7 @@ static int inet_fill_opts(inet_descriptor* desc, buf += 4; data_provided = (int) *buf++; arg_sz = get_int32(buf); - if (arg_sz > INET_MAX_BUFFER) { + if (arg_sz > INET_MAX_OPT_BUFFER) { RETURN_ERROR(); } buf += 4; @@ -5774,7 +6384,7 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen, "miscalculated buffer size"); \ } \ need = (Index) + (N); \ - if (need > INET_MAX_BUFFER/sizeof(ErlDrvTermData)) { \ + if (need > INET_MAX_OPT_BUFFER/sizeof(ErlDrvTermData)) {\ RETURN_ERROR((Spec), -ENOMEM); \ } \ if (need > spec_allocated) { \ @@ -6199,13 +6809,15 @@ static int sctp_fill_opts(inet_descriptor* desc, char* buf, int buflen, if (ap.spp_flags & SPP_PMTUD_DISABLE) { i = LOAD_ATOM (spec, i, am_pmtud_disable); n++; } - +# ifdef HAVE_STRUCT_SCTP_PADDRPARAMS_SPP_SACKDELAY + /* SPP_SACKDELAY_* not in FreeBSD 7.1 */ if (ap.spp_flags & SPP_SACKDELAY_ENABLE) { i = LOAD_ATOM (spec, i, am_sackdelay_enable); n++; } if (ap.spp_flags & SPP_SACKDELAY_DISABLE) { i = LOAD_ATOM (spec, i, am_sackdelay_disable); n++; } # endif +# endif PLACE_FOR(spec, i, LOAD_NIL_CNT + LOAD_LIST_CNT + 2*LOAD_TUPLE_CNT); @@ -6625,7 +7237,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, } } DEBUGF(("inet_ctl(%ld): GETSTAT\r\n", (long) desc->port)); - if (dstlen > INET_MAX_BUFFER) /* sanity check */ + if (dstlen > INET_MAX_OPT_BUFFER) /* sanity check */ return 0; if (dstlen > rsize) { if ((dst = (char*) ALLOC(dstlen)) == NULL) @@ -6641,7 +7253,7 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, char* dst; int dstlen = 1 /* Reply code */ + len*5; DEBUGF(("inet_ctl(%ld): INET_REQ_SUBSCRIBE\r\n", (long) desc->port)); - if (dstlen > INET_MAX_BUFFER) /* sanity check */ + if (dstlen > INET_MAX_OPT_BUFFER) /* sanity check */ return 0; if (dstlen > rsize) { if ((dst = (char*) ALLOC(dstlen)) == NULL) @@ -6676,6 +7288,13 @@ static int inet_ctl(inet_descriptor* desc, int cmd, char* buf, int len, return inet_ctl_getiflist(desc, rbuf, rsize); } + case INET_REQ_GETIFADDRS: { + DEBUGF(("inet_ctl(%ld): GETIFADDRS\r\n", (long)desc->port)); + if (!IS_OPEN(desc)) + return ctl_xerror(EXBADPORT, rbuf, rsize); + return inet_ctl_getifaddrs(desc, rbuf, rsize); + } + case INET_REQ_IFGET: { DEBUGF(("inet_ctl(%ld): IFGET\r\n", (long)desc->port)); if (!IS_OPEN(desc)) @@ -7931,7 +8550,9 @@ static int tcp_deliver(tcp_descriptor* desc, int len) len = 0; if (!desc->inet.active) { - driver_cancel_timer(desc->inet.port); + if (!desc->busy_on_send) { + driver_cancel_timer(desc->inet.port); + } sock_select(INETP(desc),(FD_READ|FD_CLOSE),0); if (desc->i_buf != NULL) tcp_restart_input(desc); diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index b19f632f52..4b3934657c 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -587,7 +587,8 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ open directory.*/ char* buffer, /* Pointer to buffer for one filename. */ - size_t size) /* Size of buffer. */ + size_t *size) /* in-out Size of buffer, length + of name. */ { DIR *dp; /* Pointer to directory structure. */ struct dirent* dirp; /* Pointer to directory entry. */ @@ -619,7 +620,8 @@ efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ if (IS_DOT_OR_DOTDOT(dirp->d_name)) continue; buffer[0] = '\0'; - strncat(buffer, dirp->d_name, size-1); + strncat(buffer, dirp->d_name, (*size)-1); + *size = strlen(dirp->d_name); return 1; } } diff --git a/erts/emulator/drivers/win32/win_con.c b/erts/emulator/drivers/win32/win_con.c index 2202ca655f..c788ad409d 100644 --- a/erts/emulator/drivers/win32/win_con.c +++ b/erts/emulator/drivers/win32/win_con.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. + * Copyright Ericsson AB 1997-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -704,6 +704,18 @@ FrameWndProc(HWND hwnd, UINT iMsg, WPARAM wParam, LPARAM lParam) } write_inbuf(&c, 1); return 0; + case WM_MOUSEWHEEL: + { + int delta = GET_WHEEL_DELTA_WPARAM(wParam); + if (delta < 0) { + PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK, + (iVscrollPos + 5)),0); + } else { + WORD pos = ((iVscrollPos - 5) < 0) ? 0 : (iVscrollPos - 5); + PostMessage(hClientWnd, WM_VSCROLL, MAKELONG(SB_THUMBTRACK,pos),0); + } + return 0; + } case WM_CHAR: c = (TCHAR)wParam; write_inbuf(&c,1); diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 04bd1139f5..3d59564f7b 100644..100755 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * Copyright Ericsson AB 1997-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -23,20 +23,20 @@ #include <windows.h> #include "sys.h" #include <ctype.h> - +#include <wchar.h> #include "erl_efile.h" /* * Microsoft-specific function to map a WIN32 error code to a Posix errno. */ -#define ISSLASH(a) ((a) == '\\' || (a) == '/') +#define ISSLASH(a) ((a) == L'\\' || (a) == L'/') #define ISDIR(st) (((st).st_mode&S_IFMT) == S_IFDIR) #define ISREG(st) (((st).st_mode&S_IFMT) == S_IFREG) #define IS_DOT_OR_DOTDOT(s) \ - (s[0] == '.' && (s[1] == '\0' || (s[1] == '.' && s[2] == '\0'))) + ((s)[0] == L'.' && ((s)[1] == L'\0' || ((s)[1] == L'.' && (s)[2] == L'\0'))) #ifndef INVALID_FILE_ATTRIBUTES #define INVALID_FILE_ATTRIBUTES ((DWORD) 0xFFFFFFFF) @@ -44,9 +44,9 @@ static int check_error(int result, Efile_error* errInfo); static int set_error(Efile_error* errInfo); -static int IsRootUNCName(const char* path); -static int extract_root(char* name); -static unsigned short dos_to_posix_mode(int attr, const char *name); +static int is_root_unc_name(const WCHAR *path); +static int extract_root(WCHAR *name); +static unsigned short dos_to_posix_mode(int attr, const WCHAR *name); static int errno_map(DWORD last_error) { @@ -196,27 +196,26 @@ win_writev(Efile_error* errInfo, int -efile_mkdir(errInfo, name) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to create. */ +efile_mkdir(Efile_error* errInfo, /* Where to return error codes. */ + char* name) /* Name of directory to create. */ { - return check_error(mkdir(name), errInfo); + return check_error(_wmkdir((WCHAR *) name), errInfo); } int -efile_rmdir(errInfo, name) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to delete. */ +efile_rmdir(Efile_error* errInfo, /* Where to return error codes. */ + char* name) /* Name of directory to delete. */ { OSVERSIONINFO os; DWORD attr; + WCHAR *wname = (WCHAR *) name; - if (RemoveDirectory(name) != FALSE) { + if (RemoveDirectoryW(wname) != FALSE) { return 1; } errno = errno_map(GetLastError()); if (errno == EACCES) { - attr = GetFileAttributes(name); + attr = GetFileAttributesW(wname); if (attr != (DWORD) -1) { if ((attr & FILE_ATTRIBUTE_DIRECTORY) == 0) { /* @@ -238,21 +237,21 @@ char* name; /* Name of directory to delete. */ GetVersionEx(&os); if (os.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) { HANDLE handle; - WIN32_FIND_DATA data; - char buffer[2*MAX_PATH]; + WIN32_FIND_DATAW data; + WCHAR buffer[2*MAX_PATH]; int len; - len = strlen(name); - strcpy(buffer, name); - if (buffer[0] && buffer[len-1] != '\\' && buffer[len-1] != '/') { - strcat(buffer, "\\"); + len = wcslen(wname); + wcscpy(buffer, wname); + if (buffer[0] && buffer[len-1] != L'\\' && buffer[len-1] != L'/') { + wcscat(buffer, L"\\"); } - strcat(buffer, "*.*"); - handle = FindFirstFile(buffer, &data); + wcscat(buffer, L"*.*"); + handle = FindFirstFileW(buffer, &data); if (handle != INVALID_HANDLE_VALUE) { while (1) { - if ((strcmp(data.cFileName, ".") != 0) - && (strcmp(data.cFileName, "..") != 0)) { + if ((wcscmp(data.cFileName, L".") != 0) + && (wcscmp(data.cFileName, L"..") != 0)) { /* * Found something in this directory. */ @@ -260,7 +259,7 @@ char* name; /* Name of directory to delete. */ errno = EEXIST; break; } - if (FindNextFile(handle, &data) == FALSE) { + if (FindNextFileW(handle, &data) == FALSE) { break; } } @@ -284,19 +283,19 @@ char* name; /* Name of directory to delete. */ } int -efile_delete_file(errInfo, name) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of file to delete. */ +efile_delete_file(Efile_error* errInfo, /* Where to return error codes. */ + char* name) /* Name of file to delete. */ { DWORD attr; + WCHAR *wname = (WCHAR *) name; - if (DeleteFile(name) != FALSE) { + if (DeleteFileW(wname) != FALSE) { return 1; } errno = errno_map(GetLastError()); if (errno == EACCES) { - attr = GetFileAttributes(name); + attr = GetFileAttributesW(wname); if (attr != (DWORD) -1) { if (attr & FILE_ATTRIBUTE_DIRECTORY) { /* @@ -308,7 +307,7 @@ char* name; /* Name of file to delete. */ } } } else if (errno == ENOENT) { - attr = GetFileAttributes(name); + attr = GetFileAttributesW(wname); if (attr != (DWORD) -1) { if (attr & FILE_ATTRIBUTE_DIRECTORY) { /* @@ -362,20 +361,21 @@ char* name; /* Name of file to delete. */ */ int -efile_rename(errInfo, src, dst) -Efile_error* errInfo; /* Where to return error codes. */ -char* src; /* Original name. */ -char* dst; /* New name. */ +efile_rename(Efile_error* errInfo, /* Where to return error codes. */ + char* src, /* Original name. */ + char* dst) /* New name. */ { DWORD srcAttr, dstAttr; + WCHAR *wsrc = (WCHAR *) src; + WCHAR *wdst = (WCHAR *) dst; - if (MoveFile(src, dst) != FALSE) { + if (MoveFileW(wsrc, wdst) != FALSE) { return 1; } errno = errno_map(GetLastError()); - srcAttr = GetFileAttributes(src); - dstAttr = GetFileAttributes(dst); + srcAttr = GetFileAttributesW(wsrc); + dstAttr = GetFileAttributesW(wdst); if (srcAttr == (DWORD) -1) { srcAttr = 0; } @@ -390,22 +390,22 @@ char* dst; /* New name. */ if (errno == EACCES) { decode: if (srcAttr & FILE_ATTRIBUTE_DIRECTORY) { - char srcPath[MAX_PATH], dstPath[MAX_PATH]; - char *srcRest, *dstRest; + WCHAR srcPath[MAX_PATH], dstPath[MAX_PATH]; + WCHAR *srcRest, *dstRest; int size; - size = GetFullPathName(src, sizeof(srcPath), srcPath, &srcRest); - if ((size == 0) || (size > sizeof(srcPath))) { + size = GetFullPathNameW(wsrc, MAX_PATH, srcPath, &srcRest); + if ((size == 0) || (size > MAX_PATH)) { return check_error(-1, errInfo); } - size = GetFullPathName(dst, sizeof(dstPath), dstPath, &dstRest); - if ((size == 0) || (size > sizeof(dstPath))) { + size = GetFullPathNameW(wdst, MAX_PATH, dstPath, &dstRest); + if ((size == 0) || (size > MAX_PATH)) { return check_error(-1, errInfo); } if (srcRest == NULL) { - srcRest = srcPath + strlen(srcPath); + srcRest = srcPath + wcslen(srcPath); } - if (strnicmp(srcPath, dstPath, srcRest - srcPath) == 0) { + if (_wcsnicmp(srcPath, dstPath, srcRest - srcPath) == 0) { /* * Trying to move a directory into itself. */ @@ -420,14 +420,14 @@ char* dst; /* New name. */ } (void) extract_root(dstPath); - if (dstPath[0] == '\0') { + if (dstPath[0] == L'\0') { /* * The filename was invalid. (Don't know why, * but play it safe.) */ errno = EINVAL; } - if (stricmp(srcPath, dstPath) != 0) { + if (_wcsicmp(srcPath, dstPath) != 0) { /* * If src is a directory and dst filesystem != src * filesystem, errno should be EXDEV. It is very @@ -463,14 +463,14 @@ char* dst; /* New name. */ * fails, it's because it wasn't empty. */ - if (RemoveDirectory(dst)) { + if (RemoveDirectoryW(wdst)) { /* * Now that that empty directory is gone, we can try * renaming again. If that fails, we'll put this empty * directory back, for completeness. */ - if (MoveFile(src, dst) != FALSE) { + if (MoveFileW(wsrc, wdst) != FALSE) { return 1; } @@ -480,8 +480,8 @@ char* dst; /* New name. */ */ errno = errno_map(GetLastError()); - CreateDirectory(dst, NULL); - SetFileAttributes(dst, dstAttr); + CreateDirectoryW(wdst, NULL); + SetFileAttributesW(wdst, dstAttr); if (errno == EACCES) { /* * Decode the EACCES to a more meaningful error. @@ -506,17 +506,17 @@ char* dst; /* New name. */ * put temp file back to old name. */ - char tempName[MAX_PATH]; + WCHAR tempName[MAX_PATH]; int result, size; - char *rest; + WCHAR *rest; - size = GetFullPathName(dst, sizeof(tempName), tempName, &rest); - if ((size == 0) || (size > sizeof(tempName)) || (rest == NULL)) { + size = GetFullPathNameW(wdst, MAX_PATH, tempName, &rest); + if ((size == 0) || (size > MAX_PATH) || (rest == NULL)) { return check_error(-1, errInfo); } - *rest = '\0'; + *rest = L'\0'; result = -1; - if (GetTempFileName(tempName, "erlr", 0, tempName) != 0) { + if (GetTempFileNameW(tempName, L"erlr", 0, tempName) != 0) { /* * Strictly speaking, need the following DeleteFile and * MoveFile to be joined as an atomic operation so no @@ -524,15 +524,15 @@ char* dst; /* New name. */ * same temp file. */ - DeleteFile(tempName); - if (MoveFile(dst, tempName) != FALSE) { - if (MoveFile(src, dst) != FALSE) { - SetFileAttributes(tempName, FILE_ATTRIBUTE_NORMAL); - DeleteFile(tempName); + DeleteFileW(tempName); + if (MoveFileW(wdst, tempName) != FALSE) { + if (MoveFileW(wsrc, wdst) != FALSE) { + SetFileAttributesW(tempName, FILE_ATTRIBUTE_NORMAL); + DeleteFileW(tempName); return 1; } else { - DeleteFile(dst); - MoveFile(tempName, dst); + DeleteFileW(wdst); + MoveFileW(tempName, wdst); } } @@ -558,11 +558,10 @@ char* dst; /* New name. */ } int -efile_chdir(errInfo, name) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to make current. */ +efile_chdir(Efile_error* errInfo, /* Where to return error codes. */ + char* name) /* Name of directory to make current. */ { - int success = check_error(chdir(name), errInfo); + int success = check_error(_wchdir((WCHAR *) name), errInfo); if (!success && errInfo->posix_errno == EINVAL) /* POSIXification of errno */ errInfo->posix_errno = ENOENT; @@ -570,59 +569,65 @@ char* name; /* Name of directory to make current. */ } int -efile_getdcwd(errInfo, drive, buffer, size) -Efile_error* errInfo; /* Where to return error codes. */ -int drive; /* 0 - current, 1 - A, 2 - B etc. */ -char* buffer; /* Where to return the current directory. */ -size_t size; /* Size of buffer. */ +efile_getdcwd(Efile_error* errInfo, /* Where to return error codes. */ + int drive, /* 0 - current, 1 - A, 2 - B etc. */ + char* buffer, /* Where to return the current directory. */ + size_t size) /* Size of buffer. */ { - if (_getdcwd(drive, buffer, size) == NULL) + WCHAR *wbuffer = (WCHAR *) buffer; + size_t wbuffer_size = size / 2; + if (_wgetdcwd(drive, wbuffer, wbuffer_size) == NULL) return check_error(-1, errInfo); - for ( ; *buffer; buffer++) - if (*buffer == '\\') - *buffer = '/'; + for ( ; *wbuffer; wbuffer++) + if (*wbuffer == L'\\') + *wbuffer = L'/'; return 1; } int -efile_readdir(errInfo, name, dir_handle, buffer, size) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to open. */ -EFILE_DIR_HANDLE* dir_handle; /* Directory handle of open directory. */ -char* buffer; /* Pointer to buffer for one filename. */ -size_t size; /* Size of buffer. */ +efile_readdir(Efile_error* errInfo, /* Where to return error codes. */ + char* name, /* Name of directory to list */ + EFILE_DIR_HANDLE* dir_handle, /* Handle of opened directory or NULL */ + char* buffer, /* Buffer to put one filename in */ + size_t *size) /* in-out size of buffer/size of filename excluding zero + termination in bytes*/ { HANDLE dir; /* Handle to directory. */ - char wildcard[MAX_PATH]; /* Wildcard to search for. */ - WIN32_FIND_DATA findData; /* Data found by FindFirstFile() or FindNext(). */ + WCHAR wildcard[MAX_PATH]; /* Wildcard to search for. */ + WIN32_FIND_DATAW findData; /* Data found by FindFirstFile() or FindNext(). */ + /* Alignment is not honored, this works on x86 because of alignment fixup by processor. + Not perfect, but faster than alinging by hand (really) */ + WCHAR *wname = (WCHAR *) name; + WCHAR *wbuffer = (WCHAR *) buffer; /* * First time we must setup everything. */ if (*dir_handle == NULL) { - int length = strlen(name); - char* s; + int length = wcslen(wname); + WCHAR* s; if (length+3 >= MAX_PATH) { errno = ENAMETOOLONG; return check_error(-1, errInfo); } - strcpy(wildcard, name); + wcscpy(wildcard, wname); s = wildcard+length-1; - if (*s != '/' && *s != '\\') - *++s = '\\'; - *++s = '*'; - *++s = '\0'; - DEBUGF(("Reading %s\n", wildcard)); - dir = FindFirstFile(wildcard, &findData); + if (*s != L'/' && *s != L'\\') + *++s = L'\\'; + *++s = L'*'; + *++s = L'\0'; + DEBUGF(("Reading %ws\n", wildcard)); + dir = FindFirstFileW(wildcard, &findData); if (dir == INVALID_HANDLE_VALUE) return set_error(errInfo); *dir_handle = (EFILE_DIR_HANDLE) dir; if (!IS_DOT_OR_DOTDOT(findData.cFileName)) { - strcpy(buffer, findData.cFileName); + wcscpy(wbuffer, findData.cFileName); + *size = wcslen(wbuffer)*2; return 1; } } @@ -635,10 +640,11 @@ size_t size; /* Size of buffer. */ dir = (HANDLE) *dir_handle; for (;;) { - if (FindNextFile(dir, &findData)) { + if (FindNextFileW(dir, &findData)) { if (IS_DOT_OR_DOTDOT(findData.cFileName)) continue; - strcpy(buffer, findData.cFileName); + wcscpy(wbuffer, findData.cFileName); + *size = wcslen(wbuffer)*2; return 1; } @@ -655,17 +661,17 @@ size_t size; /* Size of buffer. */ } int -efile_openfile(errInfo, name, flags, pfd, pSize) -Efile_error* errInfo; /* Where to return error codes. */ -char* name; /* Name of directory to open. */ -int flags; /* Flags to use for opening. */ -int* pfd; /* Where to store the file descriptor. */ -Sint64* pSize; /* Where to store the size of the file. */ +efile_openfile(Efile_error* errInfo, /* Where to return error codes. */ + char* name, /* Name of directory to open. */ + int flags, /* Flags to use for opening. */ + int* pfd, /* Where to store the file descriptor. */ + Sint64* pSize) /* Where to store the size of the file. */ { BY_HANDLE_FILE_INFORMATION fileInfo; /* File information from a handle. */ HANDLE fd; /* Handle to open file. */ DWORD access; /* Access mode: GENERIC_READ, GENERIC_WRITE. */ DWORD crFlags; + WCHAR *wname = (WCHAR *) name; switch (flags & (EFILE_MODE_READ|EFILE_MODE_WRITE)) { case EFILE_MODE_READ: @@ -692,7 +698,7 @@ Sint64* pSize; /* Where to store the size of the file. */ if (flags & EFILE_MODE_EXCL) { crFlags = CREATE_NEW; } - fd = CreateFile(name, access, + fd = CreateFileW(wname, access, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, crFlags, FILE_ATTRIBUTE_NORMAL, NULL); @@ -711,7 +717,7 @@ Sint64* pSize; /* Where to store the size of the file. */ * to EISDIR. */ if (errInfo->posix_errno && - (attr = GetFileAttributes(name)) != INVALID_FILE_ATTRIBUTES && + (attr = GetFileAttributesW(wname)) != INVALID_FILE_ATTRIBUTES && (attr & FILE_ATTRIBUTE_DIRECTORY)) { errInfo->posix_errno = EISDIR; } @@ -735,9 +741,10 @@ Sint64* pSize; /* Where to store the size of the file. */ int efile_may_openfile(Efile_error* errInfo, char *name) { + WCHAR *wname = (WCHAR *) name; DWORD attr; - if ((attr = GetFileAttributes(name)) == INVALID_FILE_ATTRIBUTES) { + if ((attr = GetFileAttributesW(wname)) == INVALID_FILE_ATTRIBUTES) { return check_error(-1, errInfo); } @@ -746,18 +753,6 @@ efile_may_openfile(Efile_error* errInfo, char *name) { return check_error(-1, errInfo); } return 1; -#if 0 - struct stat statbuf; - - if (stat(name, &statbuf)) { - return check_error(-1, errInfo); - } - if (ISDIR(statbuf)) { - errno = EISDIR; - return check_error(-1, errInfo); - } - return 1; -#endif } void @@ -792,16 +787,17 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, char* orig_name, int info_for_link) { HANDLE findhandle; /* Handle returned by FindFirstFile(). */ - WIN32_FIND_DATA findbuf; /* Data return by FindFirstFile(). */ - char name[_MAX_PATH]; + WIN32_FIND_DATAW findbuf; /* Data return by FindFirstFile(). */ + WCHAR name[_MAX_PATH]; int name_len; - char* path; - char pathbuf[_MAX_PATH]; + WCHAR *path; + WCHAR pathbuf[_MAX_PATH]; int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ + WCHAR *worig_name = (WCHAR *) orig_name; /* Don't allow wildcards to be interpreted by system */ - if (strpbrk(orig_name, "?*")) { + if (wcspbrk(worig_name, L"?*")) { enoent: errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; @@ -813,25 +809,25 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, * slash, because it causes FindFirstFile() to fail on Win95. */ - if ((name_len = strlen(orig_name)) >= _MAX_PATH) { + if ((name_len = wcslen(worig_name)) >= _MAX_PATH) { goto enoent; } else { - strcpy(name, orig_name); + wcscpy(name, worig_name); if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != ':') { - name[name_len-1] = '\0'; + name[name_len-2] != L':') { + name[name_len-1] = L'\0'; } } /* Try to get disk from name. If none, get current disk. */ - if (name[1] != ':') { + if (name[1] != L':') { drive = 0; - if (GetCurrentDirectory(sizeof(pathbuf), pathbuf) && - pathbuf[1] == ':') { - drive = tolower(pathbuf[0]) - 'a' + 1; + if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) && + pathbuf[1] == L':') { + drive = towlower(pathbuf[0]) - L'a' + 1; } - } else if (*name && name[2] == '\0') { + } else if (*name && name[2] == L'\0') { /* * X: and nothing more is an error. */ @@ -839,15 +835,15 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; } else - drive = tolower(*name) - 'a' + 1; + drive = towlower(*name) - L'a' + 1; - findhandle = FindFirstFile(name, &findbuf); + findhandle = FindFirstFileW(name, &findbuf); if (findhandle == INVALID_HANDLE_VALUE) { - if (!(strpbrk(name, "./\\") && - (path = _fullpath(pathbuf, name, _MAX_PATH)) && + if (!(wcspbrk(name, L"./\\") && + (path = _wfullpath(pathbuf, name, _MAX_PATH)) && /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ - ((strlen(path) == 3) || IsRootUNCName(path)) && - (GetDriveType(path) > 1) ) ) { + ((wcslen(path) == 3) || is_root_unc_name(path)) && + (GetDriveTypeW(path) > 1) ) ) { errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; return 0; @@ -860,8 +856,9 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, findbuf.dwFileAttributes = FILE_ATTRIBUTE_DIRECTORY; findbuf.nFileSizeHigh = 0; findbuf.nFileSizeLow = 0; - findbuf.cFileName[0] = '\0'; + findbuf.cFileName[0] = L'\0'; + pInfo->links = 1; pInfo->modifyTime.year = 1980; pInfo->modifyTime.month = 1; pInfo->modifyTime.day = 1; @@ -874,6 +871,35 @@ efile_fileinfo(Efile_error* errInfo, Efile_info* pInfo, SYSTEMTIME SystemTime; FILETIME LocalFTime; + /*first check if we are a symlink */ + if (!info_for_link && (findbuf.dwFileAttributes & + FILE_ATTRIBUTE_REPARSE_POINT)){ + /* + * given that we know this is a symlink, + we should be able to find its target */ + WCHAR target_name[_MAX_PATH]; + if (efile_readlink(errInfo, (char *) name, + (char *) target_name,256) == 1) { + FindClose(findhandle); + return efile_fileinfo(errInfo, pInfo, + (char *) target_name, info_for_link); + } + } + + /* number of links: */ + { + HANDLE handle; /* Handle returned by CreateFile() */ + BY_HANDLE_FILE_INFORMATION fileInfo; /* from CreateFile() */ + if (handle = CreateFileW(name, GENERIC_READ, 0,NULL, + OPEN_EXISTING, 0, NULL)) { + GetFileInformationByHandle(handle, &fileInfo); + pInfo->links = fileInfo.nNumberOfLinks; + CloseHandle(handle); + } else { + pInfo->links = 1; + } + } + #define GET_TIME(dst, src) \ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \ !FileTimeToSystemTime(&LocalFTime, &SystemTime)) { \ @@ -908,7 +934,10 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \ pInfo->size_low = findbuf.nFileSizeLow; pInfo->size_high = findbuf.nFileSizeHigh; - if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + if (info_for_link && (findbuf.dwFileAttributes & + FILE_ATTRIBUTE_REPARSE_POINT)) + pInfo->type = FT_SYMLINK; + else if (findbuf.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) pInfo->type = FT_DIRECTORY; else pInfo->type = FT_REGULAR; @@ -919,7 +948,6 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \ pInfo->access = FA_READ|FA_WRITE; pInfo->mode = dos_to_posix_mode(findbuf.dwFileAttributes, name); - pInfo->links = 1; pInfo->major_device = drive; pInfo->minor_device = 0; pInfo->inode = 0; @@ -930,10 +958,9 @@ if (!FileTimeToLocalFileTime(&findbuf.src, &LocalFTime) || \ } int -efile_write_info(errInfo, pInfo, name) -Efile_error* errInfo; -Efile_info* pInfo; -char* name; +efile_write_info(Efile_error* errInfo, + Efile_info* pInfo, + char* name) { SYSTEMTIME timebuf; FILETIME LocalFileTime; @@ -947,12 +974,13 @@ char* name; DWORD attr; DWORD tempAttr; BOOL modifyTime = FALSE; + WCHAR *wname = (WCHAR *) name; /* * Get the attributes for the file. */ - tempAttr = attr = GetFileAttributes((LPTSTR)name); + tempAttr = attr = GetFileAttributesW(wname); if (attr == 0xffffffff) { return set_error(errInfo); } @@ -988,8 +1016,8 @@ char* name; } \ } - MKTIME(ModifyFileTime, pInfo->accessTime, mtime); - MKTIME(AccessFileTime, pInfo->modifyTime, atime); + MKTIME(ModifyFileTime, pInfo->modifyTime, mtime); + MKTIME(AccessFileTime, pInfo->accessTime, atime); MKTIME(CreationFileTime, pInfo->cTime, ctime); #undef MKTIME @@ -1006,12 +1034,12 @@ char* name; if (tempAttr & FILE_ATTRIBUTE_READONLY) { tempAttr &= ~FILE_ATTRIBUTE_READONLY; - if (!SetFileAttributes((LPTSTR) name, tempAttr)) { + if (!SetFileAttributesW(wname, tempAttr)) { return set_error(errInfo); } } - fd = CreateFile(name, GENERIC_READ|GENERIC_WRITE, + fd = CreateFileW(wname, GENERIC_READ|GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (fd != INVALID_HANDLE_VALUE) { @@ -1029,7 +1057,7 @@ char* name; */ if (tempAttr != attr) { - if (!SetFileAttributes((LPTSTR) name, attr)) { + if (!SetFileAttributesW(wname, attr)) { return set_error(errInfo); } } @@ -1082,12 +1110,17 @@ char* buf; /* Buffer to write. */ size_t count; /* Number of bytes to write. */ { DWORD written; /* Bytes written in last operation. */ + OVERLAPPED overlapped; + OVERLAPPED* pOverlapped = NULL; if (flags & EFILE_MODE_APPEND) { - (void) SetFilePointer((HANDLE) fd, 0, NULL, FILE_END); + memset(&overlapped, 0, sizeof(overlapped)); + overlapped.Offset = 0xffffffff; + overlapped.OffsetHigh = 0xffffffff; + pOverlapped = &overlapped; } while (count > 0) { - if (!WriteFile((HANDLE) fd, buf, count, &written, NULL)) + if (!WriteFile((HANDLE) fd, buf, count, &written, pOverlapped)) return set_error(errInfo); buf += written; count -= written; @@ -1107,11 +1140,16 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ size_t size) /* Number of bytes to write */ { int cnt; /* Buffers so far written */ + OVERLAPPED overlapped; + OVERLAPPED* pOverlapped = NULL; ASSERT(iovcnt >= 0); if (flags & EFILE_MODE_APPEND) { - (void) SetFilePointer((HANDLE) fd, 0, NULL, FILE_END); + memset(&overlapped, 0, sizeof(overlapped)); + overlapped.Offset = 0xffffffff; + overlapped.OffsetHigh = 0xffffffff; + pOverlapped = &overlapped; } for (cnt = 0; cnt < iovcnt; cnt++) { if (iov[cnt].iov_base && iov[cnt].iov_len > 0) { @@ -1123,7 +1161,7 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ iov[cnt].iov_base + p, iov[cnt].iov_len - p, &w, - NULL)) + pOverlapped)) return set_error(errInfo); } } @@ -1195,7 +1233,7 @@ int flags; /* - * IsRootUNCName - returns TRUE if the argument is a UNC name specifying + * is_root_unc_name - returns TRUE if the argument is a UNC name specifying * a root share. That is, if it is of the form \\server\share\. * This routine will also return true if the argument is of the * form \\server\share (no trailing slash) but Win32 currently @@ -1205,16 +1243,16 @@ int flags; */ static int -IsRootUNCName(const char* path) +is_root_unc_name(const WCHAR *path) { /* * If a root UNC name, path will start with 2 (but not 3) slashes */ - if ((strlen(path) >= 5) /* minimum string is "//x/y" */ + if ((wcslen(path) >= 5) /* minimum string is "//x/y" */ && ISSLASH(path[0]) && ISSLASH(path[1])) { - const char * p = path + 2 ; + const WCHAR *p = path + 2; /* * find the slash between the server name and share name @@ -1257,19 +1295,19 @@ IsRootUNCName(const char* path) */ static int -extract_root(char* name) +extract_root(WCHAR* name) { - int len = strlen(name); + int len = wcslen(name); - if (isalpha(name[0]) && name[1] == ':' && ISSLASH(name[2])) { - int c = name[3]; - name[3] = '\0'; - return c == '\0'; + if (iswalpha(name[0]) && name[1] == L':' && ISSLASH(name[2])) { + WCHAR c = name[3]; + name[3] = L'\0'; + return c == L'\0'; } else if (len < 5 || !ISSLASH(name[0]) || !ISSLASH(name[1])) { goto error; } else { /* Try to find the end of the UNC name. */ - char* p; - int c; + WCHAR* p; + WCHAR c; /* * Find the slash between the server name and share name. @@ -1278,7 +1316,7 @@ extract_root(char* name) for (p = name + 2; *p; p++) if (ISSLASH(*p)) break; - if (*p == '\0') + if (*p == L'\0') goto error; /* @@ -1289,24 +1327,24 @@ extract_root(char* name) if (ISSLASH(*p)) break; c = *p; - *p = '\0'; - return c == '\0' || p[1] == '\0'; + *p = L'\0'; + return c == L'\0' || p[1] == L'\0'; } error: - *name = '\0'; + *name = L'\0'; return 1; } static unsigned short -dos_to_posix_mode(int attr, const char *name) +dos_to_posix_mode(int attr, const WCHAR *name) { register unsigned short uxmode; unsigned dosmode; - register const char *p; + register const WCHAR *p; dosmode = attr & 0xff; - if ((p = name)[1] == ':') + if ((p = name)[1] == L':') p += 2; /* check to see if this is a directory - note we must make a special @@ -1315,7 +1353,7 @@ dos_to_posix_mode(int attr, const char *name) uxmode = (unsigned short) (((ISSLASH(*p) && !p[1]) || (dosmode & FILE_ATTRIBUTE_DIRECTORY) || - *p == '\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG); + *p == L'\0') ? _S_IFDIR|_S_IEXEC : _S_IFREG); /* If attribute byte does not have read-only bit, it is read-write */ @@ -1324,11 +1362,11 @@ dos_to_posix_mode(int attr, const char *name) /* see if file appears to be executable - check extension of name */ - if (p = strrchr(name, '.')) { - if (!stricmp(p, ".exe") || - !stricmp(p, ".cmd") || - !stricmp(p, ".bat") || - !stricmp(p, ".com")) + if (p = wcsrchr(name, L'.')) { + if (!_wcsicmp(p, L".exe") || + !_wcsicmp(p, L".cmd") || + !_wcsicmp(p, L".bat") || + !_wcsicmp(p, L".com")) uxmode |= _S_IEXEC; } @@ -1343,6 +1381,60 @@ dos_to_posix_mode(int attr, const char *name) int efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) { + /* + * load dll and see if we have CreateSymbolicLink at runtime: + * (Vista only) + */ + HINSTANCE hModule = NULL; + WCHAR *wname = (WCHAR *) name; + WCHAR *wbuffer = (WCHAR *) buffer; + if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { + typedef DWORD (WINAPI * GETFINALPATHNAMEBYHANDLEPTR)( + HANDLE hFile, + LPCWSTR lpFilePath, + DWORD cchFilePath, + DWORD dwFlags); + + GETFINALPATHNAMEBYHANDLEPTR pGetFinalPathNameByHandle = + (GETFINALPATHNAMEBYHANDLEPTR)GetProcAddress(hModule, "GetFinalPathNameByHandleW"); + + if (pGetFinalPathNameByHandle == NULL) { + FreeLibrary(hModule); + } else { + /* first check if file is a symlink; {error, einval} otherwise */ + DWORD fileAttributes = GetFileAttributesW(wname); + if ((fileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { + BOOLEAN success = 0; + HANDLE h = CreateFileW(wname, GENERIC_READ, 0,NULL, OPEN_EXISTING, 0, NULL); + int len; + if(h != INVALID_HANDLE_VALUE) { + success = pGetFinalPathNameByHandle(h, wbuffer, size,0); + /* GetFinalPathNameByHandle prepends path with "\\?\": */ + len = wcslen(wbuffer); + wmemmove(wbuffer,wbuffer+4,len-3); + if (len - 4 >= 2 && wbuffer[1] == L':' && wbuffer[0] >= L'A' && + wbuffer[0] <= L'Z') { + wbuffer[0] = wbuffer[0] + L'a' - L'A'; + } + + for ( ; *wbuffer; wbuffer++) + if (*wbuffer == L'\\') + *wbuffer = L'/'; + CloseHandle(h); + } + FreeLibrary(hModule); + if (success) { + return 1; + } else { + return set_error(errInfo); + } + } else { + FreeLibrary(hModule); + errno = EINVAL; + return check_error(-1, errInfo); + } + } + } errno = ENOTSUP; return check_error(-1, errInfo); } @@ -1351,17 +1443,20 @@ efile_readlink(Efile_error* errInfo, char* name, char* buffer, size_t size) int efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) { - WIN32_FIND_DATA wfd; + WIN32_FIND_DATAW wfd; HANDLE fh; - char name[_MAX_PATH]; + WCHAR name[_MAX_PATH+1]; int name_len; - char* path; - char pathbuf[_MAX_PATH]; + WCHAR* path; + WCHAR pathbuf[_MAX_PATH+1]; /* Unclear weather GetCurrentDirectory will access one char after + _MAX_PATH */ + WCHAR *worig_name = (WCHAR *) orig_name; + WCHAR *wbuffer = (WCHAR *) buffer; int drive; /* Drive for filename (1 = A:, 2 = B: etc). */ /* Don't allow wildcards to be interpreted by system */ - if (strpbrk(orig_name, "?*")) { + if (wcspbrk(worig_name, L"?*")) { enoent: errInfo->posix_errno = ENOENT; errInfo->os_errno = ERROR_FILE_NOT_FOUND; @@ -1373,67 +1468,105 @@ efile_altname(Efile_error* errInfo, char* orig_name, char* buffer, size_t size) * slash, because it causes FindFirstFile() to fail on Win95. */ - if ((name_len = strlen(orig_name)) >= _MAX_PATH) { + if ((name_len = wcslen(worig_name)) >= _MAX_PATH) { goto enoent; } else { - strcpy(name, orig_name); + wcscpy(name, worig_name); if (name_len > 2 && ISSLASH(name[name_len-1]) && - name[name_len-2] != ':') { - name[name_len-1] = '\0'; + name[name_len-2] != L':') { + name[name_len-1] = L'\0'; } } /* Try to get disk from name. If none, get current disk. */ - if (name[1] != ':') { + if (name[1] != L':') { drive = 0; - if (GetCurrentDirectory(sizeof(pathbuf), pathbuf) && - pathbuf[1] == ':') { - drive = tolower(pathbuf[0]) - 'a' + 1; + if (GetCurrentDirectoryW(_MAX_PATH, pathbuf) && + pathbuf[1] == L':') { + drive = towlower(pathbuf[0]) - L'a' + 1; } - } else if (*name && name[2] == '\0') { + } else if (*name && name[2] == L'\0') { /* * X: and nothing more is an error. */ goto enoent; } else { - drive = tolower(*name) - 'a' + 1; + drive = towlower(*name) - L'a' + 1; } - fh = FindFirstFile(name,&wfd); + fh = FindFirstFileW(name,&wfd); if (fh == INVALID_HANDLE_VALUE) { - if (!(strpbrk(name, "./\\") && - (path = _fullpath(pathbuf, name, _MAX_PATH)) && + if (!(wcspbrk(name, L"./\\") && + (path = _wfullpath(pathbuf, name, _MAX_PATH)) && /* root dir. ('C:\') or UNC root dir. ('\\server\share\') */ - ((strlen(path) == 3) || IsRootUNCName(path)) && - (GetDriveType(path) > 1) ) ) { + ((wcslen(path) == 3) || is_root_unc_name(path)) && + (GetDriveTypeW(path) > 1) ) ) { errno = errno_map(GetLastError()); return check_error(-1, errInfo); } /* * Root directories (such as C:\ or \\server\share\ are fabricated. */ - strcpy(buffer,name); + wcscpy(wbuffer,name); return 1; } - strcpy(buffer,wfd.cAlternateFileName); - if (!*buffer) { - strcpy(buffer,wfd.cFileName); + wcscpy(wbuffer,wfd.cAlternateFileName); + if (!*wbuffer) { + wcscpy(wbuffer,wfd.cFileName); } - + FindClose(fh); return 1; } + int efile_link(Efile_error* errInfo, char* old, char* new) { - errno = ENOTSUP; - return check_error(-1, errInfo); + WCHAR *wold = (WCHAR *) old; + WCHAR *wnew = (WCHAR *) new; + if(!CreateHardLinkW(wnew, wold, NULL)) { + return set_error(errInfo); + } + return 1; } int efile_symlink(Efile_error* errInfo, char* old, char* new) { + /* + * Load dll and see if we have CreateSymbolicLink at runtime: + * (Vista only) + */ + HINSTANCE hModule = NULL; + WCHAR *wold = (WCHAR *) old; + WCHAR *wnew = (WCHAR *) new; + if ((hModule = LoadLibrary("kernel32.dll")) != NULL) { + typedef BOOLEAN (WINAPI * CREATESYMBOLICLINKFUNCPTR) ( + LPCWSTR lpSymlinkFileName, + LPCWSTR lpTargetFileName, + DWORD dwFlags); + + CREATESYMBOLICLINKFUNCPTR pCreateSymbolicLink = + (CREATESYMBOLICLINKFUNCPTR) GetProcAddress(hModule, + "CreateSymbolicLinkW"); + /* A for MBCS, W for UNICODE... char* above implies 'W'! */ + if (pCreateSymbolicLink != NULL) { + DWORD attr = GetFileAttributesW(wold); + int flag = (attr != INVALID_FILE_ATTRIBUTES && + attr & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0; + /* SYMBOLIC_LINK_FLAG_DIRECTORY = 1 */ + BOOLEAN success = pCreateSymbolicLink(wnew, wold, flag); + FreeLibrary(hModule); + + if (success) { + return 1; + } else { + return set_error(errInfo); + } + } else + FreeLibrary(hModule); + } errno = ENOTSUP; return check_error(-1, errInfo); } diff --git a/erts/emulator/hipe/hipe_arm_glue.S b/erts/emulator/hipe/hipe_arm_glue.S index 2bce01954e..8c1c55b216 100644 --- a/erts/emulator/hipe/hipe_arm_glue.S +++ b/erts/emulator/hipe/hipe_arm_glue.S @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 2a877d8ace..e7fb850530 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -450,52 +450,13 @@ BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) } /* - * Memory area for constant Erlang terms. - * - * These constants must not be forwarded by the gc. - * Therefore, the gc needs to be able to distinguish between - * collectible objects and constants. Unfortunately, an Erlang - * process' collectible objects are scattered around in two - * heaps and a list of message buffers, so testing "is X a - * collectible object?" can be expensive. - * - * Instead, constants are placed in a single contiguous area, - * which allows for an inexpensive "is X a constant?" test. - * - * XXX: Allow this area to be grown. + * Statistics on hipe constants: size of HiPE constants, in words. */ - -/* not static, needed by garbage collector */ -Eterm *hipe_constants_start = NULL; -Eterm *hipe_constants_next = NULL; -static unsigned constants_avail_words = 0; -#define CONSTANTS_BYTES (1536*1024*sizeof(Eterm)) /* 1.5 M words */ - -static Eterm *constants_alloc(unsigned nwords) -{ - Eterm *next; - - /* initialise at the first call */ - if ((next = hipe_constants_next) == NULL) { - next = (Eterm*)erts_alloc(ERTS_ALC_T_HIPE, CONSTANTS_BYTES); - hipe_constants_start = next; - hipe_constants_next = next; - constants_avail_words = CONSTANTS_BYTES / sizeof(Eterm); - } - if (nwords > constants_avail_words) { - fprintf(stderr, "Native code constants pool depleted!\r\n"); - /* Must terminate immediately. erl_exit() seems to - continue running some code which then SIGSEGVs. */ - exit(1); - } - constants_avail_words -= nwords; - hipe_constants_next = next + nwords; - return next; -} +unsigned int hipe_constants_size = 0; BIF_RETTYPE hipe_bifs_constants_size_0(BIF_ALIST_0) { - BIF_RET(make_small(hipe_constants_next - hipe_constants_start)); + BIF_RET(make_small(hipe_constants_size)); } /* @@ -526,14 +487,17 @@ static void *const_term_alloc(void *tmpl) { Eterm obj; Uint size; + Uint alloc_size; Eterm *hp; struct const_term *p; obj = (Eterm)tmpl; ASSERT(is_not_immed(obj)); size = size_object(obj); + alloc_size = size + (offsetof(struct const_term, mem)/sizeof(Eterm)); + hipe_constants_size += alloc_size; - p = (struct const_term*)constants_alloc(size + (offsetof(struct const_term, mem)/sizeof(Eterm))); + p = (struct const_term*)erts_alloc(ERTS_ALC_T_HIPE, alloc_size * sizeof(Eterm)); /* I have absolutely no idea if having a private 'off_heap' works or not. _Some_ off_heap object is required for diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index ed27d5616a..c5c1c30619 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -26,10 +26,6 @@ extern Uint *hipe_bifs_find_pc_from_mfa(Eterm mfa); -/* shared with ggc.c -- NOT an official API */ -extern Eterm *hipe_constants_start; -extern Eterm *hipe_constants_next; - extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); extern Eterm hipe_find_na_or_make_stub(Process*, Eterm, Eterm, Eterm); diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c index 8f43811537..2369ad4fa8 100644 --- a/erts/emulator/hipe/hipe_bif1.c +++ b/erts/emulator/hipe/hipe_bif1.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/hipe/hipe_bif2.c b/erts/emulator/hipe/hipe_bif2.c index f992b758be..6bcd5046e9 100644 --- a/erts/emulator/hipe/hipe_bif2.c +++ b/erts/emulator/hipe/hipe_bif2.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,7 +33,6 @@ #include "big.h" #include "hipe_debug.h" #include "hipe_mode_switch.h" -#include "hipe_bif0.h" /* hipe_constants_{start,next} */ #include "hipe_arch.h" #include "hipe_stack.h" @@ -124,18 +123,6 @@ BIF_RETTYPE hipe_bifs_show_term_1(BIF_ALIST_1) BIF_RET(am_true); } -BIF_RETTYPE hipe_bifs_show_literals_0(BIF_ALIST_0) -{ - Eterm *p; - - p = hipe_constants_start; - for (; p < hipe_constants_next; ++p) - printf("0x%0*lx: 0x%0*lx\r\n", - 2*(int)sizeof(long), (unsigned long)p, - 2*(int)sizeof(long), *p); - BIF_RET(am_true); -} - BIF_RETTYPE hipe_bifs_in_native_0(BIF_ALIST_0) { BIF_RET(am_false); diff --git a/erts/emulator/hipe/hipe_bif2.tab b/erts/emulator/hipe/hipe_bif2.tab index d8d627e370..51323ce7af 100644 --- a/erts/emulator/hipe/hipe_bif2.tab +++ b/erts/emulator/hipe/hipe_bif2.tab @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2001-2009. All Rights Reserved. +# Copyright Ericsson AB 2001-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -26,7 +26,6 @@ bif hipe_bifs:show_nstack/1 bif hipe_bifs:nstack_used_size/0 bif hipe_bifs:show_pcb/1 bif hipe_bifs:show_term/1 -bif hipe_bifs:show_literals/0 bif hipe_bifs:in_native/0 bif hipe_bifs:modeswitch_debug_on/0 bif hipe_bifs:modeswitch_debug_off/0 diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 6c9e1d9ba7..a8b6c20dd0 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,7 +28,6 @@ #include "hipe_stack.h" #include "hipe_gc.h" -#include "hipe_bif0.h" /* for hipe_constants_{start,next} */ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) { diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index 900dfc5a8a..25e21ed79e 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index e5de244d25..53ebcd4008 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -208,6 +208,8 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) #endif p->i = NULL; + /* Set current_function to undefined. stdlib hibernate tests rely on it. */ + p->current = NULL; DPRINTF("cmd == %#x (%s)", cmd, code_str(cmd)); HIPE_CHECK_PCB(p); @@ -322,20 +324,31 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) * We need to remove the BIF's parameters from the native * stack: to this end hipe_${ARCH}_glue.S stores the BIF's * arity in p->hipe.narity. + * + * If the BIF emptied the stack (typically hibernate), p->hipe.nsp is + * NULL and there is no need to get rid of stacked parameters. */ - unsigned int i, is_recursive, callee_arity; + unsigned int i, is_recursive = 0; /* Save p->arity, then update it with the original BIF's arity. Get rid of any stacked parameters in that call. */ /* XXX: hipe_call_from_native_is_recursive() copies data to reg[], which is useless in the TRAP case. Maybe write a specialised hipe_trap_from_native_is_recursive() later. */ - callee_arity = p->arity; - p->arity = p->hipe.narity; /* caller's arity */ - is_recursive = hipe_call_from_native_is_recursive(p, reg); - - p->i = (Eterm *)(p->def_arg_reg[3]); - p->arity = callee_arity; + if (p->hipe.nsp != NULL) { + unsigned int callee_arity; + callee_arity = p->arity; + p->arity = p->hipe.narity; /* caller's arity */ + is_recursive = hipe_call_from_native_is_recursive(p, reg); + + p->i = (Eterm *)(p->def_arg_reg[3]); + p->arity = callee_arity; + } + + /* If process is in P_WAITING state, we schedule the next process */ + if (p->status == P_WAITING) { + goto do_schedule; + } for (i = 0; i < p->arity; ++i) reg[i] = p->def_arg_reg[i]; @@ -592,6 +605,17 @@ void hipe_inc_nstack(Process *p) } #endif +void hipe_empty_nstack(Process *p) +{ + if (p->hipe.nstack) { + erts_free(ERTS_ALC_T_HIPE, p->hipe.nstack); + } + p->hipe.nstgraylim = NULL; + p->hipe.nsp = NULL; + p->hipe.nstack = NULL; + p->hipe.nstend = NULL; +} + static void hipe_check_nstack(Process *p, unsigned nwords) { while (hipe_nstack_avail(p) < nwords) diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h index 187b9145e2..e0c6c1b5f5 100644 --- a/erts/emulator/hipe/hipe_mode_switch.h +++ b/erts/emulator/hipe/hipe_mode_switch.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -54,6 +54,7 @@ void hipe_mode_switch_init(void); void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure); Process *hipe_mode_switch(Process*, unsigned, Eterm*); void hipe_inc_nstack(Process *p); +void hipe_empty_nstack(Process *p); void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free); Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s); diff --git a/erts/emulator/hipe/hipe_ppc_glue.S b/erts/emulator/hipe/hipe_ppc_glue.S index c010f4f047..c766099102 100644 --- a/erts/emulator/hipe/hipe_ppc_glue.S +++ b/erts/emulator/hipe/hipe_ppc_glue.S @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. + * Copyright Ericsson AB 2004-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/hipe/hipe_sparc_glue.S b/erts/emulator/hipe/hipe_sparc_glue.S index 73cefd4896..aa07137116 100644 --- a/erts/emulator/hipe/hipe_sparc_glue.S +++ b/erts/emulator/hipe/hipe_sparc_glue.S @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/hipe/hipe_x86_glue.S b/erts/emulator/hipe/hipe_x86_glue.S index 43392111fe..af2d0cb970 100644 --- a/erts/emulator/hipe/hipe_x86_glue.S +++ b/erts/emulator/hipe/hipe_x86_glue.S @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c index 0c61e7bf96..e515f1cd60 100644 --- a/erts/emulator/hipe/hipe_x86_signal.c +++ b/erts/emulator/hipe/hipe_x86_signal.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2009. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/internal_doc/dec.dat b/erts/emulator/internal_doc/dec.dat new file mode 100644 index 0000000000..771ef51baa --- /dev/null +++ b/erts/emulator/internal_doc/dec.dat @@ -0,0 +1,942 @@ +{[59],894}. +{[96],8175}. +{[180],8189}. +{[183],903}. +{[198],1236}. +{[230],1237}. +{[399],1240}. +{[415],1256}. +{[439],1248}. +{[601],1241}. +{[629],1257}. +{[658],1249}. +{[697],884}. +{[768],832}. +{[768,65],192}. +{[768,69],200}. +{[768,73],204}. +{[768,79],210}. +{[768,85],217}. +{[768,87],7808}. +{[768,89],7922}. +{[768,97],224}. +{[768,101],232}. +{[768,105],236}. +{[768,111],242}. +{[768,117],249}. +{[768,119],7809}. +{[768,121],7923}. +{[768,168],8173}. +{[768,770,65],7846}. +{[768,770,69],7872}. +{[768,770,79],7890}. +{[768,770,97],7847}. +{[768,770,101],7873}. +{[768,770,111],7891}. +{[768,772,69],7700}. +{[768,772,79],7760}. +{[768,772,101],7701}. +{[768,772,111],7761}. +{[768,774,65],7856}. +{[768,774,97],7857}. +{[768,776,85],475}. +{[768,776,117],476}. +{[768,776,953],8146}. +{[768,776,965],8162}. +{[768,787,837,913],8074}. +{[768,787,837,919],8090}. +{[768,787,837,937],8106}. +{[768,787,837,945],8066}. +{[768,787,837,951],8082}. +{[768,787,837,969],8098}. +{[768,787,913],7946}. +{[768,787,917],7962}. +{[768,787,919],7978}. +{[768,787,921],7994}. +{[768,787,927],8010}. +{[768,787,937],8042}. +{[768,787,945],7938}. +{[768,787,949],7954}. +{[768,787,951],7970}. +{[768,787,953],7986}. +{[768,787,959],8002}. +{[768,787,965],8018}. +{[768,787,969],8034}. +{[768,788,837,913],8075}. +{[768,788,837,919],8091}. +{[768,788,837,937],8107}. +{[768,788,837,945],8067}. +{[768,788,837,951],8083}. +{[768,788,837,969],8099}. +{[768,788,913],7947}. +{[768,788,917],7963}. +{[768,788,919],7979}. +{[768,788,921],7995}. +{[768,788,927],8011}. +{[768,788,933],8027}. +{[768,788,937],8043}. +{[768,788,945],7939}. +{[768,788,949],7955}. +{[768,788,951],7971}. +{[768,788,953],7987}. +{[768,788,959],8003}. +{[768,788,965],8019}. +{[768,788,969],8035}. +{[768,795,79],7900}. +{[768,795,85],7914}. +{[768,795,111],7901}. +{[768,795,117],7915}. +{[768,837,945],8114}. +{[768,837,951],8130}. +{[768,837,969],8178}. +{[768,913],8122}. +{[768,917],8136}. +{[768,919],8138}. +{[768,921],8154}. +{[768,927],8184}. +{[768,933],8170}. +{[768,937],8186}. +{[768,945],8048}. +{[768,949],8050}. +{[768,951],8052}. +{[768,953],8054}. +{[768,959],8056}. +{[768,965],8058}. +{[768,969],8060}. +{[768,8127],8141}. +{[768,8190],8157}. +{[769],833}. +{[769,65],193}. +{[769,67],262}. +{[769,69],201}. +{[769,71],500}. +{[769,73],205}. +{[769,75],7728}. +{[769,76],313}. +{[769,77],7742}. +{[769,78],323}. +{[769,79],211}. +{[769,80],7764}. +{[769,82],340}. +{[769,83],346}. +{[769,85],218}. +{[769,87],7810}. +{[769,89],221}. +{[769,90],377}. +{[769,97],225}. +{[769,99],263}. +{[769,101],233}. +{[769,103],501}. +{[769,105],237}. +{[769,107],7729}. +{[769,108],314}. +{[769,109],7743}. +{[769,110],324}. +{[769,111],243}. +{[769,112],7765}. +{[769,114],341}. +{[769,115],347}. +{[769,117],250}. +{[769,119],7811}. +{[769,121],253}. +{[769,122],378}. +{[769,168],8174}. +{[769,198],508}. +{[769,216],510}. +{[769,230],509}. +{[769,248],511}. +{[769,770,65],7844}. +{[769,770,69],7870}. +{[769,770,79],7888}. +{[769,770,97],7845}. +{[769,770,101],7871}. +{[769,770,111],7889}. +{[769,771,79],7756}. +{[769,771,85],7800}. +{[769,771,111],7757}. +{[769,771,117],7801}. +{[769,772,69],7702}. +{[769,772,79],7762}. +{[769,772,101],7703}. +{[769,772,111],7763}. +{[769,774,65],7854}. +{[769,774,97],7855}. +{[769,776,73],7726}. +{[769,776,85],471}. +{[769,776,105],7727}. +{[769,776,117],472}. +{[769,776,953],8147}. +{[769,776,965],8163}. +{[769,778,65],506}. +{[769,778,97],507}. +{[769,787,837,913],8076}. +{[769,787,837,919],8092}. +{[769,787,837,937],8108}. +{[769,787,837,945],8068}. +{[769,787,837,951],8084}. +{[769,787,837,969],8100}. +{[769,787,913],7948}. +{[769,787,917],7964}. +{[769,787,919],7980}. +{[769,787,921],7996}. +{[769,787,927],8012}. +{[769,787,937],8044}. +{[769,787,945],7940}. +{[769,787,949],7956}. +{[769,787,951],7972}. +{[769,787,953],7988}. +{[769,787,959],8004}. +{[769,787,965],8020}. +{[769,787,969],8036}. +{[769,788,837,913],8077}. +{[769,788,837,919],8093}. +{[769,788,837,937],8109}. +{[769,788,837,945],8069}. +{[769,788,837,951],8085}. +{[769,788,837,969],8101}. +{[769,788,913],7949}. +{[769,788,917],7965}. +{[769,788,919],7981}. +{[769,788,921],7997}. +{[769,788,927],8013}. +{[769,788,933],8029}. +{[769,788,937],8045}. +{[769,788,945],7941}. +{[769,788,949],7957}. +{[769,788,951],7973}. +{[769,788,953],7989}. +{[769,788,959],8005}. +{[769,788,965],8021}. +{[769,788,969],8037}. +{[769,795,79],7898}. +{[769,795,85],7912}. +{[769,795,111],7899}. +{[769,795,117],7913}. +{[769,807,67],7688}. +{[769,807,99],7689}. +{[769,837,945],8116}. +{[769,837,951],8132}. +{[769,837,959],8180}. +{[769,913],8123}. +{[769,917],8137}. +{[769,919],8139}. +{[769,921],8155}. +{[769,927],8185}. +{[769,933],8171}. +{[769,937],8187}. +{[769,945],8049}. +{[769,949],8051}. +{[769,951],8053}. +{[769,953],8055}. +{[769,959],8057}. +{[769,965],8059}. +{[769,969],8061}. +{[769,1043],1027}. +{[769,1050],1036}. +{[769,1075],1107}. +{[769,1082],1116}. +{[769,8127],8142}. +{[769,8190],8158}. +{[770,65],194}. +{[770,67],264}. +{[770,69],202}. +{[770,71],284}. +{[770,72],292}. +{[770,73],206}. +{[770,74],308}. +{[770,79],212}. +{[770,83],348}. +{[770,85],219}. +{[770,87],372}. +{[770,89],374}. +{[770,90],7824}. +{[770,97],226}. +{[770,99],265}. +{[770,101],234}. +{[770,103],285}. +{[770,104],293}. +{[770,105],238}. +{[770,106],309}. +{[770,111],244}. +{[770,115],349}. +{[770,117],251}. +{[770,119],373}. +{[770,121],375}. +{[770,122],7825}. +{[770,803,65],7852}. +{[770,803,69],7878}. +{[770,803,79],7896}. +{[770,803,97],7853}. +{[770,803,101],7879}. +{[770,803,111],7897}. +{[771,65],195}. +{[771,69],7868}. +{[771,73],296}. +{[771,78],209}. +{[771,79],213}. +{[771,85],360}. +{[771,86],7804}. +{[771,89],7928}. +{[771,97],227}. +{[771,101],7869}. +{[771,105],297}. +{[771,110],241}. +{[771,111],245}. +{[771,117],361}. +{[771,118],7805}. +{[771,121],7929}. +{[771,770,65],7850}. +{[771,770,69],7876}. +{[771,770,79],7894}. +{[771,770,97],7851}. +{[771,770,101],7877}. +{[771,770,111],7895}. +{[771,774,65],7860}. +{[771,774,97],7861}. +{[771,795,79],7904}. +{[771,795,85],7918}. +{[771,795,111],7905}. +{[771,795,117],7919}. +{[772,65],256}. +{[772,69],274}. +{[772,71],7712}. +{[772,73],298}. +{[772,79],332}. +{[772,85],362}. +{[772,97],257}. +{[772,101],275}. +{[772,103],7713}. +{[772,105],299}. +{[772,111],333}. +{[772,117],363}. +{[772,198],482}. +{[772,230],483}. +{[772,775,65],480}. +{[772,775,97],481}. +{[772,776,65],478}. +{[772,776,85],469}. +{[772,776,97],479}. +{[772,776,117],470}. +{[772,803,76],7736}. +{[772,803,82],7772}. +{[772,803,108],7737}. +{[772,803,114],7773}. +{[772,808,79],492}. +{[772,808,111],493}. +{[772,913],8121}. +{[772,921],8153}. +{[772,933],8169}. +{[772,945],8113}. +{[772,953],8145}. +{[772,965],8161}. +{[772,1048],1250}. +{[772,1059],1262}. +{[772,1080],1251}. +{[772,1091],1263}. +{[774,65],258}. +{[774,69],276}. +{[774,71],286}. +{[774,73],300}. +{[774,79],334}. +{[774,85],364}. +{[774,97],259}. +{[774,101],277}. +{[774,103],287}. +{[774,105],301}. +{[774,111],335}. +{[774,117],365}. +{[774,803,65],7862}. +{[774,803,97],7863}. +{[774,807,69],7708}. +{[774,807,101],7709}. +{[774,913],8120}. +{[774,921],8152}. +{[774,933],8168}. +{[774,945],8112}. +{[774,953],8144}. +{[774,965],8160}. +{[774,1040],1232}. +{[774,1045],1238}. +{[774,1046],1217}. +{[774,1048],1049}. +{[774,1059],1038}. +{[774,1072],1233}. +{[774,1077],1239}. +{[774,1078],1218}. +{[774,1080],1081}. +{[774,1091],1118}. +{[775,66],7682}. +{[775,67],266}. +{[775,68],7690}. +{[775,69],278}. +{[775,70],7710}. +{[775,71],288}. +{[775,72],7714}. +{[775,73],304}. +{[775,77],7744}. +{[775,78],7748}. +{[775,80],7766}. +{[775,82],7768}. +{[775,83],7776}. +{[775,84],7786}. +{[775,87],7814}. +{[775,88],7818}. +{[775,89],7822}. +{[775,90],379}. +{[775,98],7683}. +{[775,99],267}. +{[775,100],7691}. +{[775,101],279}. +{[775,102],7711}. +{[775,103],289}. +{[775,104],7715}. +{[775,109],7745}. +{[775,110],7749}. +{[775,112],7767}. +{[775,114],7769}. +{[775,115],7777}. +{[775,116],7787}. +{[775,119],7815}. +{[775,120],7819}. +{[775,121],7823}. +{[775,122],380}. +{[775,383],7835}. +{[775,769,83],7780}. +{[775,769,115],7781}. +{[775,774],784}. +{[775,780,83],7782}. +{[775,780,115],7783}. +{[775,803,83],7784}. +{[775,803,115],7785}. +{[776,65],196}. +{[776,69],203}. +{[776,72],7718}. +{[776,73],207}. +{[776,79],214}. +{[776,85],220}. +{[776,87],7812}. +{[776,88],7820}. +{[776,89],376}. +{[776,97],228}. +{[776,101],235}. +{[776,104],7719}. +{[776,105],239}. +{[776,111],246}. +{[776,116],7831}. +{[776,117],252}. +{[776,119],7813}. +{[776,120],7821}. +{[776,121],255}. +{[776,399],1242}. +{[776,415],1258}. +{[776,601],1243}. +{[776,629],1259}. +{[776,771,79],7758}. +{[776,771,111],7759}. +{[776,772,85],7802}. +{[776,772,117],7803}. +{[776,921],938}. +{[776,933],939}. +{[776,953],970}. +{[776,965],971}. +{[776,978],980}. +{[776,1030],1031}. +{[776,1040],1234}. +{[776,1045],1025}. +{[776,1046],1244}. +{[776,1047],1246}. +{[776,1048],1252}. +{[776,1054],1254}. +{[776,1059],1264}. +{[776,1063],1268}. +{[776,1067],1272}. +{[776,1072],1235}. +{[776,1077],1105}. +{[776,1078],1245}. +{[776,1079],1247}. +{[776,1080],1253}. +{[776,1086],1255}. +{[776,1091],1265}. +{[776,1095],1269}. +{[776,1099],1273}. +{[776,1110],1111}. +{[777,65],7842}. +{[777,69],7866}. +{[777,73],7880}. +{[777,79],7886}. +{[777,85],7910}. +{[777,89],7926}. +{[777,97],7843}. +{[777,101],7867}. +{[777,105],7881}. +{[777,111],7887}. +{[777,117],7911}. +{[777,121],7927}. +{[777,770,65],7848}. +{[777,770,69],7874}. +{[777,770,79],7892}. +{[777,770,97],7849}. +{[777,770,101],7875}. +{[777,770,111],7893}. +{[777,774,65],7858}. +{[777,774,97],7859}. +{[777,795,79],7902}. +{[777,795,85],7916}. +{[777,795,111],7903}. +{[777,795,117],7917}. +{[778,65],197}. +{[778,85],366}. +{[778,97],229}. +{[778,117],367}. +{[778,119],7832}. +{[778,121],7833}. +{[779,79],336}. +{[779,85],368}. +{[779,111],337}. +{[779,117],369}. +{[779,1059],1266}. +{[779,1091],1267}. +{[780,65],461}. +{[780,67],268}. +{[780,68],270}. +{[780,69],282}. +{[780,71],486}. +{[780,73],463}. +{[780,75],488}. +{[780,76],317}. +{[780,78],327}. +{[780,79],465}. +{[780,82],344}. +{[780,83],352}. +{[780,84],356}. +{[780,85],467}. +{[780,90],381}. +{[780,97],462}. +{[780,99],269}. +{[780,100],271}. +{[780,101],283}. +{[780,103],487}. +{[780,105],464}. +{[780,106],496}. +{[780,107],489}. +{[780,108],318}. +{[780,110],328}. +{[780,111],466}. +{[780,114],345}. +{[780,115],353}. +{[780,116],357}. +{[780,117],468}. +{[780,122],382}. +{[780,439],494}. +{[780,658],495}. +{[780,776,85],473}. +{[780,776,117],474}. +{[781,168],901}. +{[781,776],836}. +{[781,776,953],912}. +{[781,776,965],944}. +{[781,913],902}. +{[781,917],904}. +{[781,919],905}. +{[781,921],906}. +{[781,927],908}. +{[781,933],910}. +{[781,937],911}. +{[781,945],940}. +{[781,949],941}. +{[781,951],942}. +{[781,953],943}. +{[781,959],972}. +{[781,965],973}. +{[781,969],974}. +{[781,978],979}. +{[783,65],512}. +{[783,69],516}. +{[783,73],520}. +{[783,79],524}. +{[783,82],528}. +{[783,85],532}. +{[783,97],513}. +{[783,101],517}. +{[783,105],521}. +{[783,111],525}. +{[783,114],529}. +{[783,117],533}. +{[783,1140],1142}. +{[783,1141],1143}. +{[785,65],514}. +{[785,69],518}. +{[785,73],522}. +{[785,79],526}. +{[785,82],530}. +{[785,85],534}. +{[785,97],515}. +{[785,101],519}. +{[785,105],523}. +{[785,111],527}. +{[785,114],531}. +{[785,117],535}. +{[787],835}. +{[787,837,913],8072}. +{[787,837,919],8088}. +{[787,837,937],8104}. +{[787,837,945],8064}. +{[787,837,951],8080}. +{[787,837,969],8096}. +{[787,913],7944}. +{[787,917],7960}. +{[787,919],7976}. +{[787,921],7992}. +{[787,927],8008}. +{[787,937],8040}. +{[787,945],7936}. +{[787,949],7952}. +{[787,951],7968}. +{[787,953],7984}. +{[787,959],8000}. +{[787,961],8164}. +{[787,965],8016}. +{[787,969],8032}. +{[788,837,913],8073}. +{[788,837,919],8089}. +{[788,837,937],8105}. +{[788,837,945],8065}. +{[788,837,951],8081}. +{[788,837,969],8097}. +{[788,913],7945}. +{[788,917],7961}. +{[788,919],7977}. +{[788,921],7993}. +{[788,927],8009}. +{[788,929],8172}. +{[788,933],8025}. +{[788,937],8041}. +{[788,945],7937}. +{[788,949],7953}. +{[788,951],7969}. +{[788,953],7985}. +{[788,959],8001}. +{[788,961],8165}. +{[788,965],8017}. +{[788,969],8033}. +{[795,79],416}. +{[795,85],431}. +{[795,111],417}. +{[795,117],432}. +{[803,65],7840}. +{[803,66],7684}. +{[803,68],7692}. +{[803,69],7864}. +{[803,72],7716}. +{[803,73],7882}. +{[803,75],7730}. +{[803,76],7734}. +{[803,77],7746}. +{[803,78],7750}. +{[803,79],7884}. +{[803,82],7770}. +{[803,83],7778}. +{[803,84],7788}. +{[803,85],7908}. +{[803,86],7806}. +{[803,87],7816}. +{[803,89],7924}. +{[803,90],7826}. +{[803,97],7841}. +{[803,98],7685}. +{[803,100],7693}. +{[803,101],7865}. +{[803,104],7717}. +{[803,105],7883}. +{[803,107],7731}. +{[803,108],7735}. +{[803,109],7747}. +{[803,110],7751}. +{[803,111],7885}. +{[803,114],7771}. +{[803,115],7779}. +{[803,116],7789}. +{[803,117],7909}. +{[803,118],7807}. +{[803,119],7817}. +{[803,121],7925}. +{[803,122],7827}. +{[803,795,79],7906}. +{[803,795,85],7920}. +{[803,795,111],7907}. +{[803,795,117],7921}. +{[804,85],7794}. +{[804,117],7795}. +{[805,65],7680}. +{[805,97],7681}. +{[807,67],199}. +{[807,68],7696}. +{[807,71],290}. +{[807,72],7720}. +{[807,75],310}. +{[807,76],315}. +{[807,78],325}. +{[807,82],342}. +{[807,83],350}. +{[807,84],354}. +{[807,99],231}. +{[807,100],7697}. +{[807,103],291}. +{[807,104],7721}. +{[807,107],311}. +{[807,108],316}. +{[807,110],326}. +{[807,114],343}. +{[807,115],351}. +{[807,116],355}. +{[808,65],260}. +{[808,69],280}. +{[808,73],302}. +{[808,79],490}. +{[808,85],370}. +{[808,97],261}. +{[808,101],281}. +{[808,105],303}. +{[808,111],491}. +{[808,117],371}. +{[813,68],7698}. +{[813,69],7704}. +{[813,76],7740}. +{[813,78],7754}. +{[813,84],7792}. +{[813,85],7798}. +{[813,100],7699}. +{[813,101],7705}. +{[813,108],7741}. +{[813,110],7755}. +{[813,116],7793}. +{[813,117],7799}. +{[814,72],7722}. +{[814,104],7723}. +{[816,69],7706}. +{[816,73],7724}. +{[816,85],7796}. +{[816,101],7707}. +{[816,105],7725}. +{[816,117],7797}. +{[817,66],7686}. +{[817,68],7694}. +{[817,75],7732}. +{[817,76],7738}. +{[817,78],7752}. +{[817,82],7774}. +{[817,84],7790}. +{[817,90],7828}. +{[817,98],7687}. +{[817,100],7695}. +{[817,104],7830}. +{[817,107],7733}. +{[817,108],7739}. +{[817,110],7753}. +{[817,114],7775}. +{[817,116],7791}. +{[817,122],7829}. +{[834,168],8129}. +{[834,776,953],8151}. +{[834,776,965],8167}. +{[834,787,837,913],8078}. +{[834,787,837,919],8094}. +{[834,787,837,937],8110}. +{[834,787,837,945],8070}. +{[834,787,837,951],8086}. +{[834,787,837,969],8102}. +{[834,787,913],7950}. +{[834,787,919],7982}. +{[834,787,921],7998}. +{[834,787,937],8046}. +{[834,787,945],7942}. +{[834,787,951],7974}. +{[834,787,953],7990}. +{[834,787,965],8022}. +{[834,787,969],8038}. +{[834,788,837,913],8079}. +{[834,788,837,919],8095}. +{[834,788,837,937],8111}. +{[834,788,837,945],8071}. +{[834,788,837,951],8087}. +{[834,788,837,969],8103}. +{[834,788,913],7951}. +{[834,788,919],7983}. +{[834,788,921],7999}. +{[834,788,933],8031}. +{[834,788,937],8047}. +{[834,788,945],7943}. +{[834,788,951],7975}. +{[834,788,953],7991}. +{[834,788,965],8023}. +{[834,788,969],8039}. +{[834,837,945],8119}. +{[834,837,951],8135}. +{[834,837,969],8183}. +{[834,945],8118}. +{[834,951],8134}. +{[834,953],8150}. +{[834,965],8166}. +{[834,969],8182}. +{[834,8127],8143}. +{[834,8190],8159}. +{[837,913],8124}. +{[837,919],8140}. +{[837,937],8188}. +{[837,945],8115}. +{[837,951],8131}. +{[837,969],8179}. +{[953],8126}. +{[1463,1488],64302}. +{[1463,1522],64287}. +{[1464,1488],64303}. +{[1465,1493],64331}. +{[1468,1488],64304}. +{[1468,1489],64305}. +{[1468,1490],64306}. +{[1468,1491],64307}. +{[1468,1492],64308}. +{[1468,1493],64309}. +{[1468,1494],64310}. +{[1468,1496],64312}. +{[1468,1497],64313}. +{[1468,1498],64314}. +{[1468,1499],64315}. +{[1468,1500],64316}. +{[1468,1502],64318}. +{[1468,1504],64320}. +{[1468,1505],64321}. +{[1468,1507],64323}. +{[1468,1508],64324}. +{[1468,1510],64326}. +{[1468,1511],64327}. +{[1468,1512],64328}. +{[1468,1513],64329}. +{[1468,1514],64330}. +{[1471,1489],64332}. +{[1471,1499],64333}. +{[1471,1508],64334}. +{[1473,1468,1513],64300}. +{[1473,1513],64298}. +{[1474,1468,1513],64301}. +{[1474,1513],64299}. +{[2364,2325],2392}. +{[2364,2326],2393}. +{[2364,2327],2394}. +{[2364,2332],2395}. +{[2364,2337],2396}. +{[2364,2338],2397}. +{[2364,2344],2345}. +{[2364,2347],2398}. +{[2364,2351],2399}. +{[2364,2352],2353}. +{[2364,2355],2356}. +{[2492,2465],2524}. +{[2492,2466],2525}. +{[2492,2476],2480}. +{[2492,2479],2527}. +{[2494,2503],2507}. +{[2519,2503],2508}. +{[2620,2582],2649}. +{[2620,2583],2650}. +{[2620,2588],2651}. +{[2620,2593],2652}. +{[2620,2603],2654}. +{[2876,2849],2908}. +{[2876,2850],2909}. +{[2876,2863],2911}. +{[2878,2887],2891}. +{[2902,2887],2888}. +{[2903,2887],2892}. +{[3006,3014],3018}. +{[3006,3015],3019}. +{[3031,2962],2964}. +{[3031,3014],3020}. +{[3158,3142],3144}. +{[3266,3270],3274}. +{[3285,3263],3264}. +{[3285,3266,3270],3275}. +{[3285,3270],3271}. +{[3286,3270],3272}. +{[3390,3398],3402}. +{[3390,3399],3403}. +{[3415,3398],3404}. +{[3634,3661],3635}. +{[3762,3789],3763}. +{[3953,3954],3955}. +{[3953,3956],3957}. +{[3953,3968],3969}. +{[3953,3968,4018],3959}. +{[3953,3968,4019],3961}. +{[3968,4018],3958}. +{[3968,4019],3960}. +{[4021,3904],3945}. +{[4021,3984],4025}. +{[4023,3906],3907}. +{[4023,3916],3917}. +{[4023,3921],3922}. +{[4023,3926],3927}. +{[4023,3931],3932}. +{[4023,3986],3987}. +{[4023,3996],3997}. +{[4023,4001],4002}. +{[4023,4006],4007}. +{[4023,4011],4012}. +{[12441,12358],12436}. +{[12441,12363],12364}. +{[12441,12365],12366}. +{[12441,12367],12368}. +{[12441,12369],12370}. +{[12441,12371],12372}. +{[12441,12373],12374}. +{[12441,12375],12376}. +{[12441,12377],12378}. +{[12441,12379],12380}. +{[12441,12381],12382}. +{[12441,12383],12384}. +{[12441,12385],12386}. +{[12441,12388],12389}. +{[12441,12390],12391}. +{[12441,12392],12393}. +{[12441,12399],12400}. +{[12441,12402],12403}. +{[12441,12405],12406}. +{[12441,12408],12409}. +{[12441,12411],12412}. +{[12441,12445],12446}. +{[12441,12454],12532}. +{[12441,12459],12460}. +{[12441,12461],12462}. +{[12441,12463],12464}. +{[12441,12465],12466}. +{[12441,12467],12468}. +{[12441,12469],12470}. +{[12441,12471],12472}. +{[12441,12473],12474}. +{[12441,12475],12476}. +{[12441,12477],12478}. +{[12441,12479],12480}. +{[12441,12481],12482}. +{[12441,12484],12485}. +{[12441,12486],12487}. +{[12441,12488],12489}. +{[12441,12495],12496}. +{[12441,12498],12499}. +{[12441,12501],12502}. +{[12441,12504],12505}. +{[12441,12507],12508}. +{[12441,12527],12535}. +{[12441,12528],12536}. +{[12441,12529],12537}. +{[12441,12530],12538}. +{[12441,12541],12542}. +{[12442,12399],12401}. +{[12442,12402],12404}. +{[12442,12405],12407}. +{[12442,12408],12410}. +{[12442,12411],12413}. +{[12442,12495],12497}. +{[12442,12498],12500}. +{[12442,12501],12503}. +{[12442,12504],12506}. +{[12442,12507],12509}. diff --git a/erts/emulator/internal_doc/dec.erl b/erts/emulator/internal_doc/dec.erl new file mode 100644 index 0000000000..0315f2a52d --- /dev/null +++ b/erts/emulator/internal_doc/dec.erl @@ -0,0 +1,237 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2000-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% This program is used to generate a header file with data for +%% normalizing denormalized unicode. + +%% The C header is generated from a text file containing tuples in the +%% following format: +%% {RevList,Translation} +%% Where 'RevList' is a reversed list of the denormalized repressentation of +%% the character 'Translation'. An example would be the swedish character +%% '�', which would be represented in the file as: +%% {[776,111],246}, as the denormalized representation of codepoint 246 +%% is [111,776] (i.e an 'o' followed by the "double dot accent character 776), +%% while '�' instead is represented as {[776,97],228}, as the denormalized +%% form would be [97,776] (same accent but an 'a' instead). +%% The datafile is generated from the table on Apple's developer connection +%% http://developer.apple.com/library/mac/#technotes/tn/tn1150table.html +%% The generating is done whenever new data is present (i.e. dec.dat has +%% to be changed) and not for every build. The product (the C header) is copied +%% to $ERL_TOP/erts/beam after generation and checked in. +%% The program and the data file is included for reference. + +-module(dec). + +-compile(export_all). + +-define(HASH_SIZE_FACTOR,2). +-define(BIG_PREFIX_SIZE,392). + +-define(INPUT_FILE_NAME,"dec.dat"). +-define(OUTPUT_FILE_NAME,"erl_unicode_normalize.h"). + +read(FName) -> + {ok,L} = file:consult(FName), + [{A,B} || {A,B} <- L, + length(A) > 1% , hd(A) < 769 + ]. + +dec() -> + L = read(?INPUT_FILE_NAME), + G = group(L), + {ok,Out} = file:open(?OUTPUT_FILE_NAME,[write]), + io:format + (Out, + "/*~n" + "* %CopyrightBegin%~n" + "*~n" + "* Copyright Ericsson AB 1999-2010. All Rights Reserved.~n" + "*~n" + "* The contents of this file are subject to the Erlang Public License,~n" + "* Version 1.1, (the \"License\"); you may not use this file except in~n" + "* compliance with the License. You should have received a copy of the~n" + "* Erlang Public License along with this software. If not, it can be~n" + "* retrieved online at http://www.erlang.org/.~n" + "*~n" + "* Software distributed under the License is distributed on an " + "\"AS IS\"~n" + "* basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See~n" + "* the License for the specific language governing rights and " + "limitations~n" + "* under the License.~n" + "*~n" + "* %CopyrightEnd%~n" + "*/~n" + "/*~n" + "* This file is automatically generated by ~p.erl, " + "do not edit manually~n" + "*/~n", + [?MODULE]), + + io:format(Out, + "#define HASH_SIZE_FACTOR ~w~n" + "typedef struct _compose_entry {~n" + " Uint16 c;~n" + " Uint16 res;~n" + " Uint16 num_subs;~n" + " struct _compose_entry *subs;~n" + " int *hash;~n" + "} CompEntry;~n~n" + "static int compose_tab_size = ~p;~n", + [?HASH_SIZE_FACTOR,length(G)]), + d(Out,G,[],0), + PreTab = tuple_to_list(make_prefix_table(G,erlang:make_tuple(102,0))), + dump_prefixes(Out,PreTab), +%% Using this cuts down on the searching in the +%% actual implementation, but wastes memory with little real gain.. +%% LL = lists:flatten([PartList || {PartList,_} <- L]), +%% BigPreTab = tuple_to_list( +%% make_big_prefixes(LL, +%% erlang:make_tuple(?BIG_PREFIX_SIZE,0))), +%% dump_big_prefixes(Out,BigPreTab), + file:close(Out), + ok. + + + +d(Out,List,D,C) -> + d_sub(Out,List,D,C), + d_top_hash(Out,List,D,C), + d_top(Out,List,D,C). +d_sub(_Out,[],_D,_C) -> + ok; +d_sub(Out,[{_CP,[],_Res}|T],D,C) -> + d_sub(Out,T,D,C+1); +d_sub(Out,[{_CP,Subs,_Res0}|T],D,C) -> + d(Out,Subs,[C|D],0), + d_sub(Out,T,D,C+1). +d_top(Out,L,D,C) -> + io:format(Out,"static CompEntry ~s[] = {~n",[format_depth(D)]), + d_top_1(Out,L,D,C), + io:format(Out,"}; /* ~s */ ~n",[format_depth(D)]). + +d_top_1(_Out,[],_D,_C) -> + ok; +d_top_1(Out,[{CP,[],Res}|T],D,C) -> + io:format(Out, + "{~w, ~w, 0, NULL, NULL}",[CP,Res]), + if + T =:= [] -> + io:format(Out,"~n",[]); + true -> + io:format(Out,",~n",[]) + end, + d_top_1(Out,T,D,C+1); +d_top_1(Out,[{CP,Subs,_Res}|T],D,C) -> + io:format(Out, + "{~w, 0, ~w, ~s, ~s}",[CP,length(Subs), + format_depth([C|D]), + "hash_"++format_depth([C|D])]), + if + T =:= [] -> + io:format(Out,"~n",[]); + true -> + io:format(Out,",~n",[]) + end, + d_top_1(Out,T,D,C+1). + + +d_top_hash(Out,List,D,_C) -> + HSize = length(List)*?HASH_SIZE_FACTOR, + io:format(Out,"static int ~s[~p] = ~n",["hash_"++format_depth(D),HSize]), + Tup = d_top_hash_1(List,0,erlang:make_tuple(HSize,-1),HSize), + io:format(Out,"~p; /* ~s */ ~n",[Tup,"hash_"++format_depth(D)]). + +d_top_hash_1([],_,Hash,_HSize) -> + Hash; +d_top_hash_1([{CP,_,_}|T],Index,Hash,HSize) -> + Bucket = hash_search(Hash,HSize,CP rem HSize), + d_top_hash_1(T,Index+1,erlang:setelement(Bucket+1,Hash,Index),HSize). + +hash_search(Hash,_HSize,Bucket) when element(Bucket+1,Hash) =:= -1 -> + Bucket; +hash_search(Hash,HSize,Bucket) -> + hash_search(Hash,HSize,(Bucket + 1) rem HSize). + +format_depth(D) -> + lists:reverse(tl(lists:reverse(lists:flatten(["compose_tab_",[ integer_to_list(X) ++ "_" || X <- lists:reverse(D) ]])))). + + + + +make_prefix_table([],Table) -> + Table; +make_prefix_table([{C,_,_}|T],Table) when C =< 4023 -> + Index = (C div 32) + 1 - 24, + Pos = C rem 32, + X = element(Index,Table), + Y = X bor (1 bsl Pos), + NewTab = setelement(Index,Table,Y), + make_prefix_table(T,NewTab); +make_prefix_table([_|T],Tab) -> + make_prefix_table(T,Tab). + +dump_prefixes(Out,L) -> + io:format(Out,"#define COMP_CANDIDATE_MAP_OFFSET 24~n",[]), + io:format(Out,"static Uint32 comp_candidate_map[] = {~n",[]), + dump_prefixes_1(Out,L). +dump_prefixes_1(Out,[H]) -> + io:format(Out," 0x~8.16.0BU~n",[H]), + io:format(Out,"};~n",[]); +dump_prefixes_1(Out,[H|T]) -> + io:format(Out," 0x~8.16.0BU,~n",[H]), + dump_prefixes_1(Out,T). + +%% make_big_prefixes([],Table) -> +%% Table; +%% make_big_prefixes([C|T],Table) -> +%% Index = (C div 32) + 1, +%% Pos = C rem 32, +%% X = element(Index,Table), +%% Y = X bor (1 bsl Pos), +%% NewTab = setelement(Index,Table,Y), +%% make_big_prefixes(T,NewTab). + +%% dump_big_prefixes(Out,L) -> +%% io:format(Out,"#define BIG_COMP_CANDIDATE_SIZE ~w~n", [?BIG_PREFIX_SIZE]), +%% io:format(Out,"static Uint32 big_comp_candidate_map[] = {~n",[]), +%% dump_prefixes_1(Out,L). + +pick([],_,Acc) -> + {lists:reverse(Acc),[]}; +pick([{[H|TT],N}|T],H,Acc) -> + pick(T,H,[{TT,N}|Acc]); +pick([{[H|_],_}|_]=L,M,Acc) when H =/= M -> + {lists:reverse(Acc),L}. + + +group([]) -> + []; +group([{[H],N}|T]) -> + {Part,Rest} = pick(T,H,[]), + [{H,group(Part),N}| group(Rest)]; +group([{[H|_],_}|_]=L) -> + {Part,Rest} = pick(L,H,[]), + [{H,group(Part),0}| group(Rest)]. + + + + + diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index b1ee165489..ceb290b644 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2010. All Rights Reserved. + * Copyright Ericsson AB 2002-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -35,6 +35,7 @@ #include "global.h" #include "erl_threads.h" #include "erl_mtrace.h" +#include "erl_time.h" #include "big.h" #if HAVE_ERTS_MSEG @@ -76,8 +77,10 @@ static int atoms_initialized; static Uint cache_check_interval; +typedef struct mem_kind_t MemKind; + static void check_cache(void *unused); -static void mseg_clear_cache(void); +static void mseg_clear_cache(MemKind*); static int is_cache_check_scheduled; #ifdef ERTS_THREADS_NO_SMP static int is_cache_check_requested; @@ -122,6 +125,9 @@ static int mmap_fd; #error "Not supported" #endif /* #if HAVE_MMAP */ +#if defined(ERTS_MSEG_FAKE_SEGMENTS) && HALFWORD_HEAP +# warning "ERTS_MSEG_FAKE_SEGMENTS will only be used for high memory segments" +#endif #if defined(ERTS_MSEG_FAKE_SEGMENTS) #undef CAN_PARTLY_DESTROY @@ -159,14 +165,43 @@ static struct { CallCounter check_cache; } calls; -static cache_desc_t cache_descs[MAX_CACHE_SIZE]; -static cache_desc_t *free_cache_descs; -static cache_desc_t *cache; -static cache_desc_t *cache_end; -static Uint cache_hits; -static Uint cache_size; -static Uint min_cached_seg_size; -static Uint max_cached_seg_size; +struct mem_kind_t { + cache_desc_t cache_descs[MAX_CACHE_SIZE]; + cache_desc_t *free_cache_descs; + cache_desc_t *cache; + cache_desc_t *cache_end; + + Uint cache_size; + Uint min_cached_seg_size; + Uint max_cached_seg_size; + Uint cache_hits; + + struct { + struct { + Uint watermark; + Uint no; + Uint sz; + } current; + struct { + Uint no; + Uint sz; + } max; + struct { + Uint no; + Uint sz; + } max_ever; + } segments; + + const char* name; + MemKind* next; +};/*MemKind*/ + +#if HALFWORD_HEAP +static MemKind low_mem, hi_mem; +#else +static MemKind the_mem; +#endif +static MemKind* mk_list = NULL; static Uint max_cache_size; static Uint abs_max_cache_bad_fit; @@ -176,47 +211,32 @@ static Uint rel_max_cache_bad_fit; static Uint min_seg_size; #endif -struct { - struct { - Uint watermark; - Uint no; - Uint sz; - } current; - struct { - Uint no; - Uint sz; - } max; - struct { - Uint no; - Uint sz; - } max_ever; -} segments; -#define ERTS_MSEG_ALLOC_STAT(SZ) \ +#define ERTS_MSEG_ALLOC_STAT(C,SZ) \ do { \ - segments.current.no++; \ - if (segments.max.no < segments.current.no) \ - segments.max.no = segments.current.no; \ - if (segments.current.watermark < segments.current.no) \ - segments.current.watermark = segments.current.no; \ - segments.current.sz += (SZ); \ - if (segments.max.sz < segments.current.sz) \ - segments.max.sz = segments.current.sz; \ + C->segments.current.no++; \ + if (C->segments.max.no < C->segments.current.no) \ + C->segments.max.no = C->segments.current.no; \ + if (C->segments.current.watermark < C->segments.current.no) \ + C->segments.current.watermark = C->segments.current.no; \ + C->segments.current.sz += (SZ); \ + if (C->segments.max.sz < C->segments.current.sz) \ + C->segments.max.sz = C->segments.current.sz; \ } while (0) -#define ERTS_MSEG_DEALLOC_STAT(SZ) \ +#define ERTS_MSEG_DEALLOC_STAT(C,SZ) \ do { \ - ASSERT(segments.current.no > 0); \ - segments.current.no--; \ - ASSERT(segments.current.sz >= (SZ)); \ - segments.current.sz -= (SZ); \ + ASSERT(C->segments.current.no > 0); \ + C->segments.current.no--; \ + ASSERT(C->segments.current.sz >= (SZ)); \ + C->segments.current.sz -= (SZ); \ } while (0) -#define ERTS_MSEG_REALLOC_STAT(OSZ, NSZ) \ +#define ERTS_MSEG_REALLOC_STAT(C,OSZ, NSZ) \ do { \ - ASSERT(segments.current.sz >= (OSZ)); \ - segments.current.sz -= (OSZ); \ - segments.current.sz += (NSZ); \ + ASSERT(C->segments.current.sz >= (OSZ)); \ + C->segments.current.sz -= (OSZ); \ + C->segments.current.sz += (NSZ); \ } while (0) #define ONE_GIGA (1000000000) @@ -271,7 +291,7 @@ schedule_cache_check(void) #endif { cache_check_timer.active = 0; - erl_set_timer(&cache_check_timer, + erts_set_timer(&cache_check_timer, check_cache, NULL, NULL, @@ -302,38 +322,45 @@ check_schedule_cache_check(void) static void mseg_shutdown(void) { + MemKind* mk; erts_mtx_lock(&mseg_mutex); - mseg_clear_cache(); + for (mk=mk_list; mk; mk=mk->next) { + mseg_clear_cache(mk); + } erts_mtx_unlock(&mseg_mutex); } static ERTS_INLINE void * -mseg_create(Uint size) +mseg_create(MemKind* mk, Uint size) { void *seg; ASSERT(size % page_size == 0); -#if defined(ERTS_MSEG_FAKE_SEGMENTS) - seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size); -#elif HAVE_MMAP #if HALFWORD_HEAP - seg = pmmap(size); -#else - seg = (void *) mmap((void *) 0, (size_t) size, - MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0); - if (seg == (void *) MAP_FAILED) - seg = NULL; -#endif -#if HALFWORD_HEAP - if ((unsigned long) seg & CHECK_POINTER_MASK) { - erts_fprintf(stderr,"Pointer mask failure (0x%08lx)\n",(unsigned long) seg); - return NULL; + if (mk == &low_mem) { + seg = pmmap(size); + if ((unsigned long) seg & CHECK_POINTER_MASK) { + erts_fprintf(stderr,"Pointer mask failure (0x%08lx)\n",(unsigned long) seg); + return NULL; + } } + else #endif + { +#if defined(ERTS_MSEG_FAKE_SEGMENTS) + seg = erts_sys_alloc(ERTS_ALC_N_INVALID, NULL, size); +#elif HAVE_MMAP + { + seg = (void *) mmap((void *) 0, (size_t) size, + MMAP_PROT, MMAP_FLAGS, MMAP_FD, 0); + if (seg == (void *) MAP_FAILED) + seg = NULL; + } #else -#error "Missing mseg_create() implementation" +# error "Missing mseg_create() implementation" #endif + } INC_CC(create); @@ -341,25 +368,29 @@ mseg_create(Uint size) } static ERTS_INLINE void -mseg_destroy(void *seg, Uint size) +mseg_destroy(MemKind* mk, void *seg, Uint size) { -#if defined(ERTS_MSEG_FAKE_SEGMENTS) - erts_sys_free(ERTS_ALC_N_INVALID, NULL, seg); -#elif HAVE_MMAP + int res; -#ifdef DEBUG - int res = -#endif #if HALFWORD_HEAP - pmunmap((void *) seg, size); + if (mk == &low_mem) { + res = pmunmap((void *) seg, size); + } + else +#endif + { +#ifdef ERTS_MSEG_FAKE_SEGMENTS + erts_sys_free(ERTS_ALC_N_INVALID, NULL, seg); + res = 0; +#elif HAVE_MMAP + res = munmap((void *) seg, size); #else - munmap((void *) seg, size); +# error "Missing mseg_destroy() implementation" #endif + } + ASSERT(size % page_size == 0); ASSERT(res == 0); -#else -#error "Missing mseg_destroy() implementation" -#endif INC_CC(destroy); @@ -368,39 +399,44 @@ mseg_destroy(void *seg, Uint size) #if HAVE_MSEG_RECREATE static ERTS_INLINE void * -mseg_recreate(void *old_seg, Uint old_size, Uint new_size) +mseg_recreate(MemKind* mk, void *old_seg, Uint old_size, Uint new_size) { void *new_seg; ASSERT(old_size % page_size == 0); ASSERT(new_size % page_size == 0); -#if defined(ERTS_MSEG_FAKE_SEGMENTS) - new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size); -#elif HAVE_MREMAP #if HALFWORD_HEAP - new_seg = (void *) pmremap((void *) old_seg, - (size_t) old_size, - (size_t) new_size); -#elif defined(__NetBSD__) - new_seg = (void *) mremap((void *) old_seg, - (size_t) old_size, - NULL, - (size_t) new_size, - 0); - if (new_seg == (void *) MAP_FAILED) - new_seg = NULL; -#else - new_seg = (void *) mremap((void *) old_seg, - (size_t) old_size, - (size_t) new_size, - MREMAP_MAYMOVE); - if (new_seg == (void *) MAP_FAILED) - new_seg = NULL; + if (mk == &low_mem) { + new_seg = (void *) pmremap((void *) old_seg, + (size_t) old_size, + (size_t) new_size); + } + else #endif + { +#if defined(ERTS_MSEG_FAKE_SEGMENTS) + new_seg = erts_sys_realloc(ERTS_ALC_N_INVALID, NULL, old_seg, new_size); +#elif HAVE_MREMAP + + #if defined(__NetBSD__) + new_seg = (void *) mremap((void *) old_seg, + (size_t) old_size, + NULL, + (size_t) new_size, + 0); + #else + new_seg = (void *) mremap((void *) old_seg, + (size_t) old_size, + (size_t) new_size, + MREMAP_MAYMOVE); + #endif + if (new_seg == (void *) MAP_FAILED) + new_seg = NULL; #else #error "Missing mseg_recreate() implementation" #endif + } INC_CC(recreate); @@ -411,134 +447,142 @@ mseg_recreate(void *old_seg, Uint old_size, Uint new_size) static ERTS_INLINE cache_desc_t * -alloc_cd(void) +alloc_cd(MemKind* mk) { - cache_desc_t *cd = free_cache_descs; + cache_desc_t *cd = mk->free_cache_descs; ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); if (cd) - free_cache_descs = cd->next; + mk->free_cache_descs = cd->next; return cd; } static ERTS_INLINE void -free_cd(cache_desc_t *cd) +free_cd(MemKind* mk, cache_desc_t *cd) { ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); - cd->next = free_cache_descs; - free_cache_descs = cd; + cd->next = mk->free_cache_descs; + mk->free_cache_descs = cd; } static ERTS_INLINE void -link_cd(cache_desc_t *cd) +link_cd(MemKind* mk, cache_desc_t *cd) { ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); - if (cache) - cache->prev = cd; - cd->next = cache; + if (mk->cache) + mk->cache->prev = cd; + cd->next = mk->cache; cd->prev = NULL; - cache = cd; + mk->cache = cd; - if (!cache_end) { + if (!mk->cache_end) { ASSERT(!cd->next); - cache_end = cd; + mk->cache_end = cd; } - cache_size++; + mk->cache_size++; } +#if CAN_PARTLY_DESTROY static ERTS_INLINE void -end_link_cd(cache_desc_t *cd) +end_link_cd(MemKind* mk, cache_desc_t *cd) { ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); - if (cache_end) - cache_end->next = cd; + if (mk->cache_end) + mk->cache_end->next = cd; cd->next = NULL; - cd->prev = cache_end; - cache_end = cd; + cd->prev = mk->cache_end; + mk->cache_end = cd; - if (!cache) { + if (!mk->cache) { ASSERT(!cd->prev); - cache = cd; + mk->cache = cd; } - cache_size++; + mk->cache_size++; } +#endif static ERTS_INLINE void -unlink_cd(cache_desc_t *cd) +unlink_cd(MemKind* mk, cache_desc_t *cd) { ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); if (cd->next) cd->next->prev = cd->prev; else - cache_end = cd->prev; + mk->cache_end = cd->prev; if (cd->prev) cd->prev->next = cd->next; else - cache = cd->next; - ASSERT(cache_size > 0); - cache_size--; + mk->cache = cd->next; + ASSERT(mk->cache_size > 0); + mk->cache_size--; } static ERTS_INLINE void -check_cache_limits(void) +check_cache_limits(MemKind* mk) { cache_desc_t *cd; ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); - max_cached_seg_size = 0; - min_cached_seg_size = ~((Uint) 0); - for (cd = cache; cd; cd = cd->next) { - if (cd->size < min_cached_seg_size) - min_cached_seg_size = cd->size; - if (cd->size > max_cached_seg_size) - max_cached_seg_size = cd->size; + mk->max_cached_seg_size = 0; + mk->min_cached_seg_size = ~((Uint) 0); + for (cd = mk->cache; cd; cd = cd->next) { + if (cd->size < mk->min_cached_seg_size) + mk->min_cached_seg_size = cd->size; + if (cd->size > mk->max_cached_seg_size) + mk->max_cached_seg_size = cd->size; } - } static ERTS_INLINE void -adjust_cache_size(int force_check_limits) +adjust_cache_size(MemKind* mk, int force_check_limits) { cache_desc_t *cd; int check_limits = force_check_limits; - Sint max_cached = ((Sint) segments.current.watermark - - (Sint) segments.current.no); + Sint max_cached = ((Sint) mk->segments.current.watermark + - (Sint) mk->segments.current.no); ERTS_LC_ASSERT(erts_lc_mtx_is_locked(&mseg_mutex)); - while (((Sint) cache_size) > max_cached && ((Sint) cache_size) > 0) { - ASSERT(cache_end); - cd = cache_end; + while (((Sint) mk->cache_size) > max_cached && ((Sint) mk->cache_size) > 0) { + ASSERT(mk->cache_end); + cd = mk->cache_end; if (!check_limits && - !(min_cached_seg_size < cd->size - && cd->size < max_cached_seg_size)) { + !(mk->min_cached_seg_size < cd->size + && cd->size < mk->max_cached_seg_size)) { check_limits = 1; } if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg); - mseg_destroy(cd->seg, cd->size); - unlink_cd(cd); - free_cd(cd); + mseg_destroy(mk, cd->seg, cd->size); + unlink_cd(mk,cd); + free_cd(mk,cd); } if (check_limits) - check_cache_limits(); - + check_cache_limits(mk); } static void -check_cache(void *unused) +check_one_cache(MemKind* mk) { + if (mk->segments.current.watermark > mk->segments.current.no) + mk->segments.current.watermark--; + adjust_cache_size(mk, 0); + + if (mk->cache_size) + schedule_cache_check(); +} + +static void check_cache(void* unused) +{ + MemKind* mk; erts_mtx_lock(&mseg_mutex); is_cache_check_scheduled = 0; - if (segments.current.watermark > segments.current.no) - segments.current.watermark--; - adjust_cache_size(0); - - if (cache_size) - schedule_cache_check(); + for (mk=mk_list; mk; mk=mk->next) { + check_one_cache(mk); + } INC_CC(check_cache); @@ -546,28 +590,45 @@ check_cache(void *unused) } static void -mseg_clear_cache(void) +mseg_clear_cache(MemKind* mk) { - segments.current.watermark = 0; + mk->segments.current.watermark = 0; - adjust_cache_size(1); + adjust_cache_size(mk, 1); - ASSERT(!cache); - ASSERT(!cache_end); - ASSERT(!cache_size); + ASSERT(!mk->cache); + ASSERT(!mk->cache_end); + ASSERT(!mk->cache_size); - segments.current.watermark = segments.current.no; + mk->segments.current.watermark = mk->segments.current.no; INC_CC(clear_cache); } +static ERTS_INLINE MemKind* type2mk(ErtsAlcType_t atype) +{ +#if HALFWORD_HEAP + switch (atype) { + case ERTS_ALC_A_ETS: + case ERTS_ALC_A_BINARY: + case ERTS_ALC_A_FIXED_SIZE: + case ERTS_ALC_A_DRIVER: + return &hi_mem; + default: + return &low_mem; + } +#else + return &the_mem; +#endif +} + static void * mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) { - Uint max, min, diff_size, size; cache_desc_t *cd, *cand_cd; void *seg; + MemKind* mk = type2mk(atype); INC_CC(alloc); @@ -580,11 +641,11 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) if (!opt->cache) { create_seg: - adjust_cache_size(0); - seg = mseg_create(size); + adjust_cache_size(mk,0); + seg = mseg_create(mk, size); if (!seg) { - mseg_clear_cache(); - seg = mseg_create(size); + mseg_clear_cache(mk); + seg = mseg_create(mk, size); if (!seg) size = 0; } @@ -593,17 +654,17 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) if (seg) { if (erts_mtrace_enabled) erts_mtrace_crr_alloc(seg, atype, ERTS_MTRACE_SEGMENT_ID, size); - ERTS_MSEG_ALLOC_STAT(size); + ERTS_MSEG_ALLOC_STAT(mk,size); } return seg; } - if (size > max_cached_seg_size) + if (size > mk->max_cached_seg_size) goto create_seg; - if (size < min_cached_seg_size) { + if (size < mk->min_cached_seg_size) { - diff_size = min_cached_seg_size - size; + diff_size = mk->min_cached_seg_size - size; if (diff_size > abs_max_cache_bad_fit) goto create_seg; @@ -617,7 +678,7 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) min = ~((Uint) 0); cand_cd = NULL; - for (cd = cache; cd; cd = cd->next) { + for (cd = mk->cache; cd; cd = cd->next) { if (cd->size >= size) { if (!cand_cd) { cand_cd = cd; @@ -638,8 +699,8 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) min = cd->size; } - min_cached_seg_size = min; - max_cached_seg_size = max; + mk->min_cached_seg_size = min; + mk->max_cached_seg_size = max; if (!cand_cd) goto create_seg; @@ -648,20 +709,20 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) if (diff_size > abs_max_cache_bad_fit || 100*PAGES(diff_size) > rel_max_cache_bad_fit*PAGES(size)) { - if (max_cached_seg_size < cand_cd->size) - max_cached_seg_size = cand_cd->size; - if (min_cached_seg_size > cand_cd->size) - min_cached_seg_size = cand_cd->size; + if (mk->max_cached_seg_size < cand_cd->size) + mk->max_cached_seg_size = cand_cd->size; + if (mk->min_cached_seg_size > cand_cd->size) + mk->min_cached_seg_size = cand_cd->size; goto create_seg; } - cache_hits++; + mk->cache_hits++; size = cand_cd->size; seg = cand_cd->seg; - unlink_cd(cand_cd); - free_cd(cand_cd); + unlink_cd(mk,cand_cd); + free_cd(mk,cand_cd); *size_p = size; @@ -671,7 +732,8 @@ mseg_alloc(ErtsAlcType_t atype, Uint *size_p, const ErtsMsegOpt_t *opt) } if (seg) - ERTS_MSEG_ALLOC_STAT(size); + ERTS_MSEG_ALLOC_STAT(mk,size); + return seg; } @@ -680,41 +742,42 @@ static void mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size, const ErtsMsegOpt_t *opt) { + MemKind* mk = type2mk(atype); cache_desc_t *cd; - ERTS_MSEG_DEALLOC_STAT(size); + ERTS_MSEG_DEALLOC_STAT(mk,size); if (!opt->cache || max_cache_size == 0) { if (erts_mtrace_enabled) erts_mtrace_crr_free(atype, SEGTYPE, seg); - mseg_destroy(seg, size); + mseg_destroy(mk, seg, size); } else { int check_limits = 0; - if (size < min_cached_seg_size) - min_cached_seg_size = size; - if (size > max_cached_seg_size) - max_cached_seg_size = size; - - if (!free_cache_descs) { - cd = cache_end; - if (!(min_cached_seg_size < cd->size - && cd->size < max_cached_seg_size)) { + if (size < mk->min_cached_seg_size) + mk->min_cached_seg_size = size; + if (size > mk->max_cached_seg_size) + mk->max_cached_seg_size = size; + + if (!mk->free_cache_descs) { + cd = mk->cache_end; + if (!(mk->min_cached_seg_size < cd->size + && cd->size < mk->max_cached_seg_size)) { check_limits = 1; } if (erts_mtrace_enabled) erts_mtrace_crr_free(SEGTYPE, SEGTYPE, cd->seg); - mseg_destroy(cd->seg, cd->size); - unlink_cd(cd); - free_cd(cd); + mseg_destroy(mk, cd->seg, cd->size); + unlink_cd(mk,cd); + free_cd(mk,cd); } - cd = alloc_cd(); + cd = alloc_cd(mk); ASSERT(cd); cd->seg = seg; cd->size = size; - link_cd(cd); + link_cd(mk,cd); if (erts_mtrace_enabled) { erts_mtrace_crr_free(atype, SEGTYPE, seg); @@ -724,7 +787,7 @@ mseg_dealloc(ErtsAlcType_t atype, void *seg, Uint size, /* ASSERT(segments.current.watermark >= segments.current.no + cache_size); */ if (check_limits) - check_cache_limits(); + check_cache_limits(mk); schedule_cache_check(); @@ -737,6 +800,7 @@ static void * mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, const ErtsMsegOpt_t *opt) { + MemKind* mk = type2mk(atype); void *new_seg; Uint new_size; @@ -774,15 +838,15 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, #if CAN_PARTLY_DESTROY if (shrink_sz > min_seg_size - && free_cache_descs + && mk->free_cache_descs && opt->cache) { cache_desc_t *cd; - cd = alloc_cd(); + cd = alloc_cd(mk); ASSERT(cd); cd->seg = ((char *) seg) + new_size; cd->size = shrink_sz; - end_link_cd(cd); + end_link_cd(mk,cd); if (erts_mtrace_enabled) { erts_mtrace_crr_realloc(new_seg, @@ -801,7 +865,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, SEGTYPE, seg, new_size); - mseg_destroy(((char *) seg) + new_size, shrink_sz); + mseg_destroy(mk, ((char *) seg) + new_size, shrink_sz); } #elif HAVE_MSEG_RECREATE @@ -835,7 +899,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, #if !CAN_PARTLY_DESTROY do_recreate: #endif - new_seg = mseg_recreate((void *) seg, old_size, new_size); + new_seg = mseg_recreate(mk, (void *) seg, old_size, new_size); if (erts_mtrace_enabled) erts_mtrace_crr_realloc(new_seg, atype, SEGTYPE, seg, new_size); if (!new_seg) @@ -858,7 +922,7 @@ mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, Uint *new_size_p, *new_size_p = new_size; - ERTS_MSEG_REALLOC_STAT(old_size, new_size); + ERTS_MSEG_REALLOC_STAT(mk, old_size, new_size); return new_seg; } @@ -874,6 +938,8 @@ static struct { Eterm mcs; Eterm cci; + Eterm memkind; + Eterm name; Eterm status; Eterm cached_segments; Eterm cache_hits; @@ -923,6 +989,8 @@ init_atoms(void) #endif AM_INIT(version); + AM_INIT(memkind); + AM_INIT(name); AM_INIT(options); AM_INIT(amcbf); @@ -1133,65 +1201,88 @@ info_calls(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) } static Eterm -info_status(int *print_to_p, - void *print_to_arg, - int begin_new_max_period, - Uint **hpp, - Uint *szp) +info_status(MemKind* mk, int *print_to_p, void *print_to_arg, + int begin_new_max_period, Uint **hpp, Uint *szp) { Eterm res = THE_NON_VALUE; - if (segments.max_ever.no < segments.max.no) - segments.max_ever.no = segments.max.no; - if (segments.max_ever.sz < segments.max.sz) - segments.max_ever.sz = segments.max.sz; + if (mk->segments.max_ever.no < mk->segments.max.no) + mk->segments.max_ever.no = mk->segments.max.no; + if (mk->segments.max_ever.sz < mk->segments.max.sz) + mk->segments.max_ever.sz = mk->segments.max.sz; if (print_to_p) { int to = *print_to_p; void *arg = print_to_arg; - erts_print(to, arg, "cached_segments: %bpu\n", cache_size); - erts_print(to, arg, "cache_hits: %bpu\n", cache_hits); + erts_print(to, arg, "cached_segments: %bpu\n", mk->cache_size); + erts_print(to, arg, "cache_hits: %bpu\n", mk->cache_hits); erts_print(to, arg, "segments: %bpu %bpu %bpu\n", - segments.current.no, segments.max.no, segments.max_ever.no); + mk->segments.current.no, mk->segments.max.no, mk->segments.max_ever.no); erts_print(to, arg, "segments_size: %bpu %bpu %bpu\n", - segments.current.sz, segments.max.sz, segments.max_ever.sz); + mk->segments.current.sz, mk->segments.max.sz, mk->segments.max_ever.sz); erts_print(to, arg, "segments_watermark: %bpu\n", - segments.current.watermark); + mk->segments.current.watermark); } if (hpp || szp) { res = NIL; add_2tup(hpp, szp, &res, am.segments_watermark, - bld_unstable_uint(hpp, szp, segments.current.watermark)); + bld_unstable_uint(hpp, szp, mk->segments.current.watermark)); add_4tup(hpp, szp, &res, am.segments_size, - bld_unstable_uint(hpp, szp, segments.current.sz), - bld_unstable_uint(hpp, szp, segments.max.sz), - bld_unstable_uint(hpp, szp, segments.max_ever.sz)); + bld_unstable_uint(hpp, szp, mk->segments.current.sz), + bld_unstable_uint(hpp, szp, mk->segments.max.sz), + bld_unstable_uint(hpp, szp, mk->segments.max_ever.sz)); add_4tup(hpp, szp, &res, am.segments, - bld_unstable_uint(hpp, szp, segments.current.no), - bld_unstable_uint(hpp, szp, segments.max.no), - bld_unstable_uint(hpp, szp, segments.max_ever.no)); + bld_unstable_uint(hpp, szp, mk->segments.current.no), + bld_unstable_uint(hpp, szp, mk->segments.max.no), + bld_unstable_uint(hpp, szp, mk->segments.max_ever.no)); add_2tup(hpp, szp, &res, am.cache_hits, - bld_unstable_uint(hpp, szp, cache_hits)); + bld_unstable_uint(hpp, szp, mk->cache_hits)); add_2tup(hpp, szp, &res, am.cached_segments, - bld_unstable_uint(hpp, szp, cache_size)); + bld_unstable_uint(hpp, szp, mk->cache_size)); } if (begin_new_max_period) { - segments.max.no = segments.current.no; - segments.max.sz = segments.current.sz; + mk->segments.max.no = mk->segments.current.no; + mk->segments.max.sz = mk->segments.current.sz; } return res; } +static Eterm info_memkind(MemKind* mk, int *print_to_p, void *print_to_arg, + int begin_max_per, Uint **hpp, Uint *szp) +{ + Eterm res = THE_NON_VALUE; + Eterm atoms[3]; + Eterm values[3]; + + if (print_to_p) { + erts_print(*print_to_p, print_to_arg, "memory kind: %s\n", mk->name); + } + if (hpp || szp) { + atoms[0] = am.name; + atoms[1] = am.status; + atoms[2] = am.calls; + values[0] = erts_bld_string(hpp, szp, mk->name); + } + values[1] = info_status(mk, print_to_p, print_to_arg, begin_max_per, hpp, szp); + values[2] = info_calls(print_to_p, print_to_arg, hpp, szp); + + if (hpp || szp) + res = bld_2tup_list(hpp, szp, 3, atoms, values); + + return res; +} + + static Eterm info_version(int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { @@ -1238,6 +1329,7 @@ erts_mseg_info(int *print_to_p, Eterm res = THE_NON_VALUE; Eterm atoms[4]; Eterm values[4]; + Uint n = 0; erts_mtx_lock(&mseg_mutex); @@ -1248,17 +1340,19 @@ erts_mseg_info(int *print_to_p, atoms[0] = am.version; atoms[1] = am.options; - atoms[2] = am.status; - atoms[3] = am.calls; + atoms[2] = am.memkind; + atoms[3] = am.memkind; } - - values[0] = info_version(print_to_p, print_to_arg, hpp, szp); - values[1] = info_options("option ", print_to_p, print_to_arg, hpp, szp); - values[2] = info_status(print_to_p, print_to_arg, begin_max_per, hpp, szp); - values[3] = info_calls(print_to_p, print_to_arg, hpp, szp); - + values[n++] = info_version(print_to_p, print_to_arg, hpp, szp); + values[n++] = info_options("option ", print_to_p, print_to_arg, hpp, szp); +#if HALFWORD_HEAP + values[n++] = info_memkind(&low_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); + values[n++] = info_memkind(&hi_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); +#else + values[n++] = info_memkind(&the_mem, print_to_p, print_to_arg, begin_max_per, hpp, szp); +#endif if (hpp || szp) - res = bld_2tup_list(hpp, szp, 4, atoms, values); + res = bld_2tup_list(hpp, szp, n, atoms, values); erts_mtx_unlock(&mseg_mutex); @@ -1317,17 +1411,23 @@ erts_mseg_realloc(ErtsAlcType_t atype, void *seg, Uint old_size, void erts_mseg_clear_cache(void) { + MemKind* mk; erts_mtx_lock(&mseg_mutex); - mseg_clear_cache(); + for (mk=mk_list; mk; mk=mk->next) { + mseg_clear_cache(mk); + } erts_mtx_unlock(&mseg_mutex); } Uint erts_mseg_no(void) { - Uint n; + MemKind* mk; + Uint n = 0; erts_mtx_lock(&mseg_mutex); - n = segments.current.no; + for (mk=mk_list; mk; mk=mk->next) { + n += mk->segments.current.no; + } erts_mtx_unlock(&mseg_mutex); return n; } @@ -1338,11 +1438,43 @@ erts_mseg_unit_size(void) return page_size; } -void -erts_mseg_init(ErtsMsegInit_t *init) +static void mem_kind_init(MemKind* mk, const char* name) { unsigned i; + mk->cache = NULL; + mk->cache_end = NULL; + mk->max_cached_seg_size = 0; + mk->min_cached_seg_size = ~((Uint) 0); + mk->cache_size = 0; + mk->cache_hits = 0; + + if (max_cache_size > 0) { + for (i = 0; i < max_cache_size - 1; i++) + mk->cache_descs[i].next = &mk->cache_descs[i + 1]; + mk->cache_descs[max_cache_size - 1].next = NULL; + mk->free_cache_descs = &mk->cache_descs[0]; + } + else + mk->free_cache_descs = NULL; + + mk->segments.current.watermark = 0; + mk->segments.current.no = 0; + mk->segments.current.sz = 0; + mk->segments.max.no = 0; + mk->segments.max.sz = 0; + mk->segments.max_ever.no = 0; + mk->segments.max_ever.sz = 0; + + mk->name = name; + mk->next = mk_list; + mk_list = mk; +} + + +void +erts_mseg_init(ErtsMsegInit_t *init) +{ atoms_initialized = 0; is_init_done = 0; @@ -1385,40 +1517,33 @@ erts_mseg_init(ErtsMsegInit_t *init) min_seg_size = ~((Uint) 0); #endif - cache = NULL; - cache_end = NULL; - cache_hits = 0; - max_cached_seg_size = 0; - min_cached_seg_size = ~((Uint) 0); - cache_size = 0; + if (max_cache_size > MAX_CACHE_SIZE) + max_cache_size = MAX_CACHE_SIZE; + +#if HALFWORD_HEAP + mem_kind_init(&low_mem, "low memory"); + mem_kind_init(&hi_mem, "high memory"); +#else + mem_kind_init(&the_mem, "all memory"); +#endif is_cache_check_scheduled = 0; #ifdef ERTS_THREADS_NO_SMP is_cache_check_requested = 0; #endif +} - if (max_cache_size > MAX_CACHE_SIZE) - max_cache_size = MAX_CACHE_SIZE; - if (max_cache_size > 0) { - for (i = 0; i < max_cache_size - 1; i++) - cache_descs[i].next = &cache_descs[i + 1]; - cache_descs[max_cache_size - 1].next = NULL; - free_cache_descs = &cache_descs[0]; +static ERTS_INLINE Uint tot_cache_size(void) +{ + MemKind* mk; + Uint sz = 0; + for (mk=mk_list; mk; mk=mk->next) { + sz += mk->cache_size; } - else - free_cache_descs = NULL; - - segments.current.watermark = 0; - segments.current.no = 0; - segments.current.sz = 0; - segments.max.no = 0; - segments.max.sz = 0; - segments.max_ever.no = 0; - segments.max_ever.sz = 0; + return sz; } - /* * erts_mseg_late_init() have to be called after all allocators, * threads and timers have been initialized. @@ -1436,7 +1561,7 @@ erts_mseg_late_init(void) #ifdef ERTS_THREADS_NO_SMP async_handle = handle; #endif - if (cache_size) + if (tot_cache_size()) schedule_cache_check(); erts_mtx_unlock(&mseg_mutex); } @@ -1477,7 +1602,7 @@ erts_mseg_test(unsigned long op, case 0x406: { unsigned long res; erts_mtx_lock(&mseg_mutex); - res = (unsigned long) cache_size; + res = (unsigned long) tot_cache_size(); erts_mtx_unlock(&mseg_mutex); return res; } @@ -1568,11 +1693,14 @@ static void *do_map(void *ptr, size_t sz) return NULL; } - +#if HAVE_MMAP res = mmap(ptr, sz, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1 , 0); +#else +# error "Missing mmap support" +#endif if (res == MAP_FAILED) { #ifdef HARDDEBUG @@ -1672,10 +1800,19 @@ static int initialize_pmmap(void) MAP_NORESERVE | EXTRA_MAP_FLAGS, -1 , 0); #ifdef HARDDEBUG - printf("rsz = %ld, pages = %ld, rptr = %p\r\n", - (unsigned long) rsz, (unsigned long) (rsz / pagsz), - (void *) rptr); + printf("p=%p, rsz = %ld, pages = %ld, got range = %p -> %p\r\n", + p, (unsigned long) rsz, (unsigned long) (rsz / pagsz), + (void *) rptr, (void*)(rptr + rsz)); #endif + if ((UWord)(rptr + rsz) > RANGE_MAX) { + size_t rsz_trunc = RANGE_MAX - (UWord)rptr; +#ifdef HARDDEBUG + printf("Reducing mmap'ed memory from %lu to %lu Mb, reduced range = %p -> %p\r\n", + rsz/(1024*1024), rsz_trunc/(1024*1024), rptr, rptr+rsz_trunc); +#endif + munmap((void*)RANGE_MAX, rsz - rsz_trunc); + rsz = rsz_trunc; + } if (!do_map(rptr,pagsz)) { erl_exit(1,"Could not actually mmap first page for halfword emulator...\n"); } @@ -1756,6 +1893,7 @@ static int pmunmap(void *p, size_t size) FreeBlock *last; FreeBlock *nb = (FreeBlock *) p; + ASSERT(((unsigned long)p & CHECK_POINTER_MASK)==0); if (real_size > pagsz) { if (do_unmap(((char *) p) + pagsz,real_size - pagsz)) { return 1; diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index c17806d96c..3ae5b8d747 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -124,25 +124,11 @@ erts_smp_mtx_unlock(&(PS)->mtx) #define ERTS_POLLSET_SET_POLLED_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->polled, (long) 1)) + ((int) erts_atomic32_xchg(&(PS)->polled, (erts_aint32_t) 1)) #define ERTS_POLLSET_UNSET_POLLED(PS) \ - erts_smp_atomic_set(&(PS)->polled, (long) 0) + erts_atomic32_set(&(PS)->polled, (erts_aint32_t) 0) #define ERTS_POLLSET_IS_POLLED(PS) \ - ((int) erts_smp_atomic_read(&(PS)->polled)) - -#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) set_poller_woken_chk((PS)) -#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ -do { \ - ERTS_THR_MEMORY_BARRIER; \ - erts_smp_atomic_set(&(PS)->woken, (long) 1); \ -} while (0) -#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ -do { \ - erts_smp_atomic_set(&(PS)->woken, (long) 0); \ - ERTS_THR_MEMORY_BARRIER; \ -} while (0) -#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ - ((int) erts_smp_atomic_read(&(PS)->woken)) + ((int) erts_atomic32_read(&(PS)->polled)) #else @@ -152,69 +138,21 @@ do { \ #define ERTS_POLLSET_UNSET_POLLED(PS) #define ERTS_POLLSET_IS_POLLED(PS) 0 -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - -/* - * Ideally, the ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) operation would - * be atomic. This operation isn't, but we will do okay anyway. The - * "woken check" is only an optimization. The only requirement we have: - * If (PS)->woken is set to a value != 0 when interrupting, we have to - * write on the the wakeup pipe at least once. Multiple writes are okay. - */ -#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) ((PS)->woken++) -#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) ((PS)->woken = 1, (void) 0) -#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) ((PS)->woken = 0, (void) 0) -#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) ((PS)->woken) - -#else - -#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) 1 -#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) -#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) -#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) 1 - -#endif - #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic_set(&(PS)->have_update_requests, (long) 1) + erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 1) #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) \ - erts_smp_atomic_set(&(PS)->have_update_requests, (long) 0) + erts_smp_atomic32_set(&(PS)->have_update_requests, (erts_aint32_t) 0) #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) \ - ((int) erts_smp_atomic_read(&(PS)->have_update_requests)) + ((int) erts_smp_atomic32_read(&(PS)->have_update_requests)) #else #define ERTS_POLLSET_SET_HAVE_UPDATE_REQUESTS(PS) #define ERTS_POLLSET_UNSET_HAVE_UPDATE_REQUESTS(PS) #define ERTS_POLLSET_HAVE_UPDATE_REQUESTS(PS) 0 #endif -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP) - -#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS)) -#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) ((PS)->interrupt = 0, (void) 0) -#define ERTS_POLLSET_SET_INTERRUPTED(PS) ((PS)->interrupt = 1, (void) 0) -#define ERTS_POLLSET_IS_INTERRUPTED(PS) ((PS)->interrupt) - -#else - -#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS)) -#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ -do { \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 0); \ - ERTS_THR_MEMORY_BARRIER; \ -} while (0) -#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ -do { \ - ERTS_THR_MEMORY_BARRIER; \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 1); \ -} while (0) -#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ - ((int) erts_smp_atomic_read(&(PS)->interrupt)) - -#endif - #if ERTS_POLL_USE_FALLBACK # if ERTS_POLL_USE_POLL # define ERTS_POLL_NEED_FALLBACK(PS) ((PS)->no_poll_fds > 1) @@ -318,14 +256,12 @@ struct ErtsPollSet_ { #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE ErtsPollSetUpdateRequestsBlock update_requests; ErtsPollSetUpdateRequestsBlock *curr_upd_req_block; - erts_smp_atomic_t have_update_requests; + erts_smp_atomic32_t have_update_requests; #endif #ifdef ERTS_SMP - erts_smp_atomic_t polled; - erts_smp_atomic_t woken; + erts_atomic32_t polled; erts_smp_mtx_t mtx; #elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - volatile int woken; #endif #if ERTS_POLL_USE_WAKEUP_PIPE int wake_fds[2]; @@ -333,12 +269,12 @@ struct ErtsPollSet_ { #if ERTS_POLL_USE_FALLBACK int fallback_used; #endif -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP) - volatile int interrupt; -#else - erts_smp_atomic_t interrupt; +#ifdef ERTS_SMP + erts_atomic32_t wakeup_state; +#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + volatile int wakeup_state; #endif - erts_smp_atomic_t timeout; + erts_smp_atomic32_t timeout; #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS erts_smp_atomic_t no_avoided_wakeups; erts_smp_atomic_t no_avoided_interrupts; @@ -346,34 +282,6 @@ struct ErtsPollSet_ { #endif }; -static ERTS_INLINE int -unset_interrupted_chk(ErtsPollSet ps) -{ - int res; -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP) - /* This operation isn't atomic, but we have no need at all for an - atomic operation here... */ - res = ps->interrupt; - ps->interrupt = 0; -#else - res = (int) erts_smp_atomic_xchg(&ps->interrupt, (long) 0); - ERTS_THR_MEMORY_BARRIER; -#endif - return res; - -} - -#ifdef ERTS_SMP - -static ERTS_INLINE int -set_poller_woken_chk(ErtsPollSet ps) -{ - ERTS_THR_MEMORY_BARRIER; - return (int) erts_smp_atomic_xchg(&ps->woken, (long) 1); -} - -#endif - void erts_silence_warn_unused_result(long unused); static void fatal_error(char *format, ...); static void fatal_error_async_signal_safe(char *error_str); @@ -430,6 +338,64 @@ static void check_poll_status(ErtsPollSet ps); static void print_misc_debug_info(void); #endif +#define ERTS_POLL_NOT_WOKEN 0 +#define ERTS_POLL_WOKEN -1 +#define ERTS_POLL_WOKEN_INTR 1 + +static ERTS_INLINE void +reset_wakeup_state(ErtsPollSet ps) +{ +#ifdef ERTS_SMP + erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); + ERTS_THR_MEMORY_BARRIER; +#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + ps->wakeup_state = 0; +#endif +} + +static ERTS_INLINE int +is_woken(ErtsPollSet ps) +{ +#ifdef ERTS_SMP + return erts_atomic32_read_acqb(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN; +#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + return ps->wakeup_state != ERTS_POLL_NOT_WOKEN; +#else + return 0; +#endif +} + +static ERTS_INLINE int +is_interrupted_reset(ErtsPollSet ps) +{ +#ifdef ERTS_SMP + return (erts_atomic32_xchg(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN) + == ERTS_POLL_WOKEN_INTR); +#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + int res = ps->wakeup_state == ERTS_POLL_WOKEN_INTR; + ps->wakeup_state = ERTS_POLL_NOT_WOKEN; + return res; +#else + return 0; +#endif +} + +static ERTS_INLINE void +woke_up(ErtsPollSet ps) +{ +#ifdef ERTS_SMP + erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + if (wakeup_state == ERTS_POLL_NOT_WOKEN) + (void) erts_atomic32_cmpxchg(&ps->wakeup_state, + ERTS_POLL_WOKEN, + ERTS_POLL_NOT_WOKEN); + ASSERT(erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN); +#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + if (ps->wakeup_state == ERTS_POLL_NOT_WOKEN) + ps->wakeup_state = ERTS_POLL_WOKEN; +#endif +} + /* * --- Wakeup pipe ----------------------------------------------------------- */ @@ -437,14 +403,34 @@ static void print_misc_debug_info(void); #if ERTS_POLL_USE_WAKEUP_PIPE static ERTS_INLINE void -wake_poller(ErtsPollSet ps) +wake_poller(ErtsPollSet ps, int interrupted) { + int wake; +#ifdef ERTS_SMP + erts_aint32_t wakeup_state; + if (!interrupted) + wakeup_state = erts_atomic32_cmpxchg_relb(&ps->wakeup_state, + ERTS_POLL_WOKEN, + ERTS_POLL_NOT_WOKEN); + else { + /* + * We might unnecessarily write to the pipe, however, + * that isn't problematic. + */ + wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_INTR); + } + wake = wakeup_state == ERTS_POLL_NOT_WOKEN; +#elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT + wake = ps->wakeup_state == ERTS_POLL_NOT_WOKEN; + ps->wakeup_state = interrupted ? ERTS_POLL_WOKEN_INTR : ERTS_POLL_NOT_WOKEN; +#endif /* * NOTE: This function might be called from signal handlers in the * non-smp case; therefore, it has to be async-signal safe in * the non-smp case. */ - if (!ERTS_POLLSET_SET_POLLER_WOKEN_CHK(ps)) { + if (wake) { ssize_t res; if (ps->wake_fds[1] < 0) return; /* Not initialized yet */ @@ -1387,9 +1373,7 @@ handle_update_requests(ErtsPollSet ps) #endif /* ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE */ static ERTS_INLINE ErtsPollEvents -poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, - int *have_set_have_update_requests, - int *do_wake) +poll_control(ErtsPollSet ps, int fd, ErtsPollEvents events, int on, int *do_wake) { ErtsPollEvents new_events; @@ -1493,7 +1477,6 @@ ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps, int len) { int i; - int hshur = 0; int do_wake; int final_do_wake = 0; @@ -1505,17 +1488,17 @@ ERTS_POLL_EXPORT(erts_poll_controlv)(ErtsPollSet ps, pcev[i].fd, pcev[i].events, pcev[i].on, - &hshur, &do_wake); final_do_wake |= do_wake; } + ERTS_POLLSET_UNLOCK(ps); + #ifdef ERTS_SMP if (final_do_wake) - wake_poller(ps); + wake_poller(ps, 0); #endif /* ERTS_SMP */ - ERTS_POLLSET_UNLOCK(ps); } ErtsPollEvents @@ -1526,20 +1509,20 @@ ERTS_POLL_EXPORT(erts_poll_control)(ErtsPollSet ps, int* do_wake) /* In: Wake up polling thread */ /* Out: Poller is woken */ { - int hshur = 0; ErtsPollEvents res; ERTS_POLLSET_LOCK(ps); - res = poll_control(ps, fd, events, on, &hshur, do_wake); + res = poll_control(ps, fd, events, on, do_wake); + + ERTS_POLLSET_UNLOCK(ps); #ifdef ERTS_SMP if (*do_wake) { - wake_poller(ps); + wake_poller(ps, 0); } #endif /* ERTS_SMP */ - ERTS_POLLSET_UNLOCK(ps); return res; } @@ -1919,8 +1902,10 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res, int *ps_locked) } else { long timeout = tv->tv_sec*1000 + tv->tv_usec/1000; + if (timeout > ERTS_AINT32_T_MAX) + timeout = ERTS_AINT32_T_MAX; ASSERT(timeout >= 0); - erts_smp_atomic_set(&ps->timeout, timeout); + erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout); #if ERTS_POLL_USE_FALLBACK if (!(ps->fallback_used = ERTS_POLL_NEED_FALLBACK(ps))) { @@ -2042,15 +2027,14 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, (int) tv->tv_sec*1000 + tv->tv_usec/1000); #endif - ERTS_POLLSET_UNSET_POLLER_WOKEN(ps); if (ERTS_POLLSET_SET_POLLED_CHK(ps)) { res = EINVAL; /* Another thread is in erts_poll_wait() on this pollset... */ goto done; } - if (ERTS_POLLSET_IS_INTERRUPTED(ps)) { - /* Interrupt use zero timeout */ + if (is_woken(ps)) { + /* Use zero timeout */ itv.tv_sec = 0; itv.tv_usec = 0; tvp = &itv; @@ -2067,7 +2051,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, ps_locked = 0; res = check_fd_events(ps, tvp, no_fds, &ps_locked); - ERTS_POLLSET_SET_POLLER_WOKEN(ps); + woke_up(ps); if (res == 0) { res = ETIMEDOUT; @@ -2099,9 +2083,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, check_poll_result(pr, no_fds); #endif - res = (no_fds == 0 - ? (ERTS_POLLSET_UNSET_INTERRUPTED_CHK(ps) ? EINTR : EAGAIN) - : 0); + res = (no_fds == 0 ? (is_interrupted_reset(ps) ? EINTR : EAGAIN) : 0); *len = no_fds; } @@ -2112,7 +2094,7 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, #endif done: - erts_smp_atomic_set(&ps->timeout, LONG_MAX); + erts_smp_atomic32_set_relb(&ps->timeout, ERTS_AINT32_T_MAX); #ifdef ERTS_POLL_DEBUG_PRINT erts_printf("Leaving %s = erts_poll_wait()\n", res == 0 ? "0" : erl_errno_id(res)); @@ -2128,20 +2110,17 @@ ERTS_POLL_EXPORT(erts_poll_wait)(ErtsPollSet ps, void ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set) { +#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP) /* * NOTE: This function might be called from signal handlers in the * non-smp case; therefore, it has to be async-signal safe in * the non-smp case. */ - if (set) { - ERTS_POLLSET_SET_INTERRUPTED(ps); -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP) - wake_poller(ps); + if (!set) + reset_wakeup_state(ps); + else + wake_poller(ps, 1); #endif - } - else { - ERTS_POLLSET_UNSET_INTERRUPTED(ps); - } } /* @@ -2150,15 +2129,16 @@ ERTS_POLL_EXPORT(erts_poll_interrupt)(ErtsPollSet ps, int set) * is not guaranteed that it will timeout before 'msec' milli seconds. */ void -ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, int set, long msec) +ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, + int set, + long msec) { - if (set) { - if (erts_smp_atomic_read(&ps->timeout) > msec) { - ERTS_POLLSET_SET_INTERRUPTED(ps); #if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT || defined(ERTS_SMP) - wake_poller(ps); -#endif - } + if (!set) + reset_wakeup_state(ps); + else { + if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec) + wake_poller(ps, 1); #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS else { if (ERTS_POLLSET_IS_POLLED(ps)) @@ -2168,9 +2148,7 @@ ERTS_POLL_EXPORT(erts_poll_interrupt_timed)(ErtsPollSet ps, int set, long msec) erts_smp_atomic_inc(&ps->no_interrupt_timed); #endif } - else { - ERTS_POLLSET_UNSET_INTERRUPTED(ps); - } +#endif } int @@ -2281,14 +2259,16 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->update_requests.next = NULL; ps->update_requests.len = 0; ps->curr_upd_req_block = &ps->update_requests; - erts_smp_atomic_init(&ps->have_update_requests, 0); + erts_smp_atomic32_init(&ps->have_update_requests, 0); #endif #ifdef ERTS_SMP - erts_smp_atomic_init(&ps->polled, 0); - erts_smp_atomic_init(&ps->woken, 0); + erts_atomic32_init(&ps->polled, 0); erts_smp_mtx_init(&ps->mtx, "pollset"); +#endif +#ifdef ERTS_SMP + erts_atomic32_init(&ps->wakeup_state, (erts_aint32_t) 0); #elif ERTS_POLL_ASYNC_INTERRUPT_SUPPORT - ps->woken = 0; + ps->wakeup_state = 0; #endif #if ERTS_POLL_USE_WAKEUP_PIPE create_wakeup_pipe(ps); @@ -2310,12 +2290,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) ps->internal_fd_limit = kp_fd + 1; ps->kp_fd = kp_fd; #endif -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP) - ps->interrupt = 0; -#else - erts_smp_atomic_init(&ps->interrupt, 0); -#endif - erts_smp_atomic_init(&ps->timeout, LONG_MAX); + erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX); #ifdef ERTS_POLL_COUNT_AVOIDED_WAKEUPS erts_smp_atomic_init(&ps->no_avoided_wakeups, 0); erts_smp_atomic_init(&ps->no_avoided_interrupts, 0); diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c new file mode 100644 index 0000000000..461e763f03 --- /dev/null +++ b/erts/emulator/sys/common/erl_sys_common_misc.c @@ -0,0 +1,107 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2006-2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + + +/* + * Darwin needs conversion! + * http://developer.apple.com/library/mac/#qa/qa2001/qa1235.html + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include "sys.h" +#include "global.h" + +#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) +#define __DARWIN__ 1 +#endif + +#if !defined(__WIN32__) +#include <locale.h> +#if !defined(HAVE_SETLOCALE) || !defined(HAVE_NL_LANGINFO) || !defined(HAVE_LANGINFO_H) +#define PRIMITIVE_UTF8_CHECK 1 +#else +#include <langinfo.h> +#endif +#endif + +/* Written once and only once */ + +static int filename_encoding = ERL_FILENAME_UNKNOWN; +#if defined(__WIN32__) || defined(__DARWIN__) +static int user_filename_encoding = ERL_FILENAME_UTF8; /* Default unicode on windows */ +#else +static int user_filename_encoding = ERL_FILENAME_LATIN1; +#endif +void erts_set_user_requested_filename_encoding(int encoding) +{ + user_filename_encoding = encoding; +} + +int erts_get_user_requested_filename_encoding(void) +{ + return user_filename_encoding; +} + +void erts_init_sys_common_misc(void) +{ +#if defined(__WIN32__) + /* win_efile will totally fail if this is not set. */ + filename_encoding = ERL_FILENAME_WIN_WCHAR; +#else + if (user_filename_encoding != ERL_FILENAME_UNKNOWN) { + filename_encoding = user_filename_encoding; + } else { + char *l; + filename_encoding = ERL_FILENAME_LATIN1; +# ifdef PRIMITIVE_UTF8_CHECK + setlocale(LC_CTYPE, ""); /* Set international environment, + ignore result */ + if (((l = getenv("LC_ALL")) && *l) || + ((l = getenv("LC_CTYPE")) && *l) || + ((l = getenv("LANG")) && *l)) { + if (strstr(l, "UTF-8")) { + filename_encoding = ERL_FILENAME_UTF8; + } + } + +# else + l = setlocale(LC_CTYPE, ""); /* Set international environment */ + if (l != NULL) { + if (strcmp(nl_langinfo(CODESET), "UTF-8") == 0) { + filename_encoding = ERL_FILENAME_UTF8; + } + } +# endif + } +# if defined(__DARWIN__) + if (filename_encoding == ERL_FILENAME_UTF8) { + filename_encoding = ERL_FILENAME_UTF8_MAC; + } +# endif +#endif +} + +int erts_get_native_filename_encoding(void) +{ + return filename_encoding; +} diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index 2d5ef882f6..d8d51b192c 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. + * Copyright Ericsson AB 1997-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -329,11 +329,4 @@ extern int exit_async(void); #define ERTS_EXIT_AFTER_DUMP _exit -#ifdef ERTS_TIMER_THREAD -struct erts_iwait; /* opaque for clients */ -extern struct erts_iwait *erts_iwait_init(void); -extern void erts_iwait_wait(struct erts_iwait *iwait, struct timeval *delay); -extern void erts_iwait_interrupt(struct erts_iwait *iwait); -#endif /* ERTS_TIMER_THREAD */ - #endif /* #ifndef _ERL_UNIX_SYS_H */ diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index af4ab693dc..bafbbb0f6c 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -53,6 +53,11 @@ #define WANT_NONBLOCKING /* must define this to pull in defs from sys.h */ #include "sys.h" +#if defined(__APPLE__) && defined(__MACH__) && !defined(__DARWIN__) +#define __DARWIN__ 1 +#endif + + #ifdef USE_THREADS #include "erl_threads.h" #endif @@ -75,6 +80,7 @@ static erts_smp_rwmtx_t environ_rwmtx; #include "erl_sys_driver.h" #include "erl_check_io.h" +#include "erl_cpu_topology.h" #ifndef DISABLE_VFORK #define DISABLE_VFORK 0 @@ -159,14 +165,14 @@ static int debug_log = 0; #endif #ifdef ERTS_SMP -erts_smp_atomic_t erts_got_sigusr1; +erts_smp_atomic32_t erts_got_sigusr1; #define ERTS_SET_GOT_SIGUSR1 \ - erts_smp_atomic_set(&erts_got_sigusr1, 1) + erts_smp_atomic32_set(&erts_got_sigusr1, 1) #define ERTS_UNSET_GOT_SIGUSR1 \ - erts_smp_atomic_set(&erts_got_sigusr1, 0) -static erts_smp_atomic_t have_prepared_crash_dump; + erts_smp_atomic32_set(&erts_got_sigusr1, 0) +static erts_smp_atomic32_t have_prepared_crash_dump; #define ERTS_PREPARED_CRASH_DUMP \ - ((int) erts_smp_atomic_xchg(&have_prepared_crash_dump, 1)) + ((int) erts_smp_atomic32_xchg(&have_prepared_crash_dump, 1)) #else volatile int erts_got_sigusr1; #define ERTS_SET_GOT_SIGUSR1 (erts_got_sigusr1 = 1) @@ -234,11 +240,11 @@ static int max_files = -1; * a few variables used by the break handler */ #ifdef ERTS_SMP -erts_smp_atomic_t erts_break_requested; +erts_smp_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 1) + erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 0) + erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0) #else volatile int erts_break_requested = 0; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) @@ -399,7 +405,7 @@ typedef struct { #ifdef ERTS_THR_HAVE_SIG_FUNCS sigset_t saved_sigmask; #endif - int unbind_child; + int sched_bind_data; } erts_thr_create_data_t; /* @@ -410,15 +416,13 @@ static void * thr_create_prepare(void) { erts_thr_create_data_t *tcdp; - ErtsSchedulerData *esdp; tcdp = erts_alloc(ERTS_ALC_T_TMP, sizeof(erts_thr_create_data_t)); #ifdef ERTS_THR_HAVE_SIG_FUNCS erts_thr_sigmask(SIG_BLOCK, &thr_create_sigmask, &tcdp->saved_sigmask); #endif - esdp = erts_get_scheduler_data(); - tcdp->unbind_child = esdp && erts_is_scheduler_bound(esdp); + tcdp->sched_bind_data = erts_sched_bind_atthrcreate_prepare(); return (void *) tcdp; } @@ -430,6 +434,8 @@ thr_create_cleanup(void *vtcdp) { erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp; + erts_sched_bind_atthrcreate_parent(tcdp->sched_bind_data); + #ifdef ERTS_THR_HAVE_SIG_FUNCS /* Restore signalmask... */ erts_thr_sigmask(SIG_SETMASK, &tcdp->saved_sigmask, NULL); @@ -456,12 +462,7 @@ thr_create_prepare_child(void *vtcdp) erts_thread_disable_fpe(); #endif - if (tcdp->unbind_child) { - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); - erts_unbind_from_cpu(erts_cpuinfo); - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); - } - + erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data); } #endif /* #ifdef USE_THREADS */ @@ -508,9 +509,9 @@ erts_sys_pre_init(void) #endif } #ifdef ERTS_SMP - erts_smp_atomic_init(&erts_break_requested, 0); - erts_smp_atomic_init(&erts_got_sigusr1, 0); - erts_smp_atomic_init(&have_prepared_crash_dump, 0); + erts_smp_atomic32_init(&erts_break_requested, 0); + erts_smp_atomic32_init(&erts_got_sigusr1, 0); + erts_smp_atomic32_init(&have_prepared_crash_dump, 0); #else erts_break_requested = 0; erts_got_sigusr1 = 0; @@ -1461,9 +1462,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op CHLD_STAT_LOCK; - unbind = erts_is_scheduler_bound(NULL); - if (unbind) - erts_smp_rwmtx_rlock(&erts_cpu_bind_rwmtx); + unbind = erts_sched_bind_atfork_prepare(); #if !DISABLE_VFORK /* See fork/vfork discussion before this function. */ @@ -1476,7 +1475,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op if (pid == 0) { /* The child! Setup child... */ - if (unbind && erts_unbind_from_cpu(erts_cpuinfo) != 0) + if (erts_sched_bind_atfork_child(unbind) != 0) goto child_error; /* OBSERVE! @@ -1577,8 +1576,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op cs_argv[CS_ARGV_PROGNAME_IX] = child_setup_prog; cs_argv[CS_ARGV_WD_IX] = opts->wd ? opts->wd : "."; - cs_argv[CS_ARGV_UNBIND_IX] - = (unbind ? erts_get_unbind_from_cpu_str(erts_cpuinfo) : "false"); + cs_argv[CS_ARGV_UNBIND_IX] = erts_sched_bind_atvfork_child(unbind); cs_argv[CS_ARGV_FD_CR_IX] = fd_close_range; for (i = 0; i < CS_ARGV_NO_OF_DUP2_OPS; i++) cs_argv[CS_ARGV_DUP2_OP_IX(i)] = &dup2_op[i][0]; @@ -1627,8 +1625,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* op } #endif - if (unbind) - erts_smp_rwmtx_runlock(&erts_cpu_bind_rwmtx); + erts_sched_bind_atfork_parent(unbind); if (pid == -1) { saved_errno = errno; @@ -2997,11 +2994,27 @@ init_smp_sig_notify(void) NULL, &thr_opts); } +#ifdef __DARWIN__ + +int erts_darwin_main_thread_pipe[2]; +int erts_darwin_main_thread_result_pipe[2]; +static void initialize_darwin_main_thread_pipes(void) +{ + if (pipe(erts_darwin_main_thread_pipe) < 0 || + pipe(erts_darwin_main_thread_result_pipe) < 0) { + erl_exit(1,"Fatal error initializing Darwin main thread stealing"); + } +} + +#endif void erts_sys_main_thread(void) { erts_thread_disable_fpe(); +#ifdef __DARWIN__ + initialize_darwin_main_thread_pipes(); +#endif /* Become signal receiver thread... */ #ifdef ERTS_ENABLE_LOCK_CHECK erts_lc_set_thread_name("signal_receiver"); @@ -3010,6 +3023,27 @@ erts_sys_main_thread(void) smp_sig_notify(0); /* Notify initialized */ while (1) { /* Wait for a signal to arrive... */ +#ifdef __DARWIN__ + /* + * The wx driver needs to be able to steal the main thread for Cocoa to + * work properly. + */ + fd_set readfds; + int res; + + FD_ZERO(&readfds); + FD_SET(erts_darwin_main_thread_pipe[0], &readfds); + res = select(erts_darwin_main_thread_pipe[0] + 1, &readfds, NULL, NULL, NULL); + if (res > 0 && FD_ISSET(erts_darwin_main_thread_pipe[0],&readfds)) { + void* (*func)(void*); + void* arg; + void *resp; + read(erts_darwin_main_thread_pipe[0],&func,sizeof(void* (*)(void*))); + read(erts_darwin_main_thread_pipe[0],&arg, sizeof(void*)); + resp = (*func)(arg); + write(erts_darwin_main_thread_result_pipe[1],&resp,sizeof(void *)); + } +#else #ifdef DEBUG int res = #else @@ -3018,6 +3052,7 @@ erts_sys_main_thread(void) select(0, NULL, NULL, NULL, NULL); ASSERT(res < 0); ASSERT(errno == EINTR); +#endif } } @@ -3117,226 +3152,3 @@ erl_sys_args(int* argc, char** argv) } *argc = j; } - -#ifdef ERTS_TIMER_THREAD - -/* - * Interruptible-wait facility: low-level synchronisation state - * and methods that are implementation dependent. - * - * Constraint: Every implementation must define 'struct erts_iwait' - * with a field 'erts_smp_atomic_t state;'. - */ - -/* values for struct erts_iwait's state field */ -#define IWAIT_WAITING 0 -#define IWAIT_AWAKE 1 -#define IWAIT_INTERRUPT 2 - -#if 0 /* XXX: needs feature test in erts/configure.in */ - -/* - * This is an implementation of the interruptible wait facility on - * top of Linux-specific futexes. - */ -#include <asm/unistd.h> -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 -static int sys_futex(void *futex, int op, int val, const struct timespec *timeout) -{ - return syscall(__NR_futex, futex, op, val, timeout); -} - -struct erts_iwait { - erts_smp_atomic_t state; /* &state.counter is our futex */ -}; - -static void iwait_lowlevel_init(struct erts_iwait *iwait) { /* empty */ } - -static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay) -{ - struct timespec timeout; - int res; - - timeout.tv_sec = delay->tv_sec; - timeout.tv_nsec = delay->tv_usec * 1000; - res = sys_futex((void*)&iwait->state.counter, FUTEX_WAIT, IWAIT_WAITING, &timeout); - if (res < 0 && errno != ETIMEDOUT && errno != EWOULDBLOCK && errno != EINTR) - perror("FUTEX_WAIT"); -} - -static void iwait_lowlevel_interrupt(struct erts_iwait *iwait) -{ - int res = sys_futex((void*)&iwait->state.counter, FUTEX_WAKE, 1, NULL); - if (res < 0) - perror("FUTEX_WAKE"); -} - -#else /* using poll() or select() */ - -/* - * This is an implementation of the interruptible wait facility on - * top of pipe(), poll() or select(), read(), and write(). - */ -struct erts_iwait { - erts_smp_atomic_t state; - int read_fd; /* wait polls and reads this fd */ - int write_fd; /* interrupt writes this fd */ -}; - -static void iwait_lowlevel_init(struct erts_iwait *iwait) -{ - int fds[2]; - - if (pipe(fds) < 0) { - perror("pipe()"); - exit(1); - } - iwait->read_fd = fds[0]; - iwait->write_fd = fds[1]; -} - -#if defined(ERTS_USE_POLL) - -#include <sys/poll.h> -#define PERROR_POLL "poll()" - -static int iwait_lowlevel_poll(int read_fd, struct timeval *delay) -{ - struct pollfd pollfd; - int timeout; - - pollfd.fd = read_fd; - pollfd.events = POLLIN; - pollfd.revents = 0; - timeout = delay->tv_sec * 1000 + delay->tv_usec / 1000; - return poll(&pollfd, 1, timeout); -} - -#else /* !ERTS_USE_POLL */ - -#include <sys/select.h> -#define PERROR_POLL "select()" - -static int iwait_lowlevel_poll(int read_fd, struct timeval *delay) -{ - fd_set readfds; - - FD_ZERO(&readfds); - FD_SET(read_fd, &readfds); - return select(read_fd + 1, &readfds, NULL, NULL, delay); -} - -#endif /* !ERTS_USE_POLL */ - -static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay) -{ - int res; - char buf[64]; - - res = iwait_lowlevel_poll(iwait->read_fd, delay); - if (res > 0) - (void)read(iwait->read_fd, buf, sizeof buf); - else if (res < 0 && errno != EINTR) - perror(PERROR_POLL); -} - -static void iwait_lowlevel_interrupt(struct erts_iwait *iwait) -{ - int res = write(iwait->write_fd, "!", 1); - if (res < 0) - perror("write()"); -} - -#endif /* using poll() or select() */ - -#if 0 /* not using poll() or select() */ -/* - * This is an implementation of the interruptible wait facility on - * top of pthread_cond_timedwait(). This has two problems: - * 1. pthread_cond_timedwait() requires an absolute time point, - * so the relative delay must be converted to absolute time. - * Worse, this breaks if the machine's time is adjusted while - * we're preparing to wait. - * 2. Each cond operation requires additional mutex lock/unlock operations. - * - * Problem 2 is probably not too bad on Linux (they'll just become - * relatively cheap futex operations), but problem 1 is the real killer. - * Only use this implementation if no better alternatives are available! - */ -struct erts_iwait { - erts_smp_atomic_t state; - pthread_cond_t cond; - pthread_mutex_t mutex; -}; - -static void iwait_lowlevel_init(struct erts_iwait *iwait) -{ - iwait->cond = (pthread_cond_t) PTHREAD_COND_INITIALIZER; - iwait->mutex = (pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER; -} - -static void iwait_lowlevel_wait(struct erts_iwait *iwait, struct timeval *delay) -{ - struct timeval tmp; - struct timespec timeout; - - /* Due to pthread_cond_timedwait()'s use of absolute - time, this must be the real gettimeofday(), _not_ - the "smoothed" one beam/erl_time_sup.c implements. */ - gettimeofday(&tmp, NULL); - - tmp.tv_sec += delay->tv_sec; - tmp.tv_usec += delay->tv_usec; - if (tmp.tv_usec >= 1000*1000) { - tmp.tv_usec -= 1000*1000; - tmp.tv_sec += 1; - } - timeout.tv_sec = tmp.tv_sec; - timeout.tv_nsec = tmp.tv_usec * 1000; - pthread_mutex_lock(&iwait->mutex); - pthread_cond_timedwait(&iwait->cond, &iwait->mutex, &timeout); - pthread_mutex_unlock(&iwait->mutex); -} - -static void iwait_lowlevel_interrupt(struct erts_iwait *iwait) -{ - pthread_mutex_lock(&iwait->mutex); - pthread_cond_signal(&iwait->cond); - pthread_mutex_unlock(&iwait->mutex); -} - -#endif /* not using POLL */ - -/* - * Interruptible-wait facility. This is just a wrapper around the - * low-level synchronisation code, where we maintain our logical - * state in order to suppress some state transitions. - */ - -struct erts_iwait *erts_iwait_init(void) -{ - struct erts_iwait *iwait = malloc(sizeof *iwait); - if (!iwait) { - perror("malloc"); - exit(1); - } - iwait_lowlevel_init(iwait); - erts_smp_atomic_init(&iwait->state, IWAIT_AWAKE); - return iwait; -} - -void erts_iwait_wait(struct erts_iwait *iwait, struct timeval *delay) -{ - if (erts_smp_atomic_xchg(&iwait->state, IWAIT_WAITING) != IWAIT_INTERRUPT) - iwait_lowlevel_wait(iwait, delay); - erts_smp_atomic_set(&iwait->state, IWAIT_AWAKE); -} - -void erts_iwait_interrupt(struct erts_iwait *iwait) -{ - if (erts_smp_atomic_xchg(&iwait->state, IWAIT_INTERRUPT) == IWAIT_WAITING) - iwait_lowlevel_interrupt(iwait); -} - -#endif /* ERTS_TIMER_THREAD */ diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c index 6e9376b0f3..8ec7b31ce0 100644 --- a/erts/emulator/sys/unix/sys_float.c +++ b/erts/emulator/sys/unix/sys_float.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2010. All Rights Reserved. + * Copyright Ericsson AB 2001-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -36,11 +36,6 @@ erts_sys_init_float(void) # endif } -static ERTS_INLINE void set_current_fp_exception(unsigned long pc) -{ - /* nothing to do */ -} - #else /* !NO_FPE_SIGNALS */ #ifdef ERTS_SMP diff --git a/erts/emulator/sys/vxworks/sys.c b/erts/emulator/sys/vxworks/sys.c index 411b4b37cf..c6e7b65f32 100644 --- a/erts/emulator/sys/vxworks/sys.c +++ b/erts/emulator/sys/vxworks/sys.c @@ -85,7 +85,7 @@ EXTERN_FUNCTION(void, erl_exit, (int n, char*, _DOTS_)); EXTERN_FUNCTION(void, erl_error, (char*, va_list)); EXTERN_FUNCTION(int, driver_interrupt, (int, int)); EXTERN_FUNCTION(void, increment_time, (int)); -EXTERN_FUNCTION(int, next_time, (_VOID_)); +EXTERN_FUNCTION(int, erts_next_time, (_VOID_)); EXTERN_FUNCTION(void, set_reclaim_free_function, (FreeFunction)); EXTERN_FUNCTION(int, erl_mem_info_get, (MEM_PART_STATS *)); EXTERN_FUNCTION(void, erl_crash_dump, (char* file, int line, char* fmt, ...)); diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index a766fe9575..7662f190ef 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2007-2010. All Rights Reserved. + * Copyright Ericsson AB 2007-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -274,7 +274,6 @@ struct ErtsPollSet_ { Waiter** waiter; int allocated_waiters; /* Size ow waiter array */ int num_waiters; /* Number of waiter threads. */ - erts_atomic_t sys_io_ready; /* Tells us there is I/O ready (already). */ int restore_events; /* Tells us to restore waiters events next time around */ HANDLE event_io_ready; /* To be used when waiting for io */ @@ -282,12 +281,11 @@ struct ErtsPollSet_ { volatile int standby_wait_counter; /* Number of threads to wait for */ CRITICAL_SECTION standby_crit; /* CS to guard the counter */ HANDLE standby_wait_event; /* Event signalled when counte == 0 */ + erts_atomic32_t wakeup_state; #ifdef ERTS_SMP - erts_smp_atomic_t woken; erts_smp_mtx_t mtx; - erts_smp_atomic_t interrupt; #endif - erts_smp_atomic_t timeout; + erts_smp_atomic32_t timeout; }; #ifdef ERTS_SMP @@ -296,126 +294,24 @@ struct ErtsPollSet_ { erts_smp_mtx_lock(&(PS)->mtx) #define ERTS_POLLSET_UNLOCK(PS) \ erts_smp_mtx_unlock(&(PS)->mtx) -#define ERTS_POLLSET_SET_POLLED_CHK(PS) \ - ((int) erts_smp_atomic_xchg(&(PS)->polled, (long) 1)) -#define ERTS_POLLSET_SET_POLLED(PS) \ - erts_smp_atomic_set(&(PS)->polled, (long) 1) -#define ERTS_POLLSET_UNSET_POLLED(PS) \ - erts_smp_atomic_set(&(PS)->polled, (long) 0) -#define ERTS_POLLSET_IS_POLLED(PS) \ - ((int) erts_smp_atomic_read(&(PS)->polled)) - -#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) set_poller_woken_chk((PS)) -#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) \ -do { \ - ERTS_THR_MEMORY_BARRIER; \ - erts_smp_atomic_set(&(PS)->woken, (long) 1); \ -} while (0) -#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) \ -do { \ - erts_smp_atomic_set(&(PS)->woken, (long) 0); \ - ERTS_THR_MEMORY_BARRIER; \ -} while (0) -#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) \ - ((int) erts_smp_atomic_read(&(PS)->woken)) - -#define ERTS_POLLSET_UNSET_INTERRUPTED_CHK(PS) unset_interrupted_chk((PS)) -#define ERTS_POLLSET_UNSET_INTERRUPTED(PS) \ -do { \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 0); \ - ERTS_THR_MEMORY_BARRIER; \ -} while (0) -#define ERTS_POLLSET_SET_INTERRUPTED(PS) \ -do { \ - ERTS_THR_MEMORY_BARRIER; \ - erts_smp_atomic_set(&(PS)->interrupt, (long) 1); \ -} while (0) -#define ERTS_POLLSET_IS_INTERRUPTED(PS) \ - ((int) erts_smp_atomic_read(&(PS)->interrupt)) - -static ERTS_INLINE int -unset_interrupted_chk(ErtsPollSet ps) -{ - int res = (int) erts_smp_atomic_xchg(&ps->interrupt, (long) 0); - ERTS_THR_MEMORY_BARRIER; - return res; - -} - -static ERTS_INLINE int -set_poller_woken_chk(ErtsPollSet ps) -{ - ERTS_THR_MEMORY_BARRIER; - return (int) erts_smp_atomic_xchg(&ps->woken, (long) 1); -} #else #define ERTS_POLLSET_LOCK(PS) #define ERTS_POLLSET_UNLOCK(PS) -#define ERTS_POLLSET_SET_POLLED_CHK(PS) 0 -#define ERTS_POLLSET_UNSET_POLLED(PS) -#define ERTS_POLLSET_IS_POLLED(PS) 0 -#define ERTS_POLLSET_SET_POLLER_WOKEN_CHK(PS) 1 -#define ERTS_POLLSET_SET_POLLER_WOKEN(PS) -#define ERTS_POLLSET_UNSET_POLLER_WOKEN(PS) -#define ERTS_POLLSET_IS_POLLER_WOKEN(PS) 1 - #endif /* - * While atomics are not yet implemented for windows in the common library... - * - * MSDN doc states that SMP machines and old compilers require - * InterLockedExchange to properly read and write interlocked - * variables, otherwise the processors might reschedule - * the access and order of atomics access is destroyed... - * While they only mention it in white-papers, the problem - * in VS2003 is due to the IA64 arch, so we can still count - * on the CPU not rescheduling the access to volatile in X86 arch using - * even the slightly older compiler... - * - * So here's (hopefully) a subset of the generally working atomic - * variable access... - */ - -#if defined(__GNUC__) -# if defined(__i386__) || defined(__x86_64__) -# define VOLATILE_IN_SEQUENCE 1 -# else -# define VOLATILE_IN_SEQUENCE 0 -# endif -#elif defined(_MSC_VER) -# if _MSC_VER < 1300 -# define VOLATILE_IN_SEQUENCE 0 /* Dont trust really old compilers */ -# else -# if defined(_M_IX86) -# define VOLATILE_IN_SEQUENCE 1 -# else /* I.e. IA64 */ -# if _MSC_VER >= 1400 -# define VOLATILE_IN_SEQUENCE 1 -# else -# define VOLATILE_IN_SEQUENCE 0 -# endif -# endif -# endif -#else -# define VOLATILE_IN_SEQUENCE 0 -#endif - - - -/* * Communication with sys_interrupt */ #ifdef ERTS_SMP -extern erts_smp_atomic_t erts_break_requested; +extern erts_smp_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 1) + erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 0) + erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0) #else extern volatile int erts_break_requested; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) @@ -424,7 +320,7 @@ extern volatile int erts_break_requested; static erts_mtx_t break_waiter_lock; static HANDLE break_happened_event; -static erts_atomic_t break_waiter_state; +static erts_atomic32_t break_waiter_state; #define BREAK_WAITER_GOT_BREAK 1 #define BREAK_WAITER_GOT_HALT 2 @@ -467,29 +363,172 @@ do { \ wait_standby(PS); \ } while(0) -#if ERTS_POLL_ASYNC_INTERRUPT_SUPPORT && !defined(ERTS_SMP) +#define ERTS_POLL_NOT_WOKEN ((erts_aint32_t) 0) +#define ERTS_POLL_WOKEN_IO_READY ((erts_aint32_t) 1) +#define ERTS_POLL_WOKEN_INTR ((erts_aint32_t) 2) +#define ERTS_POLL_WOKEN_TIMEDOUT ((erts_aint32_t) 3) static ERTS_INLINE int -unset_interrupted_chk(ErtsPollSet ps) +is_io_ready(ErtsPollSet ps) { - /* This operation isn't atomic, but we have no need at all for an - atomic operation here... */ - int res = ps->interrupt; - ps->interrupt = 0; - return res; + return erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_WOKEN_IO_READY; } +static ERTS_INLINE void +woke_up(ErtsPollSet ps) +{ + if (erts_atomic32_read(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN) + erts_atomic32_cmpxchg(&ps->wakeup_state, + ERTS_POLL_WOKEN_TIMEDOUT, + ERTS_POLL_NOT_WOKEN); +#ifdef DEBUG + { + erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + switch (wakeup_state) { + case ERTS_POLL_WOKEN_IO_READY: + case ERTS_POLL_WOKEN_INTR: + case ERTS_POLL_WOKEN_TIMEDOUT: + break; + default: + ASSERT(0); + break; + } + } #endif +} + +static ERTS_INLINE int +wakeup_cause(ErtsPollSet ps) +{ + int res; + erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + switch (wakeup_state) { + case ERTS_POLL_WOKEN_IO_READY: + res = 0; + break; + case ERTS_POLL_WOKEN_INTR: + res = EINTR; + break; + case ERTS_POLL_WOKEN_TIMEDOUT: + res = ETIMEDOUT; + break; + default: + res = 0; + erl_exit(ERTS_ABORT_EXIT, + "%s:%d: Internal error: Invalid wakeup_state=%d\n", + __FILE__, __LINE__, (int) wakeup_state); + } + return res; +} + +static ERTS_INLINE DWORD +poll_wait_timeout(ErtsPollSet ps, SysTimeval *tvp) +{ + time_t timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000; + + if (timeout <= 0) { + woke_up(ps); + return (DWORD) 0; + } + + ResetEvent(ps->event_io_ready); + /* + * Since we don't know the internals of ResetEvent() we issue + * a memory barrier as a safety precaution ensuring that + * the load of wakeup_state wont be reordered with stores made + * by ResetEvent(). + */ + ERTS_THR_MEMORY_BARRIER; + if (erts_atomic32_read(&ps->wakeup_state) != ERTS_POLL_NOT_WOKEN) + return (DWORD) 0; + + if (timeout > ERTS_AINT32_T_MAX) /* Also prevents DWORD overflow */ + timeout = ERTS_AINT32_T_MAX; + + erts_smp_atomic32_set_relb(&ps->timeout, (erts_aint32_t) timeout); + return (DWORD) timeout; +} -#ifdef ERTS_SMP static ERTS_INLINE void -wake_poller(ErtsPollSet ps) +wake_poller(ErtsPollSet ps, int io_ready) { - if (!ERTS_POLLSET_SET_POLLER_WOKEN_CHK(ps)) { + erts_aint32_t wakeup_state; + if (io_ready) { + /* We may set the event multiple times. This is, however, harmless. */ + wakeup_state = erts_atomic32_read(&ps->wakeup_state); + erts_atomic32_set_relb(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); + } + else { + ERTS_THR_MEMORY_BARRIER; + wakeup_state = erts_atomic32_read(&ps->wakeup_state); + while (wakeup_state != ERTS_POLL_WOKEN_IO_READY + && wakeup_state != ERTS_POLL_WOKEN_INTR) { + erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state, + ERTS_POLL_WOKEN_INTR, + wakeup_state); + if (act == wakeup_state) { + wakeup_state = act; + break; + } + wakeup_state = act; + } + } + if (wakeup_state == ERTS_POLL_NOT_WOKEN) { + /* + * Since we don't know the internals of SetEvent() we issue + * a memory barrier as a safety precaution ensuring that + * the store we just made to wakeup_state wont be reordered + * with loads in SetEvent(). + */ + ERTS_THR_MEMORY_BARRIER; SetEvent(ps->event_io_ready); } } -#endif + +static ERTS_INLINE void +reset_io_ready(ErtsPollSet ps) +{ + erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); +} + +static ERTS_INLINE void +restore_io_ready(ErtsPollSet ps) +{ + erts_atomic32_set(&ps->wakeup_state, ERTS_POLL_WOKEN_IO_READY); +} + +/* + * notify_io_ready() is used by threads waiting for events, when + * notifying a poller thread about I/O ready. + */ +static ERTS_INLINE void +notify_io_ready(ErtsPollSet ps) +{ + wake_poller(ps, 1); +} + +static ERTS_INLINE void +reset_interrupt(ErtsPollSet ps) +{ + /* We need to keep io-ready if set */ + erts_aint32_t wakeup_state = erts_atomic32_read(&ps->wakeup_state); + while (wakeup_state != ERTS_POLL_WOKEN_IO_READY + && wakeup_state != ERTS_POLL_NOT_WOKEN) { + erts_aint32_t act = erts_atomic32_cmpxchg(&ps->wakeup_state, + ERTS_POLL_NOT_WOKEN, + wakeup_state); + if (wakeup_state == act) + break; + wakeup_state = act; + } + ERTS_THR_MEMORY_BARRIER; +} + +static ERTS_INLINE void +set_interrupt(ErtsPollSet ps) +{ + wake_poller(ps, 0); +} static void setup_standby_wait(ErtsPollSet ps, int num_threads) { @@ -653,14 +692,14 @@ static void *break_waiter(void *param) case WAIT_OBJECT_0: ResetEvent(harr[0]); erts_mtx_lock(&break_waiter_lock); - erts_atomic_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK); + erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_BREAK); SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); break; case (WAIT_OBJECT_0+1): ResetEvent(harr[1]); erts_mtx_lock(&break_waiter_lock); - erts_atomic_set(&break_waiter_state,BREAK_WAITER_GOT_HALT); + erts_atomic32_set(&break_waiter_state,BREAK_WAITER_GOT_HALT); SetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); break; @@ -767,12 +806,7 @@ event_happened: consistency_check(w); #endif ASSERT(WAIT_OBJECT_0 < i && i < WAIT_OBJECT_0+w->active_events); - if (!erts_atomic_xchg(&ps->sys_io_ready,1)) { - HARDDEBUGF(("SET EventIoReady (%d)",erts_atomic_read(&ps->sys_io_ready))); - SetEvent(ps->event_io_ready); - } else { - HARDDEBUGF(("DONT SET EventIoReady")); - } + notify_io_ready(ps); /* * The main thread wont start working on our arrays untill we're @@ -967,15 +1001,10 @@ static int cancel_driver_select(ErtsPollSet ps, HANDLE event) void erts_poll_interrupt(ErtsPollSet ps, int set /* bool */) { HARDTRACEF(("In erts_poll_interrupt(%d)",set)); -#ifdef ERTS_SMP - if (set) { - ERTS_POLLSET_SET_INTERRUPTED(ps); - wake_poller(ps); - } - else { - ERTS_POLLSET_UNSET_INTERRUPTED(ps); - } -#endif + if (!set) + reset_interrupt(ps); + else + set_interrupt(ps); HARDTRACEF(("Out erts_poll_interrupt(%d)",set)); } @@ -984,17 +1013,10 @@ void erts_poll_interrupt_timed(ErtsPollSet ps, long msec) { HARDTRACEF(("In erts_poll_interrupt_timed(%d,%ld)",set,msec)); -#ifdef ERTS_SMP - if (set) { - if (erts_smp_atomic_read(&ps->timeout) > msec) { - ERTS_POLLSET_SET_INTERRUPTED(ps); - wake_poller(ps); - } - } - else { - ERTS_POLLSET_UNSET_INTERRUPTED(ps); - } -#endif + if (!set) + reset_interrupt(ps); + else if (erts_smp_atomic32_read_acqb(&ps->timeout) > (erts_aint32_t) msec) + set_interrupt(ps); HARDTRACEF(("Out erts_poll_interrupt_timed")); } @@ -1068,10 +1090,8 @@ void erts_poll_controlv(ErtsPollSet ps, int erts_poll_wait(ErtsPollSet ps, ErtsPollResFd pr[], int *len, - SysTimeval *utvp) + SysTimeval *tvp) { - SysTimeval *tvp = utvp; - SysTimeval itv; int no_fds; DWORD timeout; EventData* ev; @@ -1084,7 +1104,7 @@ int erts_poll_wait(ErtsPollSet ps, HARDTRACEF(("In erts_poll_wait")); ERTS_POLLSET_LOCK(ps); - if (!erts_atomic_read(&ps->sys_io_ready) && ps->restore_events) { + if (!is_io_ready(ps) && ps->restore_events) { HARDDEBUGF(("Restore events: %d",ps->num_waiters)); ps->restore_events = 0; for (i = 0; i < ps->num_waiters; ++i) { @@ -1102,7 +1122,7 @@ int erts_poll_wait(ErtsPollSet ps, if (w->highwater != w->active_events) { HARDDEBUGF(("Oups!")); /* Oups, got signalled before we took the lock, can't reset */ - if(erts_atomic_read(&ps->sys_io_ready) == 0) { + if(!is_io_ready(ps)) { erl_exit(1,"Internal error: " "Inconsistent io structures in erl_poll.\n"); } @@ -1127,39 +1147,27 @@ int erts_poll_wait(ErtsPollSet ps, no_fds = ERTS_POLL_MAX_RES; #endif + timeout = poll_wait_timeout(ps, tvp); - ResetEvent(ps->event_io_ready); - ERTS_POLLSET_UNSET_POLLER_WOKEN(ps); - -#ifdef ERTS_SMP - if (ERTS_POLLSET_IS_INTERRUPTED(ps)) { - /* Interrupt use zero timeout */ - itv.tv_sec = 0; - itv.tv_usec = 0; - tvp = &itv; - } -#endif - - timeout = tvp->tv_sec * 1000 + tvp->tv_usec / 1000; /*HARDDEBUGF(("timeout = %ld",(long) timeout));*/ - erts_smp_atomic_set(&ps->timeout, timeout); - if (timeout > 0 && ! erts_atomic_read(&ps->sys_io_ready) && ! erts_atomic_read(&break_waiter_state)) { + if (timeout > 0 && !erts_atomic32_read(&break_waiter_state)) { HANDLE harr[2] = {ps->event_io_ready, break_happened_event}; int num_h = 2; - HARDDEBUGF(("Start waiting %d [%d]",num_h, (long) timeout)); + HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout)); ERTS_POLLSET_UNLOCK(ps); WaitForMultipleObjects(num_h, harr, FALSE, timeout); ERTS_POLLSET_LOCK(ps); - HARDDEBUGF(("Stop waiting %d [%d]",num_h, (long) timeout)); + HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout)); + woke_up(ps); } ERTS_UNSET_BREAK_REQUESTED; - if(erts_atomic_read(&break_waiter_state)) { + if(erts_atomic32_read(&break_waiter_state)) { erts_mtx_lock(&break_waiter_lock); - break_state = erts_atomic_read(&break_waiter_state); - erts_atomic_set(&break_waiter_state,0); + break_state = erts_atomic32_read(&break_waiter_state); + erts_atomic32_set(&break_waiter_state,0); ResetEvent(break_happened_event); erts_mtx_unlock(&break_waiter_lock); switch (break_state) { @@ -1174,15 +1182,13 @@ int erts_poll_wait(ErtsPollSet ps, } } - ERTS_POLLSET_SET_POLLER_WOKEN(ps); - - if (!erts_atomic_read(&ps->sys_io_ready)) { - res = EINTR; - HARDDEBUGF(("EINTR!")); - goto done; + res = wakeup_cause(ps); + if (res != 0) { + HARDDEBUGF(("%s!", res == EINTR ? "EINTR" : "ETIMEDOUT")); + goto done; } - erts_atomic_set(&ps->sys_io_ready,0); + reset_io_ready(ps); n = ps->num_waiters; @@ -1204,9 +1210,9 @@ int erts_poll_wait(ErtsPollSet ps, if (num >= no_fds) { w->highwater=j+1; erts_mtx_unlock(&w->mtx); - /* This might mean we still have data to report, set - back the global flag! */ - erts_atomic_set(&ps->sys_io_ready,1); + /* This might mean we still have data to report, + restore flag indicating I/O ready! */ + restore_io_ready(ps); HARDDEBUGF(("To many FD's to report!")); goto done; } @@ -1228,7 +1234,7 @@ int erts_poll_wait(ErtsPollSet ps, erts_mtx_unlock(&w->mtx); } done: - erts_smp_atomic_set(&ps->timeout, LONG_MAX); + erts_smp_atomic32_set(&ps->timeout, ERTS_AINT32_T_MAX); *len = num; ERTS_POLLSET_UNLOCK(ps); HARDTRACEF(("Out erts_poll_wait")); @@ -1306,15 +1312,13 @@ ErtsPollSet erts_poll_create_pollset(void) ps->standby_wait_counter = 0; ps->event_io_ready = CreateManualEvent(FALSE); ps->standby_wait_event = CreateManualEvent(FALSE); - erts_atomic_init(&ps->sys_io_ready,0); ps->restore_events = 0; + erts_atomic32_init(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN); #ifdef ERTS_SMP - erts_smp_atomic_init(&ps->woken, 0); erts_smp_mtx_init(&ps->mtx, "pollset"); - erts_smp_atomic_init(&ps->interrupt, 0); #endif - erts_smp_atomic_init(&ps->timeout, LONG_MAX); + erts_smp_atomic32_init(&ps->timeout, ERTS_AINT32_T_MAX); HARDTRACEF(("Out erts_poll_create_pollset")); return ps; @@ -1366,7 +1370,7 @@ void erts_poll_init(void) erts_mtx_init(&break_waiter_lock,"break_waiter_lock"); break_happened_event = CreateManualEvent(FALSE); - erts_atomic_init(&break_waiter_state, 0); + erts_atomic32_init(&break_waiter_state, 0); erts_thr_create(&thread, &break_waiter, NULL, NULL); ERTS_UNSET_BREAK_REQUESTED; diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h index 4949998abc..ecb06868d5 100644 --- a/erts/emulator/sys/win32/erl_win_dyn_driver.h +++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2009. All Rights Reserved. + * Copyright Ericsson AB 2003-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -87,15 +87,15 @@ WDD_TYPEDEF(unsigned long, erts_alc_test, (unsigned long, unsigned long, unsigned long, unsigned long)); -WDD_TYPEDEF(long, driver_binary_get_refc, (ErlDrvBinary *dbp)); -WDD_TYPEDEF(long, driver_binary_inc_refc, (ErlDrvBinary *dbp)); -WDD_TYPEDEF(long, driver_binary_dec_refc, (ErlDrvBinary *dbp)); +WDD_TYPEDEF(ErlDrvSInt, driver_binary_get_refc, (ErlDrvBinary *dbp)); +WDD_TYPEDEF(ErlDrvSInt, driver_binary_inc_refc, (ErlDrvBinary *dbp)); +WDD_TYPEDEF(ErlDrvSInt, driver_binary_dec_refc, (ErlDrvBinary *dbp)); WDD_TYPEDEF(ErlDrvPDL, driver_pdl_create, (ErlDrvPort)); WDD_TYPEDEF(void, driver_pdl_lock, (ErlDrvPDL)); WDD_TYPEDEF(void, driver_pdl_unlock, (ErlDrvPDL)); -WDD_TYPEDEF(long, driver_pdl_get_refc, (ErlDrvPDL)); -WDD_TYPEDEF(long, driver_pdl_inc_refc, (ErlDrvPDL)); -WDD_TYPEDEF(long, driver_pdl_dec_refc, (ErlDrvPDL)); +WDD_TYPEDEF(ErlDrvSInt, driver_pdl_get_refc, (ErlDrvPDL)); +WDD_TYPEDEF(ErlDrvSInt, driver_pdl_inc_refc, (ErlDrvPDL)); +WDD_TYPEDEF(ErlDrvSInt, driver_pdl_dec_refc, (ErlDrvPDL)); WDD_TYPEDEF(void, driver_system_info, (ErlDrvSysInfo *, size_t)); WDD_TYPEDEF(int, driver_get_now, (ErlDrvNowData *)); WDD_TYPEDEF(int, driver_monitor_process, (ErlDrvPort port, diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 15d4cd7361..a2159d063c 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2010. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -31,7 +31,7 @@ #include "global.h" #include "erl_threads.h" #include "../../drivers/win32/win_con.h" - +#include "erl_cpu_topology.h" void erts_sys_init_float(void); @@ -67,14 +67,17 @@ static void async_read_file(struct async_io* aio, LPVOID buf, DWORD numToRead); static int async_write_file(struct async_io* aio, LPVOID buf, DWORD numToWrite); static int get_overlapped_result(struct async_io* aio, LPDWORD pBytesRead, BOOL wait); -static BOOL CreateChildProcess(char *, HANDLE, HANDLE, +static BOOL create_child_process(char *, HANDLE, HANDLE, HANDLE, LPHANDLE, BOOL, LPVOID, LPTSTR, unsigned, char **, int *); static int create_pipe(LPHANDLE, LPHANDLE, BOOL, BOOL); -static int ApplicationType(const char* originalName, char fullPath[MAX_PATH], +static int application_type(const char* originalName, char fullPath[MAX_PATH], BOOL search_in_path, BOOL handle_quotes, int *error_return); +static int application_type_w(const WCHAR *originalName, WCHAR fullPath[MAX_PATH], + BOOL search_in_path, BOOL handle_quotes, + int *error_return); HANDLE erts_service_event; @@ -87,7 +90,7 @@ static erts_smp_atomic_t pipe_creation_counter; static erts_smp_mtx_t sys_driver_data_lock; -/* Results from ApplicationType is one of */ +/* Results from application_type(_w) is one of */ #define APPL_NONE 0 #define APPL_DOS 1 #define APPL_WIN3X 2 @@ -97,7 +100,7 @@ static int driver_write(long, HANDLE, byte*, int); static void common_stop(int); static int create_file_thread(struct async_io* aio, int mode); #ifdef ERTS_SMP -static void close_active_handles(ErlDrvPort, const HANDLE* handles, int cnt); +static void close_active_handle(ErlDrvPort, HANDLE handle); static DWORD WINAPI threaded_handle_closer(LPVOID param); #endif static DWORD WINAPI threaded_reader(LPVOID param); @@ -137,7 +140,11 @@ static BOOL win_console = FALSE; static OSVERSIONINFO int_os_version; /* Version information for Win32. */ -#ifdef ERTS_SMP +/*#define USE_CANCELIOEX + Disabled the use of CancelIoEx as its been seen to cause problem with some + drivers. Not sure what to blame; faulty drivers or some form of invalid use. +*/ +#if defined(ERTS_SMP) && defined(USE_CANCELIOEX) static BOOL (WINAPI *fpCancelIoEx)(HANDLE,LPOVERLAPPED); #endif @@ -253,7 +260,7 @@ erts_sys_prepare_crash_dump(void) } static void -init_console() +init_console(void) { char* mode = erts_read_env("ERL_CONSOLE_MODE"); @@ -273,7 +280,7 @@ init_console() erts_free_read_env(mode); } -int sys_max_files() +int sys_max_files(void) { return max_files; } @@ -289,10 +296,7 @@ int sys_max_files() */ static int -get_and_remove_option(argc, argv, option) - int* argc; /* Number of arguments. */ - char* argv[]; /* The argument vector. */ - const char* option; /* Option to search for and remove. */ +get_and_remove_option(int* argc, char* argv[], const char *option) { int i; @@ -342,9 +346,7 @@ static char *get_and_remove_option2(int *argc, char **argv, char os_type[] = "win32"; void -os_flavor(namebuf, size) -char* namebuf; /* Where to return the name. */ -unsigned size; /* Size of name buffer. */ +os_flavor(char *namebuf, unsigned size) { switch (int_os_version.dwPlatformId) { case VER_PLATFORM_WIN32_WINDOWS: @@ -617,12 +619,7 @@ struct erl_drv_entry async_driver_entry = { */ static DriverData* -new_driver_data(port_num, packet_bytes, wait_objs_required, use_threads) - int port_num; /* The port number. */ - int packet_bytes; /* Number of bytes in header. */ - int wait_objs_required; /* The number objects this port is going - /* wait for (typically 1 or 2). */ - int use_threads; /* TRUE if threads are intended to be used. */ +new_driver_data(int port_num, int packet_bytes, int wait_objs_required, int use_threads) { DriverData* dp; @@ -684,6 +681,7 @@ release_driver_data(DriverData* dp) erts_smp_mtx_lock(&sys_driver_data_lock); #ifdef ERTS_SMP +#ifdef USE_CANCELIOEX if (fpCancelIoEx != NULL) { if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { (*fpCancelIoEx)(dp->in.fd, NULL); @@ -692,10 +690,12 @@ release_driver_data(DriverData* dp) (*fpCancelIoEx)(dp->out.fd, NULL); } } - else { + else +#endif + { /* This is a workaround for the fact that CancelIo cant cancel requests issued by another thread and that we cant use - CancelIoEx as that's only availabele in Vista etc. + CancelIoEx as that's only available in Vista etc. R14: Avoid scheduler deadlock by only wait for 10ms, and then spawn a thread that will keep waiting in in order to close handles. */ HANDLE handles[2]; @@ -706,7 +706,7 @@ release_driver_data(DriverData* dp) dp->in.fd = INVALID_HANDLE_VALUE; DEBUGF(("Waiting for the in event thingie")); if (WaitForSingleObject(dp->in.ov.hEvent,timeout) == WAIT_TIMEOUT) { - handles[i++] = dp->in.ov.hEvent; + close_active_handle(dp->port_num, dp->in.ov.hEvent); dp->in.ov.hEvent = NULL; timeout = 0; } @@ -717,14 +717,11 @@ release_driver_data(DriverData* dp) dp->out.fd = INVALID_HANDLE_VALUE; DEBUGF(("Waiting for the out event thingie")); if (WaitForSingleObject(dp->out.ov.hEvent,timeout) == WAIT_TIMEOUT) { - handles[i++] = dp->out.ov.hEvent; + close_active_handle(dp->port_num, dp->out.ov.hEvent); dp->out.ov.hEvent = NULL; } DEBUGF(("...done\n")); } - if (i > 0) { - close_active_handles(dp->port_num, handles, i); - } } #else if (dp->in.thread == (HANDLE) -1 && dp->in.fd != INVALID_HANDLE_VALUE) { @@ -772,42 +769,82 @@ release_driver_data(DriverData* dp) #ifdef ERTS_SMP -struct handles_to_be_closed -{ - int cnt; - HANDLE handles[2]; +struct handles_to_be_closed { + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + unsigned cnt; }; +static struct handles_to_be_closed* htbc_curr = NULL; +CRITICAL_SECTION htbc_lock; -static void close_active_handles(ErlDrvPort port_num, const HANDLE* handles, int cnt) +static void close_active_handle(ErlDrvPort port_num, HANDLE handle) { - DWORD tid; - HANDLE thread; + struct handles_to_be_closed* htbc; int i; - struct handles_to_be_closed* htbc = erts_alloc(ERTS_ALC_T_DRV_TAB, - sizeof(struct handles_to_be_closed)); - htbc->cnt = cnt; - for (i=0; i < cnt; ++i) { - htbc->handles[i] = handles[i]; - (void) driver_select(port_num, (ErlDrvEvent)handles[i], - ERL_DRV_USE_NO_CALLBACK, 0); + EnterCriticalSection(&htbc_lock); + htbc = htbc_curr; + if (htbc == NULL || htbc->cnt >= MAXIMUM_WAIT_OBJECTS) { + DWORD tid; + HANDLE thread; + + htbc = (struct handles_to_be_closed*) erts_alloc(ERTS_ALC_T_DRV_TAB, + sizeof(*htbc)); + htbc->handles[0] = CreateAutoEvent(FALSE); + htbc->cnt = 1; + thread = (HANDLE *) _beginthreadex(NULL, 0, threaded_handle_closer, htbc, 0, &tid); + CloseHandle(thread); } - thread = (HANDLE *) _beginthreadex(NULL, 0, threaded_handle_closer, htbc, 0, &tid); - CloseHandle(thread); + htbc->handles[htbc->cnt++] = handle; + driver_select(port_num, (ErlDrvEvent)handle, ERL_DRV_USE_NO_CALLBACK, 0); + SetEvent(htbc->handles[0]); + htbc_curr = htbc; + LeaveCriticalSection(&htbc_lock); } - static DWORD WINAPI threaded_handle_closer(LPVOID param) { struct handles_to_be_closed* htbc = (struct handles_to_be_closed*) param; - int i; - DEBUGF(("threaded_handle_closer waiting for %d handles\r\n",htbc->cnt)); - WaitForMultipleObjects(htbc->cnt, htbc->handles, TRUE, INFINITE); - for (i=0; i < htbc->cnt; ++i) { - CloseHandle(htbc->handles[i]); + unsigned ix; + DWORD res; + DEBUGF(("threaded_handle_closer %p started\r\n", htbc)); + EnterCriticalSection(&htbc_lock); + for (;;) { + { + HANDLE* handles = htbc->handles; + unsigned cnt = htbc->cnt; + DWORD timeout = (htbc == htbc_curr) ? INFINITE : 10*1000; + + LeaveCriticalSection(&htbc_lock); + DEBUGF(("threaded_handle_closer %p waiting for %d handles\r\n", htbc, cnt)); + res = WaitForMultipleObjects(cnt, handles, FALSE, timeout); + } + EnterCriticalSection(&htbc_lock); + switch (res) { + case WAIT_OBJECT_0: + case WAIT_TIMEOUT: + break; /* got some more handles to wait for maybe */ + default: + ix = res - WAIT_OBJECT_0; + if (ix > 0 && ix < htbc->cnt) { + CloseHandle(htbc->handles[ix]); + htbc->handles[ix] = htbc->handles[--htbc->cnt]; + } + } + if (htbc != htbc_curr) { + if (htbc->cnt == 1) { /* no real handles left */ + break; + } + /* The thread with most free slots will be "current" */ + if (htbc->cnt < htbc_curr->cnt) { + htbc_curr = htbc; + DEBUGF(("threaded_handle_closer %p made current\r\n", htbc)); + } + } } + LeaveCriticalSection(&htbc_lock); + CloseHandle(htbc->handles[0]); erts_free(ERTS_ALC_T_DRV_TAB, htbc); - DEBUGF(("threaded_handle_closer terminating\r\n")); + DEBUGF(("threaded_handle_closer %p terminating\r\n", htbc)); return 0; } #endif /* ERTS_SMP */ @@ -820,12 +857,7 @@ threaded_handle_closer(LPVOID param) */ static ErlDrvData -set_driver_data(dp, ifd, ofd, read_write, report_exit) - DriverData* dp; - HANDLE ifd; - HANDLE ofd; - int read_write; - int report_exit; +set_driver_data(DriverData* dp, HANDLE ifd, HANDLE ofd, int read_write, int report_exit) { int index = dp - driver_data; int result; @@ -849,6 +881,31 @@ set_driver_data(dp, ifd, ofd, read_write, report_exit) return (ErlDrvData)index; } +static ErlDrvData +reuse_driver_data(DriverData *dp, HANDLE ifd, HANDLE ofd, int read_write, ErlDrvPort port_num) +{ + int index = dp - driver_data; + int result; + + dp->port_num = port_num; + dp->in.fd = ifd; + dp->out.fd = ofd; + dp->report_exit = 0; + + if (read_write & DO_READ) { + result = driver_select(dp->port_num, (ErlDrvEvent)dp->in.ov.hEvent, + ERL_DRV_READ|ERL_DRV_USE, 1); + ASSERT(result != -1); + } + + if (read_write & DO_WRITE) { + result = driver_select(dp->port_num, (ErlDrvEvent)dp->out.ov.hEvent, + ERL_DRV_WRITE|ERL_DRV_USE, 1); + ASSERT(result != -1); + } + return (ErlDrvData)index; +} + /* * Initialises an AsyncIo structure. */ @@ -922,10 +979,7 @@ release_async_io(AsyncIo* aio, ErlDrvPort port_num) */ static void -async_read_file(aio, buf, numToRead) - AsyncIo* aio; /* Pointer to driver data. */ - LPVOID buf; /* Pointer to buffer to receive data. */ - DWORD numToRead; /* Number of bytes to read. */ +async_read_file(AsyncIo* aio, LPVOID buf, DWORD numToRead) { aio->pendingError = NO_ERROR; #ifdef HARD_POLL_DEBUG @@ -976,10 +1030,9 @@ async_read_file(aio, buf, numToRead) * ---------------------------------------------------------------------- */ static int -async_write_file(aio, buf, numToWrite) - AsyncIo* aio; /* Pointer to async control block. */ - LPVOID buf; /* Pointer to buffer with data to write. */ - DWORD numToWrite; /* Number of bytes to write. */ +async_write_file(AsyncIo* aio, /* Pointer to async control block. */ + LPVOID buf, /* Pointer to buffer with data to write. */ + DWORD numToWrite) /* Number of bytes to write. */ { aio->pendingError = NO_ERROR; if (aio->thread != (HANDLE) -1) { @@ -1023,12 +1076,12 @@ async_write_file(aio, buf, numToWrite) * ---------------------------------------------------------------------- */ static int -get_overlapped_result(aio, pBytesRead, wait) - AsyncIo* aio; /* Pointer to async control block. */ - LPDWORD pBytesRead; /* Where to place the number of bytes - * transferred. - */ - BOOL wait; /* If true, wait until result is ready. */ +get_overlapped_result(AsyncIo* aio, /* Pointer to async control block. */ + LPDWORD pBytesRead, /* Where to place the number of bytes + * transferred. + */ + BOOL wait /* If true, wait until result is ready. */ + ) { DWORD error = NO_ERROR; /* Error status from last function. */ @@ -1098,14 +1151,13 @@ fd_init(void) return 0; } static int -spawn_init() +spawn_init(void) { int i; -#ifdef ERTS_SMP +#if defined(ERTS_SMP) && defined(USE_CANCELIOEX) HMODULE module = GetModuleHandle("kernel32"); - fpCancelIoEx = (module != NULL) ? - (BOOL (WINAPI *)(HANDLE,LPOVERLAPPED)) - GetProcAddress(module,"CancelIoEx") : NULL; + fpCancelIoEx = (BOOL (WINAPI *)(HANDLE,LPOVERLAPPED)) + ((module != NULL) ? GetProcAddress(module,"CancelIoEx") : NULL); DEBUGF(("fpCancelIoEx = %p\r\n", fpCancelIoEx)); #endif driver_data = (struct driver_data *) @@ -1192,8 +1244,10 @@ spawn_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts) */ DEBUGF(("Spawning \"%s\"\n", name)); - envir = win_build_environment(envir); - ok = CreateChildProcess(name, + envir = win_build_environment(envir); /* Still an ansi environment, could be + converted to unicode for spawn_executable, but + that is not done (yet) */ + ok = create_child_process(name, hChildStdin, hChildStdout, hChildStderr, @@ -1272,7 +1326,7 @@ create_file_thread(AsyncIo* aio, int mode) } /* - * A helper function used by CreateChildProcess(). + * A helper function used by create_child_process(). * Parses a command line with arguments and returns the length of the * first part containing the program name. * Example: input = "\"Program Files\"\\erl arg1 arg2" @@ -1313,24 +1367,25 @@ int parse_command(char* cmd){ return i; } -BOOL need_quotes(char *str) +static BOOL need_quotes(WCHAR *str) { int in_quote = 0; int backslashed = 0; int naked_space = 0; - while (*str != '\0') { + + while (*str != L'\0') { switch (*str) { - case '\\' : + case L'\\' : backslashed = !backslashed; break; - case '"': + case L'"': if (backslashed) { backslashed=0; } else { in_quote = !in_quote; } break; - case ' ': + case L' ': backslashed = 0; if (!(backslashed || in_quote)) { naked_space++; @@ -1349,7 +1404,7 @@ BOOL need_quotes(char *str) /* *---------------------------------------------------------------------- * - * CreateChildProcess -- + * create_child_process -- * * Create a child process that has pipes as its * standard input, output, and error. The child process runs @@ -1374,7 +1429,7 @@ BOOL need_quotes(char *str) */ static BOOL -CreateChildProcess +create_child_process ( char *origcmd, /* Command line for child process (including * name of executable). Or whole executable if st is @@ -1393,14 +1448,12 @@ CreateChildProcess ) { PROCESS_INFORMATION piProcInfo = {0}; - STARTUPINFO siStartInfo = {0}; BOOL ok = FALSE; int applType; /* Not to be changed for different types of executables */ int staticCreateFlags = GetPriorityClass(GetCurrentProcess()); int createFlags = DETACHED_PROCESS; char *newcmdline = NULL; - char execPath[MAX_PATH]; int cmdlength; char* thecommand; LPTSTR appname = NULL; @@ -1408,14 +1461,17 @@ CreateChildProcess *errno_return = -1; - siStartInfo.cb = sizeof(STARTUPINFO); - siStartInfo.dwFlags = STARTF_USESTDHANDLES; - siStartInfo.hStdInput = hStdin; - siStartInfo.hStdOutput = hStdout; - siStartInfo.hStdError = hStderr; - if (st != ERTS_SPAWN_EXECUTABLE) { + STARTUPINFO siStartInfo = {0}; + char execPath[MAX_PATH]; + + siStartInfo.cb = sizeof(STARTUPINFO); + siStartInfo.dwFlags = STARTF_USESTDHANDLES; + siStartInfo.hStdInput = hStdin; + siStartInfo.hStdOutput = hStdout; + siStartInfo.hStdError = hStderr; + /* * Parse out the program name from the command line (it can be quoted and * contain spaces). @@ -1427,9 +1483,9 @@ CreateChildProcess thecommand[cmdlength] = '\0'; DEBUGF(("spawn command: %s\n", thecommand)); - applType = ApplicationType(thecommand, execPath, TRUE, + applType = application_type(thecommand, execPath, TRUE, TRUE, errno_return); - DEBUGF(("ApplicationType returned for (%s) is %d\n", thecommand, applType)); + DEBUGF(("application_type returned for (%s) is %d\n", thecommand, applType)); erts_free(ERTS_ALC_T_TMP, (void *) thecommand); if (applType == APPL_NONE) { erts_free(ERTS_ALC_T_TMP,newcmdline); @@ -1458,126 +1514,147 @@ CreateChildProcess strcat(newcmdline, execPath); strcat(newcmdline, origcmd+cmdlength); - } else { /* ERTS_SPAWN_EXECUTABLE */ + DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags)); + ok = CreateProcessA(appname, + newcmdline, + NULL, + NULL, + TRUE, + createFlags | staticCreateFlags, + env, + wd, + &siStartInfo, + &piProcInfo); + + } else { /* ERTS_SPAWN_EXECUTABLE, filename and args are in unicode ({utf16,little}) */ int run_cmd = 0; - applType = ApplicationType(origcmd, execPath, FALSE, FALSE, - errno_return); + STARTUPINFOW siStartInfo = {0}; + WCHAR execPath[MAX_PATH]; + + + siStartInfo.cb = sizeof(STARTUPINFOW); + siStartInfo.dwFlags = STARTF_USESTDHANDLES; + siStartInfo.hStdInput = hStdin; + siStartInfo.hStdOutput = hStdout; + siStartInfo.hStdError = hStderr; + + applType = application_type_w((WCHAR *) origcmd, execPath, FALSE, FALSE, + errno_return); if (applType == APPL_NONE) { return FALSE; } if (applType == APPL_DOS) { - /* - * See comment above - */ + /* + * See comment above + */ - siStartInfo.wShowWindow = SW_HIDE; - siStartInfo.dwFlags |= STARTF_USESHOWWINDOW; - createFlags = CREATE_NEW_CONSOLE; - run_cmd = 1; + siStartInfo.wShowWindow = SW_HIDE; + siStartInfo.dwFlags |= STARTF_USESHOWWINDOW; + createFlags = CREATE_NEW_CONSOLE; + run_cmd = 1; } else if (hide) { - DEBUGF(("hiding window\n")); - siStartInfo.wShowWindow = SW_HIDE; - siStartInfo.dwFlags |= STARTF_USESHOWWINDOW; - createFlags = 0; + DEBUGF(("hiding window\n")); + siStartInfo.wShowWindow = SW_HIDE; + siStartInfo.dwFlags |= STARTF_USESHOWWINDOW; + createFlags = 0; } if (run_cmd) { - char cmdPath[MAX_PATH]; + WCHAR cmdPath[MAX_PATH]; int cmdType; - cmdType = ApplicationType("cmd.exe", cmdPath, TRUE, FALSE, errno_return); + cmdType = application_type_w(L"cmd.exe", cmdPath, TRUE, FALSE, errno_return); if (cmdType == APPL_NONE || cmdType == APPL_DOS) { return FALSE; } - appname = (char *) erts_alloc(ERTS_ALC_T_TMP, strlen(cmdPath)+1); - strcpy(appname,cmdPath); + appname = (char *) erts_alloc(ERTS_ALC_T_TMP, (wcslen(cmdPath)+1)*sizeof(WCHAR)); + wcscpy((WCHAR *) appname,cmdPath); } else { - appname = (char *) erts_alloc(ERTS_ALC_T_TMP, strlen(execPath)+1); - strcpy(appname,execPath); + appname = (char *) erts_alloc(ERTS_ALC_T_TMP, (wcslen(execPath)+1)*sizeof(WCHAR)); + wcscpy((WCHAR *) appname, execPath); } - if (argv == NULL) { + if (argv == NULL) { BOOL orig_need_q = need_quotes(execPath); - char *ptr; - int ocl = strlen(execPath); + WCHAR *ptr; + int ocl = wcslen(execPath); if (run_cmd) { newcmdline = (char *) erts_alloc(ERTS_ALC_T_TMP, - ocl + ((orig_need_q) ? 3 : 1) - + 11); - memcpy(newcmdline,"cmd.exe /c ",11); - ptr = newcmdline + 11; + (ocl + ((orig_need_q) ? 3 : 1) + + 11)*sizeof(WCHAR)); + memcpy(newcmdline,L"cmd.exe /c ",11*sizeof(WCHAR)); + ptr = (WCHAR *) (newcmdline + (11*sizeof(WCHAR))); } else { newcmdline = (char *) erts_alloc(ERTS_ALC_T_TMP, - ocl + ((orig_need_q) ? 3 : 1)); - ptr = newcmdline; + (ocl + ((orig_need_q) ? 3 : 1))*sizeof(WCHAR)); + ptr = (WCHAR *) newcmdline; } if (orig_need_q) { - *ptr++ = '"'; + *ptr++ = L'"'; } - memcpy(ptr,execPath,ocl); + memcpy(ptr,execPath,ocl*sizeof(WCHAR)); ptr += ocl; if (orig_need_q) { - *ptr++ = '"'; + *ptr++ = L'"'; } - *ptr = '\0'; + *ptr = L'\0'; } else { int sum = 1; /* '\0' */ - char **ar = argv; - char *n; + WCHAR **ar = (WCHAR **) argv; + WCHAR *n; char *save_arg0 = NULL; if (argv[0] == erts_default_arg0 || run_cmd) { save_arg0 = argv[0]; - argv[0] = execPath; + argv[0] = (char *) execPath; } if (run_cmd) { sum += 11; /* cmd.exe /c */ } while (*ar != NULL) { - sum += strlen(*ar); + sum += wcslen(*ar); if (need_quotes(*ar)) { sum += 2; /* quotes */ } sum++; /* space */ ++ar; } - ar = argv; - newcmdline = erts_alloc(ERTS_ALC_T_TMP, sum); - n = newcmdline; + ar = (WCHAR **) argv; + newcmdline = erts_alloc(ERTS_ALC_T_TMP, sum*sizeof(WCHAR)); + n = (WCHAR *) newcmdline; if (run_cmd) { - memcpy(n,"cmd.exe /c ",11); + memcpy(n,L"cmd.exe /c ",11*sizeof(WCHAR)); n += 11; } while (*ar != NULL) { int q = need_quotes(*ar); - sum = strlen(*ar); + sum = wcslen(*ar); if (q) { - *n++ = '"'; + *n++ = L'"'; } - memcpy(n,*ar,sum); + memcpy(n,*ar,sum*sizeof(WCHAR)); n += sum; if (q) { - *n++ = '"'; + *n++ = L'"'; } - *n++ = ' '; + *n++ = L' '; ++ar; } - ASSERT(n > newcmdline); - *(n-1) = '\0'; + *(n-1) = L'\0'; if (save_arg0 != NULL) { argv[0] = save_arg0; } } - } - DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags)); - ok = CreateProcess(appname, - newcmdline, - NULL, - NULL, - TRUE, - createFlags | staticCreateFlags, - env, - wd, - &siStartInfo, - &piProcInfo); - + DEBUGF(("Creating child process: %s, createFlags = %d\n", newcmdline, createFlags)); + ok = CreateProcessW((WCHAR *) appname, + (WCHAR *) newcmdline, + NULL, + NULL, + TRUE, + createFlags | staticCreateFlags, + env, + (WCHAR *) wd, + &siStartInfo, + &piProcInfo); + + } /* end SPAWN_EXECUTABLE */ if (newcmdline != NULL) { erts_free(ERTS_ALC_T_TMP,newcmdline); } @@ -1696,7 +1773,7 @@ static int create_pipe(HANDLE *phRead, HANDLE *phWrite, BOOL inheritRead, BOOL o -static int ApplicationType +static int application_type ( const char *originalName, /* Name of the application to find. */ char fullPath[MAX_PATH], /* Filled with complete path to @@ -1850,6 +1927,145 @@ static int ApplicationType return applType; } +static int application_type_w (const WCHAR *originalName, /* Name of the application to find. */ + WCHAR wfullpath[MAX_PATH],/* Filled with complete path to + * application. */ + BOOL search_in_path, /* If we should search the system wide path */ + BOOL handle_quotes, /* If we should handle quotes around executable */ + int *error_return) /* A place to put an error code */ +{ + int applType, i; + HANDLE hFile; + WCHAR *ext, *rest; + char buf[2]; + DWORD read; + IMAGE_DOS_HEADER header; + static WCHAR extensions[][5] = {L"", L".com", L".exe", L".bat"}; + int is_quoted; + int len; + WCHAR xfullpath[MAX_PATH]; + + len = wcslen(originalName); + is_quoted = handle_quotes && len > 0 && originalName[0] == L'"' && + originalName[len-1] == L'"'; + + applType = APPL_NONE; + *error_return = ENOENT; + for (i = 0; i < (int) (sizeof(extensions) / sizeof(extensions[0])); i++) { + if(is_quoted) { + lstrcpynW(xfullpath, originalName+1, MAX_PATH - 7); /* Cannot start using StringCchCopy yet, we support + older platforms */ + len = wcslen(xfullpath); + if(len > 0) { + xfullpath[len-1] = L'\0'; + } + } else { + lstrcpynW(xfullpath, originalName, MAX_PATH - 5); + } + wcscat(xfullpath, extensions[i]); + /* It seems that the Unicode version does not allow in and out parameter to overlap. */ + SearchPathW((search_in_path) ? NULL : L".", xfullpath, NULL, MAX_PATH, wfullpath, &rest); + + /* + * Ignore matches on directories or data files, return if identified + * a known type. + */ + + if (GetFileAttributesW(wfullpath) & FILE_ATTRIBUTE_DIRECTORY) { + continue; + } + + ext = wcsrchr(wfullpath, L'.'); + if ((ext != NULL) && (_wcsicmp(ext, L".bat") == 0)) { + *error_return = EACCES; + applType = APPL_DOS; + break; + } + + hFile = CreateFileW(wfullpath, GENERIC_READ, FILE_SHARE_READ, NULL, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) { + continue; + } + + *error_return = EACCES; /* If considered an error, + it's an access error */ + header.e_magic = 0; + ReadFile(hFile, (void *) &header, sizeof(header), &read, NULL); + if (header.e_magic != IMAGE_DOS_SIGNATURE) { + /* + * Doesn't have the magic number for relocatable executables. If + * filename ends with .com, assume it's a DOS application anyhow. + * Note that we didn't make this assumption at first, because some + * supposed .com files are really 32-bit executables with all the + * magic numbers and everything. + */ + + CloseHandle(hFile); + if ((ext != NULL) && (_wcsicmp(ext, L".com") == 0)) { + applType = APPL_DOS; + break; + } + continue; + } + if (header.e_lfarlc != sizeof(header)) { + /* + * All Windows 3.X and Win32 and some DOS programs have this value + * set here. If it doesn't, assume that since it already had the + * other magic number it was a DOS application. + */ + + CloseHandle(hFile); + applType = APPL_DOS; + break; + } + + /* + * The DWORD at header.e_lfanew points to yet another magic number. + */ + + buf[0] = '\0'; + SetFilePointer(hFile, header.e_lfanew, NULL, FILE_BEGIN); + ReadFile(hFile, (void *) buf, 2, &read, NULL); + CloseHandle(hFile); + + if ((buf[0] == 'L') && (buf[1] == 'E')) { + applType = APPL_DOS; + } else if ((buf[0] == 'N') && (buf[1] == 'E')) { + applType = APPL_WIN3X; + } else if ((buf[0] == 'P') && (buf[1] == 'E')) { + applType = APPL_WIN32; + } else { + continue; + } + break; + } + + if (applType == APPL_NONE) { + return APPL_NONE; + } + + if ((applType == APPL_DOS) || (applType == APPL_WIN3X)) { + /* + * Replace long path name of executable with short path name for + * 16-bit applications. Otherwise the application may not be able + * to correctly parse its own command line to separate off the + * application name from the arguments. + */ + + GetShortPathNameW(wfullpath, wfullpath, MAX_PATH); + } + if (is_quoted) { + /* restore quotes on quoted program name */ + len = wcslen(wfullpath); + memmove(wfullpath+1,wfullpath,len*sizeof(WCHAR)); + wfullpath[0]=L'"'; + wfullpath[len+1]=L'"'; + wfullpath[len+2]=L'\0'; + } + return applType; +} + /* * Thread function used to emulate overlapped reading. */ @@ -1869,9 +2085,10 @@ threaded_reader(LPVOID param) buf = OV_BUFFER_PTR(aio); numToRead = OV_NUM_TO_READ(aio); aio->pendingError = 0; - if (!ReadFile(aio->fd, buf, numToRead, &aio->bytesTransferred, NULL)) - aio->pendingError = GetLastError(); - else if (aio->flags & DF_XLAT_CR) { + if (!ReadFile(aio->fd, buf, numToRead, &aio->bytesTransferred, NULL)) { + int error = GetLastError(); + aio->pendingError = error; + } else if (aio->flags & DF_XLAT_CR) { char *s; int n; @@ -1998,56 +2215,79 @@ translate_fd(int fd) return handle; } +/* Driver level locking, start function is serialized */ +static DriverData *save_01_port = NULL; +static DriverData *save_22_port = NULL; + static ErlDrvData fd_start(ErlDrvPort port_num, char* name, SysDriverOpts* opts) { DriverData* dp; int is_std_error = (opts->ofd == 2); - - opts->ifd = (int) translate_fd(opts->ifd); - opts->ofd = (int) translate_fd(opts->ofd); - if ((dp = new_driver_data(port_num, opts->packet_bytes, 2, TRUE)) == NULL) - return ERL_DRV_ERROR_GENERAL; - - if (!create_file_thread(&dp->in, DO_READ)) { - dp->port_num = PORT_FREE; - return ERL_DRV_ERROR_GENERAL; - } - - if (!create_file_thread(&dp->out, DO_WRITE)) { - dp->port_num = PORT_FREE; - return ERL_DRV_ERROR_GENERAL; - } - - fd_driver_input = &(dp->in); - dp->in.flags = DF_XLAT_CR; - if (is_std_error) { - dp->out.flags |= DF_DROP_IF_INVH; /* Just drop messages if stderror - is an invalid handle */ + int in = opts->ifd, out = opts->ofd; + + opts->ifd = (Uint) translate_fd(in); + opts->ofd = (Uint) translate_fd(out); + if ( in == 0 && out == 1 && save_01_port != NULL) { + dp = save_01_port; + return reuse_driver_data(dp, (HANDLE) opts->ifd, (HANDLE) opts->ofd, opts->read_write, port_num); + } else if (in == 2 && out == 2 && save_22_port != NULL) { + dp = save_22_port; + return reuse_driver_data(dp, (HANDLE) opts->ifd, (HANDLE) opts->ofd, opts->read_write, port_num); + } else { + if ((dp = new_driver_data(port_num, opts->packet_bytes, 2, TRUE)) == NULL) + return ERL_DRV_ERROR_GENERAL; + + if (!create_file_thread(&dp->in, DO_READ)) { + dp->port_num = PORT_FREE; + return ERL_DRV_ERROR_GENERAL; + } + + if (!create_file_thread(&dp->out, DO_WRITE)) { + dp->port_num = PORT_FREE; + return ERL_DRV_ERROR_GENERAL; + } + + fd_driver_input = &(dp->in); + dp->in.flags = DF_XLAT_CR; + if (is_std_error) { + dp->out.flags |= DF_DROP_IF_INVH; /* Just drop messages if stderror + is an invalid handle */ + } + + if ( in == 0 && out == 1) { + save_01_port = dp; + } else if (in == 2 && out == 2) { + save_22_port = dp; + } + return set_driver_data(dp, (HANDLE) opts->ifd, (HANDLE) opts->ofd, opts->read_write, 0); } - return set_driver_data(dp, opts->ifd, opts->ofd, opts->read_write, 0); } static void fd_stop(ErlDrvData d) { int fd = (int)d; + DriverData* dp = driver_data+fd; /* - * I don't know a clean way to terminate the threads - * (TerminateThread() doesn't release the stack), - * so will we'll let the threads live. Normally, the fd - * driver is only used to support the -oldshell option, - * so this shouldn't be a problem in practice. - * - * Since we will not attempt to terminate the threads, - * better not close the input or output files either. + * There's no way we can terminate an fd port in a consistent way. + * Instead we let it live until it's opened again (which it is, + * as the only FD-drivers are for 0,1 and 2 adn the only time they + * get closed is by init:reboot). + * So - just deselect them and let everything be as is. + * They get woken up in fd_start again, where the DriverData is + * remembered. /PaN */ + if (dp->in.ov.hEvent != NULL) { + (void) driver_select(dp->port_num, + (ErlDrvEvent)dp->in.ov.hEvent, + ERL_DRV_READ, 0); + } + if (dp->out.ov.hEvent != NULL) { + (void) driver_select(dp->port_num, + (ErlDrvEvent)dp->out.ov.hEvent, + ERL_DRV_WRITE, 0); + } - driver_data[fd].in.thread = (HANDLE) -1; - driver_data[fd].out.thread = (HANDLE) -1; - driver_data[fd].in.fd = INVALID_HANDLE_VALUE; - driver_data[fd].out.fd = INVALID_HANDLE_VALUE; - - /*return */ common_stop(fd); } static ErlDrvData @@ -2139,7 +2379,6 @@ threaded_exiter(LPVOID param) * because it is an auto reset event. Therefore, always set the * exit flag and signal the event. */ - i = 0; if (dp->out.thread != (HANDLE) -1) { dp->out.flags = DF_EXIT_THREAD; @@ -2507,6 +2746,7 @@ ready_input(ErlDrvData drv_data, ErlDrvEvent ready_event) driver_failure_eof(dp->port_num); } else { /* Report real errors. */ int error = GetLastError(); + (void) driver_select(dp->port_num, ready_event, ERL_DRV_READ, 0); _dosmaperr(error); driver_failure_posix(dp->port_num, errno); @@ -2973,13 +3213,50 @@ check_supported_os_version(void) } #ifdef USE_THREADS -#ifdef ERTS_ENABLE_LOCK_COUNT + +typedef struct { + int sched_bind_data; +} erts_thr_create_data_t; + +/* + * thr_create_prepare() is called in parent thread before thread creation. + * Returned value is passed as argument to thr_create_cleanup(). + */ +static void * +thr_create_prepare(void) +{ + erts_thr_create_data_t *tcdp; + + tcdp = erts_alloc(ERTS_ALC_T_TMP, sizeof(erts_thr_create_data_t)); + tcdp->sched_bind_data = erts_sched_bind_atthrcreate_prepare(); + + return (void *) tcdp; +} + + +/* thr_create_cleanup() is called in parent thread after thread creation. */ +static void +thr_create_cleanup(void *vtcdp) +{ + erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp; + + erts_sched_bind_atthrcreate_parent(tcdp->sched_bind_data); + + erts_free(ERTS_ALC_T_TMP, tcdp); +} + static void thr_create_prepare_child(void *vtcdp) { + erts_thr_create_data_t *tcdp = (erts_thr_create_data_t *) vtcdp; + +#ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_thread_setup(); -} #endif /* ERTS_ENABLE_LOCK_COUNT */ + + erts_sched_bind_atthrcreate_child(tcdp->sched_bind_data); +} + #endif /* USE_THREADS */ void @@ -2991,9 +3268,13 @@ erts_sys_pre_init(void) #ifdef USE_THREADS { erts_thr_init_data_t eid = ERTS_THR_INIT_DATA_DEF_INITER; -#ifdef ERTS_ENABLE_LOCK_COUNT + eid.thread_create_child_func = thr_create_prepare_child; -#endif + /* Before creation in parent */ + eid.thread_create_prepare_func = thr_create_prepare; + /* After creation in parent */ + eid.thread_create_parent_func = thr_create_cleanup, + erts_thr_init(&eid); #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_init(); @@ -3027,6 +3308,7 @@ void erl_sys_init(void) #ifdef ERTS_SMP erts_smp_tsd_key_create(&win32_errstr_key); + InitializeCriticalSection(&htbc_lock); #endif erts_smp_atomic_init(&pipe_creation_counter,0); /* diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c index d2449a1bdb..943c338794 100644 --- a/erts/emulator/sys/win32/sys_interrupt.c +++ b/erts/emulator/sys/win32/sys_interrupt.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2009. All Rights Reserved. + * Copyright Ericsson AB 1997-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -31,11 +31,11 @@ #endif #ifdef ERTS_SMP -erts_smp_atomic_t erts_break_requested; +erts_smp_atomic32_t erts_break_requested; #define ERTS_SET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 1) + erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 1) #define ERTS_UNSET_BREAK_REQUESTED \ - erts_smp_atomic_set(&erts_break_requested, (long) 0) + erts_smp_atomic32_set(&erts_break_requested, (erts_aint32_t) 0) #else volatile int erts_break_requested = 0; #define ERTS_SET_BREAK_REQUESTED (erts_break_requested = 1) diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 3ab88e41d3..4d0c87bf12 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2010. All Rights Reserved. +# Copyright Ericsson AB 1997-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -83,6 +83,7 @@ MODULES= \ receive_SUITE \ ref_SUITE \ register_SUITE \ + mtx_SUITE \ save_calls_SUITE \ send_term_SUITE \ sensitive_SUITE \ @@ -121,10 +122,14 @@ NO_OPT= bs_bincomp \ bs_utf \ guard +NATIVE= hibernate NO_OPT_MODULES= $(NO_OPT:%=%_no_opt_SUITE) NO_OPT_ERL_FILES= $(NO_OPT_MODULES:%=%.erl) +NATIVE_MODULES= $(NATIVE:%=%_native_SUITE) +NATIVE_ERL_FILES= $(NATIVE_MODULES:%=%.erl) + ERL_FILES= $(MODULES:%=%.erl) TARGET_FILES = $(MODULES:%=$(EBIN)/%.$(EMULATOR)) @@ -150,7 +155,7 @@ ERL_COMPILE_FLAGS += -I$(ERL_TOP)/lib/test_server/include # Targets # ---------------------------------------------------- -make_emakefile: $(NO_OPT_ERL_FILES) +make_emakefile: $(NO_OPT_ERL_FILES) $(NATIVE_ERL_FILES) # This special rule can be removed when communication with R7B nodes # is no longer supported. $(ERL_TOP)/make/make_emakefile $(ERL_COMPILE_FLAGS) +compressed -o$(EBIN) \ @@ -159,6 +164,8 @@ make_emakefile: $(NO_OPT_ERL_FILES) $(MODULES) >> $(EMAKEFILE) $(ERL_TOP)/make/make_emakefile +no_copt +no_postopt $(ERL_COMPILE_FLAGS) \ -o$(EBIN) $(NO_OPT_MODULES) >> $(EMAKEFILE) + $(ERL_TOP)/make/make_emakefile +native $(ERL_COMPILE_FLAGS) \ + -o$(EBIN) $(NATIVE_MODULES) >> $(EMAKEFILE) tests debug opt: make_emakefile erl $(ERL_MAKE_FLAGS) -make @@ -177,6 +184,9 @@ docs: %_no_opt_SUITE.erl: %_SUITE.erl sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ +%_native_SUITE.erl: %_SUITE.erl + sed -e 's;-module($(basename $<));-module($(basename $@));' $< > $@ + # ---------------------------------------------------- # Release Target # ---------------------------------------------------- @@ -189,6 +199,7 @@ release_tests_spec: make_emakefile $(INSTALL_DATA) $(EMAKEFILE) $(TEST_SPEC_FILES) \ $(ERL_FILES) $(RELSYSDIR) $(INSTALL_DATA) $(NO_OPT_ERL_FILES) $(RELSYSDIR) + $(INSTALL_DATA) $(NATIVE_ERL_FILES) $(RELSYSDIR) chmod -R u+w $(RELSYSDIR) tar cf - *_SUITE_data | (cd $(RELSYSDIR); tar xf -) diff --git a/erts/emulator/test/a_SUITE.erl b/erts/emulator/test/a_SUITE.erl index e9d653a7c4..b541be3df6 100644 --- a/erts/emulator/test/a_SUITE.erl +++ b/erts/emulator/test/a_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. +%% Copyright Ericsson AB 2006-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -26,15 +26,32 @@ %%%------------------------------------------------------------------- -module(a_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1, long_timers/1, pollset_size/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, long_timers/1, pollset_size/1]). -all(doc) -> - []; -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [long_timers, pollset_size]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + long_timers(doc) -> []; long_timers(suite) -> diff --git a/erts/emulator/test/after_SUITE.erl b/erts/emulator/test/after_SUITE.erl index 3e1a871408..7cc329cc69 100644 --- a/erts/emulator/test/after_SUITE.erl +++ b/erts/emulator/test/after_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,27 +21,48 @@ %% Tests receive after. --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1, t_after/1, receive_after/1, receive_after_big/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + t_after/1, receive_after/1, receive_after_big/1, receive_after_errors/1, receive_var_zero/1, receive_zero/1, multi_timeout/1, receive_after_32bit/1]). --export([init_per_testcase/2, fin_per_testcase/2]). +-export([init_per_testcase/2, end_per_testcase/2]). %% Internal exports. -export([timeout_g/0]). -all(suite) -> - [t_after, receive_after, receive_after_big, receive_after_errors, - receive_var_zero, receive_zero, multi_timeout, receive_after_32bit]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [t_after, receive_after, receive_after_big, + receive_after_errors, receive_var_zero, receive_zero, + multi_timeout, receive_after_32bit]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(3)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 94766dc6e9..22b5d93983 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2009. All Rights Reserved. +%% Copyright Ericsson AB 2003-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,7 +18,8 @@ -module(alloc_SUITE). -author('[email protected]'). --export([all/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). -export([basic/1, coalesce/1, @@ -29,28 +30,40 @@ rbtree/1, mseg_clear_cache/1]). --export([init_per_testcase/2, fin_per_testcase/2]). +-export([init_per_testcase/2, end_per_testcase/2]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -define(DEFAULT_TIMETRAP_SECS, 240). -all(doc) -> []; -all(suite) -> [basic, - coalesce, - threads, - realloc_copy, - bucket_index, - bucket_mask, - rbtree, - mseg_clear_cache]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [basic, coalesce, threads, realloc_copy, bucket_index, + bucket_mask, rbtree, mseg_clear_cache]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?t:seconds(?DEFAULT_TIMETRAP_SECS)), [{watchdog, Dog},{testcase, Case}|Config]. -fin_per_testcase(_Case, Config) when is_list(Config) -> +end_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. diff --git a/erts/emulator/test/beam_SUITE.erl b/erts/emulator/test/beam_SUITE.erl index 228ff15341..02c6e19686 100644 --- a/erts/emulator/test/beam_SUITE.erl +++ b/erts/emulator/test/beam_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,16 +19,37 @@ -module(beam_SUITE). --export([all/1, packed_registers/1, apply_last/1, apply_last_bif/1, - buildo_mucho/1, heap_sizes/1, big_lists/1, fconv/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + packed_registers/1, apply_last/1, apply_last_bif/1, + buildo_mucho/1, heap_sizes/1, big_lists/1, fconv/1, + select_val/1]). -export([applied/2]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [packed_registers, apply_last, apply_last_bif, + buildo_mucho, heap_sizes, big_lists, select_val]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [packed_registers, apply_last, apply_last_bif, buildo_mucho, - heap_sizes, big_lists]. %% Verify that apply(M, F, A) is really tail recursive. @@ -302,3 +323,19 @@ do_fconv(nil, Float) when is_float(Float) -> Float + []; do_fconv(tuple_literal, Float) when is_float(Float) -> Float + {a,b}. + +select_val(Config) when is_list(Config) -> + ?line zero = do_select_val(0), + ?line big = do_select_val(1 bsl 64), + ?line integer = do_select_val(42), + ok. + +do_select_val(X) -> + case X of + 0 -> + zero; + 1 bsl 64 -> + big; + Int when is_integer(Int) -> + integer + end. diff --git a/erts/emulator/test/beam_literals_SUITE.erl b/erts/emulator/test/beam_literals_SUITE.erl index 75841adbfc..85236e4203 100644 --- a/erts/emulator/test/beam_literals_SUITE.erl +++ b/erts/emulator/test/beam_literals_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,21 +18,41 @@ %% -module(beam_literals_SUITE). --export([all/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). -export([putting/1, matching_smalls/1, matching_smalls_jt/1, matching_bigs/1, matching_more_bigs/1, matching_bigs_and_smalls/1, badmatch/1, case_clause/1, receiving/1, literal_type_tests/1, - put_list/1, fconv/1, literal_case_expression/1]). + put_list/1, fconv/1, literal_case_expression/1, + increment/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [putting, matching_smalls, matching_smalls_jt, matching_bigs, matching_more_bigs, matching_bigs_and_smalls, badmatch, case_clause, - receiving, literal_type_tests, - put_list, fconv, literal_case_expression]. + receiving, literal_type_tests, put_list, fconv, + literal_case_expression, increment]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + putting(doc) -> "Test creating lists and tuples containing big number literals."; putting(Config) when is_list(Config) -> @@ -48,6 +68,7 @@ matching_bigs(doc) -> "Test matching of a few big number literals (in Beam," matching_bigs(Config) when is_list(Config) -> a = matching1(3972907842873739), b = matching1(-389789298378939783333333333333333333784), + other = matching1(3141699999999999999999999999999999999), other = matching1(42). matching_smalls(doc) -> "Test matching small numbers (both positive and negative)."; @@ -236,14 +257,14 @@ make_test([{T,L}|Ts]) -> make_test([]) -> []. test(T, L) -> - S = lists:flatten(io_lib:format("begin io:format(\"~~p~~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L])), + S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p}]), if ~w(~w) -> true; true -> false end end. ", [T, L, T, L])), {ok,Toks,_Line} = erl_scan:string(S), {ok,E} = erl_parse:parse_exprs(Toks), {value,Val,_Bs} = erl_eval:exprs(E, []), {match,0,{atom,0,Val},hd(E)}. test(T, A, L) -> - S = lists:flatten(io_lib:format("begin io:format(\"~~p~~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ", + S = lists:flatten(io_lib:format("begin io:format(\"~~p~n\", [{~p,~p,~p}]), if ~w(~w, ~w) -> true; true -> false end end. ", [T,L,A,T,L,A])), {ok,Toks,_Line} = erl_scan:string(S), {ok,E} = erl_parse:parse_exprs(Toks), @@ -405,14 +426,51 @@ fconv_2(F) when is_float(F) -> literal_case_expression(Config) when is_list(Config) -> ?line DataDir = ?config(data_dir, Config), ?line Src = filename:join(DataDir, "literal_case_expression"), - ?line {ok,literal_case_expression=Mod,Code} = compile:file(Src, [from_asm,binary]), + ?line {ok,literal_case_expression=Mod,Code} = + compile:file(Src, [from_asm,binary]), ?line {module,Mod} = code:load_binary(Mod, Src, Code), ?line ok = Mod:x(), ?line ok = Mod:y(), + ?line ok = Mod:zi1(), + ?line ok = Mod:zi2(), + ?line ok = Mod:za1(), + ?line ok = Mod:za2(), ?line true = code:delete(Mod), ?line code:purge(Mod), ok. +%% Test the i_increment instruction. +increment(Config) when is_list(Config) -> + %% In the 32-bit emulator, Neg32 can be represented as a small, + %% but -Neg32 cannot. Therefore the i_increment instruction must + %% not be used in the subtraction that follows (since i_increment + %% cannot handle a bignum literal). + Neg32 = -(1 bsl 27), + Big32 = id(1 bsl 32), + Result32 = (1 bsl 32) + (1 bsl 27), + ?line Result32 = Big32 + (1 bsl 27), + ?line Result32 = Big32 - Neg32, + + %% Same thing, but for the 64-bit emulator. + Neg64 = -(1 bsl 59), + Big64 = id(1 bsl 64), + Result64 = (1 bsl 64) + (1 bsl 59), + ?line Result64 = Big64 + (1 bsl 59), + ?line Result64 = Big64 - Neg64, + + %% Test error handling for the i_increment instruction. + Bad = id(bad), + ?line {'EXIT',{badarith,_}} = (catch Bad + 42), + + %% Small operands, but a big result. + Res32 = 1 bsl 27, + Small32 = id(Res32-1), + ?line Res32 = Small32 + 1, + Res64 = 1 bsl 59, + Small64 = id(Res64-1), + ?line Res64 = Small64 + 1, + ok. + %% Help functions. chksum(Term) -> diff --git a/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S b/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S index c0ffe9ab53..bfdfc079dc 100644 --- a/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S +++ b/erts/emulator/test/beam_literals_SUITE_data/literal_case_expression.S @@ -1,10 +1,11 @@ {module, literal_case_expression}. %% version = 0 -{exports, [{module_info,0},{module_info,1},{x,0},{y,0}]}. +{exports, [{module_info,0},{module_info,1},{x,0},{y,0}, + {zi1,0},{zi2,0},{za1,0},{za2,0}]}. {attributes, []}. -{labels, 15}. +{labels, 32}. {function, x, 0, 2}. @@ -52,6 +53,81 @@ {label,10}. {case_end,{float,34.0000}}. +{function, zi1, 0, 16}. + {label,15}. + {func_info,{atom,literal_case_expression},{atom,zi1},0}. + {label,16}. + {test,is_integer,{f,19},[{integer,42}]}. + {select_val,{integer,42}, + {f,18}, + {list,[{integer,42}, + {f,17}, + {integer,1000}, + {f,18}]}}. + {label,17}. + {move,{atom,ok},{x,0}}. + return. + {label,18}. + {move,{atom,error},{x,0}}. + return. + {label,19}. + {case_end,{integer,42}}. + +{function, zi2, 0, 16}. + {label,20}. + {func_info,{atom,literal_case_expression},{atom,zi2},0}. + {label,21}. + {test,is_integer,{f,23},[{integer,42}]}. + {select_val,{integer,42}, + {f,23}, + {list,[{integer,42}, + {f,22}, + {integer,1000}, + {f,23}]}}. + {label,22}. + {move,{atom,ok},{x,0}}. + return. + {label,23}. + {move,{atom,error},{x,0}}. + return. + +{function, za1, 0, 25}. + {label,24}. + {func_info,{atom,literal_case_expression},{atom,za1},0}. + {label,25}. + {test,is_atom,{f,28},[{atom,x}]}. + {select_val,{atom,x}, + {f,27}, + {list,[{atom,a}, + {f,27}, + {atom,x}, + {f,26}]}}. + {label,26}. + {move,{atom,ok},{x,0}}. + return. + {label,27}. + {move,{atom,error},{x,0}}. + return. + {label,28}. + {case_end,{atom,x}}. + +{function, za2, 0, 30}. + {label,29}. + {func_info,{atom,literal_case_expression},{atom,za2},0}. + {label,30}. + {test,is_atom,{f,32},[{atom,x}]}. + {select_val,{atom,x}, + {f,32}, + {list,[{atom,a}, + {f,32}, + {atom,x}, + {f,31}]}}. + {label,31}. + {move,{atom,ok},{x,0}}. + return. + {label,32}. + {move,{atom,error},{x,0}}. + return. {function, module_info, 0, 12}. {label,11}. diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index b4ef0e6d5a..c7617d3b90 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,27 +19,74 @@ -module(bif_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, + display/1, display_huge/0, types/1, t_list_to_existing_atom/1,os_env/1,otp_7526/1, binary_to_atom/1,binary_to_existing_atom/1, atom_to_binary/1,min_max/1]). -all(suite) -> - [types,t_list_to_existing_atom,os_env,otp_7526, - atom_to_binary,binary_to_atom,binary_to_existing_atom, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [types, t_list_to_existing_atom, os_env, otp_7526, + display, + atom_to_binary, binary_to_atom, binary_to_existing_atom, min_max]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(1)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). + +display(suite) -> + []; +display(doc) -> + ["Uses erlang:display to test that erts_printf does not do deep recursion"]; +display(Config) when is_list(Config) -> + Pa = filename:dirname(code:which(?MODULE)), + {ok, Node} = test_server:start_node(display_huge_term,peer, + [{args, "-pa "++Pa}]), + true = rpc:call(Node,?MODULE,display_huge,[]), + test_server:stop_node(Node), + ok. + +display_huge() -> + erlang:display(deeep(100000)). + +deeep(0,Acc) -> + Acc; +deeep(N,Acc) -> + deeep(N-1,[Acc|[]]). + +deeep(N) -> + deeep(N,[hello]). + + types(Config) when is_list(Config) -> c:l(erl_bif_types), case erlang:function_exported(erl_bif_types, module_info, 0) of diff --git a/erts/emulator/test/big_SUITE.erl b/erts/emulator/test/big_SUITE.erl index 6cedd39009..3487917677 100644 --- a/erts/emulator/test/big_SUITE.erl +++ b/erts/emulator/test/big_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,9 +19,10 @@ -module(big_SUITE). --export([all/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). -export([t_div/1, eq_28/1, eq_32/1, eq_big/1, eq_math/1, big_literals/1, - borders/1, negative/1, big_float/1, big_float_1/1, big_float_2/1, + borders/1, negative/1, big_float_1/1, big_float_2/1, shift_limit_1/1, powmod/1, system_limit/1, otp_6692/1]). %% Internal exports. @@ -30,19 +31,38 @@ -export([fac/1, fib/1, pow/2, gcd/2, lcm/2]). --export([init_per_testcase/2, fin_per_testcase/2]). +-export([init_per_testcase/2, end_per_testcase/2]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [t_div, eq_28, eq_32, eq_big, eq_math, big_literals, + borders, negative, {group, big_float}, shift_limit_1, + powmod, system_limit, otp_6692]. + +groups() -> + [{big_float, [], [big_float_1, big_float_2]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [t_div, eq_28, eq_32, eq_big, eq_math, big_literals, borders, - negative, big_float, shift_limit_1, powmod, system_limit, otp_6692]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(3)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). @@ -260,10 +280,6 @@ big_literals(Config) when is_list(Config) -> ?line ok = Mod:t(), ok. -big_float(doc) -> - ["Test cases for mixing bignums and floats"]; -big_float(suite) -> - [big_float_1, big_float_2]. big_float_1(doc) -> ["OTP-2436, part 1"]; diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index 77d2579848..7e409f053e 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -40,9 +40,11 @@ %% phash2(Binary, N) %% --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1, init_per_testcase/2, fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2, copy_terms/1, conversions/1, deep_lists/1, deep_bitstr_lists/1, bad_list_to_binary/1, bad_binary_to_list/1, t_split_binary/1, bad_split/1, t_concat_binary/1, @@ -61,24 +63,42 @@ %% Internal exports. -export([sleeper/0]). -all(suite) -> - [copy_terms,conversions,deep_lists,deep_bitstr_lists, +suite() -> [{ct_hooks,[ts_install_cth]}, + {timetrap,{minutes,2}}]. + +all() -> + [copy_terms, conversions, deep_lists, deep_bitstr_lists, t_split_binary, bad_split, t_concat_binary, - bad_list_to_binary, bad_binary_to_list, terms, terms_float, - external_size, t_iolist_size, - bad_binary_to_term_2,safe_binary_to_term2, - bad_binary_to_term, bad_terms, t_hash, bad_size, bad_term_to_binary, - more_bad_terms, otp_5484, otp_5933, ordering, unaligned_order, - gc_test, bit_sized_binary_sizes, otp_6817, otp_8117, - deep,obsolete_funs,robustness,otp_8180]. + bad_list_to_binary, bad_binary_to_list, terms, + terms_float, external_size, t_iolist_size, + bad_binary_to_term_2, safe_binary_to_term2, + bad_binary_to_term, bad_terms, t_hash, bad_size, + bad_term_to_binary, more_bad_terms, otp_5484, otp_5933, + ordering, unaligned_order, gc_test, + bit_sized_binary_sizes, otp_6817, otp_8117, deep, + obsolete_funs, robustness, otp_8180]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> - Dog=?t:timetrap(?t:minutes(2)), - [{watchdog, Dog}|Config]. + Config. -fin_per_testcase(_Func, Config) -> - Dog=?config(watchdog, Config), - ?t:timetrap_cancel(Dog). +end_per_testcase(_Func, _Config) -> + ok. -define(heap_binary_size, 64). @@ -1041,7 +1061,7 @@ test_terms(Test_Func) -> ?line Test_Func(F = fun(A) -> 42*A end), ?line Test_Func(lists:duplicate(32, F)), - ?line Test_Func(FF = fun binary_SUITE:all/1), + ?line Test_Func(FF = fun binary_SUITE:all/0), ?line Test_Func(lists:duplicate(32, FF)), ok. @@ -1301,11 +1321,4 @@ unaligned_sub_bin(Bin0, Offs) -> <<_:Offs,Bin:Sz/binary,_:Roffs>> = id(Bin1), Bin. -hostname() -> - from($@, atom_to_list(node())). - -from(H, [H | T]) -> T; -from(H, [_ | T]) -> from(H, T); -from(_, []) -> []. - id(I) -> I. diff --git a/erts/emulator/test/bs_bincomp_SUITE.erl b/erts/emulator/test/bs_bincomp_SUITE.erl index 4e83d97689..f1c2dff560 100644 --- a/erts/emulator/test/bs_bincomp_SUITE.erl +++ b/erts/emulator/test/bs_bincomp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. +%% Copyright Ericsson AB 2006-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,15 +22,34 @@ -module(bs_bincomp_SUITE). --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, byte_aligned/1,bit_aligned/1,extended_byte_aligned/1, extended_bit_aligned/1,mixed/1,tracing/1]). --include("test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [byte_aligned, bit_aligned, extended_byte_aligned, + extended_bit_aligned, mixed, tracing]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [byte_aligned,bit_aligned,extended_byte_aligned, - extended_bit_aligned,mixed,tracing]. byte_aligned(Config) when is_list(Config) -> diff --git a/erts/emulator/test/bs_bit_binaries_SUITE.erl b/erts/emulator/test/bs_bit_binaries_SUITE.erl index 52bb925385..ff1088118d 100644 --- a/erts/emulator/test/bs_bit_binaries_SUITE.erl +++ b/erts/emulator/test/bs_bit_binaries_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. +%% Copyright Ericsson AB 2006-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,18 +22,38 @@ -module(bs_bit_binaries_SUITE). --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, misc/1,horrid_match/1,test_bitstr/1,test_bit_size/1,asymmetric_tests/1, big_asymmetric_tests/1,binary_to_and_from_list/1, big_binary_to_and_from_list/1,send_and_receive/1, send_and_receive_alot/1,append/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [misc, horrid_match, test_bitstr, test_bit_size, + asymmetric_tests, big_asymmetric_tests, + binary_to_and_from_list, big_binary_to_and_from_list, + send_and_receive, send_and_receive_alot, append]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [misc,horrid_match,test_bitstr,test_bit_size,asymmetric_tests, - big_asymmetric_tests,binary_to_and_from_list,big_binary_to_and_from_list, - send_and_receive,send_and_receive_alot,append]. misc(Config) when is_list(Config) -> ?line <<1:100>> = id(<<1:100>>), diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index 3d9b51d278..1959803385 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,22 +21,39 @@ -module(bs_construct_SUITE). --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, test1/1, test2/1, test3/1, test4/1, test5/1, testf/1, not_used/1, in_guard/1, mem_leak/1, coerce_to_float/1, bjorn/1, huge_float_field/1, huge_binary/1, system_limit/1, badarg/1, copy_writable_binary/1, kostis/1, dynamic/1, bs_add/1, - otp_7422/1]). + otp_7422/1, zero_width/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -all(suite) -> - [test1, test2, test3, test4, test5, testf, - not_used, in_guard, mem_leak, coerce_to_float, bjorn, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [test1, test2, test3, test4, test5, testf, not_used, + in_guard, mem_leak, coerce_to_float, bjorn, huge_float_field, huge_binary, system_limit, badarg, - copy_writable_binary, kostis, dynamic, bs_add, - otp_7422]. + copy_writable_binary, kostis, dynamic, bs_add, otp_7422, zero_width]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. big(1) -> 57285702734876389752897683. @@ -786,5 +803,20 @@ otp_7422_bin(N) when N < 512 -> end), otp_7422_bin(N+1); otp_7422_bin(_) -> ok. + +zero_width(Config) when is_list(Config) -> + ?line Z = id(0), + Small = id(42), + Big = id(1 bsl 128), + ?line <<>> = <<Small:Z>>, + ?line <<>> = <<Small:0>>, + ?line <<>> = <<Big:Z>>, + ?line <<>> = <<Big:0>>, + + ?line {'EXIT',{badarg,_}} = (catch <<not_a_number:0>>), + ?line {'EXIT',{badarg,_}} = (catch <<(id(not_a_number)):Z>>), + ?line {'EXIT',{badarg,_}} = (catch <<(id(not_a_number)):0>>), + + ok. id(I) -> I. diff --git a/erts/emulator/test/bs_match_bin_SUITE.erl b/erts/emulator/test/bs_match_bin_SUITE.erl index 3d054a279f..96e69dbc0b 100644 --- a/erts/emulator/test/bs_match_bin_SUITE.erl +++ b/erts/emulator/test/bs_match_bin_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,12 +19,32 @@ -module(bs_match_bin_SUITE). --export([all/1,byte_split_binary/1,bit_split_binary/1,match_huge_bin/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + byte_split_binary/1,bit_split_binary/1,match_huge_bin/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [byte_split_binary, bit_split_binary, match_huge_bin]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [byte_split_binary,bit_split_binary,match_huge_bin]. byte_split_binary(doc) -> "Tries to split a binary at all byte-aligned positions."; byte_split_binary(Config) when is_list(Config) -> diff --git a/erts/emulator/test/bs_match_int_SUITE.erl b/erts/emulator/test/bs_match_int_SUITE.erl index 99dee7c7bc..ce03ecb548 100644 --- a/erts/emulator/test/bs_match_int_SUITE.erl +++ b/erts/emulator/test/bs_match_int_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,16 +18,36 @@ -module(bs_match_int_SUITE). --export([all/1,integer/1,signed_integer/1,dynamic/1,more_dynamic/1,mml/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + integer/1,signed_integer/1,dynamic/1,more_dynamic/1,mml/1, match_huge_int/1,bignum/1,unaligned_32_bit/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -import(lists, [seq/2]). -all(suite) -> - [integer,signed_integer,dynamic,more_dynamic,mml,match_huge_int,bignum, - unaligned_32_bit]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [integer, signed_integer, dynamic, more_dynamic, mml, + match_huge_int, bignum, unaligned_32_bit]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + integer(Config) when is_list(Config) -> ?line 0 = get_int(mkbin([])), diff --git a/erts/emulator/test/bs_match_misc_SUITE.erl b/erts/emulator/test/bs_match_misc_SUITE.erl index 6de2ef67e5..b022f96740 100644 --- a/erts/emulator/test/bs_match_misc_SUITE.erl +++ b/erts/emulator/test/bs_match_misc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2009. All Rights Reserved. +%% Copyright Ericsson AB 2000-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,18 +18,38 @@ %% -module(bs_match_misc_SUITE). --export([all/1,bound_var/1,bound_tail/1,t_float/1,little_float/1,sean/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + bound_var/1,bound_tail/1,t_float/1,little_float/1,sean/1, kenneth/1,encode_binary/1,native/1,happi/1, size_var/1,wiger/1,x0_context/1,huge_float_field/1, writable_binary_matched/1,otp_7198/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [bound_var, bound_tail, t_float, little_float, sean, + kenneth, encode_binary, native, happi, size_var, wiger, + x0_context, huge_float_field, writable_binary_matched, + otp_7198]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [bound_var,bound_tail,t_float,little_float,sean, - kenneth,encode_binary,native,happi, - size_var,wiger,x0_context,huge_float_field, - writable_binary_matched,otp_7198]. bound_var(doc) -> "Test matching of bound variables."; bound_var(Config) when is_list(Config) -> diff --git a/erts/emulator/test/bs_match_tail_SUITE.erl b/erts/emulator/test/bs_match_tail_SUITE.erl index b0b0779b65..1397f2069c 100644 --- a/erts/emulator/test/bs_match_tail_SUITE.erl +++ b/erts/emulator/test/bs_match_tail_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,11 +20,31 @@ -module(bs_match_tail_SUITE). -author('[email protected]'). --export([all/1,aligned/1,unaligned/1,zero_tail/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2,aligned/1,unaligned/1,zero_tail/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [aligned, unaligned, zero_tail]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> [aligned,unaligned,zero_tail]. aligned(doc) -> "Test aligned tails."; aligned(Config) when is_list(Config) -> diff --git a/erts/emulator/test/bs_utf_SUITE.erl b/erts/emulator/test/bs_utf_SUITE.erl index 87adc5197b..72c656c400 100644 --- a/erts/emulator/test/bs_utf_SUITE.erl +++ b/erts/emulator/test/bs_utf_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2009. All Rights Reserved. +%% Copyright Ericsson AB 2008-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,13 +19,15 @@ -module(bs_utf_SUITE). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, utf8_roundtrip/1,utf16_roundtrip/1,utf32_roundtrip/1, utf8_illegal_sequences/1,utf16_illegal_sequences/1, utf32_illegal_sequences/1, bad_construction/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -define(FAIL(Expr), ?line fail_check(catch Expr, ??Expr, [])). @@ -33,14 +35,32 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(6)), [{watchdog,Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog). -all(suite) -> - [utf8_roundtrip,utf16_roundtrip,utf32_roundtrip, - utf8_illegal_sequences,utf16_illegal_sequences, - utf32_illegal_sequences,bad_construction]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [utf8_roundtrip, utf16_roundtrip, utf32_roundtrip, + utf8_illegal_sequences, utf16_illegal_sequences, + utf32_illegal_sequences, bad_construction]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + utf8_roundtrip(Config) when is_list(Config) -> ?line utf8_roundtrip(0, 16#D7FF), diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl index 7350aef4ec..8365e1c540 100644 --- a/erts/emulator/test/busy_port_SUITE.erl +++ b/erts/emulator/test/busy_port_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,21 +19,40 @@ -module(busy_port_SUITE). --export([all/1, io_to_busy/1, message_order/1, send_3/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + io_to_busy/1, message_order/1, send_3/1, system_monitor/1, no_trap_exit/1, no_trap_exit_unlinked/1, trap_exit/1, multiple_writers/1, hard_busy_driver/1, soft_busy_driver/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %% Internal exports. -export([init/2]). -all(suite) -> {req, [dynamic_loading], - [io_to_busy, message_order, send_3, - system_monitor, no_trap_exit, - no_trap_exit_unlinked, trap_exit, multiple_writers, - hard_busy_driver, soft_busy_driver]}. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [io_to_busy, message_order, send_3, system_monitor, + no_trap_exit, no_trap_exit_unlinked, trap_exit, + multiple_writers, hard_busy_driver, soft_busy_driver]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + %% Tests I/O operations to a busy port, to make sure a suspended send %% operation is correctly restarted. This used to crash Beam. diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl index e0528955b0..93fdc157f7 100644 --- a/erts/emulator/test/call_trace_SUITE.erl +++ b/erts/emulator/test/call_trace_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,7 +20,9 @@ -module(call_trace_SUITE). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, hipe/1,process_specs/1,basic/1,flags/1,errors/1,pam/1,change_pam/1, return_trace/1,exception_trace/1,on_load/1,deep_exception/1, exception_nocatch/1,bit_syntax/1]). @@ -35,25 +37,44 @@ -export([abbr/1,abbr/2]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -define(P, 20). -all(suite) -> - Common = [errors,on_load], - NotHipe = [process_specs,basic,flags,pam,change_pam,return_trace, - exception_trace,deep_exception,exception_nocatch,bit_syntax], +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + Common = [errors, on_load], + NotHipe = [process_specs, basic, flags, pam, change_pam, + return_trace, exception_trace, deep_exception, + exception_nocatch, bit_syntax], Hipe = [hipe], - case test_server:is_native(?MODULE) of + case test_server:is_native(call_trace_SUITE) of true -> Hipe ++ Common; false -> NotHipe ++ Common end. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:seconds(30)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog). diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 33351a3cc9..703a00a598 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,18 +18,38 @@ %% -module(code_SUITE). --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, new_binary_types/1,t_check_process_code/1,t_check_process_code_ets/1, external_fun/1,get_chunk/1,module_md5/1,make_stub/1, make_stub_many_funs/1,constant_pools/1, false_dependency/1,coverage/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [new_binary_types, t_check_process_code, + t_check_process_code_ets, external_fun, get_chunk, + module_md5, make_stub, make_stub_many_funs, + constant_pools, false_dependency, coverage]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [new_binary_types,t_check_process_code,t_check_process_code_ets, - external_fun,get_chunk,module_md5,make_stub,make_stub_many_funs, - constant_pools,false_dependency,coverage]. new_binary_types(Config) when is_list(Config) -> ?line Data = ?config(data_dir, Config), @@ -320,6 +340,9 @@ make_stub(Config) when is_list(Config) -> (catch code:make_stub_module(my_code_test, bit_sized_binary(Code), {[],[]})), + ?line {'EXIT',{badarg,_}} = + (catch code:make_stub_module(my_code_test_with_wrong_name, + Code, {[],[]})), ok. make_stub_many_funs(Config) when is_list(Config) -> diff --git a/erts/emulator/test/crypto_SUITE.erl b/erts/emulator/test/crypto_SUITE.erl index e3d34b923d..a82bd4fe38 100644 --- a/erts/emulator/test/crypto_SUITE.erl +++ b/erts/emulator/test/crypto_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,14 +19,34 @@ -module(crypto_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, t_md5/1,t_md5_update/1,error/1,unaligned_context/1,random_lists/1, misc_errors/1]). -all(suite) -> - [t_md5,t_md5_update,error,unaligned_context,random_lists,misc_errors]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [t_md5, t_md5_update, error, unaligned_context, + random_lists, misc_errors]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + misc_errors(doc) -> diff --git a/erts/emulator/test/crypto_reference.erl b/erts/emulator/test/crypto_reference.erl index 99107e3b57..b91535a50e 100644 --- a/erts/emulator/test/crypto_reference.erl +++ b/erts/emulator/test/crypto_reference.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2009. All Rights Reserved. +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/test/ddll_SUITE.erl b/erts/emulator/test/ddll_SUITE.erl index 79047d7de5..6e15c228cd 100644 --- a/erts/emulator/test/ddll_SUITE.erl +++ b/erts/emulator/test/ddll_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -30,7 +30,8 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --export([all/1, ddll_test/1, errors/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, ddll_test/1, errors/1, reference_count/1, kill_port/1, dont_kill_port/1]). -export([unload_on_process_exit/1, delayed_unload_with_ports/1, @@ -50,35 +51,39 @@ -import(ordsets, [subtract/2]). --include("test_server.hrl"). - -all(suite) -> - [ddll_test, errors, - reference_count, - kill_port, - dont_kill_port, - properties, - load_and_unload, - unload_on_process_exit, - delayed_unload_with_ports, +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ddll_test, errors, reference_count, kill_port, + dont_kill_port, properties, load_and_unload, + unload_on_process_exit, delayed_unload_with_ports, unload_due_to_process_exit, - no_unload_due_to_process_exit, - no_unload_due_to_process_exit_2, - unload_reload_thingie, - unload_reload_thingie_2, - unload_reload_thingie_3, - reload_pending, - load_fail_init, - reload_pending_fail_init, - reload_pending_kill, - more_error_codes, - forced_port_killing, - no_trap_exit_and_kill_ports, - monitor_demonitor, - monitor_demonitor_load, - new_interface, - lock_driver - ]. + no_unload_due_to_process_exit, + no_unload_due_to_process_exit_2, unload_reload_thingie, + unload_reload_thingie_2, unload_reload_thingie_3, + reload_pending, load_fail_init, + reload_pending_fail_init, reload_pending_kill, + more_error_codes, forced_port_killing, + no_trap_exit_and_kill_ports, monitor_demonitor, + monitor_demonitor_load, new_interface, lock_driver]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + unload_on_process_exit(suite) -> []; diff --git a/erts/emulator/test/decode_packet_SUITE.erl b/erts/emulator/test/decode_packet_SUITE.erl index d9e961be2f..c0499554eb 100644 --- a/erts/emulator/test/decode_packet_SUITE.erl +++ b/erts/emulator/test/decode_packet_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,14 +21,34 @@ -module(decode_packet_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, basic/1, packet_size/1, neg/1, http/1, line/1, ssl/1, otp_8536/1]). -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [basic, packet_size, neg, http, line, ssl, otp_8536]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Seed = {S1,S2,S3} = now(), random:seed(S1,S2,S3), @@ -36,7 +56,7 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(1)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). diff --git a/erts/emulator/test/dgawd_handler.erl b/erts/emulator/test/dgawd_handler.erl index 881354b9da..27085b7b7e 100644 --- a/erts/emulator/test/dgawd_handler.erl +++ b/erts/emulator/test/dgawd_handler.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. +%% Copyright Ericsson AB 2006-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 79252d0593..4bebae51cc 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,30 +22,30 @@ %% Tests distribution and the tcp driver. --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1, - ping/1, bulk_send/1, bulk_send_small/1, - bulk_send_big/1, - bulk_send_bigbig/1, - local_send/1, local_send_small/1, local_send_big/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + ping/1, bulk_send_small/1, + bulk_send_big/1, bulk_send_bigbig/1, + local_send_small/1, local_send_big/1, local_send_legal/1, link_to_busy/1, exit_to_busy/1, lost_exit/1, link_to_dead/1, link_to_dead_new_node/1, applied_monitor_node/1, ref_port_roundtrip/1, nil_roundtrip/1, - trap_bif/1, trap_bif_1/1, trap_bif_2/1, trap_bif_3/1, - stop_dist/1, dist_auto_connect/1, + trap_bif_1/1, trap_bif_2/1, trap_bif_3/1, + stop_dist/1, dist_auto_connect_never/1, dist_auto_connect_once/1, dist_parallel_send/1, atom_roundtrip/1, atom_roundtrip_r12b/1, contended_atom_cache_entry/1, - bad_dist_ext/1, + bad_dist_structure/1, bad_dist_ext_receive/1, bad_dist_ext_process_info/1, bad_dist_ext_control/1, bad_dist_ext_connection_id/1]). --export([init_per_testcase/2, fin_per_testcase/2]). +-export([init_per_testcase/2, end_per_testcase/2]). %% Internal exports. -export([sender/3, receiver2/2, dummy_waiter/0, dead_process/0, @@ -54,15 +54,39 @@ dist_evil_parallel_receiver/0, sendersender/4, sendersender2/4]). -all(suite) -> [ - ping, bulk_send, local_send, link_to_busy, exit_to_busy, - lost_exit, link_to_dead, link_to_dead_new_node, - applied_monitor_node, ref_port_roundtrip, nil_roundtrip, - stop_dist, trap_bif, dist_auto_connect, dist_parallel_send, - atom_roundtrip, atom_roundtrip_r12b, - contended_atom_cache_entry, - bad_dist_ext - ]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [ping, {group, bulk_send}, {group, local_send}, + link_to_busy, exit_to_busy, lost_exit, link_to_dead, + link_to_dead_new_node, applied_monitor_node, + ref_port_roundtrip, nil_roundtrip, stop_dist, + {group, trap_bif}, {group, dist_auto_connect}, + dist_parallel_send, atom_roundtrip, atom_roundtrip_r12b, + contended_atom_cache_entry, bad_dist_structure, {group, bad_dist_ext}]. + +groups() -> + [{bulk_send, [], [bulk_send_small, bulk_send_big, bulk_send_bigbig]}, + {local_send, [], + [local_send_small, local_send_big, local_send_legal]}, + {trap_bif, [], [trap_bif_1, trap_bif_2, trap_bif_3]}, + {dist_auto_connect, [], + [dist_auto_connect_never, dist_auto_connect_once]}, + {bad_dist_ext, [], + [bad_dist_ext_receive, bad_dist_ext_process_info, + bad_dist_ext_control, bad_dist_ext_connection_id]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -define(DEFAULT_TIMETRAP, 4*60*1000). @@ -70,7 +94,7 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?DEFAULT_TIMETRAP), [{watchdog, Dog},{testcase, Func}|Config]. -fin_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> +end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). @@ -118,13 +142,6 @@ ping(Config) when is_list(Config) -> ok. -bulk_send(doc) -> - ["Tests sending large amount of data to another node and measure", - "the time. This tests that a process that is suspended on a ", - "busy port will eventually be resumed."]; -bulk_send(suite) -> - [bulk_send_small, bulk_send_big, bulk_send_bigbig]. - bulk_send_small(Config) when is_list(Config) -> ?line bulk_send(64, 32). @@ -174,7 +191,7 @@ bulk_sendsend2(Terms, BinSize, BusyBufSize) -> ?line {ok, NodeRecv} = start_node(bulk_receiver), ?line Recv = spawn(NodeRecv, erlang, apply, [fun receiver/2, [0, 0]]), ?line Bin = list_to_binary(lists:duplicate(BinSize*1024, 253)), - ?line Size = Terms*size(Bin), + %%?line Size = Terms*size(Bin), %% SLF LEFT OFF HERE. %% When the caller uses small hunks, like 4k via @@ -187,7 +204,7 @@ bulk_sendsend2(Terms, BinSize, BusyBufSize) -> ?line {ok, NodeSend} = start_node(bulk_sender, "+zdbbl " ++ integer_to_list(BusyBufSize)), ?line _Send = spawn(NodeSend, erlang, apply, [fun sendersender/4, [self(), Recv, Bin, Terms]]), - ?line {Elapsed, {TermsN, SizeN}, MonitorCount} = + ?line {Elapsed, {_TermsN, SizeN}, MonitorCount} = receive {sendersender, BigRes} -> BigRes end, @@ -227,7 +244,7 @@ sendersender3(To, _Bin, 0, SendDone, MonitorCount) -> ok end, receive - {monitor, _Pid, _Type, _Info} = M -> + {monitor, _Pid, _Type, _Info} -> sendersender3(To, _Bin, 0, SendDone, MonitorCount + 1) after 0 -> if SendDone -> @@ -255,17 +272,14 @@ receiver(Terms, Size) -> end. -local_send(suite) -> - [local_send_small, local_send_big, local_send_legal]; -local_send(doc) -> - ["Tests sending small and big messages to a non-existing ", - "local registered process."]. local_send_big(doc) -> ["Sends several big message to an non-registered process on ", "the local node."]; local_send_big(Config) when is_list(Config) -> - Data0=local_send_big(doc)++local_send(doc), + Data0=local_send_big(doc)++ + ["Tests sending small and big messages to a non-existing ", + "local registered process."], Data1=[Data0,[Data0, Data0, [Data0], Data0],Data0], Data2=Data0++lists:flatten(Data1)++ list_to_binary(lists:flatten(Data1)), @@ -522,7 +536,7 @@ sink1() -> lost_exit(doc) -> "Test that EXIT and DOWN messages send to another node are not lost if " - "if the distribution port is busy."; + "the distribution port is busy."; lost_exit(Config) when is_list(Config) -> ?line {ok, Node} = start_node(lost_exit), @@ -751,9 +765,6 @@ stop_dist(Config) when is_list(Config) -> ok. -trap_bif(doc) -> - ["Verifies that BIFs which are traps to Erlang work (OTP-2680)."]; -trap_bif(suite) -> [trap_bif_1, trap_bif_2, trap_bif_3]. trap_bif_1(doc) -> [""]; @@ -790,10 +801,6 @@ tr3() -> -dist_auto_connect(doc) -> - ["Tests the kernel parameter 'dist_auto_connect'."]; -dist_auto_connect(suite) -> - [dist_auto_connect_never, dist_auto_connect_once]. % This has to be done by nodes with differrent cookies, otherwise global % will connect nodes, which is correct, but makes it hard to test. @@ -1143,8 +1150,7 @@ contended_atom_cache_entry(Config) when is_list(Config) -> ?line {ok, SNode} = start_node(Config), ?line {ok, RNode} = start_node(Config), ?line Success = make_ref(), - ?line Mstr - = spawn_link( + ?line spawn_link( SNode, fun () -> erts_debug:set_internal_state(available_internal_state, @@ -1201,13 +1207,13 @@ contended_atom_cache_entry(Config) when is_list(Config) -> ?line stop_node(RNode), ?line ok. -send_ref_atom(To, Ref, Atom, 0) -> +send_ref_atom(_To, _Ref, _Atom, 0) -> ok; send_ref_atom(To, Ref, Atom, N) -> To ! {Ref, Atom}, send_ref_atom(To, Ref, Atom, N-1). -receive_ref_atom(Ref, Atom, 0) -> +receive_ref_atom(_Ref, _Atom, 0) -> ok; receive_ref_atom(Ref, Atom, N) -> receive @@ -1242,7 +1248,7 @@ unwanted_cixs() -> nodes()). -get_conflicting_atoms(CIX, 0) -> +get_conflicting_atoms(_CIX, 0) -> []; get_conflicting_atoms(CIX, N) -> {A, B, C} = now(), @@ -1256,13 +1262,187 @@ get_conflicting_atoms(CIX, N) -> get_conflicting_atoms(CIX, N) end. +-define(COOKIE, ''). +-define(DOP_LINK, 1). +-define(DOP_SEND, 2). +-define(DOP_EXIT, 3). +-define(DOP_UNLINK, 4). +-define(DOP_REG_SEND, 6). +-define(DOP_GROUP_LEADER, 7). +-define(DOP_EXIT2, 8). + +-define(DOP_SEND_TT, 12). +-define(DOP_EXIT_TT, 13). +-define(DOP_REG_SEND_TT, 16). +-define(DOP_EXIT2_TT, 18). + +-define(DOP_MONITOR_P, 19). +-define(DOP_DEMONITOR_P, 20). +-define(DOP_MONITOR_P_EXIT, 21). + +start_monitor(Offender,P) -> + ?line Parent = self(), + ?line Q = spawn(Offender, + fun () -> + Ref = erlang:monitor(process,P), + Parent ! {self(),ref,Ref}, + receive + just_stay_alive -> ok + end + end), + ?line Ref = receive + {Q,ref,R} -> + R + after 5000 -> + error + end, + io:format("Ref is ~p~n",[Ref]), + ok. +start_link(Offender,P) -> + ?line Parent = self(), + ?line Q = spawn(Offender, + fun () -> + process_flag(trap_exit,true), + link(P), + Parent ! {self(),ref,P}, + receive + just_stay_alive -> ok + end + end), + ?line Ref = receive + {Q,ref,R} -> + R + after 5000 -> + error + end, + io:format("Ref is ~p~n",[Ref]), + ok. + +bad_dist_structure(suite) -> + []; +bad_dist_structure(doc) -> + ["Test dist messages with valid structure (binary to term ok) but malformed" + "control content"]; +bad_dist_structure(Config) when is_list(Config) -> + %process_flag(trap_exit,true), + ODog = ?config(watchdog, Config), + ?t:timetrap_cancel(ODog), + Dog = ?t:timetrap(?t:seconds(15)), + + ?line {ok, Offender} = start_node(bad_dist_structure_offender), + ?line {ok, Victim} = start_node(bad_dist_structure_victim), + ?line start_node_monitors([Offender,Victim]), + ?line Parent = self(), + ?line P = spawn(Victim, + fun () -> + process_flag(trap_exit,true), + Parent ! {self(), started}, + receive check_msgs -> ok end, + bad_dist_struct_check_msgs([one, + two]), + Parent ! {self(), messages_checked}, + receive done -> ok end + end), + ?line receive {P, started} -> ok end, + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line verify_up(Offender, Victim), + ?line true = lists:member(Offender, rpc:call(Victim, erlang, nodes, [])), + ?line start_monitor(Offender,P), + ?line P ! one, + ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P_EXIT,'replace',P,normal,normal},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_LINK},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_UNLINK,'replace'},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_UNLINK,'replace',make_ref()},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_UNLINK,make_ref(),P},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_link(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_UNLINK,normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_MONITOR_P,'replace',P,normal},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line start_monitor(Offender,P), + ?line send_bad_structure(Offender, P,{?DOP_DEMONITOR_P,'replace',P,normal},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT,'replace',P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT,make_ref(),normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT_TT,'replace',token,P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT_TT,make_ref(),token,normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT2,'replace',P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT2,make_ref(),normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT2_TT,'replace',token,P},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_EXIT2_TT,make_ref(),token,normal,normal},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace'},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace','atomic'},2), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_GROUP_LEADER,'replace',P},0), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name},2,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND_TT,'replace','',name,token},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace',''},2,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',P},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_REG_SEND,'replace','',name,{token}},2,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND_TT,'',P},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND_TT,'',name,token},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND,''},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND,'',name},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line send_bad_structure(Offender, P,{?DOP_SEND,'',P,{token}},0,{message}), + ?line pong = rpc:call(Victim, net_adm, ping, [Offender]), + ?line P ! two, + ?line P ! check_msgs, + ?line receive + {P, messages_checked} -> ok + after 5000 -> + exit(victim_is_dead) + end, + + ?line {message_queue_len, 0} + = rpc:call(Victim, erlang, process_info, [P, message_queue_len]), + + ?line unlink(P), + ?line P ! done, + ?line stop_node(Offender), + ?line stop_node(Victim), + ?t:timetrap_cancel(Dog), + ok. -bad_dist_ext(doc) -> []; -bad_dist_ext(suite) -> - [bad_dist_ext_receive, - bad_dist_ext_process_info, - bad_dist_ext_control, - bad_dist_ext_connection_id]. bad_dist_ext_receive(Config) when is_list(Config) -> @@ -1483,6 +1663,22 @@ bad_dist_ext_connection_id(Config) when is_list(Config) -> ?line stop_node(Victim). +bad_dist_struct_check_msgs([]) -> + receive + Msg -> + exit({unexpected_message, Msg}) + after 0 -> + ok + end; +bad_dist_struct_check_msgs([M|Ms]) -> + receive + {'EXIT',_,_} = EM -> + io:format("Ignoring exit message: ~p~n",[EM]), + bad_dist_struct_check_msgs([M|Ms]); + Msg -> + M = Msg, + bad_dist_struct_check_msgs(Ms) + end. bad_dist_ext_check_msgs([]) -> receive Msg -> @@ -1497,24 +1693,6 @@ bad_dist_ext_check_msgs([M|Ms]) -> bad_dist_ext_check_msgs(Ms) end. --define(COOKIE, ''). --define(DOP_LINK, 1). --define(DOP_SEND, 2). --define(DOP_EXIT, 3). --define(DOP_UNLINK, 4). --define(DOP_NODE_LINK, 5). --define(DOP_REG_SEND, 6). --define(DOP_GROUP_LEADER, 7). --define(DOP_EXIT2, 8). - --define(DOP_SEND_TT, 12). --define(DOP_EXIT_TT, 13). --define(DOP_REG_SEND_TT, 16). --define(DOP_EXIT2_TT, 18). - --define(DOP_MONITOR_P, 19). --define(DOP_DEMONITOR_P, 20). --define(DOP_MONITOR_P_EXIT, 21). dport_reg_send(Node, Name, Msg) -> DPrt = case dport(Node) of @@ -1546,6 +1724,39 @@ dport_send(To, Msg) -> ?COOKIE, To}), dmsg_ext(Msg)]). +send_bad_structure(Offender,Victim,Bad,WhereToPutSelf) -> + send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,[]). +send_bad_structure(Offender,Victim,Bad,WhereToPutSelf,PayLoad) -> + Parent = self(), + Done = make_ref(), + spawn(Offender, + fun () -> + Node = node(Victim), + pong = net_adm:ping(Node), + DPrt = dport(Node), + Bad1 = case WhereToPutSelf of + 0 -> + Bad; + N when N > 0 -> + setelement(N,Bad,self()) + end, + DData = [dmsg_hdr(), + dmsg_ext(Bad1)] ++ + case PayLoad of + [] -> []; + _Other -> [dmsg_ext(PayLoad)] + end, + port_command(DPrt, DData), + Parent ! {DData,Done} + end), + receive + {WhatSent,Done} -> + io:format("Offender sent ~p~n",[WhatSent]), + ok + after 5000 -> + exit(unable_to_send) + end. + %% send_bad_msgs(): %% Send a valid distribution header and control message @@ -1629,10 +1840,10 @@ dmsg_bad_hdr() -> 255]. % 255 atom references -dmsg_fake_hdr1() -> - A = <<"fake header atom 1">>, - [131, % Version Magic - $D, 1, 16#8, 0, size(A), A]. % Fake header +%% dmsg_fake_hdr1() -> +%% A = <<"fake header atom 1">>, +%% [131, % Version Magic +%% $D, 1, 16#8, 0, size(A), A]. % Fake header dmsg_fake_hdr2() -> A1 = <<"fake header atom 1">>, @@ -1817,7 +2028,7 @@ flush_node_changes() -> node_monitor_loop(Master) -> receive - {nodeup, Node, InfoList} = Msg -> + {nodeup, Node, _InfoList} = Msg -> Master ! {nodeup, node(), Node}, ?t:format("~p ~p: ~p~n", [node(), erlang:now(), Msg]), node_monitor_loop(Master); @@ -1854,9 +2065,9 @@ verify_no_down(A, B) -> ok end. -verify_down(A, B) -> - receive {nodedown, A, B, _} -> ok end, - receive {nodedown, B, A, _} -> ok end. +%% verify_down(A, B) -> +%% receive {nodedown, A, B, _} -> ok end, +%% receive {nodedown, B, A, _} -> ok end. verify_down(A, ReasonA, B, ReasonB) -> receive @@ -1876,11 +2087,11 @@ from(H, [H | T]) -> T; from(H, [_ | T]) -> from(H, T); from(_, []) -> []. -fun_spawn(Fun) -> - fun_spawn(Fun, []). +%% fun_spawn(Fun) -> +%% fun_spawn(Fun, []). -fun_spawn(Fun, Args) -> - spawn_link(erlang, apply, [Fun, Args]). +%% fun_spawn(Fun, Args) -> +%% spawn_link(erlang, apply, [Fun, Args]). long_or_short() -> diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 39b2ed395f..7600a44988 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -27,12 +27,12 @@ %%% - queueing -module(driver_SUITE). --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, + end_per_suite/1, init_per_group/2,end_per_group/2, init_per_testcase/2, - fin_per_testcase/2, - end_per_suite/1, + end_per_testcase/2, outputv_echo/1, - timer/1, + timer_measure/1, timer_cancel/1, timer_change/1, @@ -51,7 +51,7 @@ 'driver_system_info_ver1.1'/1, driver_system_info_current_ver/1, driver_monitor/1, - ioq_exit/1, + ioq_exit_ready_input/1, ioq_exit_ready_output/1, ioq_exit_timeout/1, @@ -78,7 +78,7 @@ -export([bin_prefix/2]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). % First byte in communication with the timer driver @@ -120,49 +120,51 @@ init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> ?line 0 = erts_debug:get_internal_state(check_io_debug), [{watchdog, Dog},{testcase, Case}|Config]. -fin_per_testcase(Case, Config) -> +end_per_testcase(Case, Config) -> Dog = ?config(watchdog, Config), - erlang:display({fin_per_testcase, Case}), + erlang:display({end_per_testcase, Case}), ?line 0 = erts_debug:get_internal_state(check_io_debug), ?t:timetrap_cancel(Dog). +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [fun_to_port, outputv_echo, queue_echo, {group, timer}, + driver_unloaded, io_ready_exit, use_fallback_pollset, + bad_fd_in_pollset, driver_event, fd_change, + steal_control, otp_6602, 'driver_system_info_ver1.0', + 'driver_system_info_ver1.1', + driver_system_info_current_ver, driver_monitor, + {group, ioq_exit}, zero_extended_marker_garb_drv, + invalid_extended_marker_drv, larger_major_vsn_drv, + larger_minor_vsn_drv, smaller_major_vsn_drv, + smaller_minor_vsn_drv, peek_non_existing_queue, + otp_6879, caller, many_events, missing_callbacks, + smp_select, driver_select_use, + thread_mseg_alloc_cache_clean]. + +groups() -> + [{timer, [], + [timer_measure, timer_cancel, timer_delay, + timer_change]}, + {ioq_exit, [], + [ioq_exit_ready_input, ioq_exit_ready_output, + ioq_exit_timeout, ioq_exit_ready_async, ioq_exit_event, + ioq_exit_ready_input_async, ioq_exit_ready_output_async, + ioq_exit_timeout_async, ioq_exit_event_async]}]. + +init_per_suite(Config) -> + Config. + end_per_suite(_Config) -> catch erts_debug:set_internal_state(available_internal_state, false). -all(suite) -> - [ - fun_to_port, - outputv_echo, - queue_echo, - timer, - driver_unloaded, - io_ready_exit, - use_fallback_pollset, - bad_fd_in_pollset, - driver_event, - fd_change, - steal_control, - otp_6602, - 'driver_system_info_ver1.0', - 'driver_system_info_ver1.1', - driver_system_info_current_ver, - driver_monitor, - ioq_exit, - zero_extended_marker_garb_drv, - invalid_extended_marker_drv, - larger_major_vsn_drv, - larger_minor_vsn_drv, - smaller_major_vsn_drv, - smaller_minor_vsn_drv, - peek_non_existing_queue, - otp_6879, - caller, - many_events, - missing_callbacks, - smp_select, - driver_select_use, - thread_mseg_alloc_cache_clean - ]. +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + fun_to_port(doc) -> "Test sending a fun to port with an outputv-capable driver."; fun_to_port(Config) when is_list(Config) -> @@ -308,7 +310,6 @@ compare(Got, Expected) -> %% Driver timer test suites %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -timer(suite) -> [timer_measure,timer_cancel,timer_delay,timer_change]. timer_measure(doc) -> ["Check that timers time out in good time."]; timer_measure(Config) when is_list(Config) -> @@ -1299,17 +1300,6 @@ driver_monitor(Config) when is_list(Config) -> ?line stop_driver(Port, Name), ?line ok. -ioq_exit(doc) -> []; -ioq_exit(suite) -> - [ioq_exit_ready_input, - ioq_exit_ready_output, - ioq_exit_timeout, - ioq_exit_ready_async, - ioq_exit_event, - ioq_exit_ready_input_async, - ioq_exit_ready_output_async, - ioq_exit_timeout_async, - ioq_exit_event_async]. -define(IOQ_EXIT_READY_INPUT, 1). -define(IOQ_EXIT_READY_OUTPUT, 2). @@ -1682,7 +1672,7 @@ smp_select0(Config) -> ProcFun = fun()-> io:format("Worker ~p starting\n",[self()]), ?line Port = open_port({spawn, DrvName}, []), smp_select_loop(Port, 100000), - sleep(500), % wait for driver to handle pending events + sleep(1000), % wait for driver to handle pending events ?line true = erlang:port_close(Port), Master ! {ok,self()}, io:format("Worker ~p finished\n",[self()]) @@ -1790,8 +1780,8 @@ mseg_alloc_ccc() -> mseg_alloc_ccc(erlang:system_info({allocator,mseg_alloc})). mseg_alloc_ccc(MsegAllocInfo) -> - ?line {value,{calls, CL}} - = lists:keysearch(calls, 1, MsegAllocInfo), + ?line {value,{memkind, MKL}} = lists:keysearch(memkind,1,MsegAllocInfo), + ?line {value,{calls, CL}} = lists:keysearch(calls, 1, MKL), ?line {value,{mseg_check_cache, GigaCCC, CCC}} = lists:keysearch(mseg_check_cache, 1, CL), ?line GigaCCC*1000000000 + CCC. @@ -1800,12 +1790,28 @@ mseg_alloc_cached_segments() -> mseg_alloc_cached_segments(erlang:system_info({allocator,mseg_alloc})). mseg_alloc_cached_segments(MsegAllocInfo) -> + MemName = case is_halfword_vm() of + true -> "high memory"; + false -> "all memory" + end, + ?line [{memkind,DrvMem}] + = lists:filter(fun(E) -> case E of + {memkind, [{name, MemName} | _]} -> true; + _ -> false + end end, MsegAllocInfo), ?line {value,{status, SL}} - = lists:keysearch(status, 1, MsegAllocInfo), + = lists:keysearch(status, 1, DrvMem), ?line {value,{cached_segments, CS}} = lists:keysearch(cached_segments, 1, SL), ?line CS. +is_halfword_vm() -> + case {erlang:system_info({wordsize, internal}), + erlang:system_info({wordsize, external})} of + {4, 8} -> true; + {WS, WS} -> false + end. + driver_alloc_sbct() -> {_, _, _, As} = erlang:system_info(allocator), case lists:keysearch(driver_alloc, 1, As) of diff --git a/erts/emulator/test/driver_SUITE_data/chkio_drv.c b/erts/emulator/test/driver_SUITE_data/chkio_drv.c index b571cb30e6..bbdb09cfcb 100644 --- a/erts/emulator/test/driver_SUITE_data/chkio_drv.c +++ b/erts/emulator/test/driver_SUITE_data/chkio_drv.c @@ -102,6 +102,7 @@ typedef struct chkio_smp_select { int write_fd; int next_read; int next_write; + int first_write; enum {Closed, Opened, Selected, Waiting} state; int wasSelected; unsigned rand_state; @@ -577,9 +578,16 @@ chkio_drv_ready_input(ErlDrvData drv_data, ErlDrvEvent event) inPipe = (pip->next_write - pip->next_read); if (inPipe == 0) { bytes = read(pip->read_fd, &word, sizeof(word)); - printf("Unexpected empty pipe, expected %u -> %u, bytes=%d, word=%d\n", - pip->next_read, pip->next_write-1, bytes, word); - abort(); + printf("Unexpected empty pipe, expected %u -> %u, bytes=%d, word=%d, written=%d\n", + pip->next_read, pip->next_write-1, bytes, word, + (pip->next_write - pip->first_write)); + /*abort(); + Allow unexpected events as it's been seen to be triggered by epoll + on Linux. Most of the time the unwanted events are filtered by + the erl_check_io layer. But when fd's are reused the events may + slip up to the driver. + */ + break; } n = rand_r(&pip->rand_state) % (inPipe*4); @@ -1252,6 +1260,7 @@ chkio_drv_control(ErlDrvData drv_data, pip->state = Opened; pip->wasSelected = 0; pip->next_write = pip->next_read = rand_r(&pip->rand_state) % 1024; + pip->first_write = pip->next_write; if (op & 1) break; op >>= 1; }/*fall through*/ diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl index 1d66b6ef70..9ac004200e 100644 --- a/erts/emulator/test/efile_SUITE.erl +++ b/erts/emulator/test/efile_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -17,12 +17,32 @@ %% %CopyrightEnd% -module(efile_SUITE). --export([all/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). -export([iter_max_files/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [iter_max_files]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> [iter_max_files]. %% %% Open as many files as possible. Do this several times and check diff --git a/erts/emulator/test/emulator.spec b/erts/emulator/test/emulator.spec index ed5bd48e84..1ea751cc3b 100644 --- a/erts/emulator/test/emulator.spec +++ b/erts/emulator/test/emulator.spec @@ -1 +1 @@ -{topcase, {dir, "../emulator_test"}}. +{suites,"../emulator_test",all}. diff --git a/erts/emulator/test/erl_drv_thread_SUITE.erl b/erts/emulator/test/erl_drv_thread_SUITE.erl index ea618e9feb..84a82cced0 100644 --- a/erts/emulator/test/erl_drv_thread_SUITE.erl +++ b/erts/emulator/test/erl_drv_thread_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2009. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,18 +19,36 @@ -module(erl_drv_thread_SUITE). -author('[email protected]'). --export([all/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). -export([basic/1, rwlock/1, tsd/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -define(DEFAULT_TIMETRAP_SECS, 240). -all(doc) -> []; -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [basic, rwlock, tsd]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% %% Testcases %% diff --git a/erts/emulator/test/erl_link_SUITE.erl b/erts/emulator/test/erl_link_SUITE.erl index 542c8dffbe..435c0872e6 100644 --- a/erts/emulator/test/erl_link_SUITE.erl +++ b/erts/emulator/test/erl_link_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,9 +28,10 @@ -author('[email protected]'). %-define(line_trace, 1). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). % Test cases -export([links/1, @@ -46,7 +47,7 @@ otp_5772_dist_monitor/1, otp_7946/1]). --export([init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]). +-export([init_per_testcase/2, end_per_testcase/2]). % Internal exports -export([test_proc/0]). @@ -77,11 +78,29 @@ -all(suite) -> [links, dist_links, monitor_nodes, process_monitors, - dist_process_monitors, busy_dist_port_monitor, - busy_dist_port_link, otp_5772_link, otp_5772_dist_link, - otp_5772_monitor, otp_5772_dist_monitor, - otp_7946]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [links, dist_links, monitor_nodes, process_monitors, + dist_process_monitors, busy_dist_port_monitor, + busy_dist_port_link, otp_5772_link, otp_5772_dist_link, + otp_5772_monitor, otp_5772_dist_monitor, otp_7946]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + catch erts_debug:set_internal_state(available_internal_state, false). + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + links(doc) -> ["Tests node local links"]; links(suite) -> []; @@ -678,13 +697,10 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> end, ?line [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> ?line Dog = ?config(watchdog, Config), ?line ?t:timetrap_cancel(Dog). -end_per_suite(_Config) -> - catch erts_debug:set_internal_state(available_internal_state, false). - tp_call(Tp, Fun) -> ?line R = make_ref(), ?line Tp ! {call, self(), R, Fun}, @@ -1050,7 +1066,6 @@ stop_node(Node) -> -define(DOP_SEND, 2). -define(DOP_EXIT, 3). -define(DOP_UNLINK, 4). --define(DOP_NODE_LINK, 5). -define(DOP_REG_SEND, 6). -define(DOP_GROUP_LEADER, 7). -define(DOP_EXIT2, 8). diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl index e60a999df1..4dc2fbaae2 100644 --- a/erts/emulator/test/erts_debug_SUITE.erl +++ b/erts/emulator/test/erts_debug_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2009. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,19 +18,40 @@ %% -module(erts_debug_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2, - flat_size/1,flat_size_big/1,df/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, + flat_size/1,flat_size_big/1,df/1, + instructions/1]). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [flat_size, flat_size_big, df, instructions]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [flat_size,flat_size_big,df]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(2)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). @@ -70,3 +91,8 @@ df(Config) when is_list(Config) -> pps() -> {erlang:ports()}. + +instructions(Config) when is_list(Config) -> + ?line Is = erts_debug:instructions(), + ?line _ = [list_to_atom(I) || I <- Is], + ok. diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl index 7fb92faf0d..2ba9375a41 100644 --- a/erts/emulator/test/estone_SUITE.erl +++ b/erts/emulator/test/estone_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,8 +18,9 @@ -module(estone_SUITE). %% Test functions --export([all/1,estone/1]). --export([init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2,estone/1]). +-export([init_per_testcase/2, end_per_testcase/2]). %% Internal exports for EStone tests -export([lists/1, @@ -44,7 +45,7 @@ run_micro/3,p1/1,ppp/3,macro/2,micros/0]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %% Test suite defines -define(default_timeout, ?t:minutes(10)). @@ -68,12 +69,31 @@ init_per_testcase(_Case, Config) -> ?line Dog=test_server:timetrap(?default_timeout), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. -all(suite) -> [estone]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [estone]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + estone(suite) -> []; diff --git a/erts/emulator/test/evil_SUITE.erl b/erts/emulator/test/evil_SUITE.erl index a8288584f4..f982b9d4ff 100644 --- a/erts/emulator/test/evil_SUITE.erl +++ b/erts/emulator/test/evil_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,7 +18,9 @@ -module(evil_SUITE). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, heap_frag/1, encode_decode_ext/1, decode_integer_ext/1, @@ -30,26 +32,37 @@ decode_pos_neg_zero/1 ]). --include("test_server.hrl"). - -all(suite) -> - [ - heap_frag, - encode_decode_ext, - decode_integer_ext, - decode_small_big_ext, - decode_large_big_ext, - decode_small_big_ext_neg, - decode_large_big_ext_neg, - decode_too_small, - decode_pos_neg_zero - ]. +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [heap_frag, encode_decode_ext, decode_integer_ext, + decode_small_big_ext, decode_large_big_ext, + decode_small_big_ext_neg, decode_large_big_ext_neg, + decode_too_small, decode_pos_neg_zero]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(_Case, Config) -> ?line Dog = test_server:timetrap(?t:minutes(0.5)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog=?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl index f1e6e004ad..9d6fc9521d 100644 --- a/erts/emulator/test/exception_SUITE.erl +++ b/erts/emulator/test/exception_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,20 +19,40 @@ -module(exception_SUITE). --export([all/1, badmatch/1, pending_errors/1, nil_arith/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + badmatch/1, pending_errors/1, nil_arith/1, stacktrace/1, nested_stacktrace/1, raise/1, gunilla/1, per/1, exception_with_heap_frag/1]). -export([bad_guy/2]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -import(lists, [foreach/2]). -all(suite) -> - [badmatch, pending_errors, nil_arith, - stacktrace, nested_stacktrace, raise, gunilla, per, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [badmatch, pending_errors, nil_arith, stacktrace, + nested_stacktrace, raise, gunilla, per, exception_with_heap_frag]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + -define(try_match(E), catch ?MODULE:bar(), {'EXIT', {{badmatch, nomatch}, _}} = (catch E = id(nomatch))). @@ -255,7 +275,16 @@ stacktrace(Conf) when is_list(Conf) -> ?line [{?MODULE,stacktrace_1,3}|_] = erase(stacktrace1), ?line St4 = erase(stacktrace2), ?line St4 = erlang:get_stacktrace(), - ok. + + try + ?line stacktrace_2() + catch + error:{badmatch,_} -> + [{?MODULE,stacktrace_2,0}, + {?MODULE,stacktrace,1}|_] = + erlang:get_stacktrace(), + ok + end. stacktrace_1(X, C1, Y) -> erase(stacktrace1), @@ -275,6 +304,9 @@ stacktrace_1(X, C1, Y) -> put(stacktrace2, erlang:get_stacktrace()) end. +stacktrace_2() -> + ok = erlang:process_info(self(), current_function), + ok. nested_stacktrace(Conf) when is_list(Conf) -> diff --git a/erts/emulator/test/float_SUITE.erl b/erts/emulator/test/float_SUITE.erl index 99e9457985..736510339f 100644 --- a/erts/emulator/test/float_SUITE.erl +++ b/erts/emulator/test/float_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,9 +19,11 @@ -module(float_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, fpe/1,fp_drv/1,fp_drv_thread/1,denormalized/1,match/1, bad_float_unpack/1]). -export([otp_7178/1]). @@ -31,18 +33,31 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(3)), [{watchdog, Dog},{testcase,Func}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog). -all(suite) -> - [fpe, - fp_drv, - fp_drv_thread, - otp_7178, - denormalized, - match, - bad_float_unpack]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [fpe, fp_drv, fp_drv_thread, otp_7178, denormalized, + match, bad_float_unpack]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + %% %% OTP-7178, list_to_float on very small numbers should give 0.0 diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl index a7889dfe90..7795efe57e 100644 --- a/erts/emulator/test/fun_SUITE.erl +++ b/erts/emulator/test/fun_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2010. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,7 +22,9 @@ -define(default_timeout, ?t:minutes(1)). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, bad_apply/1,bad_fun_call/1,badarity/1,ext_badarity/1, equality/1,ordering/1, fun_to_port/1,t_hash/1,t_phash/1,t_phash2/1,md5/1, @@ -32,19 +34,37 @@ -export([nothing/0]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [bad_apply, bad_fun_call, badarity, ext_badarity, + equality, ordering, fun_to_port, t_hash, t_phash, + t_phash2, md5, refc, refc_ets, refc_dist, + const_propagation, t_arity, t_is_function2, t_fun_info]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [bad_apply,bad_fun_call,badarity,ext_badarity,equality,ordering, - fun_to_port,t_hash,t_phash,t_phash2,md5, - refc,refc_ets,refc_dist,const_propagation, - t_arity,t_is_function2,t_fun_info]. init_per_testcase(_Case, Config) -> ?line Dog = test_server:timetrap(?default_timeout), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog=?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. diff --git a/erts/emulator/test/fun_r12_SUITE.erl b/erts/emulator/test/fun_r12_SUITE.erl index 9262731dcb..3b1dfc9825 100644 --- a/erts/emulator/test/fun_r12_SUITE.erl +++ b/erts/emulator/test/fun_r12_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,18 +20,39 @@ -module(fun_r12_SUITE). -compile(r12). --export([all/1,init_per_testcase/2,fin_per_testcase/2,dist_old_release/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2,dist_old_release/1]). -define(default_timeout, ?t:minutes(1)). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [dist_old_release]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> [dist_old_release]. init_per_testcase(_Case, Config) -> ?line Dog = test_server:timetrap(?default_timeout), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog=?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl index 066aa215b2..771d2c9a7a 100644 --- a/erts/emulator/test/gc_SUITE.erl +++ b/erts/emulator/test/gc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,15 +21,34 @@ -module(gc_SUITE). --include("test_server.hrl"). --export([all/1]). +-include_lib("test_server/include/test_server.hrl"). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). -define(default_timeout, ?t:minutes(10)). -export([grow_heap/1, grow_stack/1, grow_stack_heap/1]). -all(suite) -> - [grow_heap,grow_stack, grow_stack_heap]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [grow_heap, grow_stack, grow_stack_heap]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + grow_heap(doc) -> ["Produce a growing list of elements, ", "for X calls, then drop one item per call", diff --git a/erts/emulator/test/guard_SUITE.erl b/erts/emulator/test/guard_SUITE.erl index 8fef36dfaf..f41324c2cc 100644 --- a/erts/emulator/test/guard_SUITE.erl +++ b/erts/emulator/test/guard_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,16 +19,37 @@ -module(guard_SUITE). --export([all/1, bad_arith/1, bad_tuple/1, test_heap_guards/1, guard_bifs/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, bad_arith/1, bad_tuple/1, + test_heap_guards/1, guard_bifs/1, type_tests/1,guard_bif_binary_part/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -export([init/3]). -import(lists, [member/2]). -all(suite) -> [bad_arith, bad_tuple, test_heap_guards, guard_bifs, - type_tests, guard_bif_binary_part]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [bad_arith, bad_tuple, test_heap_guards, guard_bifs, + type_tests, guard_bif_binary_part]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + bad_arith(doc) -> "Test that a bad arithmetic operation in a guard works correctly."; bad_arith(Config) when is_list(Config) -> diff --git a/erts/emulator/test/hash_SUITE.erl b/erts/emulator/test/hash_SUITE.erl index f5d1871bfb..830ed91da9 100644 --- a/erts/emulator/test/hash_SUITE.erl +++ b/erts/emulator/test/hash_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2009. All Rights Reserved. +%% Copyright Ericsson AB 2000-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -49,7 +49,7 @@ -define(config(A,B),config(A,B)). -export([config/2]). -else. --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -endif. -ifdef(debug). @@ -69,22 +69,40 @@ config(priv_dir,_) -> ".". -else. %% When run in test server. --export([all/1,test_basic/1,test_cmp/1,test_range/1,test_spread/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + test_basic/1,test_cmp/1,test_range/1,test_spread/1, test_phash2/1,otp_5292/1,bit_level_binaries/1,otp_7127/1, - fin_per_testcase/2,init_per_testcase/2]). + end_per_testcase/2,init_per_testcase/2]). init_per_testcase(_Case, Config) -> ?line Dog=test_server:timetrap(test_server:minutes(10)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog=?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. -all(doc) -> - ["Test erlang:phash"]; -all(suite) -> - [test_basic, test_cmp, test_range, test_spread, test_phash2, otp_5292, - bit_level_binaries, otp_7127]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [test_basic, test_cmp, test_range, test_spread, + test_phash2, otp_5292, bit_level_binaries, otp_7127]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + test_basic(suite) -> []; diff --git a/erts/emulator/test/hibernate_SUITE.erl b/erts/emulator/test/hibernate_SUITE.erl index 4d36076d12..203fa6b48e 100644 --- a/erts/emulator/test/hibernate_SUITE.erl +++ b/erts/emulator/test/hibernate_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2009. All Rights Reserved. +%% Copyright Ericsson AB 2003-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,23 +19,44 @@ -module(hibernate_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2, - basic/1,min_heap_size/1,bad_args/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, + basic/1,dynamic_call/1,min_heap_size/1,bad_args/1, messages_in_queue/1,undefined_mfa/1, no_heap/1]). %% Used by test cases. --export([basic_hibernator/1,messages_in_queue_restart/2, no_heap_loop/0]). +-export([basic_hibernator/1,dynamic_call_hibernator/2,messages_in_queue_restart/2, no_heap_loop/0]). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [basic, dynamic_call, min_heap_size, bad_args, messages_in_queue, + undefined_mfa, no_heap]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [basic,min_heap_size,bad_args,messages_in_queue,undefined_mfa,no_heap]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(3)), [{watchdog,Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). @@ -138,10 +159,47 @@ whats_up_calc(A1, A2, A3, A4, A5, A6, A7, A8, A9, Acc) -> whats_up_calc(A1-1, A2+1, A3+2, A4+3, A5+4, A6+5, A7+6, A8+7, A9+8, [A1,A2|Acc]). %%% +%%% Testing a call to erlang:hibernate/3 that the compiler and loader do not +%%% translate to an instruction. +%%% + +dynamic_call(Config) when is_list(Config) -> + Ref = make_ref(), + Info = {self(),Ref}, + ExpectedHeapSz = case erlang:system_info(heap_type) of + private -> erts_debug:size([Info]); + hybrid -> erts_debug:size([a|b]) + end, + ?line Child = spawn_link(fun() -> ?MODULE:dynamic_call_hibernator(Info, hibernate) end), + ?line hibernate_wake_up(100, ExpectedHeapSz, Child), + ?line Child ! please_quit_now, + ok. + +dynamic_call_hibernator(Info, Function) -> + {catchlevel,0} = process_info(self(), catchlevel), + receive + Any -> + dynamic_call_hibernator_msg(Any, Function, Info), + dynamic_call_hibernator(Info, Function) + end. + +dynamic_call_hibernator_msg({hibernate,_}, Function, Info) -> + catch apply(erlang, Function, [?MODULE, basic_hibernator, [Info]]), + exit(hibernate_returned); +dynamic_call_hibernator_msg(Msg, _Function, Info) -> + basic_hibernator_msg(Msg, Info). + +%%% %%% Testing setting the minimum heap size. %%% min_heap_size(Config) when is_list(Config) -> + case test_server:is_native(?MODULE) of + true -> {skip, "Test case relies on trace which is not available in HiPE"}; + false -> min_heap_size_1(Config) + end. + +min_heap_size_1(Config) when is_list(Config) -> ?line erlang:trace(new, true, [call]), MFA = {?MODULE,min_hibernator,1}, ?line 1 = erlang:trace_pattern(MFA, true, [local]), diff --git a/erts/emulator/test/ignore_cores.erl b/erts/emulator/test/ignore_cores.erl index 1d738cbafd..8b1ac0fe6c 120000..100644 --- a/erts/emulator/test/ignore_cores.erl +++ b/erts/emulator/test/ignore_cores.erl @@ -1 +1,158 @@ -../../test/ignore_cores.erl
\ No newline at end of file +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%%%------------------------------------------------------------------- +%%% File : ignore_cores.erl +%%% Author : Rickard Green <[email protected]> +%%% Description : +%%% +%%% Created : 11 Feb 2008 by Rickard Green <[email protected]> +%%%------------------------------------------------------------------- + +-module(ignore_cores). + +-include_lib("test_server/include/test_server.hrl"). + +-export([init/1, fini/1, setup/3, setup/4, restore/1, dir/1]). + +-record(ignore_cores, {org_cwd, + org_path, + org_pwd_env, + ign_dir = false, + cores_dir = false}). + +%% +%% Takes a testcase config +%% + +init(Config) -> + {ok, OrgCWD} = file:get_cwd(), + [{ignore_cores, + #ignore_cores{org_cwd = OrgCWD, + org_path = code:get_path(), + org_pwd_env = os:getenv("PWD")}} + | lists:keydelete(ignore_cores, 1, Config)]. + +fini(Config) -> + #ignore_cores{org_cwd = OrgCWD, + org_path = OrgPath, + org_pwd_env = OrgPWD} = ?config(ignore_cores, Config), + ok = file:set_cwd(OrgCWD), + true = code:set_path(OrgPath), + case OrgPWD of + false -> ok; + _ -> true = os:putenv("PWD", OrgPWD) + end, + lists:keydelete(ignore_cores, 1, Config). + +setup(Suite, Testcase, Config) -> + setup(Suite, Testcase, Config, false). + +setup(Suite, Testcase, Config, SetCwd) when is_atom(Suite), + is_atom(Testcase), + is_list(Config) -> + #ignore_cores{org_cwd = OrgCWD, + org_path = OrgPath, + org_pwd_env = OrgPWD} = ?config(ignore_cores, Config), + Path = lists:map(fun (".") -> OrgCWD; (Dir) -> Dir end, OrgPath), + true = code:set_path(Path), + PrivDir = ?config(priv_dir, Config), + IgnDir = filename:join([PrivDir, + atom_to_list(Suite) + ++ "_" + ++ atom_to_list(Testcase) + ++ "_wd"]), + ok = file:make_dir(IgnDir), + case SetCwd of + false -> + ok; + _ -> + ok = file:set_cwd(IgnDir), + OrgPWD = case os:getenv("PWD") of + false -> false; + PWD -> + os:putenv("PWD", IgnDir), + PWD + end + end, + ok = file:write_file(filename:join([IgnDir, "ignore_core_files"]), <<>>), + %% cores are dumped in /cores on MacOS X + CoresDir = case {?t:os_type(), filelib:is_dir("/cores")} of + {{unix,darwin}, true} -> + filelib:fold_files("/cores", + "^core.*$", + false, + fun (C,Cs) -> [C|Cs] end, + []); + _ -> + false + end, + lists:keyreplace(ignore_cores, + 1, + Config, + {ignore_cores, + #ignore_cores{org_cwd = OrgCWD, + org_path = OrgPath, + org_pwd_env = OrgPWD, + ign_dir = IgnDir, + cores_dir = CoresDir}}). + +restore(Config) -> + #ignore_cores{org_cwd = OrgCWD, + org_path = OrgPath, + org_pwd_env = OrgPWD, + ign_dir = IgnDir, + cores_dir = CoresDir} = ?config(ignore_cores, Config), + try + case CoresDir of + false -> + ok; + _ -> + %% Move cores dumped by these testcases in /cores + %% to cwd. + lists:foreach(fun (C) -> + case lists:member(C, CoresDir) of + true -> ok; + _ -> + Dst = filename:join( + [IgnDir, + filename:basename(C)]), + {ok, _} = file:copy(C, Dst), + file:delete(C) + end + end, + filelib:fold_files("/cores", + "^core.*$", + false, + fun (C,Cs) -> [C|Cs] end, + [])) + end + after + catch file:set_cwd(OrgCWD), + catch code:set_path(OrgPath), + case OrgPWD of + false -> ok; + _ -> catch os:putenv("PWD", OrgPWD) + end + end. + + +dir(Config) -> + #ignore_cores{ign_dir = Dir} = ?config(ignore_cores, Config), + Dir. diff --git a/erts/emulator/test/list_bif_SUITE.erl b/erts/emulator/test/list_bif_SUITE.erl index 65ea88eb2f..45a44d8b43 100644 --- a/erts/emulator/test/list_bif_SUITE.erl +++ b/erts/emulator/test/list_bif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -18,21 +18,42 @@ %% -module(list_bif_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2]). -export([hd_test/1,tl_test/1,t_length/1,t_list_to_pid/1, t_list_to_float/1,t_list_to_integer/1]). -all(suite) -> - [hd_test,tl_test,t_length,t_list_to_pid,t_list_to_float,t_list_to_integer]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [hd_test, tl_test, t_length, t_list_to_pid, + t_list_to_float, t_list_to_integer]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(_Case, Config) -> ?line Dog = test_server:timetrap(test_server:seconds(60)), [{watchdog,Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog = ?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. diff --git a/erts/emulator/test/long_timers_test.erl b/erts/emulator/test/long_timers_test.erl index 28626d26fb..28a4fba9f6 100644 --- a/erts/emulator/test/long_timers_test.erl +++ b/erts/emulator/test/long_timers_test.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. +%% Copyright Ericsson AB 2006-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl index 69c89f5d2d..2b21fa58f4 100644 --- a/erts/emulator/test/match_spec_SUITE.erl +++ b/erts/emulator/test/match_spec_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,7 +19,8 @@ -module(match_spec_SUITE). --export([all/1, not_run/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, not_run/1]). -export([test_1/1, test_2/1, test_3/1, bad_match_spec_bin/1, trace_control_word/1, silent/1, silent_no_ms/1, ms_trace2/1, ms_trace3/1, boxed_and_small/1, @@ -34,29 +35,48 @@ % This test suite assumes that tracing in general works. What we test is % the match spec functionality. --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([init_per_testcase/2, fin_per_testcase/2]). +-export([init_per_testcase/2, end_per_testcase/2]). init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:seconds(10)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). -all(suite) -> - case test_server:is_native(?MODULE) of - false -> [test_1, test_2, test_3, bad_match_spec_bin, - trace_control_word, silent, silent_no_ms, - ms_trace2, ms_trace3, boxed_and_small, - destructive_in_test_bif, guard_exceptions, - unary_plus, unary_minus, fpe, moving_labels]; - true -> [not_run] +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + case test_server:is_native(match_spec_SUITE) of + false -> + [test_1, test_2, test_3, bad_match_spec_bin, + trace_control_word, silent, silent_no_ms, ms_trace2, + ms_trace3, boxed_and_small, destructive_in_test_bif, + guard_exceptions, unary_plus, unary_minus, fpe, + moving_labels]; + true -> [not_run] end. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + not_run(Config) when is_list(Config) -> {skipped, "Native Code"}. @@ -345,15 +365,15 @@ silent_no_ms(Config) when is_list(Config) -> fun () -> ?MODULE:f1(a), ?MODULE:f2(b, c), - erlang:integer_to_list(id(1)), + _ = erlang:integer_to_list(id(1)), ?MODULE:f3(d, e), ?MODULE:f1(start), ?MODULE:f2(f, g), - erlang:integer_to_list(id(2)), + _ = erlang:integer_to_list(id(2)), ?MODULE:f3(h, i), ?MODULE:f1(stop), ?MODULE:f2(j, k), - erlang:integer_to_list(id(3)), + _ = erlang:integer_to_list(id(3)), ?MODULE:f3(l, m) end, fun (Tracee) -> @@ -393,15 +413,15 @@ silent_no_ms(Config) when is_list(Config) -> fun () -> ?MODULE:f1(a), ?MODULE:f2(b, c), - erlang:integer_to_list(id(1)), + _ = erlang:integer_to_list(id(1)), ?MODULE:f3(d, e), ?MODULE:f1(start), ?MODULE:f2(f, g), - erlang:integer_to_list(id(2)), + _ = erlang:integer_to_list(id(2)), ?MODULE:f3(h, i), ?MODULE:f1(stop), ?MODULE:f2(j, k), - erlang:integer_to_list(id(3)), + _ = erlang:integer_to_list(id(3)), ?MODULE:f3(l, m) end, fun (Tracee) -> @@ -455,18 +475,18 @@ ms_trace2(Config) when is_list(Config) -> fun () -> ?MODULE:f1(a), ?MODULE:f2(b, c), - erlang:integer_to_list(id(1)), + _ = erlang:integer_to_list(id(1)), ?MODULE:f3(d, e), fn([all], [call,return_to,{tracer,Tracer}]), ?MODULE:f1(f), f2(g, h), f1(i), - erlang:integer_to_list(id(2)), + _ = erlang:integer_to_list(id(2)), ?MODULE:f3(j, k), fn([call,return_to], []), ?MODULE:f1(l), ?MODULE:f2(m, n), - erlang:integer_to_list(id(3)), + _ = erlang:integer_to_list(id(3)), ?MODULE:f3(o, p) end, fun (Tracee) -> @@ -551,26 +571,26 @@ ms_trace3(Config) when is_list(Config) -> register(TraceeName, self()), ?MODULE:f1(a), ?MODULE:f2(b, c), - erlang:integer_to_list(id(1)), + _ = erlang:integer_to_list(id(1)), ?MODULE:f3(d, e), Controller ! {self(),Tag,start}, receive {Controller,Tag,started} -> ok end, ?MODULE:f1(f), f2(g, h), f1(i), - erlang:integer_to_list(id(2)), + _ = erlang:integer_to_list(id(2)), ?MODULE:f3(j, k), Controller ! {self(),Tag,stop_1}, receive {Controller,Tag,stopped_1} -> ok end, ?MODULE:f1(l), ?MODULE:f2(m, n), - erlang:integer_to_list(id(3)), + _ = erlang:integer_to_list(id(3)), ?MODULE:f3(o, p), Controller ! {self(),Tag,stop_2}, receive {Controller,Tag,stopped_2} -> ok end, ?MODULE:f1(q), ?MODULE:f2(r, s), - erlang:integer_to_list(id(4)), + _ = erlang:integer_to_list(id(4)), ?MODULE:f3(t, u) end, diff --git a/erts/emulator/test/module_info_SUITE.erl b/erts/emulator/test/module_info_SUITE.erl index f34a2b496c..8a63d9fe3e 100644 --- a/erts/emulator/test/module_info_SUITE.erl +++ b/erts/emulator/test/module_info_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2009. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,9 +19,11 @@ -module(module_info_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,end_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, exports/1,functions/1,native/1]). %%-compile(native). @@ -29,8 +31,29 @@ %% Helper. -export([native_proj/1,native_filter/1]). -all(suite) -> - [exports,functions,native]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + modules(). + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + +modules() -> + [exports, functions, native]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(3)), @@ -42,14 +65,18 @@ end_per_testcase(_Func, Config) -> %% Should return all functions exported from this module. (local) all_exported() -> - All = add_arity(all(suite)), - lists:sort([{all,1},{init_per_testcase,2},{end_per_testcase,2}, + All = add_arity(modules()), + lists:sort([{all,0},{suite,0},{groups,0}, + {init_per_suite,1},{end_per_suite,1}, + {init_per_group,2},{end_per_group,2}, + {init_per_testcase,2},{end_per_testcase,2}, {module_info,0},{module_info,1},{native_proj,1}, {native_filter,1}|All]). %% Should return all functions in this module. (local) all_functions() -> - Locals = [{add_arity,1},{add_arity,2},{all_exported,0},{all_functions,0}], + Locals = [{add_arity,1},{add_arity,2},{all_exported,0},{all_functions,0}, + {modules,0}], lists:sort(Locals++all_exported()). %% Test that the list of exported functions from this module is correct. diff --git a/erts/emulator/test/monitor_SUITE.erl b/erts/emulator/test/monitor_SUITE.erl index 68e378dfec..aec59867d8 100644 --- a/erts/emulator/test/monitor_SUITE.erl +++ b/erts/emulator/test/monitor_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,29 +19,49 @@ -module(monitor_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, case_1/1, case_1a/1, case_2/1, case_2a/1, mon_e_1/1, demon_e_1/1, demon_1/1, - demon_2/1, demon_3/1, demonitor_flush/1, remove_monitor/1, + demon_2/1, demon_3/1, demonitor_flush/1, local_remove_monitor/1, remote_remove_monitor/1, mon_1/1, mon_2/1, large_exit/1, list_cleanup/1, mixer/1, named_down/1, otp_5827/1]). --export([init_per_testcase/2, fin_per_testcase/2]). +-export([init_per_testcase/2, end_per_testcase/2]). -export([y2/1, g/1, g0/0, g1/0, large_exit_sub/1]). -all(suite) -> - [case_1, case_1a, case_2, case_2a, mon_e_1, demon_e_1, demon_1, mon_1, - mon_2, demon_2, demon_3, demonitor_flush, remove_monitor, - large_exit, list_cleanup, mixer, named_down, - otp_5827]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [case_1, case_1a, case_2, case_2a, mon_e_1, demon_e_1, + demon_1, mon_1, mon_2, demon_2, demon_3, + demonitor_flush, {group, remove_monitor}, large_exit, + list_cleanup, mixer, named_down, otp_5827]. + +groups() -> + [{remove_monitor, [], + [local_remove_monitor, remote_remove_monitor]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(15)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). @@ -315,8 +335,6 @@ demonitor_flush_test(Node) -> -define(RM_MON_GROUPS, 100). -define(RM_MON_GPROCS, 100). -remove_monitor(suite) -> - [local_remove_monitor, remote_remove_monitor]. local_remove_monitor(Config) when is_list(Config) -> Gs = generate(fun () -> start_remove_monitor_group(node()) end, diff --git a/erts/emulator/test/mtx_SUITE.erl b/erts/emulator/test/mtx_SUITE.erl new file mode 100644 index 0000000000..e0a7878bd8 --- /dev/null +++ b/erts/emulator/test/mtx_SUITE.erl @@ -0,0 +1,479 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% +%% Stress tests of rwmutex implementation. +%% +%% Author: Rickard Green +%% +-module(mtx_SUITE). + +%%-define(line_trace,true). + +-include_lib("common_test/include/ct.hrl"). + +-export([all/0,suite/0,groups/0, + init_per_group/2,end_per_group/2, init_per_suite/1, + end_per_suite/1, init_per_testcase/2, end_per_testcase/2]). + +-export([long_rwlock/1, + hammer_ets_rwlock/1, + hammer_rwlock/1, + hammer_rwlock_check/1, + hammer_tryrwlock/1, + hammer_tryrwlock_check/1, + hammer_sched_long_rwlock/1, + hammer_sched_long_rwlock_check/1, + hammer_sched_long_freqread_rwlock/1, + hammer_sched_long_freqread_rwlock_check/1, + hammer_sched_long_tryrwlock/1, + hammer_sched_long_tryrwlock_check/1, + hammer_sched_long_freqread_tryrwlock/1, + hammer_sched_long_freqread_tryrwlock_check/1, + hammer_sched_rwlock/1, + hammer_sched_rwlock_check/1, + hammer_sched_freqread_rwlock/1, + hammer_sched_freqread_rwlock_check/1, + hammer_sched_tryrwlock/1, + hammer_sched_tryrwlock_check/1, + hammer_sched_freqread_tryrwlock/1, + hammer_sched_freqread_tryrwlock_check/1]). + +init_per_suite(Config) when is_list(Config) -> + DataDir = ?config(data_dir, Config), + Lib = filename:join([DataDir, atom_to_list(?MODULE)]), + ok = erlang:load_nif(Lib, none), + Config. + +end_per_suite(Config) when is_list(Config) -> + Config. + +init_per_testcase(_Case, Config) -> + Dog = ?t:timetrap(?t:minutes(15)), + [{watchdog, Dog}|Config]. + +end_per_testcase(_Func, Config) -> + Dog = ?config(watchdog, Config), + ?t:timetrap_cancel(Dog). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [long_rwlock, hammer_rwlock_check, hammer_rwlock, + hammer_tryrwlock_check, hammer_tryrwlock, + hammer_ets_rwlock, hammer_sched_long_rwlock_check, + hammer_sched_long_rwlock, + hammer_sched_long_freqread_rwlock_check, + hammer_sched_long_freqread_rwlock, + hammer_sched_long_tryrwlock_check, + hammer_sched_long_tryrwlock, + hammer_sched_long_freqread_tryrwlock_check, + hammer_sched_long_freqread_tryrwlock, + hammer_sched_rwlock_check, hammer_sched_rwlock, + hammer_sched_freqread_rwlock_check, + hammer_sched_freqread_rwlock, + hammer_sched_tryrwlock_check, hammer_sched_tryrwlock, + hammer_sched_freqread_tryrwlock_check, + hammer_sched_freqread_tryrwlock]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + +long_rwlock(Config) when is_list(Config) -> + statistics(runtime), + LLRes = long_rw_test(), + {_, RunTime} = statistics(runtime), + %% A very short run time is expected, since + %% threads in the test mostly wait + ?t:format("RunTime=~p~n", [RunTime]), + ?line true = RunTime < 100, + ?line RunTimeStr = "Run-time during test was "++integer_to_list(RunTime)++" ms.", + case LLRes of + ok -> + {comment, RunTimeStr}; + {comment, Comment} -> + {comment, Comment ++ " " ++ RunTimeStr} + end. + +hammer_rwlock(Config) when is_list(Config) -> + hammer_rw_test(false). + +hammer_rwlock_check(Config) when is_list(Config) -> + hammer_rw_test(true). + +hammer_tryrwlock(Config) when is_list(Config) -> + hammer_tryrw_test(false). + +hammer_tryrwlock_check(Config) when is_list(Config) -> + hammer_tryrw_test(true). + +hammer_sched_rwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, false, true, 0, 0). + +hammer_sched_rwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, true, true, 0, 0). + +hammer_sched_freqread_rwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, false, true, 0, 0). + +hammer_sched_freqread_rwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, true, true, 0, 0). + +hammer_sched_tryrwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, false, false, 0, 100). + +hammer_sched_tryrwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, true, false, 0, 100). + +hammer_sched_freqread_tryrwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, false, false, 0, 100). + +hammer_sched_freqread_tryrwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, true, false, 0, 100). + +hammer_sched_long_rwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, false, true, 100, 0). + +hammer_sched_long_rwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, true, true, 100, 0). + +hammer_sched_long_freqread_rwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, false, true, 100, 0). + +hammer_sched_long_freqread_rwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, true, true, 100, 0). + +hammer_sched_long_tryrwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, false, false, 100, 100). + +hammer_sched_long_tryrwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(false, true, false, 100, 100). + +hammer_sched_long_freqread_tryrwlock(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, false, false, 100, 100). + +hammer_sched_long_freqread_tryrwlock_check(Config) when is_list(Config) -> + hammer_sched_rwlock_test(true, true, false, 100, 100). + +hammer_sched_rwlock_test(FreqRead, LockCheck, Blocking, WaitLocked, WaitUnlocked) -> + case create_rwlock(FreqRead, LockCheck) of + enotsup -> + {skipped, "Not supported."}; + RWLock -> + Onln = erlang:system_info(schedulers_online), + NWPs = case Onln div 3 of + 1 -> case Onln < 4 of + true -> 1; + false -> 2 + end; + X -> X + end, + NRPs = Onln - NWPs, + NoLockOps = ((((50000000 div Onln) + div case {Blocking, WaitLocked} of + {false, 0} -> 1; + _ -> 10 + end) + div (case WaitLocked == 0 of + true -> 1; + false -> WaitLocked*250 + end)) + div handicap()), + ?t:format("NoLockOps=~p~n", [NoLockOps]), + Sleep = case Blocking of + true -> NoLockOps; + false -> NoLockOps div 10 + end, + WPs = lists:map( + fun (Sched) -> + spawn_opt( + fun () -> + io:format("Writer on scheduler ~p.~n", + [Sched]), + Sched = erlang:system_info(scheduler_id), + receive go -> gone end, + hammer_sched_rwlock_proc(RWLock, + Blocking, + true, + WaitLocked, + WaitUnlocked, + NoLockOps, + Sleep), + Sched = erlang:system_info(scheduler_id) + end, + [link, {scheduler, Sched}]) + end, + lists:seq(1, NWPs)), + RPs = lists:map( + fun (Sched) -> + spawn_opt( + fun () -> + io:format("Reader on scheduler ~p.~n", + [Sched]), + Sched = erlang:system_info(scheduler_id), + receive go -> gone end, + hammer_sched_rwlock_proc(RWLock, + Blocking, + false, + WaitLocked, + WaitUnlocked, + NoLockOps, + Sleep), + Sched = erlang:system_info(scheduler_id) + end, + [link, {scheduler, Sched}]) + end, + lists:seq(NWPs + 1, NWPs + NRPs)), + Procs = WPs ++ RPs, + case {Blocking, WaitLocked} of + {_, 0} -> ok; + {false, _} -> ok; + _ -> statistics(runtime) + end, + lists:foreach(fun (P) -> P ! go end, Procs), + lists:foreach(fun (P) -> + M = erlang:monitor(process, P), + receive + {'DOWN', M, process, P, _} -> + ok + end + end, + Procs), + case {Blocking, WaitLocked} of + {_, 0} -> ok; + {false, _} -> ok; + _ -> + {_, RunTime} = statistics(runtime), + ?t:format("RunTime=~p~n", [RunTime]), + ?line true = RunTime < 500, + {comment, + "Run-time during test was " + ++ integer_to_list(RunTime) + ++ " ms."} + end + end. + +hammer_sched_rwlock_proc(_RWLock, + _Blocking, + _WriteOp, + _WaitLocked, + _WaitUnlocked, + 0, + _Sleep) -> + ok; +hammer_sched_rwlock_proc(RWLock, + Blocking, + WriteOp, + WaitLocked, + WaitUnlocked, + Times, + Sleep) when Times rem Sleep == 0 -> + rwlock_op(RWLock, Blocking, WriteOp, WaitLocked, WaitUnlocked), + hammer_sched_rwlock_proc(RWLock, + Blocking, + WriteOp, + WaitLocked, + WaitUnlocked, + Times - 1, + Sleep); +hammer_sched_rwlock_proc(RWLock, + Blocking, + WriteOp, + WaitLocked, + WaitUnlocked, + Times, + Sleep) -> + rwlock_op(RWLock, Blocking, WriteOp, WaitLocked, 0), + hammer_sched_rwlock_proc(RWLock, + Blocking, + WriteOp, + WaitLocked, + WaitUnlocked, + Times - 1, + Sleep). + +-define(HAMMER_ETS_RWLOCK_REPEAT_TIMES, 1). +-define(HAMMER_ETS_RWLOCK_TSIZE, 500). + +hammer_ets_rwlock(Config) when is_list(Config) -> + {Ops, Procs} = case handicap() of + 1 -> {20000, 500}; + 2 -> {20000, 50}; + 3 -> {2000, 50}; + _ -> {200, 50} + end, + ?t:format("Procs=~p~nOps=~p~n", [Procs, Ops]), + lists:foreach(fun (XOpts) -> + ?t:format("Running with extra opts: ~p", [XOpts]), + hammer_ets_rwlock_test(XOpts, true, 2, Ops, + Procs, false) + end, + [[], + [{read_concurrency, true}], + [{write_concurrency, true}], + [{read_concurrency, true},{write_concurrency, true}]]), + ok. + +%% Aux funcs + +long_rw_test() -> + exit(no_nif_implementation). + +hammer_rw_test(_Arg) -> + exit(no_nif_implementation). + +hammer_tryrw_test(_Arg) -> + exit(no_nif_implementation). + +create_rwlock(_FreqRead, _LockCheck) -> + exit(no_nif_implementation). + +rwlock_op(_RWLock, _Blocking, _WriteOp, _WaitLocked, _WaitUnlocked) -> + exit(no_nif_implementation). + +hammer_ets_rwlock_put_data() -> + put(?MODULE, {"here are some", data, "to store", make_ref()}). + +hammer_ets_rwlock_get_data() -> + get(?MODULE). + +hammer_ets_rwlock_ops(_T, _UW, _N, _C, _SC, 0) -> + ok; +hammer_ets_rwlock_ops(T, UW, N, C, SC, Tot) when N >= ?HAMMER_ETS_RWLOCK_TSIZE -> + hammer_ets_rwlock_ops(T, UW, 0, C, SC, Tot); +hammer_ets_rwlock_ops(T, UW, N, 0, SC, Tot) -> + case UW of + true -> + true = ets:insert(T, {N, Tot, hammer_ets_rwlock_get_data()}); + false -> + [{N, _, _}] = ets:lookup(T, N) + end, + hammer_ets_rwlock_ops(T, UW, N+1, SC, SC, Tot-1); +hammer_ets_rwlock_ops(T, UW, N, C, SC, Tot) -> + case UW of + false -> + true = ets:insert(T, {N, Tot, hammer_ets_rwlock_get_data()}); + true -> + [{N, _, _}] = ets:lookup(T, N) + end, + hammer_ets_rwlock_ops(T, UW, N+1, C-1, SC, Tot-1). + +hammer_ets_rwlock_init(T, N) when N < ?HAMMER_ETS_RWLOCK_TSIZE -> + ets:insert(T, {N, N, N}), + hammer_ets_rwlock_init(T, N+1); +hammer_ets_rwlock_init(_T, _N) -> + ok. + +hammer_ets_rwlock_test(XOpts, UW, C, N, NP, SC) -> + receive after 100 -> ok end, + {TP, TM} = spawn_monitor( + fun () -> + _L = repeat_list( + fun () -> + Caller = self(), + T = fun () -> + Parent = self(), + hammer_ets_rwlock_put_data(), + T=ets:new(x, [public | XOpts]), + hammer_ets_rwlock_init(T, 0), + Ps0 = repeat_list( + fun () -> + spawn_link( + fun () -> + hammer_ets_rwlock_put_data(), + receive go -> ok end, + hammer_ets_rwlock_ops(T, UW, N, C, C, N), + Parent ! {done, self()}, + receive after infinity -> ok end + end) + end, + NP - case SC of + false -> 0; + _ -> 1 + end), + Ps = case SC of + false -> Ps0; + _ -> [spawn_link(fun () -> + hammer_ets_rwlock_put_data(), + receive go -> ok end, + hammer_ets_rwlock_ops(T, UW, N, SC, SC, N), + Parent ! {done, self()}, + receive after infinity -> ok end + end) | Ps0] + end, + Start = now(), + lists:foreach(fun (P) -> P ! go end, Ps), + lists:foreach(fun (P) -> receive {done, P} -> ok end end, Ps), + Stop = now(), + lists:foreach(fun (P) -> + unlink(P), + exit(P, bang), + M = erlang:monitor(process, P), + receive + {'DOWN', M, process, P, _} -> ok + end + end, Ps), + Res = timer:now_diff(Stop, Start)/1000000, + Caller ! {?MODULE, self(), Res} + end, + TP = spawn_link(T), + receive + {?MODULE, TP, Res} -> + Res + end + end, + ?HAMMER_ETS_RWLOCK_REPEAT_TIMES) + end), + receive + {'DOWN', TM, process, TP, _} -> ok + end. + +repeat_list(Fun, N) -> + repeat_list(Fun, N, []). + +repeat_list(_Fun, 0, Acc) -> + Acc; +repeat_list(Fun, N, Acc) -> + repeat_list(Fun, N-1, [Fun()|Acc]). + + +handicap() -> + X0 = case catch (erlang:system_info(logical_processors_available) >= + erlang:system_info(schedulers_online)) of + true -> 1; + _ -> 2 + end, + case erlang:system_info(build_type) of + opt -> + X0; + ReallySlow when ReallySlow == debug; + ReallySlow == valgrind; + ReallySlow == purify -> + X0*3; + _Slow -> + X0*2 + end. + diff --git a/erts/emulator/test/mtx_SUITE_data/Makefile.src b/erts/emulator/test/mtx_SUITE_data/Makefile.src new file mode 100644 index 0000000000..b6c843269c --- /dev/null +++ b/erts/emulator/test/mtx_SUITE_data/Makefile.src @@ -0,0 +1,30 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2010. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +include @erts_lib_include_internal_generated@@[email protected] +include @erts_lib_include_internal_generated@@DS@erts_internal.mk + +NIF_LIBS = mtx_SUITE@dll@ + +SHLIB_EXTRA_CFLAGS = $(ETHR_DEFS) -I@erts_lib_include_internal@ -I@erts_lib_include_internal_generated@ +LIBS = @ERTS_LIBS@ + +all: $(NIF_LIBS) + +@SHLIB_RULES@ diff --git a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c new file mode 100644 index 0000000000..818023211c --- /dev/null +++ b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c @@ -0,0 +1,692 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Stress tests of rwmutex implementation. + * + * Author: Rickard Green + */ + +#include "erl_nif.h" + +#ifdef __WIN32__ +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include <windows.h> +#else +# include "ethread.h" +# include "erl_misc_utils.h" +# include <unistd.h> +#endif + +#include <errno.h> +#include <stdio.h> + +static int +fail(const char *file, int line, const char *function, const char *assertion); + +#undef ASSERT +#define ASSERT(X) ((void) ((X) ? 1 : fail(__FILE__, __LINE__, __func__, #X))) + +#ifdef __WIN32__ +/* + * We cannot access the ethread symbols directly; test + * what we got in the nif api instead... + */ +#define HAVE_FREQREAD_SUPPORT 0 +#define RWMUTEX_T ErlNifRWLock +#define RWMUTEX_CREATE(FR) enif_rwlock_create("dummy") +#define RWMUTEX_DESTROY enif_rwlock_destroy +#define RWMUTEX_WLOCK enif_rwlock_rwlock +#define RWMUTEX_TRYWLOCK enif_rwlock_tryrwlock +#define RWMUTEX_WUNLOCK enif_rwlock_rwunlock +#define RWMUTEX_TRYRLOCK enif_rwlock_tryrlock +#define RWMUTEX_RLOCK enif_rwlock_rlock +#define RWMUTEX_RUNLOCK enif_rwlock_runlock +#define THR_ID ErlNifTid +#define THR_CREATE(A, B, C, D) enif_thread_create("dummy", (A), (B), (C), (D)) +#define THR_JOIN enif_thread_join +#define ATOMIC_T volatile LONG +#define ATOMIC_INIT(VarP, Val) (*(VarP) = (Val)) +#define ATOMIC_SET(VarP, Val) (*(VarP) = (Val)) +#define ATOMIC_READ(VarP) (*(VarP)) +#define ATOMIC_INC InterlockedIncrement +#define ATOMIC_DEC InterlockedDecrement + +#else + +#ifdef ETHR_USE_OWN_RWMTX_IMPL__ +# define HAVE_FREQREAD_SUPPORT 1 +#else +# define HAVE_FREQREAD_SUPPORT 0 +#endif + +#define RWMUTEX_T ethr_rwmutex +static ethr_rwmutex * +RWMUTEX_CREATE(int freqread) +{ + ethr_rwmutex *rwmtx = enif_alloc(sizeof(ethr_rwmutex)); + ethr_rwmutex_opt rwmtx_opt = ETHR_RWMUTEX_OPT_DEFAULT_INITER; + if (freqread) + rwmtx_opt.type = ETHR_RWMUTEX_TYPE_FREQUENT_READ; + ASSERT(rwmtx); + ASSERT(ethr_rwmutex_init_opt(rwmtx, &rwmtx_opt) == 0); + return rwmtx; +} +static void +RWMUTEX_DESTROY(ethr_rwmutex *rwmtx) +{ + ASSERT(ethr_rwmutex_destroy(rwmtx) == 0); + enif_free(rwmtx); +} +#define RWMUTEX_TRYWLOCK ethr_rwmutex_tryrwlock +#define RWMUTEX_WLOCK ethr_rwmutex_rwlock +#define RWMUTEX_WUNLOCK ethr_rwmutex_rwunlock +#define RWMUTEX_TRYRLOCK ethr_rwmutex_tryrlock +#define RWMUTEX_RLOCK ethr_rwmutex_rlock +#define RWMUTEX_RUNLOCK ethr_rwmutex_runlock +#define THR_ID ethr_tid +#define THR_CREATE ethr_thr_create +#define THR_JOIN ethr_thr_join +#define ATOMIC_T ethr_atomic_t +#define ATOMIC_INIT ethr_atomic_init +#define ATOMIC_SET ethr_atomic_set +#define ATOMIC_READ ethr_atomic_read +#define ATOMIC_INC ethr_atomic_inc +#define ATOMIC_DEC ethr_atomic_dec + +#endif + + +#if !defined(__func__) +# if !defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L +# if !defined(__GNUC__) || __GNUC__ < 2 +# define __func__ "[unknown_function]" +# else +# define __func__ __FUNCTION__ +# endif +# endif +#endif + +static void milli_sleep(int ms); +static int get_bool(ErlNifEnv* env, ERL_NIF_TERM term); + +/* + * Long rwlock testcase + */ + +#define LONG_RW_NO_W_THREADS 6 +#define LONG_RW_NO_THREADS 20 +#define LONG_RW_NO_WLOCK_COUNT 100 + +typedef struct { + RWMUTEX_T *rwlock; + ATOMIC_T *is_wlocked; + ATOMIC_T *is_rlocked; + int *stop; + int *count; + int sleep; +} long_rw_t; + +static void * +long_rw_w(void *varg) +{ + long_rw_t *arg = varg; + int stop = 0; + do { + RWMUTEX_WLOCK(arg->rwlock); + ASSERT(!ATOMIC_READ(arg->is_wlocked)); + ATOMIC_SET(arg->is_wlocked, 1); + ASSERT(!ATOMIC_READ(arg->is_rlocked)); + milli_sleep(arg->sleep); + if (++(*arg->count) > LONG_RW_NO_WLOCK_COUNT) + stop = *arg->stop = 1; + ATOMIC_SET(arg->is_wlocked, 0); + ASSERT(!ATOMIC_READ(arg->is_rlocked)); + RWMUTEX_WUNLOCK(arg->rwlock); + } while (!stop); + return NULL; +} + +static void * +long_rw_r(void *varg) +{ + long_rw_t *arg = varg; + int stop; + do { + RWMUTEX_RLOCK(arg->rwlock); + ASSERT(!ATOMIC_READ(arg->is_wlocked)); + ATOMIC_INC(arg->is_rlocked); + milli_sleep(arg->sleep); + stop = *arg->stop; + ATOMIC_DEC(arg->is_rlocked); + ASSERT(!ATOMIC_READ(arg->is_wlocked)); + RWMUTEX_RUNLOCK(arg->rwlock); + } while (!stop); + return NULL; +} + + +static ERL_NIF_TERM long_rw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + int res, freqread, i, count, stop; + ATOMIC_T is_wlocked, is_rlocked; + THR_ID tid[LONG_RW_NO_THREADS]; + long_rw_t arg; + long_rw_t targ[LONG_RW_NO_THREADS]; + + ATOMIC_INIT(&is_wlocked, 0); + ATOMIC_INIT(&is_rlocked, 0); + + freqread = 0; + + arg.is_wlocked = &is_wlocked; + arg.is_rlocked = &is_rlocked; + arg.count = &count; + arg.stop = &stop; + + restart: + + stop = 0; + count = 0; + + arg.rwlock = RWMUTEX_CREATE(freqread); + + ASSERT(arg.rwlock); + + for (i = 0; i < LONG_RW_NO_W_THREADS; i++) { + targ[i] = arg; + targ[i].sleep = 100 + i*10; + ASSERT(THR_CREATE(&tid[i], long_rw_w, &targ[i], NULL) == 0); + } + for (; i < LONG_RW_NO_THREADS; i++) { + targ[i] = arg; + targ[i].sleep = 100; + ASSERT(THR_CREATE(&tid[i], long_rw_r, &targ[i], NULL) == 0); + } + for (i = 0; i < LONG_RW_NO_THREADS; i++) + ASSERT(THR_JOIN(tid[i], NULL) == 0); + + ASSERT(!ATOMIC_READ(arg.is_wlocked)); + ASSERT(!ATOMIC_READ(arg.is_rlocked)); + + RWMUTEX_DESTROY(arg.rwlock); + + if (HAVE_FREQREAD_SUPPORT && !freqread) { + freqread = 1; + goto restart; + } + + if (freqread) + return enif_make_atom(env, "ok"); + else + return enif_make_tuple2(env, + enif_make_atom(env, + "comment"), + enif_make_string(env, + "No frequent read test made.", + ERL_NIF_LATIN1)); +} + +/* + * Hammer rwlock testcase + */ + +#define HAMMER_RW_NO_W_THREADS 6 +#define HAMMER_RW_NO_THREADS 20 +#define HAMMER_RW_NO_WLOCK_COUNT 1000000 + +typedef struct { + RWMUTEX_T *rwlock; + ATOMIC_T is_locked; + int lock_check; + int stop; + int count; +} hammer_rw_t; + +static void * +hammer_rw_w(void *varg) +{ + hammer_rw_t *arg = varg; + int stop = 0; + do { + RWMUTEX_WLOCK(arg->rwlock); + if (arg->lock_check) { + ASSERT(!ATOMIC_READ(&arg->is_locked)); + ATOMIC_SET(&arg->is_locked, -1); + } + if (++arg->count > HAMMER_RW_NO_WLOCK_COUNT) + stop = arg->stop = 1; + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) == -1); + ATOMIC_SET(&arg->is_locked, 0); + } + RWMUTEX_WUNLOCK(arg->rwlock); + } while (!stop); + return NULL; +} + +static void * +hammer_rw_r(void *varg) +{ + hammer_rw_t *arg = varg; + int stop; + do { + RWMUTEX_RLOCK(arg->rwlock); + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) >= 0); + ATOMIC_INC(&arg->is_locked); + } + stop = arg->stop; + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) > 0); + ATOMIC_DEC(&arg->is_locked); + } + RWMUTEX_RUNLOCK(arg->rwlock); + } while (!stop); + return NULL; +} + + +static ERL_NIF_TERM hammer_rw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + hammer_rw_t arg; + char buf[10]; + int res, freqread, i; + THR_ID tid[HAMMER_RW_NO_THREADS]; + + if (argc != 1) + goto badarg; + + arg.lock_check = get_bool(env, argv[0]); + if (arg.lock_check < 0) + goto badarg; + + ATOMIC_INIT(&arg.is_locked, 0); + + freqread = 0; + + restart: + arg.stop = 0; + arg.count = 0; + + arg.rwlock = RWMUTEX_CREATE(freqread); + + ASSERT(arg.rwlock); + + for (i = 0; i < HAMMER_RW_NO_W_THREADS; i++) + ASSERT(THR_CREATE(&tid[i], hammer_rw_w, &arg, NULL) == 0); + for (; i < HAMMER_RW_NO_THREADS; i++) + ASSERT(THR_CREATE(&tid[i], hammer_rw_r, &arg, NULL) == 0); + for (i = 0; i < HAMMER_RW_NO_THREADS; i++) + ASSERT(THR_JOIN(tid[i], NULL) == 0); + + ASSERT(!ATOMIC_READ(&arg.is_locked)); + + RWMUTEX_DESTROY(arg.rwlock); + + if (HAVE_FREQREAD_SUPPORT && !freqread) { + freqread = 1; + goto restart; + } + + if (freqread) + return enif_make_atom(env, "ok"); + else + return enif_make_tuple2(env, + enif_make_atom(env, + "comment"), + enif_make_string(env, + "No frequent read test made.", + ERL_NIF_LATIN1)); + badarg: + return enif_make_badarg(env); +} + +/* + * Hammer try rwlock testcase + */ + +#define HAMMER_TRYRW_NO_W_THREADS 10 +#define HAMMER_TRYRW_NO_THREADS 20 +#define HAMMER_TRYRW_NO_WLOCK_COUNT 10000000 +#define HAMMER_TRYRW_NO_RLOCK_COUNT 10000000 +#define HAMMER_TRYRW_NO_WLOCK_WAIT_COUNT ((10*HAMMER_TRYRW_NO_WLOCK_COUNT)/8) +#define HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT ((10*HAMMER_TRYRW_NO_RLOCK_COUNT)/8) + +typedef struct { + RWMUTEX_T *rwlock; + ATOMIC_T is_locked; + int lock_check; + int w_count; + ATOMIC_T r_count; +} hammer_tryrw_t; + +static void * +hammer_tryrw_w(void *varg) +{ + hammer_tryrw_t *arg = varg; + int stop = 0; + int wait = 0; + do { + while (EBUSY == RWMUTEX_TRYWLOCK(arg->rwlock)); + if (arg->lock_check) { + ASSERT(!ATOMIC_READ(&arg->is_locked)); + ATOMIC_SET(&arg->is_locked, -1); + } + if (++arg->w_count > HAMMER_TRYRW_NO_WLOCK_COUNT) + stop = 1; + else if (arg->w_count > HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT) + wait = 1; + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) == -1); + ATOMIC_SET(&arg->is_locked, 0); + } + RWMUTEX_WUNLOCK(arg->rwlock); + if (wait) + milli_sleep(1); + } while (!stop); + return NULL; +} + +static void * +hammer_tryrw_r(void *varg) +{ + hammer_tryrw_t *arg = varg; + long r_count; + int stop = 0; + int wait = 0; + do { + while (EBUSY == RWMUTEX_TRYRLOCK(arg->rwlock)); + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) >= 0); + ATOMIC_INC(&arg->is_locked); + } + ATOMIC_INC(&arg->r_count); + r_count = ATOMIC_READ(&arg->r_count); + if (r_count > HAMMER_TRYRW_NO_RLOCK_COUNT) + stop = 1; + else if (r_count > HAMMER_TRYRW_NO_RLOCK_WAIT_COUNT) + wait = 1; + if (arg->lock_check) { + ASSERT(ATOMIC_READ(&arg->is_locked) > 0); + ATOMIC_DEC(&arg->is_locked); + } + RWMUTEX_RUNLOCK(arg->rwlock); + if (wait) + milli_sleep(1); + } while (!stop); + return NULL; +} + + +static ERL_NIF_TERM hammer_tryrw_test(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + hammer_tryrw_t arg; + char buf[10]; + int res, freqread, i; + THR_ID tid[HAMMER_TRYRW_NO_THREADS]; + + if (argc != 1) + goto badarg; + + arg.lock_check = get_bool(env, argv[0]); + if (arg.lock_check < 0) + goto badarg; + + ATOMIC_INIT(&arg.is_locked, 0); + freqread = 0; + + restart: + + arg.w_count = 0; + ATOMIC_INIT(&arg.r_count, 0); + + arg.rwlock = RWMUTEX_CREATE(freqread); + + ASSERT(arg.rwlock); + + for (i = 0; i < HAMMER_TRYRW_NO_W_THREADS; i++) + ASSERT(THR_CREATE(&tid[i], hammer_tryrw_w, &arg, NULL) == 0); + for (; i < HAMMER_TRYRW_NO_THREADS; i++) + ASSERT(THR_CREATE(&tid[i], hammer_tryrw_r, &arg, NULL) == 0); + for (i = 0; i < HAMMER_TRYRW_NO_THREADS; i++) + ASSERT(THR_JOIN(tid[i], NULL) == 0); + + ASSERT(!ATOMIC_READ(&arg.is_locked)); + + RWMUTEX_DESTROY(arg.rwlock); + + if (HAVE_FREQREAD_SUPPORT && !freqread) { + freqread = 1; + goto restart; + } + + if (freqread) + return enif_make_atom(env, "ok"); + else + return enif_make_tuple2(env, + enif_make_atom(env, + "comment"), + enif_make_string(env, + "No frequent read test made.", + ERL_NIF_LATIN1)); + badarg: + return enif_make_badarg(env); +} + +typedef struct { + int lock_check; + ATOMIC_T is_locked; + RWMUTEX_T *rwlock; +} rwlock_resource_t; + +static void +rwlock_destructor(ErlNifEnv* env, void* obj) +{ + rwlock_resource_t *rwlr = obj; + if (rwlr->lock_check) + ASSERT(!ATOMIC_READ(&rwlr->is_locked)); + RWMUTEX_DESTROY(rwlr->rwlock); +} + +/* + * create_rwlock(FreqRead, LockCheck) + */ + +static ERL_NIF_TERM +create_rwlock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + int lock_check, freqread; + ERL_NIF_TERM rwlock_term; + rwlock_resource_t *rwlr; + char buf[100]; + + if (argc != 2) + goto badarg; + + freqread = get_bool(env, argv[0]); + if (freqread < 0) + goto badarg; + + if (!HAVE_FREQREAD_SUPPORT && freqread) + return enif_make_atom(env, "enotsup"); + + lock_check = get_bool(env, argv[1]); + if (lock_check < 0) + goto badarg; + + rwlr = enif_alloc_resource(enif_priv_data(env), sizeof(rwlock_resource_t)); + rwlr->lock_check = lock_check; + ATOMIC_INIT(&rwlr->is_locked, 0); + rwlr->rwlock = RWMUTEX_CREATE(freqread); + rwlock_term = enif_make_resource(env, rwlr); + enif_release_resource(rwlr); + return rwlock_term; + + badarg: + return enif_make_badarg(env); +} + +/* + * rwlock_op(RWLock, Blocking, WriteOp, WaitTime) + */ + +static ERL_NIF_TERM +rwlock_op(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + rwlock_resource_t *rwlr; + int blocking, write, wait_locked, wait_unlocked; + + if (argc != 5) + goto badarg; + + if (!enif_get_resource(env, argv[0], enif_priv_data(env), (void **) &rwlr)) + goto badarg; + + blocking = get_bool(env, argv[1]); + if (blocking < 0) + goto badarg; + + write = get_bool(env, argv[2]); + if (write < 0) + goto badarg; + + if (!enif_get_int(env, argv[3], &wait_locked)) + goto badarg; + if (wait_locked < 0) + goto badarg; + + if (!enif_get_int(env, argv[4], &wait_unlocked)) + goto badarg; + if (wait_unlocked < 0) + goto badarg; + + if (write) { + if (blocking) + RWMUTEX_WLOCK(rwlr->rwlock); + else + while (EBUSY == RWMUTEX_TRYWLOCK(rwlr->rwlock)); + if (rwlr->lock_check) { + ASSERT(!ATOMIC_READ(&rwlr->is_locked)); + ATOMIC_SET(&rwlr->is_locked, -1); + } + } + else { + if (blocking) + RWMUTEX_RLOCK(rwlr->rwlock); + else + while (EBUSY == RWMUTEX_TRYRLOCK(rwlr->rwlock)); + if (rwlr->lock_check) { + ASSERT(ATOMIC_READ(&rwlr->is_locked) >= 0); + ATOMIC_INC(&rwlr->is_locked); + } + } + + if (wait_locked) + milli_sleep(wait_locked); + + if (write) { + if (rwlr->lock_check) { + ASSERT(ATOMIC_READ(&rwlr->is_locked) == -1); + ATOMIC_SET(&rwlr->is_locked, 0); + } + RWMUTEX_WUNLOCK(rwlr->rwlock); + } + else { + if (rwlr->lock_check) { + ASSERT(ATOMIC_READ(&rwlr->is_locked) > 0); + ATOMIC_DEC(&rwlr->is_locked); + } + RWMUTEX_RUNLOCK(rwlr->rwlock); + } + + if (wait_unlocked) + milli_sleep(wait_unlocked); + + return enif_make_atom(env, "ok"); + badarg: + return enif_make_badarg(env); +} + +static int load_nif_lib(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + *priv_data = enif_open_resource_type(env, + NULL, + "rwlock_resource", + rwlock_destructor, + ERL_NIF_RT_CREATE, + NULL); + if (*priv_data) + return 0; + else + return -1; +} + +/* + * 0 -> false + * >0 -> true + * <0 -> error + */ + +static int +get_bool(ErlNifEnv* env, ERL_NIF_TERM term) +{ + int res; + char buf[10]; + + res = enif_get_atom(env, term, buf, sizeof(buf), ERL_NIF_LATIN1); + if (res == 0) + return -1; + if (strcmp("false", buf) == 0) + return 0; + else if (strcmp("true", buf) == 0) + return 1; + else + return -1; +} + +static int +fail(const char *file, int line, const char *function, const char *assertion) +{ + fprintf(stderr, "%s:%d: Assertion failed in %s(): %s\n", + file, line, function, assertion); + abort(); +} + +static void +milli_sleep(int ms) +{ +#ifdef __WIN32__ + Sleep(ms); +#else + while (erts_milli_sleep(ms) != 0); +#endif +} + +static ErlNifFunc nif_funcs[] = { + {"long_rw_test", 0, long_rw_test}, + {"hammer_rw_test", 1, hammer_rw_test}, + {"hammer_tryrw_test", 1, hammer_tryrw_test}, + {"create_rwlock", 2, create_rwlock}, + {"rwlock_op", 5, rwlock_op} +}; + +ERL_NIF_INIT(mtx_SUITE, nif_funcs, load_nif_lib, NULL, NULL, NULL) diff --git a/erts/emulator/test/nested_SUITE.erl b/erts/emulator/test/nested_SUITE.erl index 310892424e..2cd67ebaae 100644 --- a/erts/emulator/test/nested_SUITE.erl +++ b/erts/emulator/test/nested_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,11 +19,33 @@ -module(nested_SUITE). --export([all/1, case_in_case/1, case_in_after/1, catch_in_catch/1, bif_in_bif/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + case_in_case/1, case_in_after/1, catch_in_catch/1, bif_in_bif/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [case_in_case, case_in_after, catch_in_catch, + bif_in_bif]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> [case_in_case, case_in_after, catch_in_catch, bif_in_bif]. case_in_case(suite) -> []; case_in_case(Config) when is_list(Config) -> diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index f45cfa3e4a..b79c30d8d9 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009-2010. All Rights Reserved. +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -23,14 +23,18 @@ -define(CHECK(Exp,Got), check(Exp,Got,?LINE)). %%-define(CHECK(Exp,Got), ?line Exp = Got). --include("test_server.hrl"). - --export([all/1, - %%init_per_testcase/2, - fin_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1, - types/1, many_args/1, binaries/1, get_string/1, get_atom/1, api_macros/1, - from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, resource_takeover/1, - threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, is_checks/1, +-include_lib("test_server/include/test_server.hrl"). + +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, + end_per_testcase/2, basic/1, reload/1, upgrade/1, heap_frag/1, + types/1, many_args/1, binaries/1, get_string/1, get_atom/1, + api_macros/1, + from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, + resource_takeover/1, + threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, + is_checks/1, get_length/1, make_atom/1, make_string/1]). -export([many_args_100/100]). @@ -40,7 +44,7 @@ %% list_seq/1,type_test/0,tuple_2_list/1,is_identical/2,compare/2, %% clone_bin/1,make_sub_bin/3,string_to_bin/2,atom_to_bin/2,macros/1, %% tuple_2_list_and_tuple/1,iolist_2_bin/1,get_resource_type/1,alloc_resource/2, -%% make_resource/1,get_resource/2,release_resource/1,last_resource_dtor_call/0, +%% make_resource/1,get_resource/2,release_resource/1,last_resource_dtor_call/0, suite/0, %% make_new_resource/2,make_new_resource_binary/1,send_list_seq/2,send_new_blob/2, %% alloc_msgenv/0,clear_msgenv/1,grow_blob/2,send_blob/2,send_blob_thread/3, %% join_send_thread/1]). @@ -48,17 +52,37 @@ -define(nif_stub,nif_stub_error(?LINE)). -all(suite) -> - [basic, reload, upgrade, heap_frag, types, many_args, binaries, get_string, - get_atom, api_macros, from_array, iolist_as_binary, resource, resource_binary, - resource_takeover, threading, send, send2, send3, send_threaded, neg, is_checks, - get_length, make_atom, make_string]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [basic, reload, upgrade, heap_frag, types, many_args, + binaries, get_string, get_atom, api_macros, from_array, + iolist_as_binary, resource, resource_binary, + resource_takeover, threading, send, send2, send3, + send_threaded, neg, is_checks, get_length, make_atom, + make_string]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + -%%init_per_testcase(_Case, Config) -> -%% ?line Dog = ?t:timetrap(?t:seconds(60*60*24)), -%% [{watchdog, Dog}|Config]. +init_per_testcase(_Case, Config) -> +% ?line Dog = ?t:timetrap(?t:seconds(60*60*24)), + Config. -fin_per_testcase(_Func, _Config) -> +end_per_testcase(_Func, _Config) -> %%Dog = ?config(watchdog, Config), %%?t:timetrap_cancel(Dog), P1 = code:purge(nif_mod), @@ -73,7 +97,7 @@ basic(Config) when is_list(Config) -> ?line true = (lib_version() =/= undefined), ?line [{load,1,1,101},{lib_version,1,2,102}] = call_history(), ?line [] = call_history(), - ?line [?MODULE] = erlang:system_info(taints), + ?line true = lists:member(?MODULE, erlang:system_info(taints)), ok. reload(doc) -> ["Test reload callback in nif lib"]; @@ -107,7 +131,8 @@ reload(Config) when is_list(Config) -> ?line true = erlang:purge_module(nif_mod), ?line [{unload,1,3,103}] = nif_mod_call_history(), - ?line [?MODULE, nif_mod] = erlang:system_info(taints), + ?line true = lists:member(?MODULE, erlang:system_info(taints)), + ?line true = lists:member(nif_mod, erlang:system_info(taints)), ?line verify_tmpmem(TmpMem), ok. @@ -197,7 +222,8 @@ upgrade(Config) when is_list(Config) -> ?line true = erlang:purge_module(nif_mod), ?line [{unload,2,4,204}] = nif_mod_call_history(), - ?line [?MODULE, nif_mod] = erlang:system_info(taints), + ?line true = lists:member(?MODULE, erlang:system_info(taints)), + ?line true = lists:member(nif_mod, erlang:system_info(taints)), ?line verify_tmpmem(TmpMem), ok. @@ -727,7 +753,8 @@ resource_takeover(Config) when is_list(Config) -> ?line ok = forget_resource(AN4), ?line [] = nif_mod_call_history(), - ?line [?MODULE, nif_mod] = erlang:system_info(taints), + ?line true = lists:member(?MODULE, erlang:system_info(taints)), + ?line true = lists:member(nif_mod, erlang:system_info(taints)), ?line verify_tmpmem(TmpMem), ok. diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.erl b/erts/emulator/test/nif_SUITE_data/nif_mod.erl index 7888a589e7..6634624698 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_mod.erl +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2009. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,7 +19,7 @@ -module(nif_mod). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -export([load_nif_lib/2, load_nif_lib/3, start/0, lib_version/0, call_history/0, get_priv_data_ptr/0, make_new_resource/2, get_resource/2]). diff --git a/erts/emulator/test/nif_SUITE_data/tester.erl b/erts/emulator/test/nif_SUITE_data/tester.erl index 9df2158200..b393e29b82 100644 --- a/erts/emulator/test/nif_SUITE_data/tester.erl +++ b/erts/emulator/test/nif_SUITE_data/tester.erl @@ -1,6 +1,6 @@ -module(tester). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -export([load_nif_lib/2, run/0]). diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl index f3d9eb783b..aa83459ef8 100644 --- a/erts/emulator/test/node_container_SUITE.erl +++ b/erts/emulator/test/node_container_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,10 +29,12 @@ %-define(line_trace, 1). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %-compile(export_all). --export([all/1, init_per_testcase/2, fin_per_testcase/2, end_per_suite/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, init_per_testcase/2, + end_per_testcase/2, node_container_refc_check/1]). -export([term_to_binary_to_term_eq/1, @@ -55,25 +57,30 @@ -define(DEFAULT_TIMEOUT, ?t:minutes(10)). -all(doc) -> []; -all(suite) -> - [term_to_binary_to_term_eq, - round_trip_eq, - cmp, - ref_eq, - node_table_gc, - dist_link_refc, - dist_monitor_refc, - node_controller_refc, - ets_refc, - match_spec_refc, - timer_refc, - otp_4715, - pid_wrap, - port_wrap, - bad_nc, - unique_pid, - iter_max_procs]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [term_to_binary_to_term_eq, round_trip_eq, cmp, ref_eq, + node_table_gc, dist_link_refc, dist_monitor_refc, + node_controller_refc, ets_refc, match_spec_refc, + timer_refc, otp_4715, pid_wrap, port_wrap, bad_nc, + unique_pid, iter_max_procs]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + available_internal_state(false). + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + available_internal_state(Bool) when Bool == true; Bool == false -> case {Bool, @@ -95,14 +102,11 @@ init_per_testcase(_Case, Config) when is_list(Config) -> available_internal_state(true), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) when is_list(Config) -> +end_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. -end_per_suite(_Config) -> - available_internal_state(false). - %%% %%% The test cases ------------------------------------------------------------- %%% diff --git a/erts/emulator/test/nofrag_SUITE.erl b/erts/emulator/test/nofrag_SUITE.erl index ece55f433c..6b6ac28e2e 100644 --- a/erts/emulator/test/nofrag_SUITE.erl +++ b/erts/emulator/test/nofrag_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2009. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,9 +19,11 @@ -module(nofrag_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,end_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, error_handler/1,error_handler_apply/1, error_handler_fixed_apply/1,error_handler_fun/1, error_handler_tuple_fun/1, @@ -30,9 +32,28 @@ %% Exported functions for an error_handler module. -export([undefined_function/3,undefined_lambda/3,breakpoint/3]). -all(suite) -> - [error_handler,error_handler_apply,error_handler_fixed_apply, - error_handler_fun,error_handler_tuple_fun,debug_breakpoint]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [error_handler, error_handler_apply, + error_handler_fixed_apply, error_handler_fun, + error_handler_tuple_fun, debug_breakpoint]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(3)), diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl index d009994e2d..4459732257 100644 --- a/erts/emulator/test/num_bif_SUITE.erl +++ b/erts/emulator/test/num_bif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,7 +19,7 @@ -module(num_bif_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %% Tests the BIFs: %% abs/1 @@ -31,15 +31,36 @@ %% round/1 %% trunc/1 --export([all/1, t_abs/1, t_float/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, t_abs/1, t_float/1, t_float_to_list/1, t_integer_to_list/1, t_list_to_integer/1, - t_list_to_float/1, t_list_to_float_safe/1, t_list_to_float_risky/1, + t_list_to_float_safe/1, t_list_to_float_risky/1, t_round/1, t_trunc/1]). -all(suite) -> [t_abs, t_float, t_float_to_list, t_integer_to_list, - t_list_to_float, t_list_to_integer, - t_round, t_trunc]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [t_abs, t_float, t_float_to_list, t_integer_to_list, + {group, t_list_to_float}, t_list_to_integer, t_round, + t_trunc]. + +groups() -> + [{t_list_to_float, [], + [t_list_to_float_safe, t_list_to_float_risky]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + t_abs(Config) when is_list(Config) -> %% Floats. @@ -140,7 +161,6 @@ t_integer_to_list(Config) when is_list(Config) -> %% Tests list_to_float/1. -t_list_to_float(suite) -> [t_list_to_float_safe, t_list_to_float_risky]. t_list_to_float_safe(Config) when is_list(Config) -> ?line 0.0 = list_to_float(id("0.0")), diff --git a/erts/emulator/test/old_mod.erl b/erts/emulator/test/old_mod.erl index 6c47ba6f8f..124842390a 100644 --- a/erts/emulator/test/old_mod.erl +++ b/erts/emulator/test/old_mod.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2009. All Rights Reserved. +%% Copyright Ericsson AB 2003-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/test/old_scheduler_SUITE.erl b/erts/emulator/test/old_scheduler_SUITE.erl index 70348f64db..262536a068 100644 --- a/erts/emulator/test/old_scheduler_SUITE.erl +++ b/erts/emulator/test/old_scheduler_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2009. All Rights Reserved. +%% Copyright Ericsson AB 2004-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,24 +19,44 @@ -module(old_scheduler_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). -export([equal/1, many_low/1, few_low/1, max/1, high/1]). -define(default_timeout, ?t:minutes(11)). -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> case catch erlang:system_info(modified_timing_level) of Level when is_integer(Level) -> {skipped, - "Modified timing (level " ++ integer_to_list(Level) - ++ ") is enabled. Testcases gets messed up by modfied " - "timing."}; - _ -> - [equal, many_low, few_low, max, high] + "Modified timing (level " ++ + integer_to_list(Level) ++ + ") is enabled. Testcases gets messed " + "up by modfied timing."}; + _ -> [equal, many_low, few_low, max, high] end. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %%----------------------------------------------------------------------------------- %% TEST SUITE DESCRIPTION %% @@ -63,7 +83,7 @@ init_per_testcase(_Case, Config) -> ?line MS = erlang:system_flag(multi_scheduling, block), [{prio,Prio},{watchdog,Dog},{multi_scheduling, MS}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> erlang:system_flag(multi_scheduling, unblock), Dog=?config(watchdog, Config), Prio=?config(prio, Config), diff --git a/erts/emulator/test/op_SUITE.erl b/erts/emulator/test/op_SUITE.erl index 55d8d9ab0f..ef4689b850 100644 --- a/erts/emulator/test/op_SUITE.erl +++ b/erts/emulator/test/op_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,22 +19,43 @@ -module(op_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, bsl_bsr/1,logical/1,t_not/1,relop_simple/1,relop/1,complex_relop/1]). -export([]). -import(lists, [foldl/3,flatmap/2]). -all(suite) -> - [bsl_bsr,logical,t_not,relop_simple,relop,complex_relop]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [bsl_bsr, logical, t_not, relop_simple, relop, + complex_relop]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(Case, Config) when is_atom(Case), is_list(Config) -> Dog=?t:timetrap(?t:minutes(3)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index a7476ca9bb..eac56a867d 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -73,18 +73,19 @@ %% --export([all/1, init_per_testcase/2, fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2, init_per_suite/1, end_per_suite/1, - stream/1, stream_small/1, stream_big/1, + stream_small/1, stream_big/1, basic_ping/1, slow_writes/1, bad_packet/1, bad_port_messages/1, - multiple_packets/1, mul_basic/1, mul_slow_writes/1, + mul_basic/1, mul_slow_writes/1, dying_port/1, port_program_with_path/1, open_input_file_port/1, open_output_file_port/1, iter_max_ports/1, eof/1, input_only/1, output_only/1, name1/1, - t_binary/1, options/1, parallell/1, t_exit/1, + t_binary/1, parallell/1, t_exit/1, env/1, bad_env/1, cd/1, exit_status/1, - tps/1, tps_16_bytes/1, tps_1K/1, line/1, stderr_to_stdout/1, + tps_16_bytes/1, tps_1K/1, line/1, stderr_to_stdout/1, otp_3906/1, otp_4389/1, win_massive/1, win_massive_client/1, mix_up_ports/1, otp_5112/1, otp_5119/1, otp_6224/1, exit_status_multi_scheduling_block/1, ports/1, @@ -98,31 +99,42 @@ -export([otp_3906_forker/5, otp_3906_start_forker_starter/4]). -export([env_slave_main/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -include_lib("kernel/include/file.hrl"). -all(suite) -> - [ - otp_6224, stream, basic_ping, slow_writes, bad_packet, - bad_port_messages, options, multiple_packets, parallell, - dying_port, port_program_with_path, - open_input_file_port, open_output_file_port, - name1, - env, bad_env, cd, exit_status, - iter_max_ports, t_exit, tps, line, stderr_to_stdout, - otp_3906, otp_4389, win_massive, mix_up_ports, - otp_5112, otp_5119, - exit_status_multi_scheduling_block, - ports, spawn_driver, spawn_executable, close_deaf_port, - unregister_name - ]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [otp_6224, {group, stream}, basic_ping, slow_writes, + bad_packet, bad_port_messages, {group, options}, + {group, multiple_packets}, parallell, dying_port, + port_program_with_path, open_input_file_port, + open_output_file_port, name1, env, bad_env, cd, + exit_status, iter_max_ports, t_exit, {group, tps}, line, + stderr_to_stdout, otp_3906, otp_4389, win_massive, + mix_up_ports, otp_5112, otp_5119, + exit_status_multi_scheduling_block, ports, spawn_driver, + spawn_executable, close_deaf_port, unregister_name]. + +groups() -> + [{stream, [], [stream_small, stream_big]}, + {options, [], [t_binary, eof, input_only, output_only]}, + {multiple_packets, [], [mul_basic, mul_slow_writes]}, + {tps, [], [tps_16_bytes, tps_1K]}]. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + -define(DEFAULT_TIMEOUT, ?t:minutes(5)). init_per_testcase(Case, Config) -> [{testcase, Case} |Config]. -fin_per_testcase(_Case, _Config) -> +end_per_testcase(_Case, _Config) -> ok. init_per_suite(Config) when is_list(Config) -> @@ -191,7 +203,6 @@ win_massive_loop(P,N) -> -stream(suite) -> [stream_small, stream_big]. %% Test that we can send a stream of bytes and get it back. %% We will send only a small amount of data, to avoid deadlock. @@ -304,7 +315,6 @@ bad_message(PortTest, Message) -> %% Tests various options (stream and {packet, Number} are implicitly %% tested in other test cases). -options(suite) -> [t_binary, eof, input_only, output_only]. %% Tests the 'binary' option for a port. @@ -416,7 +426,6 @@ output_and_verify(Config, Filename, Options, Data) -> %% Test that receiving several packages written in the same %% write operation works. -multiple_packets(suite) -> [mul_basic, mul_slow_writes]. %% Basic test of receiving multiple packages, written in %% one operation by the other end. @@ -740,7 +749,6 @@ suicide_port(Config) when is_list(Config) -> ?line exit(Port, die), ?line receive after infinity -> ok end. -tps(suite) -> [tps_16_bytes, tps_1K]. tps_16_bytes(doc) -> ""; tps_16_bytes(suite) -> []; @@ -1049,8 +1057,10 @@ otp_3906(Config) when is_list(Config) -> -define(OTP_3906_MAX_CONC_OSP, 50). otp_3906(Config, OSName) -> - ?line TSDir = filename:dirname(code:which(test_server)), - ?line {ok, Variables} = file:consult(filename:join(TSDir, "variables")), + ?line DataDir = filename:dirname(proplists:get_value(data_dir,Config)), + ?line {ok, Variables} = file:consult( + filename:join([DataDir,"..","..", + "test_server","variables"])), case lists:keysearch('CC', 1, Variables) of {value,{'CC', CC}} -> SuiteDir = filename:dirname(code:which(?MODULE)), @@ -2302,14 +2312,35 @@ load_driver(Dir, Driver) -> end. -close_deaf_port(doc) -> ["Send data to port program that does not read it, then close port."]; +close_deaf_port(doc) -> ["Send data to port program that does not read it, then close port." + "Primary targeting Windows to test threaded_handle_closer in sys.c"]; close_deaf_port(suite) -> []; close_deaf_port(Config) when is_list(Config) -> ?line Dog = test_server:timetrap(test_server:seconds(100)), ?line DataDir = ?config(data_dir, Config), ?line DeadPort = os:find_executable("dead_port", DataDir), - ?line Port = open_port({spawn,DeadPort++" 60"},[]), ?line erlang:port_command(Port,"Hello, can you hear me!?!?"), ?line port_close(Port), - ok. + + Res = close_deaf_port_1(0, DeadPort), + io:format("Waiting for OS procs to terminate...\n"), + receive after 5*1000 -> ok end, + ?line test_server:timetrap_cancel(Dog), + Res. + +close_deaf_port_1(1000, _) -> + ok; +close_deaf_port_1(N, Cmd) -> + Timeout = integer_to_list(random:uniform(5*1000)), + ?line try open_port({spawn_executable,Cmd},[{args,[Timeout]}]) of + Port -> + ?line erlang:port_command(Port,"Hello, can you hear me!?!?"), + ?line port_close(Port), + close_deaf_port_1(N+1, Cmd) + catch + _:eagain -> + {comment, "Could not spawn more than " ++ integer_to_list(N) ++ " OS processes."} + end. + + diff --git a/erts/emulator/test/port_SUITE_data/dead_port.c b/erts/emulator/test/port_SUITE_data/dead_port.c index 6fa77112be..68e96fbf14 100644 --- a/erts/emulator/test/port_SUITE_data/dead_port.c +++ b/erts/emulator/test/port_SUITE_data/dead_port.c @@ -72,14 +72,14 @@ char *argv[]; { int x; if (argc < 2) { - fprintf(stderr,"Usage %s <seconds>\n",argv[0]); + fprintf(stderr,"Usage %s <milliseconds>\n",argv[0]); return 1; } if ((x = atoi(argv[1])) <= 0) { - fprintf(stderr,"Usage %s <seconds>\n",argv[0]); + fprintf(stderr,"Usage %s <milliseconds>\n",argv[0]); return 1; } - delay(x*1000); + delay(x); return 0; } diff --git a/erts/emulator/test/port_bif_SUITE.erl b/erts/emulator/test/port_bif_SUITE.erl index f4e0bb9fa8..d9c82aba0e 100644 --- a/erts/emulator/test/port_bif_SUITE.erl +++ b/erts/emulator/test/port_bif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,25 +20,47 @@ -module(port_bif_SUITE). --export([all/1, command/1, command_e/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, command/1, command_e_1/1, command_e_2/1, command_e_3/1, command_e_4/1, - port_info/1, port_info1/1, port_info2/1, + port_info1/1, port_info2/1, connect/1, control/1, echo_to_busy/1]). -export([do_command_e_1/1, do_command_e_2/1, do_command_e_4/1]). --export([init_per_testcase/2, fin_per_testcase/2]). +-export([init_per_testcase/2, end_per_testcase/2]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [command, {group, port_info}, connect, control, + echo_to_busy]. + +groups() -> + [{command_e, [], + [command_e_1, command_e_2, command_e_3, command_e_4]}, + {port_info, [], [port_info1, port_info2]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [command, port_info, connect, control, echo_to_busy]. init_per_testcase(_Func, Config) when is_list(Config) -> Dog=test_server:timetrap(test_server:minutes(10)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) when is_list(Config) -> +end_per_testcase(_Func, Config) when is_list(Config) -> Dog=?config(watchdog, Config), test_server:timetrap_cancel(Dog). @@ -69,11 +91,6 @@ do_command(P, Data) -> end. -command_e(suite) -> [command_e_1, - command_e_2, - command_e_3, - command_e_4]; -command_e(doc) -> "Tests port_command/2 with errors". %% port_command/2: badarg 1st arg command_e_1(Config) when is_list(Config) -> @@ -161,7 +178,6 @@ do_command_e_4(Program) -> ?line erlang:port_command(P, Data), exit(survived). -port_info(suite) -> [port_info1, port_info2]. %% Tests the port_info/1 BIF port_info1(Config) when is_list(Config) -> diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 77f850d0fb..a731f09e4c 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -25,12 +25,13 @@ %% process_info/1,2 %% register/2 (partially) --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -define(heap_binary_size, 64). --export([all/1, spawn_with_binaries/1, - t_exit_1/1, t_exit_2/1, t_exit_2_other/1, t_exit_2_other_normal/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, spawn_with_binaries/1, + t_exit_1/1, t_exit_2_other/1, t_exit_2_other_normal/1, self_exit/1, normal_suicide_exit/1, abnormal_suicide_exit/1, t_exit_2_catch/1, trap_exit_badarg/1, trap_exit_badarg_in_bif/1, exit_and_timeout/1, exit_twice/1, @@ -46,39 +47,67 @@ processes_large_tab/1, processes_default_tab/1, processes_small_tab/1, processes_this_tab/1, processes_apply_trap/1, processes_last_call_trap/1, processes_gc_trap/1, - processes_term_proc_list/1, processes_bif/1, - otp_7738/1, otp_7738_waiting/1, otp_7738_suspended/1, + processes_term_proc_list/1, + otp_7738_waiting/1, otp_7738_suspended/1, otp_7738_resume/1]). -export([prio_server/2, prio_client/2]). --export([init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]). +-export([init_per_testcase/2, end_per_testcase/2]). -export([hangaround/2, processes_bif_test/0, do_processes/1, processes_term_proc_list_test/1]). -all(suite) -> - [spawn_with_binaries, t_exit_1, t_exit_2, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [spawn_with_binaries, t_exit_1, {group, t_exit_2}, trap_exit_badarg, trap_exit_badarg_in_bif, - t_process_info, process_info_other_msg, process_info_other_dist_msg, - process_info_2_list, - process_info_lock_reschedule, process_info_lock_reschedule2, - process_status_exiting, - bump_reductions, low_prio, yield, yield2, otp_4725, bad_register, - garbage_collect, process_info_messages, process_flag_badarg, process_flag_heap_size, - spawn_opt_heap_size, otp_6237, processes_bif, otp_7738]. + t_process_info, process_info_other_msg, + process_info_other_dist_msg, process_info_2_list, + process_info_lock_reschedule, + process_info_lock_reschedule2, process_status_exiting, + bump_reductions, low_prio, yield, yield2, otp_4725, + bad_register, garbage_collect, process_info_messages, + process_flag_badarg, process_flag_heap_size, + spawn_opt_heap_size, otp_6237, {group, processes_bif}, + {group, otp_7738}]. + +groups() -> + [{t_exit_2, [], + [t_exit_2_other, t_exit_2_other_normal, self_exit, + normal_suicide_exit, abnormal_suicide_exit, + t_exit_2_catch, exit_and_timeout, exit_twice]}, + {processes_bif, [], + [processes_large_tab, processes_default_tab, + processes_small_tab, processes_this_tab, + processes_last_call_trap, processes_apply_trap, + processes_gc_trap, processes_term_proc_list]}, + {otp_7738, [], + [otp_7738_waiting, otp_7738_suspended, + otp_7738_resume]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + catch erts_debug:set_internal_state(available_internal_state, false), + Config. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(10)), [{watchdog, Dog},{testcase, Func}|Config]. -fin_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> +end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). -end_per_suite(Config) -> - catch erts_debug:set_internal_state(available_internal_state, false), - Config. - fun_spawn(Fun) -> spawn_link(erlang, apply, [Fun, []]). @@ -117,10 +146,6 @@ t_exit_1() -> {'EXIT', Pid, Garbage} -> ok end. -t_exit_2(suite) -> [t_exit_2_other, t_exit_2_other_normal, - self_exit, normal_suicide_exit, - abnormal_suicide_exit, t_exit_2_catch, - exit_and_timeout, exit_twice]. %% Tests exit/2 with a lot of data in the exit message. t_exit_2_other(Config) when is_list(Config) -> @@ -1227,17 +1252,6 @@ otp_6237_select_loop() -> otp_6237_select_loop(). -processes_bif(doc) -> - []; -processes_bif(suite) -> - [processes_large_tab, - processes_default_tab, - processes_small_tab, - processes_this_tab, - processes_last_call_trap, - processes_apply_trap, - processes_gc_trap, - processes_term_proc_list]. -define(NoTestProcs, 10000). -record(processes_bif_info, {min_start_reds, @@ -1965,10 +1979,6 @@ processes_term_proc_list_test(MustChk) -> ?line erlang:system_flag(multi_scheduling, unblock), ?line as_expected. -otp_7738(doc) -> - []; -otp_7738(suite) -> - [otp_7738_waiting, otp_7738_suspended, otp_7738_resume]. otp_7738_waiting(doc) -> []; diff --git a/erts/emulator/test/pseudoknot_SUITE.erl b/erts/emulator/test/pseudoknot_SUITE.erl index 907204cf93..5a7cdcecd5 100644 --- a/erts/emulator/test/pseudoknot_SUITE.erl +++ b/erts/emulator/test/pseudoknot_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2001-2009. All Rights Reserved. +%% Copyright Ericsson AB 2001-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,9 +19,29 @@ -module(pseudoknot_SUITE). --export([all/1,test/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2,test/1]). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [test]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> [test]. test(Config) when is_list(Config) -> statistics(runtime), @@ -3274,13 +3294,13 @@ most_distant_atom(Sols) -> maximum(map(sol_most_distant_atom, Sols)). maximum([H|T]) -> - max(T,H). + max1(T,H). -max([H|T],M) when is_float(H), is_float(M), H > M -> - max(T,H); -max([_|T],M) -> - max(T,M); -max([],M) -> M. +max1([H|T],M) when is_float(H), is_float(M), H > M -> + max1(T,H); +max1([_|T],M) -> + max1(T,M); +max1([],M) -> M. map(_Func,[]) -> []; map(Func,[H|T]) -> diff --git a/erts/emulator/test/random_iolist.erl b/erts/emulator/test/random_iolist.erl index 4bce347d9a..8f21b5a3b3 100644 --- a/erts/emulator/test/random_iolist.erl +++ b/erts/emulator/test/random_iolist.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2009. All Rights Reserved. +%% Copyright Ericsson AB 2008-2010. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl index 40ebf2bd21..b070e2b986 100644 --- a/erts/emulator/test/receive_SUITE.erl +++ b/erts/emulator/test/receive_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010. All Rights Reserved. +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,21 +21,40 @@ %% Tests receive after. --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, call_with_huge_message_queue/1,receive_in_between/1]). --export([init_per_testcase/2,fin_per_testcase/2]). +-export([init_per_testcase/2,end_per_testcase/2]). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [call_with_huge_message_queue, receive_in_between]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. -all(suite) -> - [call_with_huge_message_queue,receive_in_between]. init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(3)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). diff --git a/erts/emulator/test/ref_SUITE.erl b/erts/emulator/test/ref_SUITE.erl index fa77095efd..e13dfa1575 100644 --- a/erts/emulator/test/ref_SUITE.erl +++ b/erts/emulator/test/ref_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,23 +19,44 @@ -module(ref_SUITE). --export([all/1,init_per_testcase/2,fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2]). -export([wrap_1/1]). -export([loop_ref/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). init_per_testcase(_, Config) -> ?line Dog=test_server:timetrap(test_server:minutes(2)), [{watchdog, Dog}|Config]. -fin_per_testcase(_, Config) -> +end_per_testcase(_, Config) -> Dog=?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. -all(suite) -> [wrap_1]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [wrap_1]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + wrap_1(doc) -> "Check that refs don't wrap around easily."; wrap_1(Config) when is_list(Config) -> diff --git a/erts/emulator/test/register_SUITE.erl b/erts/emulator/test/register_SUITE.erl index c03ee23b2e..9953df3458 100644 --- a/erts/emulator/test/register_SUITE.erl +++ b/erts/emulator/test/register_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009. All Rights Reserved. +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,24 +22,43 @@ %-define(line_trace, 1). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %-compile(export_all). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). -export([otp_8099/1]). -define(DEFAULT_TIMEOUT, ?t:minutes(2)). -all(doc) -> []; -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [otp_8099]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + init_per_testcase(Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), [{watchdog, Dog}, {testcase, Case} | Config]. -fin_per_testcase(_Case, Config) when is_list(Config) -> +end_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. diff --git a/erts/emulator/test/save_calls_SUITE.erl b/erts/emulator/test/save_calls_SUITE.erl index b56c4ad0b0..390b49b604 100644 --- a/erts/emulator/test/save_calls_SUITE.erl +++ b/erts/emulator/test/save_calls_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,17 +19,36 @@ -module(save_calls_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). -export([save_calls_1/1,dont_break_reductions/1]). -export([do_bopp/1, do_bipp/0, do_bepp/0]). -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [save_calls_1, dont_break_reductions]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + dont_break_reductions(suite) -> []; dont_break_reductions(doc) -> diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index 06442bfad6..f16d0ea429 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -30,10 +30,12 @@ %-define(line_trace, 1). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %-compile(export_all). --export([all/1, init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2, end_per_suite/1]). -export([equal/1, few_low/1, @@ -44,7 +46,7 @@ equal_with_high/1, equal_with_high_max/1, bound_process/1, - scheduler_bind/1, + scheduler_bind_types/1, cpu_topology/1, update_cpu_info/1, @@ -57,21 +59,35 @@ -define(MIN_SCHEDULER_TEST_TIMEOUT, ?t:minutes(1)). -all(doc) -> []; -all(suite) -> - [equal, - few_low, - many_low, - equal_with_part_time_high, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [equal, few_low, many_low, equal_with_part_time_high, equal_with_part_time_max, - equal_and_high_with_part_time_max, - equal_with_high, - equal_with_high_max, - bound_process, - scheduler_bind, - scheduler_suspend, + equal_and_high_with_part_time_max, equal_with_high, + equal_with_high_max, bound_process, + {group, scheduler_bind}, scheduler_suspend, reader_groups]. +groups() -> + [{scheduler_bind, [], + [scheduler_bind_types, cpu_topology, update_cpu_info, + sct_cmd, sbt_cmd]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(Config) -> + catch erts_debug:set_internal_state(available_internal_state, false), + Config. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + init_per_testcase(Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), process_flag(priority, max), @@ -79,15 +95,11 @@ init_per_testcase(Case, Config) when is_list(Config) -> OkRes = ok, [{watchdog, Dog}, {testcase, Case}, {ok_res, OkRes} |Config]. -fin_per_testcase(_Case, Config) when is_list(Config) -> +end_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. -end_per_suite(Config) -> - catch erts_debug:set_internal_state(available_internal_state, false), - Config. - -define(ERTS_RUNQ_CHECK_BALANCE_REDS_PER_SCHED, (2000*2000)). -define(DEFAULT_TEST_REDS_PER_SCHED, 200000000). @@ -247,12 +259,6 @@ bound_loop(NS, N, M, Sched) -> Sched = erlang:system_info(scheduler_id), bound_loop(NS, N-1, M, Sched). -scheduler_bind(suite) -> - [scheduler_bind_types, - cpu_topology, - update_cpu_info, - sct_cmd, - sbt_cmd]. -define(TOPOLOGY_A_CMD, "+sct" @@ -856,9 +862,9 @@ get_affinity_mask(Port, Status, Affinity) when Status == unknown; {Port,{exit_status,S}} -> get_affinity_mask(Port, S, Affinity) end; -get_affinity_mask(Port, Status, bad) -> +get_affinity_mask(_Port, _Status, bad) -> unknown; -get_affinity_mask(Port, Status, Affinity) -> +get_affinity_mask(_Port, _Status, Affinity) -> Affinity. get_affinity_mask() -> @@ -1383,67 +1389,6 @@ reader_groups_map(CPUT, Groups) -> %% Utils %% -tilera_cpu_topology() -> - [{processor,[{node,[{core,{logical,0}}, - {core,{logical,1}}, - {core,{logical,2}}, - {core,{logical,8}}, - {core,{logical,9}}, - {core,{logical,10}}, - {core,{logical,11}}, - {core,{logical,16}}, - {core,{logical,17}}, - {core,{logical,18}}, - {core,{logical,19}}, - {core,{logical,24}}, - {core,{logical,25}}, - {core,{logical,27}}, - {core,{logical,29}}]}, - {node,[{core,{logical,3}}, - {core,{logical,4}}, - {core,{logical,5}}, - {core,{logical,6}}, - {core,{logical,7}}, - {core,{logical,12}}, - {core,{logical,13}}, - {core,{logical,14}}, - {core,{logical,15}}, - {core,{logical,20}}, - {core,{logical,21}}, - {core,{logical,22}}, - {core,{logical,23}}, - {core,{logical,28}}, - {core,{logical,30}}]}, - {node,[{core,{logical,31}}, - {core,{logical,36}}, - {core,{logical,37}}, - {core,{logical,38}}, - {core,{logical,44}}, - {core,{logical,45}}, - {core,{logical,46}}, - {core,{logical,47}}, - {core,{logical,51}}, - {core,{logical,52}}, - {core,{logical,53}}, - {core,{logical,54}}, - {core,{logical,55}}, - {core,{logical,60}}, - {core,{logical,61}}]}, - {node,[{core,{logical,26}}, - {core,{logical,32}}, - {core,{logical,33}}, - {core,{logical,34}}, - {core,{logical,35}}, - {core,{logical,39}}, - {core,{logical,40}}, - {core,{logical,41}}, - {core,{logical,42}}, - {core,{logical,43}}, - {core,{logical,48}}, - {core,{logical,49}}, - {core,{logical,50}}, - {core,{logical,58}}]}]}]. - l(Id) -> {logical, Id}. diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl index 5fd01a9ac5..6615873392 100644 --- a/erts/emulator/test/send_term_SUITE.erl +++ b/erts/emulator/test/send_term_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2009. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,24 +19,43 @@ -module(send_term_SUITE). --export([all/1,basic/1]). --export([init_per_testcase/2,fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2,basic/1]). +-export([init_per_testcase/2,end_per_testcase/2]). -export([generate_external_terms_files/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(3)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog). -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [basic]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + basic(Config) when is_list(Config) -> Drv = "send_term_drv", ?line P = start_driver(Config, Drv), @@ -61,7 +80,7 @@ basic(Config) when is_list(Config) -> ?line ExpectExt2Term = term(P, 5), %% ERL_DRV_INT, ERL_DRV_UINT - ?line case erlang:system_info(wordsize) of + ?line case erlang:system_info({wordsize, external}) of 4 -> ?line {-1, 4294967295} = term(P, 6); 8 -> diff --git a/erts/emulator/test/sensitive_SUITE.erl b/erts/emulator/test/sensitive_SUITE.erl index 458275af81..634df367ca 100644 --- a/erts/emulator/test/sensitive_SUITE.erl +++ b/erts/emulator/test/sensitive_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2009. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,9 +19,11 @@ -module(sensitive_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, stickiness/1,send_trace/1,recv_trace/1,proc_trace/1,call_trace/1, meta_trace/1,running_trace/1,gc_trace/1,seq_trace/1, t_process_info/1,t_process_display/1,save_calls/1]). @@ -34,14 +36,33 @@ init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:minutes(5)), [{watchdog,Dog}|Config]. -fin_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> +end_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog). -all(suite) -> - [stickiness,send_trace,recv_trace,proc_trace,call_trace, - meta_trace,running_trace,gc_trace,seq_trace, - t_process_info,t_process_display,save_calls]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [stickiness, send_trace, recv_trace, proc_trace, + call_trace, meta_trace, running_trace, gc_trace, + seq_trace, t_process_info, t_process_display, + save_calls]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + stickiness(Config) when is_list(Config) -> ?line {Tracer,Mref} = spawn_monitor(fun() -> diff --git a/erts/emulator/test/signal_SUITE.erl b/erts/emulator/test/signal_SUITE.erl index e9103ca3c1..736dfe5b56 100644 --- a/erts/emulator/test/signal_SUITE.erl +++ b/erts/emulator/test/signal_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. +%% Copyright Ericsson AB 2006-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -30,8 +30,9 @@ -define(DEFAULT_TIMEOUT_SECONDS, 120). %-define(line_trace, 1). --include("test_server.hrl"). --export([all/1]). +-include_lib("test_server/include/test_server.hrl"). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). % Test cases -export([xm_sig_order/1, @@ -49,38 +50,48 @@ pending_exit_group_leader/1, exit_before_pending_exit/1]). --export([init_per_testcase/2, fin_per_testcase/2, end_per_suite/1]). +-export([init_per_testcase/2, end_per_testcase/2]). init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> ?line Dog = ?t:timetrap(?t:seconds(?DEFAULT_TIMEOUT_SECONDS)), available_internal_state(true), ?line [{testcase, Func},{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> ?line Dog = ?config(watchdog, Config), ?line ?t:timetrap_cancel(Dog). +init_per_suite(Config) -> + Config. + end_per_suite(_Config) -> available_internal_state(true), - erts_debug:set_internal_state(not_running_optimization, true), + catch erts_debug:set_internal_state(not_running_optimization, true), available_internal_state(false). -all(suite) -> - [xm_sig_order, - pending_exit_unlink_process, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [xm_sig_order, pending_exit_unlink_process, pending_exit_unlink_dist_process, - pending_exit_unlink_port, - pending_exit_trap_exit, - pending_exit_receive, - pending_exit_trap_exit, - pending_exit_gc, - pending_exit_is_process_alive, + pending_exit_unlink_port, pending_exit_trap_exit, + pending_exit_receive, pending_exit_trap_exit, + pending_exit_gc, pending_exit_is_process_alive, pending_exit_process_display, pending_exit_process_info_1, - pending_exit_process_info_2, - pending_exit_group_leader, + pending_exit_process_info_2, pending_exit_group_leader, exit_before_pending_exit]. +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + xm_sig_order(doc) -> ["Test that exit signals and messages are received " "in correct order"]; xm_sig_order(suite) -> []; diff --git a/erts/emulator/test/statistics_SUITE.erl b/erts/emulator/test/statistics_SUITE.erl index 898908c40f..0392312a6f 100644 --- a/erts/emulator/test/statistics_SUITE.erl +++ b/erts/emulator/test/statistics_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -21,13 +21,14 @@ %% Tests the statistics/1 bif. --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, init_per_testcase/2, - fin_per_testcase/2, - wall_clock/1, wall_clock_zero_diff/1, wall_clock_update/1, - runtime/1, runtime_zero_diff/1, + end_per_testcase/2, + wall_clock_zero_diff/1, wall_clock_update/1, + runtime_zero_diff/1, runtime_update/1, runtime_diff/1, - run_queue/1, run_queue_one/1, + run_queue_one/1, reductions/1, reductions_big/1, garbage_collection/1, io/1, badarg/1]). @@ -35,24 +36,47 @@ -export([hog/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). init_per_testcase(_, Config) -> ?line Dog = test_server:timetrap(test_server:seconds(300)), [{watchdog, Dog}|Config]. -fin_per_testcase(_, Config) -> +end_per_testcase(_, Config) -> Dog = ?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. -all(suite) -> [wall_clock, runtime, reductions, reductions_big, run_queue, - garbage_collection, io, badarg]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [{group, wall_clock}, {group, runtime}, reductions, + reductions_big, {group, run_queue}, garbage_collection, + io, badarg]. + +groups() -> + [{wall_clock, [], + [wall_clock_zero_diff, wall_clock_update]}, + {runtime, [], + [runtime_zero_diff, runtime_update, runtime_diff]}, + {run_queue, [], [run_queue_one]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + %%% Testing statistics(wall_clock). -wall_clock(suite) -> [wall_clock_zero_diff, wall_clock_update]. wall_clock_zero_diff(doc) -> @@ -99,7 +123,6 @@ wall_clock_update1(0) -> %%% Test statistics(runtime). -runtime(suite) -> [runtime_zero_diff, runtime_update, runtime_diff]. runtime_zero_diff(doc) -> "Tests that the difference between the times returned from two consectuitive " @@ -225,7 +248,6 @@ reductions_big_loop() -> %%% Tests of statistics(run_queue). -run_queue(suite) -> [run_queue_one]. run_queue_one(doc) -> "Tests that statistics(run_queue) returns 1 if we start a " diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index ba433d4e11..9b782b35a2 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -30,23 +30,44 @@ %-define(line_trace, 1). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %-compile(export_all). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). -export([process_count/1, system_version/1, misc_smoke_tests/1, heap_size/1, wordsize/1]). -define(DEFAULT_TIMEOUT, ?t:minutes(2)). -all(doc) -> []; -all(suite) -> [process_count, system_version, misc_smoke_tests, heap_size, wordsize]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [process_count, system_version, misc_smoke_tests, + heap_size, wordsize]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) when is_list(Config) -> +end_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. @@ -132,6 +153,7 @@ misc_smoke_tests(Config) when is_list(Config) -> ?line true = is_binary(erlang:system_info(procs)), ?line true = is_binary(erlang:system_info(loaded)), ?line true = is_binary(erlang:system_info(dist)), + ?line ok = try erlang:system_info({cpu_topology,erts_get_cpu_topology_error_case}), fail catch error:badarg -> ok end, ?line ok. diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl index 7b0d6d19fe..32089e8872 100644 --- a/erts/emulator/test/system_profile_SUITE.erl +++ b/erts/emulator/test/system_profile_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2009. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,35 +22,52 @@ -module(system_profile_SUITE). --export([all/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, system_profile_on_and_off/1, runnable_procs/1, runnable_ports/1, scheduler/1 ]). --export([init_per_testcase/2, fin_per_testcase/2]). +-export([init_per_testcase/2, end_per_testcase/2]). --export([profiler_process/1, ring_loop/1, port_echo_start/0, list_load/0, run_load/2]). +-export([profiler_process/1, ring_loop/1, port_echo_start/0, + list_load/0, run_load/2]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -define(default_timeout, ?t:minutes(1)). init_per_testcase(_Case, Config) -> ?line Dog=?t:timetrap(?default_timeout), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog=?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. -all(suite) -> - %% Test specification on test suite level - [system_profile_on_and_off, - runnable_procs, - runnable_ports, - scheduler]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [system_profile_on_and_off, runnable_procs, + runnable_ports, scheduler]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + %% No specification clause needed for an init function in a conf case!!! diff --git a/erts/emulator/test/time_SUITE.erl b/erts/emulator/test/time_SUITE.erl index 095e9dd1af..bd48a0a7db 100644 --- a/erts/emulator/test/time_SUITE.erl +++ b/erts/emulator/test/time_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,14 +29,15 @@ %% now/0 %% --export([all/1, univ_to_local/1, local_to_univ/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, univ_to_local/1, local_to_univ/1, bad_univ_to_local/1, bad_local_to_univ/1, consistency/1, - now/1, now_unique/1, now_update/1, timestamp/1]). + now_unique/1, now_update/1, timestamp/1]). -export([local_to_univ_utc/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -export([linear_time/1]). @@ -54,10 +55,28 @@ -define(dst_timezone, 2). -all(suite) -> [univ_to_local, local_to_univ, - local_to_univ_utc, - bad_univ_to_local, bad_local_to_univ, - consistency, now, timestamp]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [univ_to_local, local_to_univ, local_to_univ_utc, + bad_univ_to_local, bad_local_to_univ, consistency, + {group, now}, timestamp]. + +groups() -> + [{now, [], [now_unique, now_update]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + local_to_univ_utc(suite) -> []; @@ -283,7 +302,6 @@ repeating_timestamp_check(N) -> %% Test now/0. -now(suite) -> [now_unique, now_update]. %% Tests that successive calls to now/0 returns different values. %% Also returns a comment string with the median difference between diff --git a/erts/emulator/test/timer_bif_SUITE.erl b/erts/emulator/test/timer_bif_SUITE.erl index 9ac5afcc45..7ff7449ff5 100644 --- a/erts/emulator/test/timer_bif_SUITE.erl +++ b/erts/emulator/test/timer_bif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2009. All Rights Reserved. +%% Copyright Ericsson AB 1998-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,7 +19,9 @@ -module(timer_bif_SUITE). --export([all/1,init_per_testcase/2,fin_per_testcase/2,end_per_suite/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2]). -export([start_timer_1/1, send_after_1/1, send_after_2/1, send_after_3/1, cancel_timer_1/1, start_timer_big/1, send_after_big/1, @@ -27,7 +29,7 @@ read_timer_trivial/1, read_timer/1, cleanup/1, evil_timers/1, registered_process/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). init_per_testcase(_Case, Config) -> ?line Dog=test_server:timetrap(test_server:seconds(30)), @@ -37,19 +39,35 @@ init_per_testcase(_Case, Config) -> end, [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog = ?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. +init_per_suite(Config) -> + Config. + end_per_suite(_Config) -> catch erts_debug:set_internal_state(available_internal_state, false). -all(suite) -> - [start_timer_1, send_after_1, send_after_2, cancel_timer_1, - start_timer_e, send_after_e, cancel_timer_e, - start_timer_big, send_after_big, read_timer_trivial, read_timer, - cleanup, evil_timers, registered_process]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [start_timer_1, send_after_1, send_after_2, + cancel_timer_1, start_timer_e, send_after_e, + cancel_timer_e, start_timer_big, send_after_big, + read_timer_trivial, read_timer, cleanup, evil_timers, + registered_process]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + start_timer_1(doc) -> ["Basic start_timer/3 functionality"]; start_timer_1(Config) when is_list(Config) -> diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl index e9713fcf0f..221b65309a 100644 --- a/erts/emulator/test/trace_SUITE.erl +++ b/erts/emulator/test/trace_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -23,7 +23,8 @@ %%% Tests the trace BIF. %%% --export([all/1, receive_trace/1, self_send/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, receive_trace/1, self_send/1, timeout_trace/1, send_trace/1, procs_trace/1, dist_procs_trace/1, suspend/1, mutual_suspend/1, suspend_exit/1, suspender_exit/1, @@ -35,22 +36,39 @@ system_monitor_large_heap_1/1, system_monitor_large_heap_2/1, bad_flag/1, trace_delivered/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %%% Internal exports -export([process/1]). -all(suite) -> - [cpu_timestamp, receive_trace, self_send, timeout_trace, send_trace, - procs_trace, dist_procs_trace, - suspend, mutual_suspend, suspend_exit, suspender_exit, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [cpu_timestamp, receive_trace, self_send, timeout_trace, + send_trace, procs_trace, dist_procs_trace, suspend, + mutual_suspend, suspend_exit, suspender_exit, suspend_system_limit, suspend_opts, suspend_waiting, - new_clear, existing_clear, - set_on_spawn, set_on_first_spawn, - system_monitor_args, more_system_monitor_args, - system_monitor_long_gc_1, system_monitor_long_gc_2, - system_monitor_large_heap_1, system_monitor_large_heap_2, - bad_flag, trace_delivered]. + new_clear, existing_clear, set_on_spawn, + set_on_first_spawn, system_monitor_args, + more_system_monitor_args, system_monitor_long_gc_1, + system_monitor_long_gc_2, system_monitor_large_heap_1, + system_monitor_large_heap_2, bad_flag, trace_delivered]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + %% No longer testing anything, just reporting whether cpu_timestamp diff --git a/erts/emulator/test/trace_bif_SUITE.erl b/erts/emulator/test/trace_bif_SUITE.erl index 3f91f8dc08..2c78aa394f 100644 --- a/erts/emulator/test/trace_bif_SUITE.erl +++ b/erts/emulator/test/trace_bif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2009. All Rights Reserved. +%% Copyright Ericsson AB 1998-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,24 +19,44 @@ -module(trace_bif_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1]). --export([trace_bif/1, trace_bif_timestamp/1, trace_on_and_off/1, trace_bif_local/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). +-export([trace_bif/1, trace_bif_timestamp/1, trace_on_and_off/1, + trace_bif_local/1, trace_bif_timestamp_local/1, trace_bif_return/1, not_run/1, trace_info_old_code/1]). -export([bif_process/0]). -all(suite) -> - case test_server:is_native(?MODULE) of +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + case test_server:is_native(trace_bif_SUITE) of true -> [not_run]; false -> [trace_bif, trace_bif_timestamp, trace_on_and_off, - trace_bif_local, trace_bif_timestamp_local, + trace_bif_local, trace_bif_timestamp_local, trace_bif_return, trace_info_old_code] end. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + not_run(Config) when is_list(Config) -> {skipped,"Native code"}. diff --git a/erts/emulator/test/trace_call_count_SUITE.erl b/erts/emulator/test/trace_call_count_SUITE.erl index 07aa7c8d8d..2ac58493ff 100644 --- a/erts/emulator/test/trace_call_count_SUITE.erl +++ b/erts/emulator/test/trace_call_count_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -42,7 +42,7 @@ -define(config(A,B),config(A,B)). -export([config/2]). -else. --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -endif. -ifdef(debug). @@ -62,7 +62,9 @@ config(priv_dir,_) -> ".". -else. %% When run in test server. --export([all/1, init_per_testcase/2, fin_per_testcase/2, not_run/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2, not_run/1]). -export([basic/1, on_and_off/1, info/1, pause_and_restart/1, combo/1]). @@ -70,7 +72,7 @@ init_per_testcase(_Case, Config) -> ?line Dog=test_server:timetrap(test_server:seconds(30)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_count]), erlang:trace_pattern(on_load, false, [local,meta,call_count]), erlang:trace(all, false, [all]), @@ -78,15 +80,31 @@ fin_per_testcase(_Case, Config) -> test_server:timetrap_cancel(Dog), ok. -all(doc) -> - ["Test call count tracing of local function calls."]; -all(suite) -> - case test_server:is_native(?MODULE) of +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + case test_server:is_native(trace_call_count_SUITE) of true -> [not_run]; - false -> [basic, on_and_off, info, - pause_and_restart, combo] + false -> + [basic, on_and_off, info, pause_and_restart, combo] end. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + not_run(Config) when is_list(Config) -> {skipped,"Native code"}. diff --git a/erts/emulator/test/trace_call_time_SUITE.erl b/erts/emulator/test/trace_call_time_SUITE.erl index 7bc91addde..5dfa87bbee 100644 --- a/erts/emulator/test/trace_call_time_SUITE.erl +++ b/erts/emulator/test/trace_call_time_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010. All Rights Reserved. +%% Copyright Ericsson AB 2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -57,12 +57,15 @@ %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %% When run in test server. --export([all/1, init_per_testcase/2, fin_per_testcase/2, not_run/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2, not_run/1]). -export([basic/1, on_and_off/1, info/1, - pause_and_restart/1, scheduling/1, called_function/1, combo/1, bif/1, nif/1]). + pause_and_restart/1, scheduling/1, called_function/1, combo/1, + bif/1, nif/1]). init_per_testcase(_Case, Config) -> ?line Dog=test_server:timetrap(test_server:seconds(400)), @@ -71,7 +74,7 @@ init_per_testcase(_Case, Config) -> timer:now_diff(now(),now()), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> erlang:trace_pattern({'_','_','_'}, false, [local,meta,call_time,call_count]), erlang:trace_pattern(on_load, false, [local,meta,call_time,call_count]), erlang:trace(all, false, [all]), @@ -79,15 +82,32 @@ fin_per_testcase(_Case, Config) -> test_server:timetrap_cancel(Dog), ok. -all(doc) -> - ["Test call count tracing of local function calls."]; -all(suite) -> - case test_server:is_native(?MODULE) of +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + case test_server:is_native(trace_call_time_SUITE) of true -> [not_run]; - false -> [basic, on_and_off, info, - pause_and_restart, scheduling, combo, bif, nif, called_function] + false -> + [basic, on_and_off, info, pause_and_restart, scheduling, + combo, bif, nif, called_function] end. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + not_run(Config) when is_list(Config) -> {skipped,"Native code"}. @@ -407,7 +427,7 @@ nif(Config) when is_list(Config) -> ?line 1 = erlang:trace_pattern({?MODULE, nif_dec, '_'}, true, [call_time]), ?line 1 = erlang:trace_pattern({?MODULE, with_nif, '_'}, true, [call_time]), ?line Pid = setup(), - ?line {L, T1} = execute(Pid, fun() -> with_nif(M) end), + ?line {_, T1} = execute(Pid, fun() -> with_nif(M) end), % the nif is called M - 1 times, the last time the function with 'with_nif' % returns ok and does not call the nif. @@ -486,7 +506,7 @@ with_nif(N) -> with_nif(?MODULE:nif_dec(N)). -nif_dec(N) -> 0. +nif_dec(_) -> 0. dec(N) -> loaded(10000), diff --git a/erts/emulator/test/trace_local_SUITE.erl b/erts/emulator/test/trace_local_SUITE.erl index 24005774ba..091e960610 100644 --- a/erts/emulator/test/trace_local_SUITE.erl +++ b/erts/emulator/test/trace_local_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2009. All Rights Reserved. +%% Copyright Ericsson AB 2000-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -45,7 +45,7 @@ -export([config/2]). -define(DEFAULT_RECEIVE_TIMEOUT, 1000). -else. --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -define(DEFAULT_RECEIVE_TIMEOUT, infinity). -endif. @@ -68,7 +68,8 @@ config(priv_dir,_) -> %%% When run in test server %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% --export([all/1, basic/1, bit_syntax/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, basic/1, bit_syntax/1, return/1, on_and_off/1, stack_grow/1,info/1, delete/1, exception/1, exception_apply/1, exception_function/1, exception_apply_function/1, @@ -79,34 +80,51 @@ config(priv_dir,_) -> exception_meta_nocatch/1, exception_meta_nocatch_apply/1, exception_meta_nocatch_function/1, exception_meta_nocatch_apply_function/1, - init_per_testcase/2, fin_per_testcase/2]). + init_per_testcase/2, end_per_testcase/2]). init_per_testcase(_Case, Config) -> ?line Dog=test_server:timetrap(test_server:minutes(2)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> shutdown(), Dog=?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. -all(doc) -> - ["Test tracing of local function calls and return traces."]; -all(suite) -> - case test_server:is_native(?MODULE) of +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + case test_server:is_native(trace_local_SUITE) of true -> [not_run]; - false -> [basic, bit_syntax, return, on_and_off, stack_grow, info, delete, - exception, exception_apply, - exception_function, exception_apply_function, - exception_nocatch, exception_nocatch_apply, - exception_nocatch_function, - exception_nocatch_apply_function, - exception_meta, exception_meta_apply, - exception_meta_function, exception_meta_apply_function, - exception_meta_nocatch, exception_meta_nocatch_apply, - exception_meta_nocatch_function, - exception_meta_nocatch_apply_function] + false -> + [basic, bit_syntax, return, on_and_off, stack_grow, + info, delete, exception, exception_apply, + exception_function, exception_apply_function, + exception_nocatch, exception_nocatch_apply, + exception_nocatch_function, + exception_nocatch_apply_function, exception_meta, + exception_meta_apply, exception_meta_function, + exception_meta_apply_function, exception_meta_nocatch, + exception_meta_nocatch_apply, + exception_meta_nocatch_function, + exception_meta_nocatch_apply_function] end. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + not_run(Config) when is_list(Config) -> {skipped,"Native code"}. @@ -796,9 +814,6 @@ loop(D1,D2,D3,0) -> loop(D1,D2,D3,N) -> max(N,loop(D1,D2,D3,N-1)). -max(A, B) when A > B -> A; -max(_, B) -> B. - exported_wrap(Val) -> exported(Val). diff --git a/erts/emulator/test/trace_meta_SUITE.erl b/erts/emulator/test/trace_meta_SUITE.erl index d84cb3cdf2..45987cc319 100644 --- a/erts/emulator/test/trace_meta_SUITE.erl +++ b/erts/emulator/test/trace_meta_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2002-2009. All Rights Reserved. +%% Copyright Ericsson AB 2002-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -45,7 +45,7 @@ -define(config(A,B),config(A,B)). -export([config/2]). -else. --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -endif. -ifdef(debug). @@ -65,7 +65,9 @@ config(priv_dir,_) -> ".". -else. %% When run in test server. --export([all/1, init_per_testcase/2, fin_per_testcase/2, not_run/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2, not_run/1]). -export([basic/1, return/1, on_and_off/1, stack_grow/1, info/1, tracer/1, combo/1, nosilent/1]). @@ -73,19 +75,36 @@ init_per_testcase(_Case, Config) -> ?line Dog=test_server:timetrap(test_server:minutes(5)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> shutdown(), Dog=?config(watchdog, Config), test_server:timetrap_cancel(Dog), ok. -all(doc) -> - ["Test meta tracing of local function calls and return trace."]; -all(suite) -> - case test_server:is_native(?MODULE) of - true -> [not_run]; - false -> [basic, return, on_and_off, stack_grow, - info, tracer, combo, nosilent] - end. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> +case test_server:is_native(trace_meta_SUITE) of + true -> [not_run]; + false -> + [basic, return, on_and_off, stack_grow, info, tracer, + combo, nosilent] +end. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + not_run(Config) when is_list(Config) -> {skipped,"Native code"}. @@ -594,11 +613,6 @@ loop(D1,D2,D3,0) -> loop(D1,D2,D3,N) -> max(N,loop(D1,D2,D3,N-1)). -max(A,B) when A > B -> - A; -max(_A,B) -> - B. - id(X) -> X. diff --git a/erts/emulator/test/trace_nif_SUITE.erl b/erts/emulator/test/trace_nif_SUITE.erl index 587cc08979..a7484a22fd 100644 --- a/erts/emulator/test/trace_nif_SUITE.erl +++ b/erts/emulator/test/trace_nif_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2009. All Rights Reserved. +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,9 +19,10 @@ -module(trace_nif_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). --export([all/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2]). -export([trace_nif/1, trace_nif_timestamp/1, trace_nif_local/1, @@ -32,19 +33,33 @@ -export([nif_process/0, nif/0, nif/1]). -all(suite) -> - case test_server:is_native(?MODULE) of +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + case test_server:is_native(trace_nif_SUITE) of true -> [not_run]; false -> - [trace_nif, - trace_nif_timestamp, - trace_nif_local, - trace_nif_meta, - trace_nif_timestamp_local, - trace_nif_return - ] + [trace_nif, trace_nif_timestamp, trace_nif_local, + trace_nif_meta, trace_nif_timestamp_local, + trace_nif_return] end. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + not_run(Config) when is_list(Config) -> {skipped,"Native code"}. diff --git a/erts/emulator/test/trace_port_SUITE.erl b/erts/emulator/test/trace_port_SUITE.erl index 5febe177f9..0026da4979 100644 --- a/erts/emulator/test/trace_port_SUITE.erl +++ b/erts/emulator/test/trace_port_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1999-2009. All Rights Reserved. +%% Copyright Ericsson AB 1999-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,7 +20,9 @@ -module(trace_port_SUITE). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, call_trace/1, return_trace/1, send/1, @@ -34,29 +36,42 @@ gc/1, default_tracer/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -test_cases() -> - [call_trace, - return_trace, - send, - receive_trace, - process_events, - schedule, - fake_schedule, +test_cases() -> + [call_trace, return_trace, send, receive_trace, + process_events, schedule, fake_schedule, fake_schedule_after_register, fake_schedule_after_getting_linked, - fake_schedule_after_getting_unlinked, - gc, + fake_schedule_after_getting_unlinked, gc, default_tracer]. -all(suite) -> test_cases(). +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + test_cases(). + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog = ?t:timetrap(?t:seconds(30)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog). diff --git a/erts/emulator/test/tuple_SUITE.erl b/erts/emulator/test/tuple_SUITE.erl index c4edb16d68..bfc3910742 100644 --- a/erts/emulator/test/tuple_SUITE.erl +++ b/erts/emulator/test/tuple_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2009. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -17,11 +17,13 @@ %% %CopyrightEnd% %% -module(tuple_SUITE). --export([all/1, t_size/1, t_tuple_size/1, t_element/1, t_setelement/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + t_size/1, t_tuple_size/1, t_element/1, t_setelement/1, t_list_to_tuple/1, t_tuple_to_list/1, t_make_tuple_2/1, t_make_tuple_3/1, t_append_element/1, build_and_match/1, tuple_with_case/1, tuple_in_guard/1]). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %% Tests tuples and the BIFs: %% @@ -33,13 +35,30 @@ %% make_tuple/2 %% -all(suite) -> - [build_and_match, t_size, t_tuple_size, - t_list_to_tuple, t_tuple_to_list, - t_element, t_setelement, t_make_tuple_2, - t_make_tuple_3, t_append_element, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [build_and_match, t_size, t_tuple_size, t_list_to_tuple, + t_tuple_to_list, t_element, t_setelement, + t_make_tuple_2, t_make_tuple_3, t_append_element, tuple_with_case, tuple_in_guard]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + build_and_match(Config) when is_list(Config) -> ?line {} = id({}), ?line {1} = id({1}), @@ -80,7 +99,7 @@ t_tuple_size(Config) when is_list(Config) -> ludicrous_tuple_size(T) when tuple_size(T) =:= 16#7777777777777777777777777777777777 -> ok; -ludicrous_tuple_size(T) -> error. +ludicrous_tuple_size(_) -> error. %% Tests element/2. diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl index 67d2b288a2..4b3075a164 100644 --- a/erts/emulator/test/z_SUITE.erl +++ b/erts/emulator/test/z_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2006-2009. All Rights Reserved. +%% Copyright Ericsson AB 2006-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,10 +29,12 @@ %-define(line_trace, 1). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). %-compile(export_all). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, init_per_testcase/2, + end_per_testcase/2]). -export([schedulers_alive/1, node_container_refc_check/1, long_timers/1, pollset_size/1, @@ -40,19 +42,33 @@ -define(DEFAULT_TIMEOUT, ?t:minutes(5)). -all(doc) -> []; -all(suite) -> - [schedulers_alive, - node_container_refc_check, - long_timers, - pollset_size, - check_io_debug]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [schedulers_alive, node_container_refc_check, + long_timers, pollset_size, check_io_debug]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) when is_list(Config) -> +end_per_testcase(_Case, Config) when is_list(Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index de19a2e35b..e7c57142c0 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -27,6 +27,7 @@ my $outdir = "."; # Directory for output files. my $verbose = 0; my $hot = 1; my $num_file_opcodes = 0; +my $wordsize = 32; # This is shift counts and mask for the packer. my $WHOLE_WORD = ''; @@ -36,12 +37,20 @@ my @pack_mask; $pack_instr[2] = ['6', 'i']; $pack_instr[3] = ['0', '0', 'i']; +$pack_instr[4] = ['6', '6', '6', 'i']; # Only for 64 bit wordsize $pack_shift[2] = ['0', 'BEAM_LOOSE_SHIFT']; $pack_shift[3] = ['0', 'BEAM_TIGHT_SHIFT', '(2*BEAM_TIGHT_SHIFT)']; +$pack_shift[4] = ['0', 'BEAM_LOOSE_SHIFT', # Only for 64 bit wordsize + '(2*BEAM_LOOSE_SHIFT)', + '(3*BEAM_LOOSE_SHIFT)']; $pack_mask[2] = ['BEAM_LOOSE_MASK', $WHOLE_WORD]; $pack_mask[3] = ['BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK', 'BEAM_TIGHT_MASK']; +$pack_mask[4] = ['BEAM_LOOSE_MASK', # Only for 64 bit wordsize + 'BEAM_LOOSE_MASK', + 'BEAM_LOOSE_MASK', + $WHOLE_WORD]; # There are two types of instructions: generic and specific. # The generic instructions are those generated by the Beam compiler. @@ -80,6 +89,8 @@ my %cold_code; my @unnumbered_generic; my %unnumbered; +my %is_transformed; + # # Code transformations. # @@ -118,7 +129,8 @@ my %arg_size = ('r' => 0, # x(0) - x register zero 't' => 1, # untagged integer -- can be packed 'b' => 1, # pointer to bif 'A' => 1, # arity value - 'P' => 1, # byte offset into tuple + 'P' => 1, # byte offset into tuple or stack + 'Q' => 1, # like 'P', but packable 'h' => 1, # character 'l' => 1, # float reg 'q' => 1, # literal term @@ -157,6 +169,7 @@ my @tag_type; $type_bit{'U'} = $type_bit{'u'}; $type_bit{'e'} = $type_bit{'u'}; $type_bit{'P'} = $type_bit{'u'}; + $type_bit{'Q'} = $type_bit{'u'}; } # @@ -169,6 +182,7 @@ while (@ARGV && $ARGV[0] =~ /^-(.*)/) { ($target = \&emulator_output), next if /^emulator/; ($target = \&compiler_output), next if /^compiler/; ($outdir = shift), next if /^outdir/; + ($wordsize = shift), next if /^wordsize/; ($verbose = 1), next if /^v/; die "$0: Bad option: -$_\n"; } @@ -474,8 +488,9 @@ sub emulator_output { $gen_transform_offset{$key} : -1; my($spec_op) = $gen_to_spec{$key}; my($num_specific) = $num_specific{$key}; - defined $spec_op or $tr != -1 or + defined $spec_op or $obsolete[$gen_opnum{$name,$arity}] or + $is_transformed{$name,$arity} or error("instruction $key has no specific instruction"); $spec_op = -1 unless defined $spec_op; &init_item($name, $arity, $spec_op, $num_specific, $tr, $min_window{$key}); @@ -498,12 +513,14 @@ sub emulator_output { print "#define NUM_SPECIFIC_OPS ", scalar(@op_to_name), "\n"; print "\n"; print "#ifdef ARCH_64\n"; + print "# define BEAM_WIDE_MASK 0xFFFFUL\n"; print "# define BEAM_LOOSE_MASK 0x1FFFUL\n"; print "#if HALFWORD_HEAP\n"; print "# define BEAM_TIGHT_MASK 0x1FFCUL\n"; print "#else\n"; print "# define BEAM_TIGHT_MASK 0x1FF8UL\n"; print "#endif\n"; + print "# define BEAM_WIDE_SHIFT 32\n"; print "# define BEAM_LOOSE_SHIFT 16\n"; print "# define BEAM_TIGHT_SHIFT 16\n"; print "#else\n"; @@ -796,6 +813,7 @@ sub basic_generator { 'I' => 1, 't' => 1, 'P' => 1, + 'Q' => 1, ); # Pick up the macro to use and its flags (if any). @@ -916,7 +934,18 @@ sub basic_generator { $var_decls .= "BeamInstr tmp_packed2;" if $macro_code =~ /tmp_packed2/; if ($flags =~ /-nonext/) { - $code = "$macro_code\n"; + $code = join("\n", + "{ $var_decls", + $macro_code, + "}"); + } elsif ($flags =~ /-goto:(\S*)/) { + my $goto = $1; + $code = join("\n", + "{ $var_decls", + $macro_code, + "I += $size + 1;", + "goto $goto;", + "}"); } else { $code = join("\n", "{ $var_decls", @@ -935,18 +964,31 @@ sub basic_generator { sub do_pack { my(@args) = @_; - my($i); my($packable_args) = 0; + my @is_packable; # Packability (boolean) for each argument. + my $wide_packing = 0; # # Count the number of packable arguments. If we encounter any 's' or 'd' # arguments, packing is not possible. # - for ($i = 0; $i < @args; $i++) { - if ($args[$i] =~ /[xyt]/) { + my $packable_types = "xytQ"; + foreach my $arg (@args) { + if ($arg =~ /^[$packable_types]/) { $packable_args++; - } elsif ($args[$i] =~ /[sd]/) { + push @is_packable, 1; + } elsif ($arg =~ /^I/ and $wordsize == 64 and $packable_args < 2) { + $wide_packing = 1; + push @is_packable, 1; + if (++$packable_args == 2) { + # We can only pack two arguments. Turn off packing + # for the rest of the arguments. + $packable_types = "\xFF"; + } + } elsif ($arg =~ /^[sd]/) { return ('', '', @args); + } else { + push @is_packable, 0; } } @@ -962,10 +1004,27 @@ sub do_pack { # beginning). my($up) = ''; # Pack commands (storing back while # moving forward). - my($args_per_word) = $packable_args < 4 ? $packable_args : 2; - my(@shift) = @{$pack_shift[$args_per_word]}; - my(@mask) = @{$pack_mask[$args_per_word]}; - my(@pack_instr) = @{$pack_instr[$args_per_word]}; + my $args_per_word; + if ($packable_args < 4 or $wordsize == 64) { + $args_per_word = $packable_args; + } else { + # 4 packable argument, 32 bit wordsize. Need 2 words. + $args_per_word = 2; + } + + my @shift; + my @mask; + my @instr; + + if ($wide_packing) { + @shift = ('0', 'BEAM_WIDE_SHIFT'); + @mask = ('BEAM_WIDE_MASK', $WHOLE_WORD); + @instr = ('w', 'i'); + } else { + @shift = @{$pack_shift[$args_per_word]}; + @mask = @{$pack_mask[$args_per_word]}; + @instr = @{$pack_instr[$args_per_word]}; + } # # Now generate the packing instructions. One complication is that @@ -979,10 +1038,10 @@ sub do_pack { my($ap) = 0; # Argument number within word. my($tmpnum) = 1; # Number of temporary variable. my($expr) = ''; - for ($i = 0; $i < @args; $i++) { + for (my $i = 0; $i < @args; $i++) { my($reg) = $args[$i]; my($this_size) = $arg_size{$reg}; - if ($reg =~ /[xyt]/) { + if ($is_packable[$i]) { $this_size = 0; $did_some_packing = 1; @@ -993,7 +1052,7 @@ sub do_pack { $this_size = 1; } - $down = "$pack_instr[$ap]$down"; + $down = "$instr[$ap]$down"; my($unpack) = &make_unpack($tmpnum, $shift[$ap], $mask[$ap]); $args[$i] = "pack:$this_size:$reg" . "b($unpack)"; @@ -1103,6 +1162,10 @@ sub compile_transform { if ($obsolete[$gen_opnum{$name,$arity}]) { error("obsolete function must not be used in transformations"); } + + if ($src) { + $is_transformed{$name,$arity} = 1; + } [$name,$arity,@ops]; } @@ -1291,13 +1354,28 @@ sub tr_gen_from { my($var, $type, $type_val, $cond, $val) = @$op; if ($type ne '' && $type ne '*') { - my($types) = ''; - my($type_mask) = 0; - foreach (split('', $type)) { - $types .= "$_ "; - $type_mask |= $type_bit{$_}; + # + # The is_bif, is_not_bif, and is_func instructions have + # their own built-in type test and don't need to + # be guarded with a type test instruction. + # + unless ($cond eq 'is_bif' or + $cond eq 'is_not_bif' or + $cond eq 'is_func') { + my($types) = ''; + my($type_mask) = 0; + foreach (split('', $type)) { + $types .= "$_ "; + $type_mask |= $type_bit{$_}; + } + if ($cond ne 'is_eq') { + push(@code, &make_op($types, 'is_type', $type_mask)); + } else { + $cond = ''; + push(@code, &make_op($types, 'is_type_eq', + $type_mask, $val)); + } } - push(@code, &make_op($types, 'is_type', $type_mask)); } if ($cond eq 'is_func') { diff --git a/erts/emulator/utils/count b/erts/emulator/utils/count new file mode 100755 index 0000000000..617f5c25e8 --- /dev/null +++ b/erts/emulator/utils/count @@ -0,0 +1,127 @@ +%% -*- erlang -*- +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-mode(compile). + +main(_) -> + DisDir = "./dis", + ok = filelib:ensure_dir(filename:join(DisDir, "dummy")), + io:format("Dissambling to ~s\n", [DisDir]), + ok = file:set_cwd(DisDir), + Path = code:get_path() -- ["."], + Beams0 = [filelib:wildcard(filename:join(Dir, "*.beam")) || + Dir <- Path], + Beams = lists:append(Beams0), + Mods0 = [list_to_atom(filename:rootname(filename:basename(F))) || + F <- Beams], + Mods = lists:usort(Mods0), + start_sem(), + Ps = [begin + {_,Ref} = spawn_monitor(fun() -> count(M) end), + Ref + end || M <- Mods], + [put(list_to_atom(I), 0) || I <- erts_debug:instructions()], + Res = wait_for_all(Ps, 1), + OutFile = "count", + {ok,Out} = file:open(OutFile, [write]), + [io:format(Out, "~s ~p\n", [I,C]) || {I,C} <- Res], + ok = file:close(Out), + io:format("\nResult written to ~s\n", + [filename:join(DisDir, OutFile)]), + ok. + +wait_for_all([], _) -> + lists:reverse(lists:keysort(2, get())); +wait_for_all([_|_]=Ps, I) -> + receive + {'DOWN',Ref,process,_,Result} -> + io:format("\r~p", [I]), + [increment(Key, Count) || {Key,Count} <- Result], + wait_for_all(Ps -- [Ref], I+1) + end. + +count(M) -> + down(), + erts_debug:df(M), + {ok,Fd} = file:open(atom_to_list(M) ++ ".dis", [read,raw]), + count_is(Fd), + ok = file:close(Fd), + exit(get()). + +count_is(Fd) -> + case file:read_line(Fd) of + {ok,Line} -> + count_instr(Line), + count_is(Fd); + eof -> + ok + end. + +count_instr([$\s|T]) -> + count_instr_1(T, []); +count_instr([_|T]) -> + count_instr(T); +count_instr([]) -> + %% Empty line. + ok. + +count_instr_1([$\s|_], Acc) -> + Instr = list_to_atom(lists:reverse(Acc)), + increment(Instr, 1); +count_instr_1([H|T], Acc) -> + count_instr_1(T, [H|Acc]). + +increment(Key, Inc) -> + case get(Key) of + undefined -> + put(Key, Inc); + Count -> + put(Key, Count+Inc) + end. + +%%% +%%% Counting sempahore to limit the number of processes that +%%% can run concurrently. +%%% + +down() -> + sem ! {down,self()}, + receive + sem_taken -> ok + end. + +start_sem() -> + spawn(fun() -> + register(sem, self()), + process_flag(trap_exit, true), + do_sem(erlang:system_info(schedulers)+1) end). + +do_sem(0) -> + receive + {'EXIT',_,_} -> + do_sem(1) + end; +do_sem(C) -> + receive + {down,Pid} -> + link(Pid), + Pid ! sem_taken, + do_sem(C-1) + end. diff --git a/erts/emulator/utils/loaded b/erts/emulator/utils/loaded new file mode 100644 index 0000000000..d124a64a78 --- /dev/null +++ b/erts/emulator/utils/loaded @@ -0,0 +1,44 @@ +%% -*- erlang -*- +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1998-2011. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +%% Run like: +%% $ERL_TOP/bin/escript erts/emulator/utils/loaded + +-mode(compile). + +main(_) -> + LibDir = code:lib_dir(), + io:format("Library root is ~s\n", [LibDir]), + Wc = filename:join(LibDir, "*/ebin/*.beam"), + Beams = filelib:wildcard(Wc), + BeamFileSize = lists:sum([filelib:file_size(Beam) || Beam <- Beams]), + io:format("~w BEAM files containing ~w bytes\n", + [length(Beams),BeamFileSize]), + Ms = [list_to_atom(filename:rootname(filename:basename(Beam))) || + Beam <- Beams], + [{module,_} = code:ensure_loaded(M) || M <- Ms], + <<"Current code: ",T/binary>> = erlang:system_info(loaded), + Digits = grab_digits(T), + io:format("~w modules comprising ~s words when loaded\n", + [length(Ms),Digits]). + +grab_digits(<<H,T/binary>>) when $0 =< H, H =< $9 -> + [H|grab_digits(T)]; +grab_digits(<<$\n,_/binary>>) -> []. diff --git a/erts/emulator/zlib/zutil.h b/erts/emulator/zlib/zutil.h index d560382691..a8872e1c88 100644 --- a/erts/emulator/zlib/zutil.h +++ b/erts/emulator/zlib/zutil.h @@ -142,6 +142,7 @@ extern const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #ifdef WIN32 # ifndef __CYGWIN__ /* Cygwin is Unix, not Win32 */ # define OS_CODE 0x0b +# define F_OPEN(name, mode) _wfopen((WCHAR *)(name), (WCHAR *)(mode)) /* Unicode */ # endif #endif diff --git a/erts/epmd/src/epmd.c b/erts/epmd/src/epmd.c index 65ff0cd6b2..e94533f0ba 100644 --- a/erts/epmd/src/epmd.c +++ b/erts/epmd/src/epmd.c @@ -433,7 +433,7 @@ static void usage(EpmdVars *g) fprintf(stderr, " List names registered with the currently " "running epmd\n"); fprintf(stderr, " -kill\n"); - fprintf(stderr, " Kill the currently runniing epmd\n"); + fprintf(stderr, " Kill the currently running epmd\n"); fprintf(stderr, " (only allowed if -names show empty database or\n"); fprintf(stderr, " -relaxed_command_check was given when epmd was started).\n"); fprintf(stderr, " -stop Name\n"); diff --git a/erts/epmd/src/epmd_srv.c b/erts/epmd/src/epmd_srv.c index ef471a473a..3499ab2934 100644 --- a/erts/epmd/src/epmd_srv.c +++ b/erts/epmd/src/epmd_srv.c @@ -2,7 +2,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2010. All Rights Reserved. + * Copyright Ericsson AB 1998-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -157,8 +157,10 @@ void run(EpmdVars *g) dbg_printf(g,2,"starting"); - listen(listensock, SOMAXCONN); - + if(listen(listensock, SOMAXCONN) < 0) { + dbg_perror(g,"failed to listen on socket"); + epmd_cleanup_exit(g,1); + } FD_ZERO(&g->orig_read_mask); FD_SET(listensock,&g->orig_read_mask); diff --git a/erts/epmd/test/epmd.spec b/erts/epmd/test/epmd.spec index 0e2496bc72..e72272cf94 100644 --- a/erts/epmd/test/epmd.spec +++ b/erts/epmd/test/epmd.spec @@ -1 +1 @@ -{topcase, {dir, "../epmd_test"}}. +{suites,"../epmd_test",all}. diff --git a/erts/epmd/test/epmd_SUITE.erl b/erts/epmd/test/epmd_SUITE.erl index da69412e12..72c890503d 100644 --- a/erts/epmd/test/epmd_SUITE.erl +++ b/erts/epmd/test/epmd_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -17,7 +17,7 @@ %% %CopyrightEnd% %% -module(epmd_SUITE). --include("test_server.hrl"). +-include_lib("test_server/include/test_server.hrl"). -include_lib("kernel/include/file.hrl"). @@ -35,7 +35,9 @@ -record(node_info, {port, node_type, prot, lvsn, hvsn, node_name, extra}). % Test server specific exports --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). -export( [ @@ -64,7 +66,7 @@ returns_valid_empty_extra/1, returns_valid_populated_extra_with_nulls/1, - buffer_overrun/1, + buffer_overrun_1/1, buffer_overrun_2/1, no_nonlocal_register/1, @@ -101,42 +103,37 @@ %% all/1 %% -all(suite) -> - [ - register_name, - register_names_1, - register_names_2, - register_duplicate_name, - get_port_nr, - slow_get_port_nr, - unregister_others_name_1, - unregister_others_name_2, - register_overflow, - name_with_null_inside, - name_null_terminated, - stupid_names_req, - - no_data, - one_byte, - two_bytes, - partial_packet, - zero_length, - too_large, - alive_req_too_small_1, - alive_req_too_small_2, - alive_req_too_large, - - returns_valid_empty_extra, +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [register_name, register_names_1, register_names_2, + register_duplicate_name, get_port_nr, slow_get_port_nr, + unregister_others_name_1, unregister_others_name_2, + register_overflow, name_with_null_inside, + name_null_terminated, stupid_names_req, no_data, + one_byte, two_bytes, partial_packet, zero_length, + too_large, alive_req_too_small_1, alive_req_too_small_2, + alive_req_too_large, returns_valid_empty_extra, returns_valid_populated_extra_with_nulls, + {group, buffer_overrun}, no_nonlocal_register, + no_nonlocal_kill, no_live_killing]. + +groups() -> + [{buffer_overrun, [], + [buffer_overrun_1, buffer_overrun_2]}]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. - buffer_overrun, - %buffer_overrun_1, - %buffer_overrun_2, +end_per_group(_GroupName, Config) -> + Config. - no_nonlocal_register, - no_nonlocal_kill, - no_live_killing - ]. %% %% Run before and after each test case @@ -147,7 +144,7 @@ init_per_testcase(_Func, Config) -> cleanup(), [{watchdog, Dog} | Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> cleanup(), Dog = ?config(watchdog, Config), catch test_server:timetrap_cancel(Dog), % We may have canceled already @@ -725,8 +722,6 @@ returns_valid_populated_extra_with_nulls(Config) when is_list(Config) -> ok. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -buffer_overrun(suite) -> - [buffer_overrun_1,buffer_overrun_2]. buffer_overrun_1(suite) -> []; diff --git a/erts/etc/common/Makefile.in b/erts/etc/common/Makefile.in index e866511153..4754328c0b 100644 --- a/erts/etc/common/Makefile.in +++ b/erts/etc/common/Makefile.in @@ -178,7 +178,7 @@ MC_OUTPUTS= \ MT_FLAG="-MD" endif INET_GETHOST = $(BINDIR)/inet_gethost.exe -INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe $(BINDIR)/run_test.exe +INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer.exe $(BINDIR)/dialyzer.exe $(BINDIR)/erlc.exe $(BINDIR)/start_erl.exe $(BINDIR)/escript.exe $(BINDIR)/ct_run.exe INSTALL_SRC = $(WINETC)/start_erl.c $(WINETC)/Nmakefile.start_erl ERLEXECDIR=. INSTALL_LIBS = @@ -211,7 +211,7 @@ ERLSRV_OBJECTS= MC_OUTPUTS= INET_GETHOST = $(BINDIR)/inet_gethost@EXEEXT@ INSTALL_EMBEDDED_PROGS += $(BINDIR)/typer@EXEEXT@ $(BINDIR)/dialyzer@EXEEXT@ \ - $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ $(BINDIR)/run_test@EXEEXT@ \ + $(BINDIR)/erlc@EXEEXT@ $(BINDIR)/escript@EXEEXT@ $(BINDIR)/ct_run@EXEEXT@ \ $(BINDIR)/run_erl $(BINDIR)/to_erl $(BINDIR)/dyn_erl INSTALL_EMBEDDED_DATA = ../unix/start.src ../unix/start_erl.src INSTALL_TOP = Install @@ -274,7 +274,7 @@ endif rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/dyn_erl.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/safe_string.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/typer.o - rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/run_test.o + rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/ct_run.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/vxcall.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/erl.o rm -f $(ERL_TOP)/erts/obj*/$(TARGET)/werl.o @@ -350,11 +350,11 @@ $(BINDIR)/escript@EXEEXT@: $(OBJDIR)/escript.o $(OBJDIR)/escript.o: escript.c $(CC) $(CFLAGS) -o $@ -c escript.c -$(BINDIR)/run_test@EXEEXT@: $(OBJDIR)/run_test.o - $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/run_test.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) +$(BINDIR)/ct_run@EXEEXT@: $(OBJDIR)/ct_run.o + $(PURIFY) $(LD) $(LDFLAGS) -o $@ $(OBJDIR)/ct_run.o -L$(OBJDIR) $(LIBS) $(ERTS_INTERNAL_LIBS) -$(OBJDIR)/run_test.o: run_test.c - $(CC) $(CFLAGS) -o $@ -c run_test.c +$(OBJDIR)/ct_run.o: ct_run.c + $(CC) $(CFLAGS) -o $@ -c ct_run.c #------------------------------------------------------------------------ diff --git a/erts/etc/common/run_test.c b/erts/etc/common/ct_run.c index 042b8571ca..7aaab716f7 100644 --- a/erts/etc/common/run_test.c +++ b/erts/etc/common/ct_run.c @@ -85,6 +85,7 @@ static char* strsave(char* string); static void push_words(char* src); static int run_erlang(char* name, char** argv); static char* get_default_emulator(char* progname); +static void print_deprecation_warning(char *progname); #ifdef __WIN32__ static char* possibly_quote(char* arg); #endif @@ -131,6 +132,8 @@ main(int argc, char** argv) int erl_args; char** argv0 = argv; + print_deprecation_warning(argv[0]); + emulator = get_default_emulator(argv[0]); /* @@ -391,7 +394,7 @@ run_erlang(char* progname, char** argv) status = my_spawnvp(argv)/*_spawnvp(_P_WAIT,progname,argv)*/; if (status == -1) { - fprintf(stderr, "run_test: Error executing '%s': %d", progname, + fprintf(stderr, "ct_run: Error executing '%s': %d", progname, GetLastError()); } return status; @@ -411,7 +414,7 @@ error(char* format, ...) va_start(ap, format); erts_vsnprintf(sbuf, sizeof(sbuf), format, ap); va_end(ap); - fprintf(stderr, "run_test: %s\n", sbuf); + fprintf(stderr, "ct_run: %s\n", sbuf); exit(1); } @@ -432,6 +435,27 @@ strsave(char* string) return p; } +/* Instead of making sure basename exists, we do our own */ +static char *simple_basename(char *path) +{ + char *ptr; + for (ptr = path; *ptr != '\0'; ++ptr) { + if (*ptr == '/' || *ptr == '\\') { + path = ptr + 1; + } + } + return path; +} + +static void print_deprecation_warning(char* progpath) +{ + char *basename = simple_basename(progpath); + if(strcmp(basename,"run_test") == 0 || + strcmp(basename, "run_test.exe") == 0) { + printf("---***---\nDeprecated: run_test is deprecated and will be removed in R16B,\n please use ct_run instead\n---***---\n"); + } +} + static char* get_default_emulator(char* progname) { diff --git a/erts/etc/common/dialyzer.c b/erts/etc/common/dialyzer.c index 4453e63f1c..04e9199ef3 100644 --- a/erts/etc/common/dialyzer.c +++ b/erts/etc/common/dialyzer.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/etc/common/erlc.c b/erts/etc/common/erlc.c index cd137435d1..35c360a99d 100644 --- a/erts/etc/common/erlc.c +++ b/erts/etc/common/erlc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2010. All Rights Reserved. + * Copyright Ericsson AB 1997-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -28,6 +28,7 @@ #include <winbase.h> /* FIXE ME config_win32.h? */ #define HAVE_STRERROR 1 +#define snprintf _snprintf #endif #include <ctype.h> @@ -260,6 +261,95 @@ main(int argc, char** argv) case 'I': PUSH2("@i", process_opt(&argc, &argv, 0)); break; + case 'M': + { + char *buf, *key, *val; + size_t buf_len; + + if (argv[1][2] == '\0') { /* -M */ + /* Push the following options: + * o 'makedep' + * o {makedep_output, standard_io} + */ + buf = strsave("makedep"); + PUSH2("@option", buf); + + key = "makedep_output"; + val = "standard_io"; + buf_len = 1 + strlen(key) + 1 + strlen(val) + 1 + 1; + buf = emalloc(buf_len); + snprintf(buf, buf_len, "{%s,%s}", key, val); + PUSH2("@option", buf); + } else if (argv[1][3] == '\0') { + switch(argv[1][2]) { + case 'D': /* -MD */ + /* Push the following options: + * o 'makedep' + */ + buf = strsave("makedep"); + PUSH2("@option", buf); + break; + case 'F': /* -MF <file> */ + /* Push the following options: + * o 'makedep' + * o {makedep_output, <file>} + */ + buf = strsave("makedep"); + PUSH2("@option", buf); + + key = "makedep_output"; + val = process_opt(&argc, &argv, 1); + buf_len = 1 + strlen(key) + 2 + strlen(val) + 2 + 1; + buf = emalloc(buf_len); + snprintf(buf, buf_len, "{%s,\"%s\"}", key, val); + PUSH2("@option", buf); + break; + case 'T': /* -MT <target> */ + /* Push the following options: + * o {makedep_target, <target>} + */ + key = "makedep_target"; + val = process_opt(&argc, &argv, 1); + buf_len = 1 + strlen(key) + 2 + strlen(val) + 2 + 1; + buf = emalloc(buf_len); + snprintf(buf, buf_len, "{%s,\"%s\"}", key, val); + PUSH2("@option", buf); + break; + case 'Q': /* -MQ <target> */ + /* Push the following options: + * o {makedep_target, <target>} + * o makedep_quote_target + */ + key = "makedep_target"; + val = process_opt(&argc, &argv, 1); + buf_len = 1 + strlen(key) + 2 + strlen(val) + 2 + 1; + buf = emalloc(buf_len); + snprintf(buf, buf_len, "{%s,\"%s\"}", key, val); + PUSH2("@option", buf); + + buf = strsave("makedep_quote_target"); + PUSH2("@option", buf); + break; + case 'G': /* -MG */ + /* Push the following options: + * o makedep_add_missing + */ + buf = strsave("makedep_add_missing"); + PUSH2("@option", buf); + break; + case 'P': /* -MP */ + /* Push the following options: + * o makedep_phony + */ + buf = strsave("makedep_add_missing"); + PUSH2("@option", buf); + break; + default: + goto error; + } + } + } + break; case 'o': PUSH2("@outdir", process_opt(&argc, &argv, 0)); break; @@ -561,6 +651,15 @@ usage(void) {"-hybrid", "compile using hybrid-heap emulator"}, {"-help", "shows this help text"}, {"-I path", "where to search for include files"}, + {"-M", "generate a rule for make(1) describing the dependencies"}, + {"-MF file", "write the dependencies to 'file'"}, + {"-MT target", "change the target of the rule emitted by dependency " + "generation"}, + {"-MQ target", "same as -MT but quote characters special to make(1)"}, + {"-MG", "consider missing headers as generated files and add them to " + "the dependencies"}, + {"-MP", "add a phony target for each dependency"}, + {"-MD", "same as -M -MT file (with default 'file')"}, {"-o name", "name output directory or file"}, {"-pa path", "add path to the front of Erlang's code path"}, {"-pz path", "add path to the end of Erlang's code path"}, diff --git a/erts/etc/common/heart.c b/erts/etc/common/heart.c index 3e19e5f386..778b3569c7 100644 --- a/erts/etc/common/heart.c +++ b/erts/etc/common/heart.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2009. All Rights Reserved. + * Copyright Ericsson AB 1996-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/etc/common/inet_gethost.c b/erts/etc/common/inet_gethost.c index d3ff4874ac..8bd9368aa1 100644 --- a/erts/etc/common/inet_gethost.c +++ b/erts/etc/common/inet_gethost.c @@ -59,15 +59,14 @@ #define WIN32_LEAN_AND_MEAN #include <winsock2.h> #include <windows.h> +#include <ws2tcpip.h> #include <process.h> #include <stdio.h> #include <stdlib.h> /* These are not used even if they would exist which they should not */ -#undef HAVE_GETADDRINFO #undef HAVE_GETIPNODEBYNAME #undef HAVE_GETHOSTBYNAME2 -#undef HAVE_GETNAMEINFO #undef HAVE_GETIPNODEBYADDR #else /* Unix */ @@ -1761,7 +1760,7 @@ static int worker_loop(void) struct addrinfo hints; memset(&hints, 0, sizeof(hints)); - hints.ai_flags = (AI_CANONNAME|AI_V4MAPPED|AI_ADDRCONFIG); + hints.ai_flags = AI_CANONNAME; hints.ai_socktype = SOCK_STREAM; hints.ai_family = AF_INET6; DEBUGF(5, ("Starting getaddrinfo(%s, ...)", data)); diff --git a/erts/etc/common/typer.c b/erts/etc/common/typer.c index de48daf002..c95959d52d 100644 --- a/erts/etc/common/typer.c +++ b/erts/etc/common/typer.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2009. All Rights Reserved. + * Copyright Ericsson AB 2006-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/etc/unix/Install.src b/erts/etc/unix/Install.src index 7dead62ab0..8f40c43874 100644 --- a/erts/etc/unix/Install.src +++ b/erts/etc/unix/Install.src @@ -89,9 +89,12 @@ cp -p $ERL_ROOT/erts-%I_VSN%/bin/erl . cp -p $ERL_ROOT/erts-%I_VSN%/bin/erlc . cp -p $ERL_ROOT/erts-%I_VSN%/bin/dialyzer . cp -p $ERL_ROOT/erts-%I_VSN%/bin/typer . -cp -p $ERL_ROOT/erts-%I_VSN%/bin/run_test . +cp -p $ERL_ROOT/erts-%I_VSN%/bin/ct_run . cp -p $ERL_ROOT/erts-%I_VSN%/bin/escript . +# Remove in R16B +ln -s ct_run run_test + # # Set a soft link to epmd # This should not be done for an embedded system! diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 73b1bafbe0..0355f2629f 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -2,7 +2,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2010. All Rights Reserved. +# Copyright Ericsson AB 2003-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -228,17 +228,30 @@ if [ $run_valgrind != yes ]; then fi if [ "x$GDB" = "x" ]; then if [ $run_valgrind = yes ]; then + valversion=`valgrind --version` + valmajor=`echo $valversion | sed 's,[a-z]*\-\([0-9]*\).*,\1,'` + valminor=`echo $valversion | sed 's,[a-z]*\-[0-9]*.\([0-9]*\).*,\1,'` emu_xargs=`echo $xargs | sed "s|+|-|g"` - if [ "x$VALGRIND_LOG_DIR" = "x" ]; then - valgrind_log= - else - valgrind_log="--log-file=$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU.log" - fi if [ "x$VALGRIND_LOG_XML" = "x" ]; then valgrind_xml= + log_file_prefix="--log-file=" else export VALGRIND_LOG_XML valgrind_xml="--xml=yes" + if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then + log_file_prefix="--xml-file=" + else + log_file_prefix="--log-file=" + fi + fi + if [ "x$VALGRIND_LOG_DIR" = "x" ]; then + valgrind_log= + else + if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then + valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU.log.$$" + else + valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU.log" + fi fi if [ "x$VALGRIND_MISC_FLAGS" = "x" ]; then valgrind_misc_flags= diff --git a/erts/etc/unix/format_man_pages b/erts/etc/unix/format_man_pages index 2c4f6eee4f..93dcdcd8fa 100644 --- a/erts/etc/unix/format_man_pages +++ b/erts/etc/unix/format_man_pages @@ -3,7 +3,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1996-2009. All Rights Reserved. +# Copyright Ericsson AB 1996-2010. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -59,34 +59,21 @@ esac # Create the 'cat' directories (probably not needed) # -cd $ERL_ROOT +cd $ERL_ROOT/man -if [ ! -d man/cat1 ] -then - mkdir man/cat1 -fi +for d in 0 1 2 3 4 5 6 7 8 9 +do + if [ ! -d cat$d ] + then + mkdir cat$d + fi -if [ ! -d man/cat3 ] -then - mkdir man/cat3 -fi - -if [ ! -d man/cat4 ] -then - mkdir man/cat4 -fi - -if [ ! -d man/cat6 ] -then - mkdir man/cat6 -fi +done # # Cleanup old formatting # -cd $ERL_ROOT/man - rm -f whatis windex # Remove old cat files diff --git a/erts/etc/win32/Install.c b/erts/etc/win32/Install.c index ca814e3f80..6e60512f6d 100644 --- a/erts/etc/win32/Install.c +++ b/erts/etc/win32/Install.c @@ -46,7 +46,7 @@ int main(int argc, char **argv) HANDLE module = GetModuleHandle(NULL); char *binaries[] = { "erl.exe", "werl.exe", "erlc.exe", "dialyzer.exe", "typer.exe", - "escript.exe", "run_test.exe", NULL }; + "escript.exe", "ct_run.exe", NULL }; char *scripts[] = { "start_clean.boot", "start_sasl.boot", NULL }; char fromname[MAX_PATH]; char toname[MAX_PATH]; @@ -172,6 +172,20 @@ int main(int argc, char **argv) } } + // Remove in R16B + sprintf(fromname,"%s\\%s",bin_dir,"ct_run.exe"); + sprintf(toname,"%s\\%s",bin_dir,"run_test.exe"); + if (GetFileAttributes(fromname) == 0xFFFFFFFF) { + fprintf(stderr,"Could not find file %s\n", + fromname); + exit(1); + } + if (!CopyFile(fromname,toname,FALSE)) { + fprintf(stderr,"Could not copy file %s to %s\n", + fromname,toname); + fprintf(stderr,"Continuing installation anyway...\n"); + } + for (i = 0; scripts[i] != NULL; ++i) { sprintf(fromname,"%s\\%s",release_dir,scripts[i]); sprintf(toname,"%s\\%s",bin_dir,scripts[i]); diff --git a/erts/etc/win32/cygwin_tools/vc/ld.sh b/erts/etc/win32/cygwin_tools/vc/ld.sh index b04935ed9b..406c63ffee 100755 --- a/erts/etc/win32/cygwin_tools/vc/ld.sh +++ b/erts/etc/win32/cygwin_tools/vc/ld.sh @@ -53,7 +53,7 @@ while test -n "$1" ; do STDLIB_FORCED=true; STDLIB=LIBCMTD.LIB;; -lsocket) - DEFAULT_LIBRARIES="$DEFAULT_LIBRARIES WS2_32.LIB";; + DEFAULT_LIBRARIES="$DEFAULT_LIBRARIES WS2_32.LIB IPHLPAPI.LIB";; -l*) y=`echo $x | sed 's,^-l\(.*\),\1,g'`; MPATH=`cygpath -m $y`; @@ -158,7 +158,7 @@ else fi p=$$ -CMD="$linktype -nologo -incremental:no $CMD $STDLIB $DEFAULT_LIBRARIES" +CMD="$linktype -nologo -incremental:no -largeaddressaware $CMD $STDLIB $DEFAULT_LIBRARIES" if [ "X$LD_SH_DEBUG_LOG" != "X" ]; then echo ld.sh "$SAVE" >>$LD_SH_DEBUG_LOG echo link.exe $CMD >>$LD_SH_DEBUG_LOG @@ -168,6 +168,7 @@ RES=$? CMANIFEST=`cygpath $MANIFEST` if [ "$RES" = "0" -a -f "$CMANIFEST" ]; then # Add stuff to manifest to turn off "virtualization" + sed -n -i '1h;1!H;${;g;s,<trustInfo.*</trustInfo>.,,g;p;}' $CMANIFEST sed -i "s/<\/assembly>/ <ms_asmv2:trustInfo xmlns:ms_asmv2=\"urn:schemas-microsoft-com:asm.v2\">\n <ms_asmv2:security>\n <ms_asmv2:requestedPrivileges>\n <ms_asmv2:requestedExecutionLevel level=\"AsInvoker\" uiAccess=\"false\"\/>\n <\/ms_asmv2:requestedPrivileges>\n <\/ms_asmv2:security>\n <\/ms_asmv2:trustInfo>\n<\/assembly>/" $CMANIFEST eval mt.exe -nologo -manifest "$MANIFEST" -outputresource:"$OUTPUTRES" >>/tmp/link.exe.${p}.1 2>>/tmp/link.exe.${p}.2 diff --git a/erts/etc/win32/erlsrv/erlsrv_service.c b/erts/etc/win32/erlsrv/erlsrv_service.c index a58ee862c5..8891379643 100644 --- a/erts/etc/win32/erlsrv/erlsrv_service.c +++ b/erts/etc/win32/erlsrv/erlsrv_service.c @@ -523,7 +523,7 @@ static BOOL start_a_service(ServerInfo *srvi){ srvi->keys[WorkDir].data.bytes : NULL, &start, &(srvi->info))){ - sprintf(errbuff,"Could not start erlang service" + sprintf(errbuff,"Could not start erlang service " "with commandline \"%s\".", service_name, execbuff @@ -924,7 +924,7 @@ static VOID WINAPI service_main_loop(DWORD argc, char **argv){ } else { DWORD ecode = NO_ERROR; if(success_wait == NO_SUCCESS_WAIT){ - log_warning("Erlang machine volountarily stopped. " + log_warning("Erlang machine voluntarily stopped. " "The service is not restarted as OnFail " "is set to ignore."); } else { diff --git a/erts/etc/win32/nsis/Makefile b/erts/etc/win32/nsis/Makefile index ebb3ad9a96..ae2343b420 100644 --- a/erts/etc/win32/nsis/Makefile +++ b/erts/etc/win32/nsis/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2009. All Rights Reserved. +# Copyright Ericsson AB 2003-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -45,6 +45,7 @@ WTARGET_DIR=$(shell (cygpath -d $(TARGET_DIR) 2>/dev/null || cygpath -d $(TARGET REDIST_FILE=$(shell (sh ./find_redist.sh || echo "")) REDIST_DLL_VERSION=$(shell (sh ./dll_version_helper.sh || echo "")) +REDIST_DLL_NAME=$(shell (sh ./dll_version_helper.sh -n || echo "")) release_spec: @NSIS_VER=`makensis /hdrinfo | head -1 | awk '{print $$2}'`; \ @@ -73,6 +74,7 @@ release_spec: cp $(REDIST_FILE) $(RELEASE_PATH)/vcredist_x86.exe;\ echo '!define HAVE_REDIST_FILE 1' >> $(VERSION_HEADER); \ echo '!define REDIST_DLL_VERSION "$(REDIST_DLL_VERSION)"' >> $(VERSION_HEADER);\ + echo '!define REDIST_DLL_NAME "$(REDIST_DLL_NAME)"' >> $(VERSION_HEADER);\ fi;\ if [ -f $(RELEASE_PATH)/docs/doc/index.html ];\ then \ diff --git a/erts/etc/win32/nsis/dll_version_helper.sh b/erts/etc/win32/nsis/dll_version_helper.sh index e0047dea8b..eecd4a72b5 100755 --- a/erts/etc/win32/nsis/dll_version_helper.sh +++ b/erts/etc/win32/nsis/dll_version_helper.sh @@ -2,7 +2,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2007-2009. All Rights Reserved. +# Copyright Ericsson AB 2007-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -26,6 +26,7 @@ # exit 0 cat > hello.c <<EOF +#include <windows.h> #include <stdio.h> int main(void) @@ -35,15 +36,81 @@ int main(void) } EOF -cl /MD hello.c > /dev/null 2>&1 +cl /MD hello.c > /dev/null 2>&1 if [ '!' -f hello.exe.manifest ]; then - echo "This compiler does not generate manifest files - OK if using mingw" >&2 - exit 0 + # Gah - VC 2010 changes the way it handles DLL's and manifests... Again... + # need another way of getting the version + DLLNAME=`dumpbin.exe /imports hello.exe | egrep MSVCR.*dll` + DLLNAME=`echo $DLLNAME` + cat > helper.c <<EOF +#include <windows.h> +#include <stdio.h> + +#define REQ_MODULE "$DLLNAME" + +int main(void) +{ + DWORD dummy; + DWORD versize; + int i,n; + unsigned char *versinfo; + char buff[100]; + + char *vs_verinfo; + unsigned int vs_ver_size; + + struct LANGANDCODEPAGE { + WORD language; + WORD codepage; + } *translate; + + unsigned int tr_size; + + if (!(versize = GetFileVersionInfoSize(REQ_MODULE,&dummy))) { + fprintf(stderr,"No version info size in %s!\n",REQ_MODULE); + exit(1); + } + versinfo=malloc(versize); + if (!GetFileVersionInfo(REQ_MODULE,dummy,versize,versinfo)) { + fprintf(stderr,"No version info in %s!\n",REQ_MODULE); + exit(2); + } + if (!VerQueryValue(versinfo,"\\\\VarFileInfo\\\\Translation",&translate,&tr_size)) { + fprintf(stderr,"No translation info in %s!\n",REQ_MODULE); + exit(3); + } + n = tr_size/sizeof(translate); + for(i=0; i < n; ++i) { + sprintf(buff,"\\\\StringFileInfo\\\\%04x%04x\\\\FileVersion", + translate[i].language,translate[i].codepage); + if (VerQueryValue(versinfo,buff,&vs_verinfo,&vs_ver_size)) { + printf("%s\n",(char *) vs_verinfo); + return 0; + } + } + fprintf(stderr,"Failed to find file version of %s\n",REQ_MODULE); + return 0; +} +EOF + cl /MD helper.c version.lib > /dev/null 2>&1 + if [ '!' -f helper.exe ]; then + echo "Failed to build helper program." >&2 + exit 1 + fi + NAME=$DLLNAME + VERSION=`./helper` +else + VERSION=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*version=.\([0-9\.]*\).*,\1,g' | grep -v '<'` + NAME=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*name=.[A-Za-z\.]*\([0-9]*\).*,msvcr\1.dll,g' | grep -v '<'` +fi +#rm -f hello.c hello.obj hello.exe hello.exe.manifest helper.c helper.obj helper.exe helper.exe.manifest +if [ "$1" = "-n" ]; then + ASKEDFOR=$NAME +else + ASKEDFOR=$VERSION fi -VERSION=`grep '<assemblyIdentity' hello.exe.manifest | sed 's,.*version=.\([0-9\.]*\).*,\1,g' | grep -v '<'` -rm -f hello.c hello.obj hello.exe hello.exe.manifest -if [ -z "$VERSION" ]; then +if [ -z "$ASKEDFOR" ]; then exit 1 fi -echo $VERSION +echo $ASKEDFOR exit 0 diff --git a/erts/etc/win32/nsis/erlang20.nsi b/erts/etc/win32/nsis/erlang20.nsi index 43e5d91604..941e8e6f5d 100644 --- a/erts/etc/win32/nsis/erlang20.nsi +++ b/erts/etc/win32/nsis/erlang20.nsi @@ -311,23 +311,23 @@ FunctionEnd Function .onInit
SectionGetFlags 0 $MYTEMP
-; MessageBox MB_YESNO "Found $SYSDIR\msvcr80.dll" IDYES FoundLbl
- IfFileExists $SYSDIR\msvcr80.dll MaybeFoundInSystemLbl
+ ;MessageBox MB_YESNO "Found $SYSDIR\${REDIST_DLL_NAME}" IDYES FoundLbl
+ IfFileExists $SYSDIR\${REDIST_DLL_NAME} MaybeFoundInSystemLbl
SearchSxsLbl:
FindFirst $0 $1 $WINDIR\WinSxS\x86*
LoopLbl:
StrCmp $1 "" NotFoundLbl
- IfFileExists $WINDIR\WinSxS\$1\msvcr80.dll MaybeFoundInSxsLbl
+ IfFileExists $WINDIR\WinSxS\$1\${REDIST_DLL_NAME} MaybeFoundInSxsLbl
FindNext $0 $1
Goto LoopLbl
MaybeFoundInSxsLbl:
- GetDllVersion $WINDIR\WinSxS\$1\msvcr80.dll $R0 $R1
+ GetDllVersion $WINDIR\WinSxS\$1\${REDIST_DLL_NAME} $R0 $R1
Call DllVersionGoodEnough
FindNext $0 $1
IntCmp 2 $R0 LoopLbl
Goto FoundLbl
MaybeFoundInSystemLbl:
- GetDllVersion $SYSDIR\msvcr80.dll $R0 $R1
+ GetDllVersion $SYSDIR\${REDIST_DLL_NAME} $R0 $R1
Call DllVersionGoodEnough
IntCmp 2 $R0 SearchSxSLbl
FoundLbl:
diff --git a/erts/etc/win32/nsis/find_redist.sh b/erts/etc/win32/nsis/find_redist.sh index c5572839c5..328811a0d7 100755 --- a/erts/etc/win32/nsis/find_redist.sh +++ b/erts/etc/win32/nsis/find_redist.sh @@ -2,7 +2,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2007-2009. All Rights Reserved. +# Copyright Ericsson AB 2007-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -107,16 +107,70 @@ for x in cl bin vc; do fi BPATH="$NBPATH" done -#echo $BPATH -for x in sdk v2.0 bootstrapper packages vcredist_x86 vcredist_x86.exe; do - #echo "x=$x" - #echo "BPATH=$BPATH" - NBPATH=`add_path_element $x "$BPATH"` - if [ "$NBPATH" = "$BPATH" ]; then - echo "Failed to locate vcredist_x86.exe because directory structure was unexpected" >&2 - exit 3 +BPATH_LIST=$BPATH + +# rc.exe is in the Microsoft SDK directory of VS2008 +RCPATH=`lookup_prog_in_path rc` +fail=false +if [ '!' -z "$RCPATH" ]; then + BPATH=$RCPATH + allow_fail=false + for x in rc bin @ANY v6.0A v7.0A v7.1; do + if [ $x = @ANY ]; then + allow_fail=true + else + NBPATH=`remove_path_element $x "$BPATH"` + if [ $allow_fail = false -a "$NBPATH" = "$BPATH" ]; then + fail=true + break; + fi + BPATH="$NBPATH" + fi + done + if [ $fail = false ]; then + BPATH_LIST="$BPATH_LIST $BPATH" + fi +fi + +# Frantic search through two roots with different +# version directories. We want to be very specific about the +# directory structures as we wouldnt want to find the wrong +# redistributables... + +#echo $BPATH_LIST +for BP in $BPATH_LIST; do + #echo "BP=$BP" + for verdir in "sdk v2.0" "sdk v3.5" "v6.0A" "v7.0A" "v7.1"; do + BPATH=$BP + fail=false + allow_fail=false + for x in $verdir @ANY bootstrapper packages vcredist_x86 Redist VC @ALL vcredist_x86.exe; do + #echo "x=$x" + #echo "BPATH=$BPATH" + #echo "allow_fail=$allow_fail" + if [ $x = @ANY ]; then + allow_fail=true + elif [ $x = @ALL ]; then + allow_fail=false + else + NBPATH=`add_path_element $x "$BPATH"` + if [ $allow_fail = false -a "$NBPATH" = "$BPATH" ]; then + fail=true + break; + fi + BPATH="$NBPATH" + fi + done + if [ $fail = false ]; then + break; + fi + done + if [ $fail = false ]; then + echo $BPATH + exit 0 fi - BPATH="$NBPATH" done -echo $BPATH -exit 0
\ No newline at end of file + +echo "Failed to locate vcredist_x86.exe because directory structure was unexpected" >&2 +exit 3 + diff --git a/erts/example/next_perm.cc b/erts/example/next_perm.cc index ee81cb0404..1427cd3979 100644 --- a/erts/example/next_perm.cc +++ b/erts/example/next_perm.cc @@ -120,7 +120,7 @@ static void ready_async(ErlDrvData drv_data, ErlDrvThreadData async_data) ErlDrvPort port = reinterpret_cast<ErlDrvPort>(drv_data); our_async_data* d = reinterpret_cast<our_async_data*>(async_data); int n = d->data.size(), result_n = n*2 + 5; - ErlDrvTermData* result = new ErlDrvTermData[result_n], * rp = result; + ErlDrvTermData *result = new ErlDrvTermData[result_n], *rp = result; *rp++ = ERL_DRV_PORT; *rp++ = driver_mk_port(port); for (vector<int>::iterator i = d->data.begin(); diff --git a/erts/include/internal/ethr_atomics.h b/erts/include/internal/ethr_atomics.h new file mode 100644 index 0000000000..1caf4d0567 --- /dev/null +++ b/erts/include/internal/ethr_atomics.h @@ -0,0 +1,726 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: The ethread atomic API + * Author: Rickard Green + */ + +#ifndef ETHR_ATOMIC_H__ +#define ETHR_ATOMIC_H__ + +#if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) +# define ETHR_NEED_ATOMIC_PROTOTYPES__ +#endif + +#ifndef ETHR_HAVE_NATIVE_ATOMICS +/* + * No native atomic implementation available. :( + * Use fallback... + */ +typedef ethr_sint32_t ethr_atomic32_t; +typedef ethr_sint_t ethr_atomic_t; +#else +/* + * Map ethread native atomics to ethread API atomics. + * + * We do at least have a native atomic implementation that + * can handle integers of a size larger than or equal to + * the size of pointers. + */ + +/* -- Pointer size atomics -- */ + +#undef ETHR_NAINT_T__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_NATMC_ADDR_FUNC__ +#if ETHR_SIZEOF_PTR == 8 +# if defined(ETHR_HAVE_NATIVE_ATOMIC64) +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr +typedef ethr_native_atomic64_t ethr_atomic_t; +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# else +# error "Missing native atomic implementation" +# endif +#elif ETHR_SIZEOF_PTR == 4 +# define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr +# ifdef ETHR_HAVE_NATIVE_ATOMIC32 +typedef ethr_native_atomic32_t ethr_atomic_t; +# define ETHR_NAINT_T__ ethr_sint32_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +# elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +typedef ethr_native_atomic64_t ethr_atomic_t; +# define ETHR_NATMC_T__ ethr_native_atomic64_t +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +# else +# error "Missing native atomic implementation" +# endif +#endif + +/* -- 32-bit atomics -- */ + +#undef ETHR_NAINT32_T__ +#undef ETHR_NATMC32_FUNC__ +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +typedef ethr_native_atomic32_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint32_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +typedef ethr_native_atomic64_t ethr_atomic32_t; +# define ETHR_NAINT32_T__ ethr_sint64_t +# define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X +#else +# error "Missing native atomic implementation" +#endif + +#endif + +#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ +ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *); +void ethr_atomic_init(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_set(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read(ethr_atomic_t *); +ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *); +ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *); +void ethr_atomic_inc(ethr_atomic_t *); +void ethr_atomic_dec(ethr_atomic_t *); +ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_add(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *, ethr_sint_t); +ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *); +void ethr_atomic_set_relb(ethr_atomic_t *, ethr_sint_t); +void ethr_atomic_dec_relb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *); +ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); +ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *, ethr_sint_t, ethr_sint_t); + +ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *); +void ethr_atomic32_init(ethr_atomic32_t *, ethr_sint32_t); +void ethr_atomic32_set(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *); +void ethr_atomic32_inc(ethr_atomic32_t *); +void ethr_atomic32_dec(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *, ethr_sint32_t); +void ethr_atomic32_add(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *, ethr_sint32_t); +ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *, + ethr_sint32_t, + ethr_sint32_t); +ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *); +void ethr_atomic32_set_relb(ethr_atomic32_t *, ethr_sint32_t); +void ethr_atomic32_dec_relb(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *); +ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *, + ethr_sint32_t, + ethr_sint32_t); +ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *, + ethr_sint32_t, + ethr_sint32_t); +#endif + +int ethr_init_atomics(void); + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +#ifndef ETHR_HAVE_NATIVE_ATOMICS +/* + * Fallbacks for atomics used in absence of a native implementation. + */ + +#define ETHR_ATOMIC_ADDR_BITS 10 +#define ETHR_ATOMIC_ADDR_SHIFT 6 + +typedef struct { + union { + ethr_spinlock_t lck; + char buf[ETHR_CACHE_LINE_SIZE]; + } u; +} ethr_atomic_protection_t; + +extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; + +#define ETHR_ATOMIC_PTR2LCK__(PTR) \ +(ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ + & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) + + +#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ +do { \ + ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ + ethr_spin_lock(slp__); \ + { EXPS; } \ + ethr_spin_unlock(slp__); \ +} while (0) + +#endif + +/* + * --- Pointer size atomics --------------------------------------------------- + */ + +static ETHR_INLINE ethr_sint_t * +ETHR_INLINE_FUNC_NAME_(ethr_atomic_addr)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t *) ETHR_NATMC_ADDR_FUNC__(var); +#else + return (ethr_sint_t *) var; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(init)(var, (ETHR_NAINT_T__) i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(set)(var, (ETHR_NAINT_T__) i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(read)(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, ethr_sint_t incr) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(add)(var, (ETHR_NAINT_T__) incr); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, ethr_sint_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(add_return)(var, (ETHR_NAINT_T__) i); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(inc)(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(dec)(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return)(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return)(var); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, + ethr_sint_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(and_retold)(var, + (ETHR_NAINT_T__) mask); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, + ethr_sint_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(or_retold)(var, + (ETHR_NAINT_T__) mask); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, ethr_sint_t new) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(xchg)(var, + (ETHR_NAINT_T__) new); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); + return res; +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg)(var, + (ETHR_NAINT_T__) new, + (ETHR_NAINT_T__) exp); +#else + ethr_sint_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, + { + res = *var; + if (__builtin_expect(res == exp, 1)) + *var = new; + }); + return res; +#endif +} + +/* + * Important memory barrier requirements. + * + * The following atomic operations *must* supply a memory barrier of + * at least the type specified by its suffix: + * _acqb = acquire barrier + * _relb = release barrier + */ + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(read_acqb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(inc_return_acqb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, + ethr_sint_t val) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(set_relb)(var, (ETHR_NAINT_T__) val); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC_FUNC__(dec_relb)(var); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(dec_return_relb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_acqb)(var, + (ETHR_NAINT_T__) new, + (ETHR_NAINT_T__) exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#endif +} + +static ETHR_INLINE ethr_sint_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, + ethr_sint_t new, + ethr_sint_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint_t) ETHR_NATMC_FUNC__(cmpxchg_relb)(var, + (ETHR_NAINT_T__) new, + (ETHR_NAINT_T__) exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); +#endif +} + +/* + * --- 32-bit atomics --------------------------------------------------------- + */ + +static ETHR_INLINE ethr_sint32_t * +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_addr)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return ethr_native_atomic32_addr(var); +#else + return (ethr_sint32_t *) var; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_init)(ethr_atomic32_t *var, + ethr_sint32_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(init)(var, (ETHR_NAINT32_T__) i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(ethr_atomic32_t *var, ethr_sint32_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(set)(var, (ETHR_NAINT32_T__) i); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read)(var); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add)(ethr_atomic32_t *var, + ethr_sint32_t incr) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(add)(var, (ETHR_NAINT32_T__) incr); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_add_read)(ethr_atomic32_t *var, + ethr_sint32_t i) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) + ETHR_NATMC32_FUNC__(add_return)(var, (ETHR_NAINT32_T__) i); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); + return res; +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(inc)(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(dec)(var); +#else + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return)(var); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return)(var); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var)); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_band)(ethr_atomic32_t *var, + ethr_sint32_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) + ETHR_NATMC32_FUNC__(and_retold)(var, (ETHR_NAINT32_T__) mask); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_bor)(ethr_atomic32_t *var, + ethr_sint32_t mask) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return + (ethr_sint32_t) ETHR_NATMC32_FUNC__(or_retold)(var, + (ETHR_NAINT32_T__) mask); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_xchg)(ethr_atomic32_t *var, + ethr_sint32_t new) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(xchg)(var, + (ETHR_NAINT32_T__) new); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); + return res; +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(cmpxchg)(var, + (ETHR_NAINT32_T__) new, + (ETHR_NAINT32_T__) exp); +#else + ethr_sint32_t res; + ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, + { + res = *var; + if (__builtin_expect(res == exp, 1)) + *var = new; + }); + return res; +#endif +} + +/* + * Important memory barrier requirements. + * + * The following atomic operations *must* supply a memory barrier of + * at least the type specified by its suffix: + * _acqb = acquire barrier + * _relb = release barrier + */ + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read_acqb)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(read_acqb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read_acqb)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(inc_return_acqb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_inc_read)(var); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set_relb)(ethr_atomic32_t *var, + ethr_sint32_t val) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(set_relb)(var, (ETHR_NAINT32_T__) val); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic32_set)(var, val); +#endif +} + +static ETHR_INLINE void +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_relb)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + ETHR_NATMC32_FUNC__(dec_relb)(var); +#else + ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec)(var); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read_relb)(ethr_atomic32_t *var) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) ETHR_NATMC32_FUNC__(dec_return_relb)(var); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_dec_read)(var); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_acqb)(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) + ETHR_NATMC32_FUNC__(cmpxchg_acqb)(var, + (ETHR_NAINT32_T__) new, + (ETHR_NAINT32_T__) exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp); +#endif +} + +static ETHR_INLINE ethr_sint32_t +ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg_relb)(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ +#ifdef ETHR_HAVE_NATIVE_ATOMICS + return (ethr_sint32_t) + ETHR_NATMC32_FUNC__(cmpxchg_relb)(var, + (ETHR_NAINT32_T__) new, + (ETHR_NAINT32_T__) exp); +#else + return ETHR_INLINE_FUNC_NAME_(ethr_atomic32_cmpxchg)(var, new, exp); +#endif +} + + +#endif /* ETHR_TRY_INLINE_FUNCS */ + +#undef ETHR_NAINT_T__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_NATMC_ADDR_FUNC__ + +#undef ETHR_NAINT32_T__ +#undef ETHR_NATMC32_FUNC__ + +#endif diff --git a/erts/include/internal/ethr_mutex.h b/erts/include/internal/ethr_mutex.h index 8d9d5e3d08..fadaf1e2a4 100644 --- a/erts/include/internal/ethr_mutex.h +++ b/erts/include/internal/ethr_mutex.h @@ -33,6 +33,13 @@ # define ETHR_MTX_HARD_DEBUG #endif +#if 0 +# define ETHR_MTX_CHK_EXCL +#if 1 +# define ETHR_MTX_CHK_NON_EXCL +#endif +#endif + #ifdef ETHR_MTX_HARD_DEBUG # ifdef __GNUC__ # warning ETHR_MTX_HARD_DEBUG @@ -49,6 +56,15 @@ #if defined(ETHR_USE_OWN_RWMTX_IMPL__) || defined(ETHR_USE_OWN_MTX_IMPL__) +#ifdef ETHR_DEBUG +# ifndef ETHR_MTX_CHK_EXCL +# define ETHR_MTX_CHK_EXCL +# endif +# ifndef ETHR_MTX_CHK_NON_EXCL +# define ETHR_MTX_CHK_NON_EXCL +# endif +#endif + #if 0 # define ETHR_MTX_Q_LOCK_SPINLOCK__ # define ETHR_MTX_QLOCK_TYPE__ ethr_spinlock_t @@ -62,14 +78,14 @@ # error Need a qlock implementation #endif -#define ETHR_RWMTX_W_FLG__ (((long) 1) << 31) -#define ETHR_RWMTX_W_WAIT_FLG__ (((long) 1) << 30) -#define ETHR_RWMTX_R_WAIT_FLG__ (((long) 1) << 29) +#define ETHR_RWMTX_W_FLG__ (((ethr_sint32_t) 1) << 31) +#define ETHR_RWMTX_W_WAIT_FLG__ (((ethr_sint32_t) 1) << 30) +#define ETHR_RWMTX_R_WAIT_FLG__ (((ethr_sint32_t) 1) << 29) /* frequent read kind */ -#define ETHR_RWMTX_R_FLG__ (((long) 1) << 28) -#define ETHR_RWMTX_R_PEND_UNLCK_MASK__ (ETHR_RWMTX_R_FLG__ - 1) -#define ETHR_RWMTX_R_MASK__ (ETHR_RWMTX_R_WAIT_FLG__ - 1) +#define ETHR_RWMTX_R_FLG__ (((ethr_sint32_t) 1) << 28) +#define ETHR_RWMTX_R_ABRT_UNLCK_FLG__ (((ethr_sint32_t) 1) << 27) +#define ETHR_RWMTX_R_PEND_UNLCK_MASK__ (ETHR_RWMTX_R_ABRT_UNLCK_FLG__ - 1) /* normal kind */ #define ETHR_RWMTX_RS_MASK__ (ETHR_RWMTX_R_WAIT_FLG__ - 1) @@ -79,20 +95,39 @@ #define ETHR_CND_WAIT_FLG__ ETHR_RWMTX_R_WAIT_FLG__ +#ifdef ETHR_DEBUG +#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V) \ + ETHR_ASSERT(!((V) & ~(ETHR_RWMTX_W_FLG__ \ + | ETHR_RWMTX_W_WAIT_FLG__ \ + | ETHR_RWMTX_R_WAIT_FLG__ \ + | ETHR_RWMTX_RS_MASK__))) +#else +#define ETHR_DBG_CHK_UNUSED_FLG_BITS(V) +#endif + +#define ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(MTX) \ + ETHR_DBG_CHK_UNUSED_FLG_BITS(ethr_atomic32_read(&(MTX)->mtxb.flgs)) + struct ethr_mutex_base_ { #ifdef ETHR_MTX_HARD_DEBUG_FENCE long pre_fence; #endif - ethr_atomic_t flgs; - ETHR_MTX_QLOCK_TYPE__ qlck; - ethr_ts_event *q; + ethr_atomic32_t flgs; short aux_scnt; short main_scnt; + ETHR_MTX_QLOCK_TYPE__ qlck; + ethr_ts_event *q; #ifdef ETHR_MTX_HARD_DEBUG_WSQ int ws; #endif +#ifdef ETHR_MTX_CHK_EXCL + ethr_atomic32_t exclusive; +#endif +#ifdef ETHR_MTX_CHK_NON_EXCL + ethr_atomic32_t non_exclusive; +#endif #ifdef ETHR_MTX_HARD_DEBUG_LFS - ethr_atomic_t hdbg_lfs; + ethr_atomic32_t hdbg_lfs; #endif }; @@ -201,7 +236,7 @@ typedef struct { typedef union { struct { - ethr_atomic_t readers; + ethr_atomic32_t readers; int waiting_readers; int byte_offset; ethr_rwmutex_lived lived; @@ -263,13 +298,13 @@ void ethr_rwmutex_rwunlock(ethr_rwmutex *); #ifdef ETHR_MTX_HARD_DEBUG_LFS # define ETHR_MTX_HARD_DEBUG_LFS_INIT(MTXB) \ do { \ - ethr_atomic_init(&(MTXB)->hdbg_lfs, 0); \ + ethr_atomic32_init(&(MTXB)->hdbg_lfs, 0); \ } while (0) # define ETHR_MTX_HARD_DEBUG_LFS_RLOCK(MTXB) \ do { \ - long val__; \ + ethr_sint32_t val__; \ ETHR_COMPILER_BARRIER; \ - val__ = ethr_atomic_inc_read(&(MTXB)->hdbg_lfs); \ + val__ = ethr_atomic32_inc_read(&(MTXB)->hdbg_lfs); \ ETHR_MTX_HARD_ASSERT(val__ > 0); \ } while (0) # define ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(MTXB, RES) \ @@ -282,15 +317,15 @@ do { \ } while (0) # define ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(MTXB) \ do { \ - long val__ = ethr_atomic_dec_read(&(MTXB)->hdbg_lfs); \ + ethr_sint32_t val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs); \ ETHR_MTX_HARD_ASSERT(val__ >= 0); \ ETHR_COMPILER_BARRIER; \ } while (0) # define ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(MTXB) \ do { \ - long val__; \ + ethr_sint32_t val__; \ ETHR_COMPILER_BARRIER; \ - val__ = ethr_atomic_dec_read(&(MTXB)->hdbg_lfs); \ + val__ = ethr_atomic32_dec_read(&(MTXB)->hdbg_lfs); \ ETHR_MTX_HARD_ASSERT(val__ == -1); \ } while (0) # define ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(MTXB, RES) \ @@ -303,7 +338,7 @@ do { \ } while (0) # define ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(MTXB) \ do { \ - long val__ = ethr_atomic_inctest(&(MTXB)->hdbg_lfs); \ + ethr_sint32_t val__ = ethr_atomic32_inctest(&(MTXB)->hdbg_lfs); \ ETHR_MTX_HARD_ASSERT(val__ == 0); \ ETHR_COMPILER_BARRIER; \ } while (0) @@ -344,6 +379,116 @@ do { \ #define ETHR_MTX_HARD_DEBUG_FENCE_INIT(X) #endif +#ifdef ETHR_MTX_CHK_EXCL + +#if !defined(ETHR_DEBUG) && defined(__GNUC__) +#warning "check exclusive is enabled" +#endif + +# define ETHR_MTX_CHK_EXCL_INIT__(MTXB) \ + ethr_atomic32_init(&(MTXB)->exclusive, 0) + +# define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if (!ethr_atomic32_read(&(MTXB)->exclusive)) \ + ethr_assert_failed(__FILE__, __LINE__, __func__,\ + "is exclusive"); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if (ethr_atomic32_read(&(MTXB)->exclusive)) \ + ethr_assert_failed(__FILE__, __LINE__, __func__,\ + "is not exclusive"); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB) \ +do { \ + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL((MTXB)); \ + ethr_atomic32_set(&(MTXB)->exclusive, 1); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB) \ +do { \ + ETHR_MTX_CHK_EXCL_IS_EXCL((MTXB)); \ + ethr_atomic32_set(&(MTXB)->exclusive, 0); \ + ETHR_COMPILER_BARRIER; \ +} while (0) + +#ifdef ETHR_MTX_CHK_NON_EXCL + +#if !defined(ETHR_DEBUG) && defined(__GNUC__) +#warning "check non-exclusive is enabled" +#endif + +# define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB) \ + ethr_atomic32_init(&(MTXB)->non_exclusive, 0) +# define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if (!ethr_atomic32_read(&(MTXB)->non_exclusive)) \ + ethr_assert_failed(__FILE__, __LINE__, __func__,\ + "is non-exclusive"); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + if (ethr_atomic32_read(&(MTXB)->non_exclusive)) \ + ethr_assert_failed(__FILE__, __LINE__, __func__,\ + "is not non-exclusive"); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + ethr_atomic32_inc(&(MTXB)->non_exclusive); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO) \ +do { \ + ETHR_COMPILER_BARRIER; \ + ethr_atomic32_add(&(MTXB)->non_exclusive, (NO)); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +# define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB) \ +do { \ + ETHR_COMPILER_BARRIER; \ + ethr_atomic32_dec(&(MTXB)->non_exclusive); \ + ETHR_COMPILER_BARRIER; \ +} while (0) +#else +# define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB) +#endif + +#else +# define ETHR_MTX_CHK_EXCL_INIT__(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_SET_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_UNSET_EXCL(MTXB) +# define ETHR_MTX_CHK_NON_EXCL_INIT__(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL_NO(MTXB, NO) +# define ETHR_MTX_CHK_EXCL_SET_NON_EXCL(MTXB) +# define ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(MTXB) +#endif + +# define ETHR_MTX_CHK_EXCL_INIT(MTXB) \ +do { \ + ETHR_MTX_CHK_EXCL_INIT__((MTXB)); \ + ETHR_MTX_CHK_NON_EXCL_INIT__((MTXB)); \ +} while (0) + + #ifdef ETHR_USE_OWN_MTX_IMPL__ #define ETHR_MTX_DEFAULT_MAIN_SPINCOUNT_MAX 2000 @@ -356,21 +501,28 @@ do { \ #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_MUTEX_IMPL__) -void ethr_mutex_lock_wait__(ethr_mutex *, long); -void ethr_mutex_unlock_wake__(ethr_mutex *, long); +void ethr_mutex_lock_wait__(ethr_mutex *, ethr_sint32_t); +void ethr_mutex_unlock_wake__(ethr_mutex *, ethr_sint32_t); static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) { - long act; + ethr_sint32_t act; int res; ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); - act = ethr_atomic_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); + act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); res = (act == 0) ? 0 : EBUSY; +#ifdef ETHR_MTX_CHK_EXCL + if (res == 0) + ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb); +#endif + ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&mtx->mtxb, res); ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); ETHR_COMPILER_BARRIER; return res; @@ -379,15 +531,19 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_trylock)(ethr_mutex *mtx) static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) { - long act; + ethr_sint32_t act; ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); - act = ethr_atomic_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); + act = ethr_atomic32_cmpxchg_acqb(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__, 0); if (act != 0) ethr_mutex_lock_wait__(mtx, act); + ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb); + ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb); ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); ETHR_COMPILER_BARRIER; } @@ -395,16 +551,20 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_lock)(ethr_mutex *mtx) static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) { - long act; + ethr_sint32_t act; ETHR_COMPILER_BARRIER; ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&mtx->mtxb); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); + + ETHR_MTX_CHK_EXCL_UNSET_EXCL(&mtx->mtxb); - act = ethr_atomic_cmpxchg_relb(&mtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__); + act = ethr_atomic32_cmpxchg_relb(&mtx->mtxb.flgs, 0, ETHR_RWMTX_W_FLG__); if (act != ETHR_RWMTX_W_FLG__) ethr_mutex_unlock_wake__(mtx, act); ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(mtx); } #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/ethr_optimized_fallbacks.h b/erts/include/internal/ethr_optimized_fallbacks.h index 2f9f987d0b..8e04692856 100644 --- a/erts/include/internal/ethr_optimized_fallbacks.h +++ b/erts/include/internal/ethr_optimized_fallbacks.h @@ -71,36 +71,46 @@ ethr_opt_spin_lock(ethr_opt_spinlock_t *lock) #define ETHR_HAVE_NATIVE_SPINLOCKS 1 #define ETHR_HAVE_OPTIMIZED_SPINLOCKS 1 -typedef ethr_native_atomic_t ethr_native_spinlock_t; +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +typedef ethr_native_atomic32_t ethr_native_spinlock_t; +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +typedef ethr_native_atomic64_t ethr_native_spinlock_t; +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#else +# error "Missing native atomic implementation" +#endif #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) static ETHR_INLINE void ethr_native_spinlock_init(ethr_native_spinlock_t *lock) { - ethr_native_atomic_init((ethr_native_atomic_t *) lock, 0); + ETHR_NATMC_FUNC__(init)(lock, 0); } static ETHR_INLINE void ethr_native_spin_unlock(ethr_native_spinlock_t *lock) { ETHR_COMPILER_BARRIER; - ETHR_ASSERT(ethr_native_atomic_read((ethr_native_atomic_t *) lock) == 1); - ethr_native_atomic_set_relb((ethr_native_atomic_t *) lock, 0); + ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == 1); + ETHR_NATMC_FUNC__(set_relb)(lock, 0); } static ETHR_INLINE void ethr_native_spin_lock(ethr_native_spinlock_t *lock) { - while (ethr_native_atomic_cmpxchg_acqb((ethr_native_atomic_t *) lock, - (long) 1, (long) 0) != 0) { - ETHR_SPIN_BODY; + while (ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, 1, 0) != 0) { + while (ETHR_NATMC_FUNC__(read)(lock) != 0) + ETHR_SPIN_BODY; } ETHR_COMPILER_BARRIER; } #endif +#undef ETHR_NATMC_FUNC__ + #endif @@ -111,16 +121,26 @@ ethr_native_spin_lock(ethr_native_spinlock_t *lock) #define ETHR_HAVE_NATIVE_RWSPINLOCKS 1 #define ETHR_HAVE_OPTIMIZED_RWSPINLOCKS 1 -typedef ethr_native_atomic_t ethr_native_rwlock_t; +#if defined(ETHR_HAVE_NATIVE_ATOMIC32) +typedef ethr_native_atomic32_t ethr_native_rwlock_t; +# define ETHR_NAINT_T__ ethr_sint32_t +# define ETHR_WLOCK_FLAG__ (((ethr_sint32_t) 1) << 30) +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#elif defined(ETHR_HAVE_NATIVE_ATOMIC64) +typedef ethr_native_atomic64_t ethr_native_rwlock_t; +# define ETHR_NAINT_T__ ethr_sint64_t +# define ETHR_WLOCK_FLAG__ (((ethr_sint64_t) 1) << 62) +# define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#else +# error "Missing native atomic implementation" +#endif #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) -#define ETHR_WLOCK_FLAG__ (((long) 1) << 30) - static ETHR_INLINE void ethr_native_rwlock_init(ethr_native_rwlock_t *lock) { - ethr_native_atomic_init((ethr_native_atomic_t *) lock, 0); + ETHR_NATMC_FUNC__(init)(lock, 0); } static ETHR_INLINE void @@ -128,22 +148,24 @@ ethr_native_read_unlock(ethr_native_rwlock_t *lock) { ETHR_COMPILER_BARRIER; #ifdef DEBUG - ETHR_ASSERT(ethr_native_atomic_read((ethr_native_atomic_t *) lock) >= 0); + ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) >= 0); #endif - ethr_native_atomic_dec_relb((ethr_native_atomic_t *) lock); + ETHR_NATMC_FUNC__(dec_relb)(lock); } static ETHR_INLINE void ethr_native_read_lock(ethr_native_rwlock_t *lock) { - long act, exp = 0; + ETHR_NAINT_T__ act, exp = 0; while (1) { - act = ethr_native_atomic_cmpxchg_acqb((ethr_native_atomic_t *) lock, - exp+1, exp); + act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp+1, exp); if (act == exp) break; - ETHR_SPIN_BODY; - exp = (act & ETHR_WLOCK_FLAG__) ? 0 : act; + while (act & ETHR_WLOCK_FLAG__) { + ETHR_SPIN_BODY; + act = ETHR_NATMC_FUNC__(read)(lock); + } + exp = act; } ETHR_COMPILER_BARRIER; } @@ -152,18 +174,16 @@ static ETHR_INLINE void ethr_native_write_unlock(ethr_native_rwlock_t *lock) { ETHR_COMPILER_BARRIER; - ETHR_ASSERT(ethr_native_atomic_read((ethr_native_atomic_t *) lock) - == ETHR_WLOCK_FLAG__); - ethr_native_atomic_set_relb((ethr_native_atomic_t *) lock, 0); + ETHR_ASSERT(ETHR_NATMC_FUNC__(read)(lock) == ETHR_WLOCK_FLAG__); + ETHR_NATMC_FUNC__(set_relb)(lock, 0); } static ETHR_INLINE void ethr_native_write_lock(ethr_native_rwlock_t *lock) { - long act, exp = 0; + ETHR_NAINT_T__ act, exp = 0; while (1) { - act = ethr_native_atomic_cmpxchg_acqb((ethr_native_atomic_t *) lock, - exp|ETHR_WLOCK_FLAG__, exp); + act = ETHR_NATMC_FUNC__(cmpxchg_acqb)(lock, exp|ETHR_WLOCK_FLAG__, exp); if (act == exp) break; ETHR_SPIN_BODY; @@ -173,13 +193,17 @@ ethr_native_write_lock(ethr_native_rwlock_t *lock) /* Wait for readers to leave */ while (act != ETHR_WLOCK_FLAG__) { ETHR_SPIN_BODY; - act = ethr_native_atomic_read_acqb((ethr_native_atomic_t *) lock); + act = ETHR_NATMC_FUNC__(read_acqb)(lock); } ETHR_COMPILER_BARRIER; } #endif +#undef ETHR_NAINT_T__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_WLOCK_FLAG__ + #endif #endif diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 4a205699bd..4cd95faf6a 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -37,11 +37,6 @@ #undef ETHR_HAVE_OPTIMIZED_SPINLOCK #undef ETHR_HAVE_OPTIMIZED_RWSPINLOCK -typedef struct { - long tv_sec; - long tv_nsec; -} ethr_timeval; - #if defined(DEBUG) # define ETHR_DEBUG #endif @@ -73,7 +68,7 @@ typedef struct { #endif /* Assume 64-byte cache line size */ -#define ETHR_CACHE_LINE_SIZE 64L +#define ETHR_CACHE_LINE_SIZE ((ethr_uint_t) 64) #define ETHR_CACHE_LINE_MASK (ETHR_CACHE_LINE_SIZE - 1) #define ETHR_CACHE_LINE_ALIGN_SIZE(SZ) \ @@ -171,6 +166,22 @@ typedef pthread_key_t ethr_tsd_key; # undef WIN32_LEAN_AND_MEAN #endif +#if defined(_MSC_VER) + +#if ETHR_SIZEOF_LONG == 4 +#define ETHR_HAVE_INT32_T 1 +typedef long ethr_sint32_t; +typedef unsigned long ethr_uint32_t; +#endif + +#if ETHR_SIZEOF___INT64 == 8 +#define ETHR_HAVE_INT64_T 1 +typedef __int64 ethr_sint64_t; +typedef unsigned __int64 ethr_uint64_t; +#endif + +#endif + struct ethr_join_data_; /* Types */ @@ -198,12 +209,48 @@ typedef DWORD ethr_tsd_key; #endif -#ifdef SIZEOF_LONG -#if SIZEOF_LONG < ETHR_SIZEOF_PTR -#error size of long currently needs to be at least the same as size of void * +#ifndef ETHR_HAVE_INT32_T +#if ETHR_SIZEOF_INT == 4 +#define ETHR_HAVE_INT32_T 1 +typedef int ethr_sint32_t; +typedef unsigned int ethr_uint32_t; +#elif ETHR_SIZEOF_LONG == 4 +#define ETHR_HAVE_INT32_T 1 +typedef long ethr_sint32_t; +typedef unsigned long ethr_uint32_t; #endif #endif +#ifndef ETHR_HAVE_INT64_T +#if ETHR_SIZEOF_INT == 8 +#define ETHR_HAVE_INT64_T 1 +typedef int ethr_sint64_t; +typedef unsigned int ethr_uint64_t; +#elif ETHR_SIZEOF_LONG == 8 +#define ETHR_HAVE_INT64_T 1 +typedef long ethr_sint64_t; +typedef unsigned long ethr_uint64_t; +#elif ETHR_SIZEOF_LONG_LONG == 8 +#define ETHR_HAVE_INT64_T 1 +typedef long long ethr_sint64_t; +typedef unsigned long long ethr_uint64_t; +#endif +#endif + +#if ETHR_SIZEOF_PTR == 4 +#ifndef ETHR_HAVE_INT32_T +#error "No 32-bit integer type found" +#endif +typedef ethr_sint32_t ethr_sint_t; +typedef ethr_uint32_t ethr_uint_t; +#elif ETHR_SIZEOF_PTR == 8 +#ifndef ETHR_HAVE_INT64_T +#error "No 64-bit integer type found" +#endif +typedef ethr_sint64_t ethr_sint_t; +typedef ethr_uint64_t ethr_uint_t; +#endif + /* __builtin_expect() is needed by both native atomics code * and the fallback code */ #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) @@ -239,6 +286,8 @@ typedef DWORD ethr_tsd_key; # include "gcc/ethread.h" # include "libatomic_ops/ethread.h" # endif +# elif defined(ETHR_HAVE_LIBATOMIC_OPS) +# include "libatomic_ops/ethread.h" # elif defined(ETHR_WIN32_THREADS) # include "win/ethread.h" # endif @@ -384,7 +433,6 @@ typedef struct { #if !defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) # define ETHR_NEED_SPINLOCK_PROTOTYPES__ # define ETHR_NEED_RWSPINLOCK_PROTOTYPES__ -# define ETHR_NEED_ATOMIC_PROTOTYPES__ #endif int ethr_init(ethr_init_data *); @@ -397,7 +445,6 @@ void ethr_thr_exit(void *); ethr_tid ethr_self(void); int ethr_equal_tids(ethr_tid, ethr_tid); -int ethr_time_now(ethr_timeval *); int ethr_tsd_key_create(ethr_tsd_key *); int ethr_tsd_key_delete(ethr_tsd_key); int ethr_tsd_set(ethr_tsd_key, void *); @@ -500,312 +547,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_spin_lock)(ethr_spinlock_t *lock) #endif /* ETHR_TRY_INLINE_FUNCS */ -#ifdef ETHR_HAVE_NATIVE_ATOMICS -/* - * Map ethread native atomics to ethread API atomics. - */ -typedef ethr_native_atomic_t ethr_atomic_t; -#else -typedef long ethr_atomic_t; -#endif - -#ifdef ETHR_NEED_ATOMIC_PROTOTYPES__ -void ethr_atomic_init(ethr_atomic_t *, long); -void ethr_atomic_set(ethr_atomic_t *, long); -long ethr_atomic_read(ethr_atomic_t *); -long ethr_atomic_inc_read(ethr_atomic_t *); -long ethr_atomic_dec_read(ethr_atomic_t *); -void ethr_atomic_inc(ethr_atomic_t *); -void ethr_atomic_dec(ethr_atomic_t *); -long ethr_atomic_add_read(ethr_atomic_t *, long); -void ethr_atomic_add(ethr_atomic_t *, long); -long ethr_atomic_read_band(ethr_atomic_t *, long); -long ethr_atomic_read_bor(ethr_atomic_t *, long); -long ethr_atomic_xchg(ethr_atomic_t *, long); -long ethr_atomic_cmpxchg(ethr_atomic_t *, long, long); -long ethr_atomic_read_acqb(ethr_atomic_t *); -long ethr_atomic_inc_read_acqb(ethr_atomic_t *); -void ethr_atomic_set_relb(ethr_atomic_t *, long); -void ethr_atomic_dec_relb(ethr_atomic_t *); -long ethr_atomic_dec_read_relb(ethr_atomic_t *); -long ethr_atomic_cmpxchg_acqb(ethr_atomic_t *, long, long); -long ethr_atomic_cmpxchg_relb(ethr_atomic_t *, long, long); -#endif - -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) - -#ifndef ETHR_HAVE_NATIVE_ATOMICS -/* - * Fallbacks for atomics used in absence of a native implementation. - */ - -#define ETHR_ATOMIC_ADDR_BITS 10 -#define ETHR_ATOMIC_ADDR_SHIFT 6 - -typedef struct { - union { - ethr_spinlock_t lck; - char buf[ETHR_CACHE_LINE_SIZE]; - } u; -} ethr_atomic_protection_t; - -extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; - -#define ETHR_ATOMIC_PTR2LCK__(PTR) \ -(ðr_atomic_protection__[((((unsigned long) (PTR)) >> ETHR_ATOMIC_ADDR_SHIFT) \ - & ((1 << ETHR_ATOMIC_ADDR_BITS) - 1))].u.lck) - - -#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \ -do { \ - ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \ - ethr_spin_lock(slp__); \ - { EXPS; } \ - ethr_spin_unlock(slp__); \ -} while (0) - -#endif - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_init)(ethr_atomic_t *var, long i) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_init(var, i); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(ethr_atomic_t *var, long i) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_set(var, i); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = i); -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_read(var); -#else - long res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (long) *var); - return res; -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add)(ethr_atomic_t *var, long incr) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_add(var, incr); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += incr); -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_add_read)(ethr_atomic_t *var, long i) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_add_return(var, i); -#else - long res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += i; res = *var); - return res; -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_inc(var); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var)); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_dec(var); -#else - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var)); -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_inc_return(var); -#else - long res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (long) ++(*var)); - return res; -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_dec_return(var); -#else - long res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (long) --(*var)); - return res; -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_band)(ethr_atomic_t *var, - long mask) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_and_retold(var, mask); -#else - long res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= mask); - return res; -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_bor)(ethr_atomic_t *var, - long mask) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_or_retold(var, mask); -#else - long res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= mask); - return res; -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_xchg)(ethr_atomic_t *var, - long new) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_xchg(var, new); -#else - long res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = new); - return res; -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(ethr_atomic_t *var, - long new, - long exp) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_cmpxchg(var, new, exp); -#else - long res; - ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, - { - res = *var; - if (__builtin_expect(res == exp, 1)) - *var = new; - }); - return res; -#endif -} - -/* - * Important memory barrier requirements. - * - * The following atomic operations *must* supply a memory barrier of - * at least the type specified by its suffix: - * _acqb = acquire barrier - * _relb = release barrier - */ - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_read_acqb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_read_acqb(var); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_read)(var); -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read_acqb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_inc_return_acqb(var); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_inc_read)(var); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_set_relb)(ethr_atomic_t *var, long val) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_set_relb(var, val); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_set)(var, val); -#endif -} - -static ETHR_INLINE void -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_relb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - ethr_native_atomic_dec_relb(var); -#else - ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec)(var); -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read_relb)(ethr_atomic_t *var) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_dec_return_relb(var); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_dec_read)(var); -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_acqb)(ethr_atomic_t *var, - long new, - long exp) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_cmpxchg_acqb(var, new, exp); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); -#endif -} - -static ETHR_INLINE long -ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg_relb)(ethr_atomic_t *var, - long new, - long exp) -{ -#ifdef ETHR_HAVE_NATIVE_ATOMICS - return ethr_native_atomic_cmpxchg_relb(var, new, exp); -#else - return ETHR_INLINE_FUNC_NAME_(ethr_atomic_cmpxchg)(var, new, exp); -#endif -} - -#endif /* ETHR_TRY_INLINE_FUNCS */ +#include "ethr_atomics.h" typedef struct ethr_ts_event_ ethr_ts_event; /* Needed by ethr_mutex.h */ @@ -823,7 +565,7 @@ struct ethr_ts_event_ { ethr_ts_event *prev; ethr_event event; void *udata; - ethr_atomic_t uaflgs; + ethr_atomic32_t uaflgs; unsigned uflgs; unsigned iflgs; /* for ethr lib only */ short rgix; /* for ethr lib only */ diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index 5debb44756..f394d790d2 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -20,6 +20,21 @@ /* Define to the size of pointers */ #undef ETHR_SIZEOF_PTR +/* Define to the size of int */ +#undef ETHR_SIZEOF_INT + +/* Define to the size of long */ +#undef ETHR_SIZEOF_LONG + +/* Define to the size of long long */ +#undef ETHR_SIZEOF_LONG_LONG + +/* Define to the size of __int64 */ +#undef ETHR_SIZEOF___INT64 + +/* Define if bigendian */ +#undef ETHR_BIGENDIAN + /* Define if you want to disable native ethread implementations */ #undef ETHR_DISABLE_NATIVE_IMPLS @@ -100,6 +115,27 @@ /* Define to the size of AO_t if libatomic_ops is used */ #undef ETHR_SIZEOF_AO_T +/* Define if you have _InterlockedCompareExchange64() */ +#undef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 + +/* Define if you have _InterlockedDecrement64() */ +#undef ETHR_HAVE__INTERLOCKEDDECREMENT64 + +/* Define if you have _InterlockedIncrement64() */ +#undef ETHR_HAVE__INTERLOCKEDINCREMENT64 + +/* Define if you have _InterlockedExchangeAdd64() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 + +/* Define if you have _InterlockedExchange64() */ +#undef ETHR_HAVE__INTERLOCKEDEXCHANGE64 + +/* Define if you have _InterlockedAnd64() */ +#undef ETHR_HAVE__INTERLOCKEDAND64 + +/* Define if you have _InterlockedOr64() */ +#undef ETHR_HAVE__INTERLOCKEDOR64 + /* Define if you want to turn on extra sanity checking in the ethread library */ #undef ETHR_XCHK diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h index e8e529dd48..16935084b1 100644 --- a/erts/include/internal/gcc/ethr_atomic.h +++ b/erts/include/internal/gcc/ethr_atomic.h @@ -22,24 +22,35 @@ * Author: Rickard Green */ -#ifndef ETHR_GCC_ATOMIC_H__ -#define ETHR_GCC_ATOMIC_H__ +#undef ETHR_INCLUDE_ATOMIC_IMPL__ +#if !defined(ETHR_GCC_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +#define ETHR_GCC_ATOMIC32_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_GCC_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +#define ETHR_GCC_ATOMIC64_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#endif + +#ifdef ETHR_INCLUDE_ATOMIC_IMPL__ -#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_GCC_ATOMIC_OPS) -#define ETHR_HAVE_NATIVE_ATOMICS 1 +#ifndef ETHR_GCC_ATOMIC_COMMON__ +#define ETHR_GCC_ATOMIC_COMMON__ -#define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 -/* Enable immediate read/write on platforms where we know it is safe */ +#define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 0 #if defined(__i386__) || defined(__x86_64__) || defined(__sparc__) \ || defined(__powerpc__) || defined(__ppc__) || defined(__mips__) -# undef ETHR_IMMED_ATOMIC_SET_GET_SAFE__ -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 +# undef ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ +# define ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ 1 #endif -typedef struct { - volatile long counter; -} ethr_native_atomic_t; - +#if defined(__x86_64__) || (defined(__i386__) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 +#else +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 +#endif /* * According to the documentation this is what we want: @@ -47,34 +58,73 @@ typedef struct { * However, __sync_synchronize() is known to erroneously be * a noop on at least some platforms with some gcc versions. * This has suposedly been fixed in some gcc version, but we - * don't know from which version. Therefore, we use the - * workaround implemented below on all gcc versions except - * for gcc 4.2 or above for MIPS, where it's been verified. + * don't know from which version. Therefore, we only use + * it when it has been verified to work. Otherwise + * we use a workaround. */ #if defined(__mips__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2)) +/* __sync_synchronize() has been verified to work here */ #define ETHR_MEMORY_BARRIER __sync_synchronize() +#define ETHR_READ_DEPEND_MEMORY_BARRIER __sync_synchronize() +#elif defined(__x86_64__) || (defined(__i386__) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +/* Use fence instructions directly instead of workaround */ +#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") +#define ETHR_WRITE_MEMORY_BARRIER __asm__ __volatile__("sfence" : : : "memory") +#define ETHR_READ_MEMORY_BARRIER __asm__ __volatile__("lfence" : : : "memory") +#define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("" : : : "memory") #else +/* Workaround */ #define ETHR_MEMORY_BARRIER \ do { \ - volatile long x___ = 0; \ - (void) __sync_val_compare_and_swap(&x___, (long) 0, (long) 1); \ + volatile ethr_sint32_t x___ = 0; \ + (void) __sync_val_compare_and_swap(&x___, (ethr_sint32_t) 0, (ethr_sint32_t) 1); \ } while (0) -#endif #define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#endif + +#define ETHR_COMPILER_BARRIER __asm__ __volatile__("" : : : "memory") -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#endif /* ETHR_GCC_ATOMIC_COMMON__ */ + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +#define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t +#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t +#else +#error "Unsupported integer size" +#endif + +typedef struct { + volatile ETHR_AINT_T__ counter; +} ETHR_ATMC_T__; + + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->counter; +} static ETHR_INLINE void -ethr_native_atomic_set(ethr_native_atomic_t *var, long value) +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ +#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ var->counter = value; #else /* * Unfortunately no __sync_store() or similar exist in the gcc atomic * op interface. We therefore have to simulate it this way... */ - long act = 0, exp; + ETHR_AINT_T__ act = 0, exp; do { exp = act; act = __sync_val_compare_and_swap(&var->counter, exp, value); @@ -82,80 +132,86 @@ ethr_native_atomic_set(ethr_native_atomic_t *var, long value) #endif } -#define ethr_native_atomic_init ethr_native_atomic_set +static ETHR_INLINE void +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +{ + ETHR_NATMC_FUNC__(set)(var, value); +} -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ +#if ETHR_READ_AND_SET_WITHOUT_SYNC_OP__ return var->counter; #else /* * Unfortunately no __sync_fetch() or similar exist in the gcc atomic * op interface. We therefore have to simulate it this way... */ - return __sync_add_and_fetch(&var->counter, (long) 0); + return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0); #endif } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { (void) __sync_add_and_fetch(&var->counter, incr); } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { return __sync_add_and_fetch(&var->counter, incr); } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) { - (void) __sync_add_and_fetch(&var->counter, (long) 1); + (void) __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1); } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) { - (void) __sync_sub_and_fetch(&var->counter, (long) 1); + (void) __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1); } -static ETHR_INLINE long -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { - return __sync_add_and_fetch(&var->counter, (long) 1); + return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 1); } -static ETHR_INLINE long -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { - return __sync_sub_and_fetch(&var->counter, (long) 1); + return __sync_sub_and_fetch(&var->counter, (ETHR_AINT_T__) 1); } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { return __sync_fetch_and_and(&var->counter, mask); } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - return (long) __sync_fetch_and_or(&var->counter, mask); + return (ETHR_AINT_T__) __sync_fetch_and_or(&var->counter, mask); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { return __sync_val_compare_and_swap(&var->counter, old, new); } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { - long exp, act = 0; + ETHR_AINT_T__ exp, act = 0; do { exp = act; act = __sync_val_compare_and_swap(&var->counter, exp, new); @@ -167,22 +223,68 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) * Atomic ops with at least specified barriers. */ -static ETHR_INLINE long -ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) { - return __sync_add_and_fetch(&var->counter, (long) 0); +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_AINT_T__ val = var->counter; + ETHR_COMPILER_BARRIER; + return val; +#else + return __sync_add_and_fetch(&var->counter, (ETHR_AINT_T__) 0); +#endif } -#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return -#define ethr_native_atomic_set_relb ethr_native_atomic_xchg -#define ethr_native_atomic_dec_relb ethr_native_atomic_dec_return -#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_COMPILER_BARRIER; + var->counter = i; +#else + (void) ETHR_NATMC_FUNC__(xchg)(var, i); +#endif +} -#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg -#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +{ + return ETHR_NATMC_FUNC__(inc_return)(var); +} -#endif +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +{ + ETHR_NATMC_FUNC__(dec)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) +{ + return ETHR_NATMC_FUNC__(dec_return)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} #endif +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ +#undef ETHR_AINT_SUFFIX__ + #endif diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h index bb378e31e0..392a1aa2b2 100644 --- a/erts/include/internal/gcc/ethread.h +++ b/erts/include/internal/gcc/ethread.h @@ -25,6 +25,16 @@ #ifndef ETHREAD_GCC_H__ #define ETHREAD_GCC_H__ +#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_GCC_ATOMIC_OPS) +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "ethr_atomic.h" +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_ATOMIC_WANT_64BIT_IMPL__ +# include "ethr_atomic.h" +#endif + +#endif #endif diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h index f28258059f..4e402f261a 100644 --- a/erts/include/internal/i386/atomic.h +++ b/erts/include/internal/i386/atomic.h @@ -23,14 +23,24 @@ * * This code requires a 486 or newer processor. */ -#ifndef ETHREAD_I386_ATOMIC_H -#define ETHREAD_I386_ATOMIC_H -/* An atomic is an aligned long accessed via locked operations. - */ -typedef struct { - volatile long counter; -} ethr_native_atomic_t; +#undef ETHR_INCLUDE_ATOMIC_IMPL__ +#if !defined(ETHR_X86_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +#define ETHR_X86_ATOMIC32_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_X86_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +#define ETHR_X86_ATOMIC64_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#endif + +#ifdef ETHR_INCLUDE_ATOMIC_IMPL__ + +#ifndef ETHR_X86_ATOMIC_COMMON__ +#define ETHR_X86_ATOMIC_COMMON__ + +#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 #if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) #define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") @@ -40,123 +50,161 @@ typedef struct { #else #define ETHR_MEMORY_BARRIER \ do { \ - volatile long x___ = 0; \ + volatile ethr_sint32_t x___ = 0; \ __asm__ __volatile__("lock; incl %0" : "=m"(x___) : "m"(x___) : "memory"); \ } while (0) #endif -#define ETHR_ATOMIC_HAVE_INC_DEC_INSTRUCTIONS 1 - -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#endif /* ETHR_X86_ATOMIC_COMMON__ */ -#ifdef __x86_64__ -#define LONG_SUFFIX "q" +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +#define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t +#define ETHR_AINT_SUFFIX__ "l" +#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t +#define ETHR_AINT_SUFFIX__ "q" #else -#define LONG_SUFFIX "l" +#error "Unsupported integer size" #endif +/* An atomic is an aligned ETHR_AINT_T__ accessed via locked operations. + */ +typedef struct { + volatile ETHR_AINT_T__ counter; +} ETHR_ATMC_T__; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->counter; +} + static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { var->counter = i; } -#define ethr_native_atomic_set(v, i) ethr_native_atomic_init((v), (i)) -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + var->counter = i; +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { return var->counter; } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { __asm__ __volatile__( - "lock; add" LONG_SUFFIX " %1, %0" + "lock; add" ETHR_AINT_SUFFIX__ " %1, %0" : "=m"(var->counter) : "ir"(incr), "m"(var->counter)); } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) { __asm__ __volatile__( - "lock; inc" LONG_SUFFIX " %0" + "lock; inc" ETHR_AINT_SUFFIX__ " %0" : "=m"(var->counter) : "m"(var->counter)); } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) { __asm__ __volatile__( - "lock; dec" LONG_SUFFIX " %0" + "lock; dec" ETHR_AINT_SUFFIX__ " %0" : "=m"(var->counter) : "m"(var->counter)); } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - long tmp; + ETHR_AINT_T__ tmp; tmp = incr; __asm__ __volatile__( - "lock; xadd" LONG_SUFFIX " %0, %1" /* xadd didn't exist prior to the 486 */ + "lock; xadd" ETHR_AINT_SUFFIX__ " %0, %1" /* xadd didn't exist prior to the 486 */ : "=r"(tmp) : "m"(var->counter), "0"(tmp)); /* now tmp is the atomic's previous value */ return tmp + incr; } -#define ethr_native_atomic_inc_return(var) ethr_native_atomic_add_return((var), 1) -#define ethr_native_atomic_dec_return(var) ethr_native_atomic_add_return((var), -1) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) +{ + return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) 1); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) +{ + return ETHR_NATMC_FUNC__(add_return)(var, (ETHR_AINT_T__) -1); +} -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { __asm__ __volatile__( - "lock; cmpxchg" LONG_SUFFIX " %2, %3" + "lock; cmpxchg" ETHR_AINT_SUFFIX__ " %2, %3" : "=a"(old), "=m"(var->counter) : "r"(new), "m"(var->counter), "0"(old) : "cc", "memory"); /* full memory clobber to make this a compiler barrier */ return old; } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - long tmp, old; + ETHR_AINT_T__ tmp, old; tmp = var->counter; do { old = tmp; - tmp = ethr_native_atomic_cmpxchg(var, tmp & mask, tmp); + tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp & mask, tmp); } while (__builtin_expect(tmp != old, 0)); /* now tmp is the atomic's previous value */ return tmp; } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - long tmp, old; + ETHR_AINT_T__ tmp, old; tmp = var->counter; do { old = tmp; - tmp = ethr_native_atomic_cmpxchg(var, tmp | mask, tmp); + tmp = ETHR_NATMC_FUNC__(cmpxchg)(var, tmp | mask, tmp); } while (__builtin_expect(tmp != old, 0)); /* now tmp is the atomic's previous value */ return tmp; } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) { - long tmp = val; + ETHR_AINT_T__ tmp = val; __asm__ __volatile__( - "xchg" LONG_SUFFIX " %0, %1" + "xchg" ETHR_AINT_SUFFIX__ " %0, %1" : "=r"(tmp) : "m"(var->counter), "0"(tmp)); /* now tmp is the atomic's previous value */ @@ -167,20 +215,73 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) * Atomic ops with at least specified barriers. */ -#define ethr_native_atomic_read_acqb ethr_native_atomic_read -#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) +{ + ETHR_AINT_T__ val; #if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) -#define ethr_native_atomic_set_relb ethr_native_atomic_set + val = var->counter; #else -#define ethr_native_atomic_set_relb ethr_native_atomic_xchg + val = ETHR_NATMC_FUNC__(add_return)(var, 0); #endif -#define ethr_native_atomic_dec_relb ethr_native_atomic_dec -#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return -#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg -#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg + __asm__ __volatile__("" : : : "memory"); + return val; +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + __asm__ __volatile__("" : : : "memory"); +#if defined(__x86_64__) || !defined(ETHR_PRE_PENTIUM4_COMPAT) + var->counter = i; +#else + (void) ETHR_NATMC_FUNC__(xchg)(var, i); +#endif +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +{ + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); + __asm__ __volatile__("" : : : "memory"); + return res; +} -#undef LONG_SUFFIX +static ETHR_INLINE void +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) +{ + __asm__ __volatile__("" : : : "memory"); + ETHR_NATMC_FUNC__(dec)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) +{ + __asm__ __volatile__("" : : : "memory"); + return ETHR_NATMC_FUNC__(dec_return)(var); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) +{ + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} #endif /* ETHR_TRY_INLINE_FUNCS */ -#endif /* ETHREAD_I386_ATOMIC_H */ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ +#undef ETHR_AINT_SUFFIX__ + +#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/i386/ethread.h b/erts/include/internal/i386/ethread.h index ed43e77279..b5a17caefb 100644 --- a/erts/include/internal/i386/ethread.h +++ b/erts/include/internal/i386/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,7 +24,12 @@ #ifndef ETHREAD_I386_ETHREAD_H #define ETHREAD_I386_ETHREAD_H +#define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "atomic.h" +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_ATOMIC_WANT_64BIT_IMPL__ +# include "atomic.h" +#endif #include "spinlock.h" #include "rwlock.h" diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h index a6eb43a0bd..d56693dbf8 100644 --- a/erts/include/internal/libatomic_ops/ethr_atomic.h +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -46,17 +46,39 @@ * - AO_store() * - AO_compare_and_swap() * - * The `AO_t' type also have to be at least as large as - * `void *' and `long' types. + * The `AO_t' type also have to be at least as large as the `void *' type. */ #if ETHR_SIZEOF_AO_T < ETHR_SIZEOF_PTR #error The AO_t type is too small #endif +#if ETHR_SIZEOF_AO_T == 4 +#define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t +#define ETHR_AINT_SUFFIX__ "l" +#elif ETHR_SIZEOF_AO_T == 8 +#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t +#define ETHR_AINT_SUFFIX__ "q" +#else +#error "Unsupported integer size" +#endif + +#if ETHR_SIZEOF_AO_T == 8 +typedef union { + volatile AO_t counter; + ethr_sint32_t sint32[2]; +} ETHR_ATMC_T__; +#else typedef struct { volatile AO_t counter; -} ethr_native_atomic_t; +} ETHR_ATMC_T__; +#endif #define ETHR_MEMORY_BARRIER AO_nop_full() #ifdef AO_HAVE_nop_write @@ -72,123 +94,151 @@ typedef struct { #ifdef AO_NO_DD_ORDERING # define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER #else -# define ETHR_READ_DEPEND_MEMORY_BARRIER __asm__ __volatile__("":::"memory") +# define ETHR_READ_DEPEND_MEMORY_BARRIER AO_compiler_barrier() +#endif + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->counter; +} + +#if ETHR_SIZEOF_AO_T == 8 +/* + * We also need to provide an ethr_native_atomic32_addr(), since + * this 64-bit implementation will be used implementing 32-bit + * native atomics. + */ + +static ETHR_INLINE ethr_sint32_t * +ethr_native_atomic32_addr(ETHR_ATMC_T__ *var) +{ + ETHR_ASSERT(((void *) &var->sint32[0]) == ((void *) &var->counter)); +#ifdef ETHR_BIGENDIAN + return &var->sint32[1]; +#else + return &var->sint32[0]; #endif +} -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#endif /* ETHR_SIZEOF_AO_T == 8 */ static ETHR_INLINE void -ethr_native_atomic_set(ethr_native_atomic_t *var, long value) +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) { AO_store(&var->counter, (AO_t) value); } static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, long value) +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) { - ethr_native_atomic_set(var, value); + ETHR_NATMC_FUNC__(set)(var, value); } -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { - return (long) AO_load(&var->counter); + return (ETHR_AINT_T__) AO_load(&var->counter); } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { #ifdef AO_HAVE_fetch_and_add - return ((long) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr; + return ((ETHR_AINT_T__) AO_fetch_and_add(&var->counter, (AO_t) incr)) + incr; #else while (1) { AO_t exp = AO_load(&var->counter); AO_t new = exp + (AO_t) incr; if (AO_compare_and_swap(&var->counter, exp, new)) - return (long) new; + return (ETHR_AINT_T__) new; } #endif } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - (void) ethr_native_atomic_add_return(var, incr); + (void) ETHR_NATMC_FUNC__(add_return)(var, incr); } -static ETHR_INLINE long -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { #ifdef AO_HAVE_fetch_and_add1 - return ((long) AO_fetch_and_add1(&var->counter)) + 1; + return ((ETHR_AINT_T__) AO_fetch_and_add1(&var->counter)) + 1; #else - return ethr_native_atomic_add_return(var, 1); + return ETHR_NATMC_FUNC__(add_return)(var, 1); #endif } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) { - (void) ethr_native_atomic_inc_return(var); + (void) ETHR_NATMC_FUNC__(inc_return)(var); } -static ETHR_INLINE long -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { #ifdef AO_HAVE_fetch_and_sub1 - return ((long) AO_fetch_and_sub1(&var->counter)) - 1; + return ((ETHR_AINT_T__) AO_fetch_and_sub1(&var->counter)) - 1; #else - return ethr_native_atomic_add_return(var, -1); + return ETHR_NATMC_FUNC__(add_return)(var, -1); #endif } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) { - (void) ethr_native_atomic_dec_return(var); + (void) ETHR_NATMC_FUNC__(dec_return)(var); } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { while (1) { AO_t exp = AO_load(&var->counter); AO_t new = exp & ((AO_t) mask); if (AO_compare_and_swap(&var->counter, exp, new)) - return (long) exp; + return (ETHR_AINT_T__) exp; } } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { while (1) { AO_t exp = AO_load(&var->counter); AO_t new = exp | ((AO_t) mask); if (AO_compare_and_swap(&var->counter, exp, new)) - return (long) exp; + return (ETHR_AINT_T__) exp; } } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long exp) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) { - long act; + ETHR_AINT_T__ act; do { if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new)) return exp; - act = (long) AO_load(&var->counter); + act = (ETHR_AINT_T__) AO_load(&var->counter); } while (act == exp); return act; } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { while (1) { AO_t exp = AO_load(&var->counter); if (AO_compare_and_swap(&var->counter, exp, (AO_t) new)) - return (long) exp; + return (ETHR_AINT_T__) exp; } } @@ -196,97 +246,105 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) * Atomic ops with at least specified barriers. */ -static ETHR_INLINE long -ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) { #ifdef AO_HAVE_load_acquire - return (long) AO_load_acquire(&var->counter); + return (ETHR_AINT_T__) AO_load_acquire(&var->counter); #else - long res = ethr_native_atomic_read(var); + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); ETHR_MEMORY_BARRIER; return res; #endif } -static ETHR_INLINE long -ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { #ifdef AO_HAVE_fetch_and_add1_acquire - return ((long) AO_fetch_and_add1_acquire(&var->counter)) + 1; + return ((ETHR_AINT_T__) AO_fetch_and_add1_acquire(&var->counter)) + 1; #else - long res = ethr_native_atomic_add_return(var, 1); + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(add_return)(var, 1); ETHR_MEMORY_BARRIER; return res; #endif } static ETHR_INLINE void -ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long value) +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) { #ifdef AO_HAVE_store_release AO_store_release(&var->counter, (AO_t) value); #else ETHR_MEMORY_BARRIER; - ethr_native_atomic_set(var, value); + ETHR_NATMC_FUNC__(set)(var, value); #endif } -static ETHR_INLINE long -ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { #ifdef AO_HAVE_fetch_and_sub1_release - return ((long) AO_fetch_and_sub1_release(&var->counter)) - 1; + return ((ETHR_AINT_T__) AO_fetch_and_sub1_release(&var->counter)) - 1; #else ETHR_MEMORY_BARRIER; - return ethr_native_atomic_dec_return(var); + return ETHR_NATMC_FUNC__(dec_return)(var); #endif } static ETHR_INLINE void -ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) { - (void) ethr_native_atomic_dec_return_relb(var); + (void) ETHR_NATMC_FUNC__(dec_return_relb)(var); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long exp) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) { #ifdef AO_HAVE_compare_and_swap_acquire - long act; + ETHR_AINT_T__ act; do { if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new)) return exp; - act = (long) AO_load(&var->counter); + act = (ETHR_AINT_T__) AO_load(&var->counter); } while (act == exp); AO_nop_full(); return act; #else - long act = ethr_native_atomic_cmpxchg(var, new, exp); + ETHR_AINT_T__ act = ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); ETHR_MEMORY_BARRIER; return act; #endif } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long exp) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) { #ifdef AO_HAVE_compare_and_swap_release - long act; + ETHR_AINT_T__ act; do { if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new)) return exp; - act = (long) AO_load(&var->counter); + act = (ETHR_AINT_T__) AO_load(&var->counter); } while (act == exp); return act; #else ETHR_MEMORY_BARRIER; - return ethr_native_atomic_cmpxchg(var, new, exp); + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, exp); #endif } -#endif +#endif /* ETHR_TRY_INLINE_FUNCS */ -#endif +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ -#endif +#endif /* !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_LIBATOMIC_OPS) */ + +#endif /* ETHR_LIBATOMIC_OPS_ATOMIC_H__ */ diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index f21f7c9588..522f433649 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -28,31 +28,39 @@ #ifndef ETHREAD_PPC_ATOMIC_H #define ETHREAD_PPC_ATOMIC_H +#define ETHR_HAVE_NATIVE_ATOMIC32 1 + typedef struct { - volatile int counter; -} ethr_native_atomic_t; + volatile ethr_sint32_t counter; +} ethr_native_atomic32_t; #define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory") -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ethr_sint32_t * +ethr_native_atomic32_addr(ethr_native_atomic32_t *var) +{ + return (ethr_sint32_t *) &var->counter; +} static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, int i) +ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i) { var->counter = i; } -#define ethr_native_atomic_set(v, i) ethr_native_atomic_init((v), (i)) +#define ethr_native_atomic32_set(v, i) ethr_native_atomic32_init((v), (i)) -static ETHR_INLINE int -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_read(ethr_native_atomic32_t *var) { return var->counter; } -static ETHR_INLINE int -ethr_native_atomic_add_return(ethr_native_atomic_t *var, int incr) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { - int tmp; + ethr_sint32_t tmp; __asm__ __volatile__( "eieio\n\t" @@ -69,16 +77,16 @@ ethr_native_atomic_add_return(ethr_native_atomic_t *var, int incr) } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, int incr) +ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) { /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic_add_return(var, incr); + (void)ethr_native_atomic32_add_return(var, incr); } -static ETHR_INLINE int -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var) { - int tmp; + ethr_sint32_t tmp; __asm__ __volatile__( "eieio\n\t" @@ -95,16 +103,16 @@ ethr_native_atomic_inc_return(ethr_native_atomic_t *var) } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ethr_native_atomic32_inc(ethr_native_atomic32_t *var) { /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic_inc_return(var); + (void)ethr_native_atomic32_inc_return(var); } -static ETHR_INLINE int -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) { - int tmp; + ethr_sint32_t tmp; __asm__ __volatile__( "eieio\n\t" @@ -121,16 +129,16 @@ ethr_native_atomic_dec_return(ethr_native_atomic_t *var) } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ethr_native_atomic32_dec(ethr_native_atomic32_t *var) { /* XXX: could use weaker version here w/o eieio+isync */ - (void)ethr_native_atomic_dec_return(var); + (void)ethr_native_atomic32_dec_return(var); } -static ETHR_INLINE int -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, int mask) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - int old, new; + ethr_sint32_t old, new; __asm__ __volatile__( "eieio\n\t" @@ -146,10 +154,10 @@ ethr_native_atomic_and_retold(ethr_native_atomic_t *var, int mask) return old; } -static ETHR_INLINE int -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, int mask) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { - int old, new; + ethr_sint32_t old, new; __asm__ __volatile__( "eieio\n\t" @@ -165,10 +173,10 @@ ethr_native_atomic_or_retold(ethr_native_atomic_t *var, int mask) return old; } -static ETHR_INLINE int -ethr_native_atomic_xchg(ethr_native_atomic_t *var, int val) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) { - int tmp; + ethr_sint32_t tmp; __asm__ __volatile__( "eieio\n\t" @@ -183,10 +191,12 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, int val) return tmp; } -static ETHR_INLINE int -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, int new, int expected) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t expected) { - int old; + ethr_sint32_t old; __asm__ __volatile__( "eieio\n\t" @@ -210,20 +220,20 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, int new, int expected) */ static ETHR_INLINE long -ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) { - long res = ethr_native_atomic_read(var); + long res = ethr_native_atomic32_read(var); ETHR_MEMORY_BARRIER; return res; } -#define ethr_native_atomic_set_relb ethr_native_atomic_xchg -#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return -#define ethr_native_atomic_dec_relb ethr_native_atomic_dec_return -#define ethr_native_atomic_dec_return_relb ethr_native_atomic_dec_return +#define ethr_native_atomic32_set_relb ethr_native_atomic32_xchg +#define ethr_native_atomic32_inc_return_acqb ethr_native_atomic32_inc_return +#define ethr_native_atomic32_dec_relb ethr_native_atomic32_dec_return +#define ethr_native_atomic32_dec_return_relb ethr_native_atomic32_dec_return -#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg -#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg +#define ethr_native_atomic32_cmpxchg_acqb ethr_native_atomic32_cmpxchg +#define ethr_native_atomic32_cmpxchg_relb ethr_native_atomic32_cmpxchg #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/ppc32/ethread.h b/erts/include/internal/ppc32/ethread.h index 12efc1b653..3b619e9d01 100644 --- a/erts/include/internal/ppc32/ethread.h +++ b/erts/include/internal/ppc32/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/include/internal/pthread/ethr_event.h b/erts/include/internal/pthread/ethr_event.h index 104ec287e0..4c29b28536 100644 --- a/erts/include/internal/pthread/ethr_event.h +++ b/erts/include/internal/pthread/ethr_event.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009-2010. All Rights Reserved. + * Copyright Ericsson AB 2009-2011. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -30,31 +30,9 @@ #include <linux/futex.h> #include <sys/time.h> -/* - * Note: Linux futexes operate on 32-bit integers, but - * ethr_native_atomic_t are 64-bits on 64-bit - * platforms. This has to be taken into account. - * Therefore, in each individual value used each - * byte look the same. - */ - -#if ETHR_SIZEOF_PTR == 8 - -#define ETHR_EVENT_OFF_WAITER__ 0xffffffffffffffffL -#define ETHR_EVENT_OFF__ 0x7777777777777777L -#define ETHR_EVENT_ON__ 0L - -#elif ETHR_SIZEOF_PTR == 4 - -#define ETHR_EVENT_OFF_WAITER__ 0xffffffffL -#define ETHR_EVENT_OFF__ 0x77777777L -#define ETHR_EVENT_ON__ 0L - -#else - -#error ehrm... - -#endif +#define ETHR_EVENT_OFF_WAITER__ ((ethr_sint32_t) -1) +#define ETHR_EVENT_OFF__ ((ethr_sint32_t) 1) +#define ETHR_EVENT_ON__ ((ethr_sint32_t) 0) #if defined(FUTEX_WAIT_PRIVATE) && defined(FUTEX_WAKE_PRIVATE) # define ETHR_FUTEX_WAIT__ FUTEX_WAIT_PRIVATE @@ -65,11 +43,17 @@ #endif typedef struct { - ethr_atomic_t futex; + ethr_atomic32_t futex; } ethr_event; -#define ETHR_FUTEX__(FTX, OP, VAL) \ - (-1 == syscall(__NR_futex, (void *) (FTX), (OP), (int) (VAL), NULL, NULL, 0)\ +#define ETHR_FUTEX__(FTX, OP, VAL) \ + (-1 == syscall(__NR_futex, \ + (void *) ethr_atomic32_addr((FTX)), \ + (OP), \ + (int) (VAL), \ + NULL, \ + NULL, \ + 0) \ ? errno : 0) #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) @@ -77,9 +61,9 @@ typedef struct { static void ETHR_INLINE ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { - long val; - ETHR_WRITE_MEMORY_BARRIER; - val = ethr_atomic_xchg(&e->futex, ETHR_EVENT_ON__); + ethr_sint32_t val; + ETHR_MEMORY_BARRIER; + val = ethr_atomic32_xchg(&e->futex, ETHR_EVENT_ON__); if (val == ETHR_EVENT_OFF_WAITER__) { int res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAKE__, 1); if (res != 0) @@ -90,7 +74,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) static void ETHR_INLINE ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) { - ethr_atomic_set(&e->futex, ETHR_EVENT_OFF__); + ethr_atomic32_set(&e->futex, ETHR_EVENT_OFF__); ETHR_MEMORY_BARRIER; } @@ -100,7 +84,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) /* --- Posix mutex/cond implementation of events ---------------------------- */ typedef struct { - ethr_atomic_t state; + ethr_atomic32_t state; pthread_mutex_t mtx; pthread_cond_t cnd; } ethr_event; @@ -114,9 +98,9 @@ typedef struct { static void ETHR_INLINE ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { - long val; - ETHR_WRITE_MEMORY_BARRIER; - val = ethr_atomic_xchg(&e->state, ETHR_EVENT_ON__); + ethr_sint32_t val; + ETHR_MEMORY_BARRIER; + val = ethr_atomic32_xchg(&e->state, ETHR_EVENT_ON__); if (val == ETHR_EVENT_OFF_WAITER__) { int res = pthread_mutex_lock(&e->mtx); if (res != 0) @@ -133,7 +117,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) static void ETHR_INLINE ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) { - ethr_atomic_set(&e->state, ETHR_EVENT_OFF__); + ethr_atomic32_set(&e->state, ETHR_EVENT_OFF__); ETHR_MEMORY_BARRIER; } diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index 2a995d4465..00380dbf07 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -21,49 +21,86 @@ * Native ethread atomics on SPARC V9. * Author: Mikael Pettersson. */ -#ifndef ETHR_SPARC32_ATOMIC_H -#define ETHR_SPARC32_ATOMIC_H -typedef struct { - volatile long counter; -} ethr_native_atomic_t; +#undef ETHR_INCLUDE_ATOMIC_IMPL__ +#if !defined(ETHR_SPARC_V9_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +#define ETHR_SPARC_V9_ATOMIC32_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_SPARC_V9_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +#define ETHR_SPARC_V9_ATOMIC64_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#endif + +#ifdef ETHR_INCLUDE_ATOMIC_IMPL__ + +#ifndef ETHR_SPARC_V9_ATOMIC_COMMON__ +#define ETHR_SPARC_V9_ATOMIC_COMMON__ #define ETHR_MEMORY_BARRIER \ __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \ : : : "memory") -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#endif /* ETHR_SPARC_V9_ATOMIC_COMMON__ */ -#if defined(__arch64__) -#define CASX "casx" +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 +#define ETHR_HAVE_NATIVE_ATOMIC32 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t +#define ETHR_CAS__ "cas" +#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 +#define ETHR_HAVE_NATIVE_ATOMIC64 1 +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t +#define ETHR_CAS__ "casx" #else -#define CASX "cas" +#error "Unsupported integer size" #endif +typedef struct { + volatile ETHR_AINT_T__ counter; +} ETHR_ATMC_T__; + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->counter; +} + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) +{ + var->counter = i; +} + static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { var->counter = i; } -#define ethr_native_atomic_set(v, i) ethr_native_atomic_init((v), (i)) -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { return var->counter; } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - long old, tmp; + ETHR_AINT_T__ old, tmp; __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); do { old = var->counter; tmp = old+incr; __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(tmp) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); @@ -73,46 +110,46 @@ ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - (void)ethr_native_atomic_add_return(var, incr); + (void)ETHR_NATMC_FUNC__(add_return)(var, incr); } -static ETHR_INLINE long -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { - return ethr_native_atomic_add_return(var, 1); + return ETHR_NATMC_FUNC__(add_return)(var, 1); } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) { - (void)ethr_native_atomic_add_return(var, 1); + (void)ETHR_NATMC_FUNC__(add_return)(var, 1); } -static ETHR_INLINE long -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { - return ethr_native_atomic_add_return(var, -1); + return ETHR_NATMC_FUNC__(add_return)(var, -1); } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) { - (void)ethr_native_atomic_add_return(var, -1); + (void)ETHR_NATMC_FUNC__(add_return)(var, -1); } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - long old, tmp; + ETHR_AINT_T__ old, tmp; __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); do { old = var->counter; tmp = old & mask; __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(tmp) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); @@ -121,17 +158,17 @@ ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) return old; } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - long old, tmp; + ETHR_AINT_T__ old, tmp; __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); do { old = var->counter; tmp = old | mask; __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(tmp) : "r"(old), "r"(&var->counter), "0"(tmp) : "memory"); @@ -140,17 +177,17 @@ ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) return old; } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ val) { - long old, new; + ETHR_AINT_T__ old, new; __asm__ __volatile__("membar #LoadLoad|#StoreLoad"); do { old = var->counter; new = val; __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(new) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); @@ -159,12 +196,12 @@ ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) return old; } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) { __asm__ __volatile__("membar #LoadLoad|#StoreLoad\n"); __asm__ __volatile__( - CASX " [%2], %1, %0" + ETHR_CAS__ " [%2], %1, %0" : "=&r"(new) : "r"(old), "r"(&var->counter), "0"(new) : "memory"); @@ -176,39 +213,65 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) * Atomic ops with at least specified barriers. */ -static ETHR_INLINE long -ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +/* TODO: relax acquire barriers */ + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) { - long res = ethr_native_atomic_read(var); - __asm__ __volatile__("membar #StoreLoad|#StoreStore"); + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(read)(var); + __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore" : : : "memory"); return res; } static ETHR_INLINE void -ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - __asm__ __volatile__("membar #LoadStore|#StoreStore"); - ethr_native_atomic_set(var, i); + __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); + ETHR_NATMC_FUNC__(set)(var, i); +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) +{ + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(inc_return)(var); + __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); + return res; } static ETHR_INLINE void -ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) { - __asm__ __volatile__("membar #LoadStore|#StoreStore"); - ethr_native_atomic_dec(var); + __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); + ETHR_NATMC_FUNC__(dec)(var); } -static ETHR_INLINE long -ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { - __asm__ __volatile__("membar #LoadStore|#StoreStore"); - return ethr_native_atomic_dec_return(var); + __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); + return ETHR_NATMC_FUNC__(dec_return)(var); } -#define ethr_native_atomic_inc_return_acqb ethr_native_atomic_inc_return -#define ethr_native_atomic_cmpxchg_acqb ethr_native_atomic_cmpxchg -#define ethr_native_atomic_cmpxchg_relb ethr_native_atomic_cmpxchg +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) +{ + ETHR_AINT_T__ res = ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); + __asm__ __volatile__("membar #LoadLoad|#LoadStore" : : : "memory"); + return res; +} + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ old) +{ + __asm__ __volatile__("membar #LoadStore|#StoreStore" : : : "memory"); + return ETHR_NATMC_FUNC__(cmpxchg)(var, new, old); +} #endif /* ETHR_TRY_INLINE_FUNCS */ -#endif /* ETHR_SPARC32_ATOMIC_H */ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ +#undef ETHR_CAS__ + +#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/sparc32/ethread.h b/erts/include/internal/sparc32/ethread.h index dca113b4d6..aea9794390 100644 --- a/erts/include/internal/sparc32/ethread.h +++ b/erts/include/internal/sparc32/ethread.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. + * Copyright Ericsson AB 2005-2010. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -24,7 +24,12 @@ #ifndef ETHREAD_SPARC32_ETHREAD_H #define ETHREAD_SPARC32_ETHREAD_H +#define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "atomic.h" +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_ATOMIC_WANT_64BIT_IMPL__ +# include "atomic.h" +#endif #include "spinlock.h" #include "rwlock.h" diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 69569d82d1..48e4c0c6c8 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -24,92 +24,102 @@ #ifndef ETHREAD_TILE_ATOMIC_H #define ETHREAD_TILE_ATOMIC_H +#define ETHR_HAVE_NATIVE_ATOMIC32 1 + #include <atomic.h> /* An atomic is an aligned int accessed via locked operations. */ typedef struct { - volatile long counter; -} ethr_native_atomic_t; + volatile ethr_sint32_t counter; +} ethr_native_atomic32_t; #define ETHR_MEMORY_BARRIER __insn_mf() -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + +static ETHR_INLINE ethr_sint32_t * +ethr_native_atomic32_addr(ethr_native_atomic32_t *var) +{ + return (ethr_sint32_t *) &var->counter; +} static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +ethr_native_atomic32_init(ethr_native_atomic32_t *var, ethr_sint32_t i) { var->counter = i; } static ETHR_INLINE void -ethr_native_atomic_set(ethr_native_atomic_t *var, long i) +ethr_native_atomic32_set(ethr_native_atomic32_t *var, ethr_sint32_t i) { atomic_exchange_acq(&var->counter, i); } -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_read(ethr_native_atomic32_t *var) { return var->counter; } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ethr_native_atomic32_add(ethr_native_atomic32_t *var, ethr_sint32_t incr) { atomic_add(&var->counter, incr); } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ethr_native_atomic32_inc(ethr_native_atomic32_t *var) { atomic_increment(&var->counter); } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ethr_native_atomic32_dec(ethr_native_atomic32_t *var) { atomic_decrement(&var->counter); } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_add_return(ethr_native_atomic32_t *var, ethr_sint32_t incr) { return atomic_exchange_and_add(&var->counter, incr) + incr; } -static ETHR_INLINE long -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_inc_return(ethr_native_atomic32_t *var) { - return ethr_native_atomic_add_return(var, 1); + return ethr_native_atomic32_add_return(var, 1); } -static ETHR_INLINE long -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_dec_return(ethr_native_atomic32_t *var) { - return ethr_native_atomic_add_return(var, -1); + return ethr_native_atomic32_add_return(var, -1); } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_and_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { return atomic_and_val(&var->counter, mask); } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_or_retold(ethr_native_atomic32_t *var, ethr_sint32_t mask) { return atomic_or_val(&var->counter, mask); } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long val) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_xchg(ethr_native_atomic32_t *var, ethr_sint32_t val) { return atomic_exchange_acq(&var->counter, val); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long expected) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t expected) { return atomic_compare_and_exchange_val_acq(&var->counter, new, expected); } @@ -118,54 +128,58 @@ ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long expected) * Atomic ops with at least specified barriers. */ -static ETHR_INLINE long -ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_read_acqb(ethr_native_atomic32_t *var) { - long res = ethr_native_atomic_read(var); + ethr_sint32_t res = ethr_native_atomic32_read(var); ETHR_MEMORY_BARRIER; return res; } -static ETHR_INLINE long -ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var) { - long res = ethr_native_atomic_inc_return(var); + ethr_sint32_t res = ethr_native_atomic32_inc_return(var); ETHR_MEMORY_BARRIER; return res; } static ETHR_INLINE void -ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long val) +ethr_native_atomic32_set_relb(ethr_native_atomic32_t *var, ethr_sint32_t val) { ETHR_MEMORY_BARRIER; - ethr_native_atomic_set(var, val); + ethr_native_atomic32_set(var, val); } static ETHR_INLINE void -ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +ethr_native_atomic32_dec_relb(ethr_native_atomic32_t *var) { ETHR_MEMORY_BARRIER; - ethr_native_atomic_dec(var); + ethr_native_atomic32_dec(var); } -static ETHR_INLINE long -ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var) { ETHR_MEMORY_BARRIER; - return ethr_native_atomic_dec_return(var); + return ethr_native_atomic32_dec_return(var); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long exp) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) { - return ethr_native_atomic_cmpxchg(var, new, exp); + return ethr_native_atomic32_cmpxchg(var, new, exp); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long exp) +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) { ETHR_MEMORY_BARRIER; - return ethr_native_atomic_cmpxchg(var, new, exp); + return ethr_native_atomic32_cmpxchg(var, new, exp); } #endif /* ETHR_TRY_INLINE_FUNCS */ diff --git a/erts/include/internal/win/ethr_atomic.h b/erts/include/internal/win/ethr_atomic.h index 500459dd6c..60def01a7e 100644 --- a/erts/include/internal/win/ethr_atomic.h +++ b/erts/include/internal/win/ethr_atomic.h @@ -22,223 +22,394 @@ * Author: Rickard Green */ -#ifndef ETHR_WIN_ATOMIC_H__ -#define ETHR_WIN_ATOMIC_H__ - -#ifdef _MSC_VER -# if _MSC_VER < 1300 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 /* Dont trust really old compilers */ -# else -# if defined(_M_IX86) -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else /* I.e. IA64 */ -# if _MSC_VER >= 1400 -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 -# else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 -# endif -# endif -# endif -# if _MSC_VER >= 1400 -# include <intrin.h> -# undef ETHR_COMPILER_BARRIER -# define ETHR_COMPILER_BARRIER _ReadWriteBarrier() -# endif -#pragma intrinsic(_ReadWriteBarrier) -#pragma intrinsic(_InterlockedAnd) -#pragma intrinsic(_InterlockedOr) +#undef ETHR_INCLUDE_ATOMIC_IMPL__ +#if !defined(ETHR_WIN_ATOMIC32_H__) && defined(ETHR_ATOMIC_WANT_32BIT_IMPL__) +#define ETHR_WIN_ATOMIC32_H__ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 4 +#undef ETHR_ATOMIC_WANT_32BIT_IMPL__ +#elif !defined(ETHR_WIN_ATOMIC64_H__) && defined(ETHR_ATOMIC_WANT_64BIT_IMPL__) +#define ETHR_WIN_ATOMIC64_H__ +#ifdef ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64 +/* _InterlockedCompareExchange64() required... */ +#define ETHR_INCLUDE_ATOMIC_IMPL__ 8 +#endif +#undef ETHR_ATOMIC_WANT_64BIT_IMPL__ +#endif + +#ifdef ETHR_INCLUDE_ATOMIC_IMPL__ + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + +#ifndef ETHR_WIN_ATOMIC_COMMON__ +#define ETHR_WIN_ATOMIC_COMMON__ + +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#if defined(_M_IX86) || defined(_M_AMD64) || defined(_M_IA64) +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 1 #else -# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 +# define ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ 0 #endif +#if defined(_M_AMD64) || (defined(_M_IX86) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 1 +#else +# define ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ 0 +#endif /* - * No configure test checking for _Interlocked*_{acq,rel} and - * Interlocked*{Acquire,Release} have been written yet... + * No configure test checking for interlocked acquire/release + * versions have been written, yet. It should define + * ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS if, and + * only if, all used interlocked operations with barriers + * exists. * * Note, that these are pure optimizations for the itanium * processor. */ -#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_ACQ -#pragma intrinsic(_InterlockedCompareExchange_acq) +#include <intrin.h> +#undef ETHR_COMPILER_BARRIER +#define ETHR_COMPILER_BARRIER _ReadWriteBarrier() +#pragma intrinsic(_ReadWriteBarrier) +#pragma intrinsic(_InterlockedCompareExchange) + +#if defined(_M_AMD64) || (defined(_M_IX86) \ + && !defined(ETHR_PRE_PENTIUM4_COMPAT)) +#include <emmintrin.h> +#include <mmintrin.h> +#pragma intrinsic(_mm_mfence) +#define ETHR_MEMORY_BARRIER _mm_mfence() +#pragma intrinsic(_mm_sfence) +#define ETHR_WRITE_MEMORY_BARRIER _mm_sfence() +#pragma intrinsic(_mm_lfence) +#define ETHR_READ_MEMORY_BARRIER _mm_lfence() +#define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER + +#else + +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile long x___ = 0; \ + _InterlockedCompareExchange(&x___, (long) 1, (long) 0); \ +} while (0) + #endif -#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_REL + +#endif /* ETHR_WIN_ATOMIC_COMMON__ */ + +#if ETHR_INCLUDE_ATOMIC_IMPL__ == 4 + +#define ETHR_HAVE_NATIVE_ATOMIC32 1 + +/* + * All used operations available as 32-bit intrinsics + */ + +#pragma intrinsic(_InterlockedDecrement) +#pragma intrinsic(_InterlockedIncrement) +#pragma intrinsic(_InterlockedExchangeAdd) +#pragma intrinsic(_InterlockedExchange) +#pragma intrinsic(_InterlockedAnd) +#pragma intrinsic(_InterlockedOr) +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#pragma intrinsic(_InterlockedExchangeAdd_acq) +#pragma intrinsic(_InterlockedIncrement_acq) +#pragma intrinsic(_InterlockedDecrement_rel) +#pragma intrinsic(_InterlockedCompareExchange_acq) #pragma intrinsic(_InterlockedCompareExchange_rel) #endif +#define ETHR_ILCKD__(X) _Interlocked ## X +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## _acq +#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## _rel +#else +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X +#define ETHR_ILCKD_REL__(X) _Interlocked ## X +#endif + +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic32_t +#define ETHR_AINT_T__ ethr_sint32_t + +#elif ETHR_INCLUDE_ATOMIC_IMPL__ == 8 + +#define ETHR_HAVE_NATIVE_ATOMIC64 1 + +/* + * _InterlockedCompareExchange64() is required. The other may not + * be available, but if so, we can generate them. + */ +#pragma intrinsic(_InterlockedCompareExchange64) + +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ +#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) *(PTR) +#else +#define ETHR_OWN_ILCKD_INIT_VAL__(PTR) (__int64) 0 +#endif + +#define ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, PTR, NEW, ACT, EXP, OPS, RET) \ +{ \ + __int64 NEW, ACT, EXP; \ + ACT = ETHR_OWN_ILCKD_INIT_VAL__(PTR); \ + do { \ + EXP = ACT; \ + { OPS; } \ + ACT = _InterlockedCompareExchange64(PTR, NEW, EXP); \ + } while (ACT != EXP); \ + return RET; \ +} + +#define ETHR_OWN_ILCKD_1_IMPL__(FUNC, NEW, ACT, EXP, OPS, RET) \ +static __forceinline __int64 \ +FUNC(__int64 volatile *ptr) \ +ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) + +#define ETHR_OWN_ILCKD_2_IMPL__(FUNC, NEW, ACT, EXP, OPS, ARG, RET) \ +static __forceinline __int64 \ +FUNC(__int64 volatile *ptr, __int64 ARG) \ +ETHR_OWN_ILCKD_BODY_IMPL__(FUNC, ptr, NEW, ACT, EXP, OPS, RET) + + +#ifdef ETHR_HAVE__INTERLOCKEDDECREMENT64 +#pragma intrinsic(_InterlockedDecrement64) +#else +ETHR_OWN_ILCKD_1_IMPL__(_InterlockedDecrement64, new, act, exp, + new = act - 1, new) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDINCREMENT64 +#pragma intrinsic(_InterlockedIncrement64) +#else +ETHR_OWN_ILCKD_1_IMPL__(_InterlockedIncrement64, new, act, exp, + new = act + 1, new) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGEADD64 +#pragma intrinsic(_InterlockedExchangeAdd64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchangeAdd64, new, act, exp, + new = act + arg, arg, act) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDEXCHANGE64 +#pragma intrinsic(_InterlockedExchange64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedExchange64, new, act, exp, + new = arg, arg, act) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDAND64 +#pragma intrinsic(_InterlockedAnd64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedAnd64, new, act, exp, + new = act & arg, arg, act) +#endif +#ifdef ETHR_HAVE__INTERLOCKEDOR64 +#pragma intrinsic(_InterlockedOr64) +#else +ETHR_OWN_ILCKD_2_IMPL__(_InterlockedOr64, new, act, exp, + new = act | arg, arg, act) +#endif +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#pragma intrinsic(_InterlockedExchangeAdd64_acq) +#pragma intrinsic(_InterlockedIncrement64_acq) +#pragma intrinsic(_InterlockedDecrement64_rel) +#pragma intrinsic(_InterlockedCompareExchange64_acq) +#pragma intrinsic(_InterlockedCompareExchange64_rel) +#endif + +#define ETHR_ILCKD__(X) _Interlocked ## X ## 64 +#ifdef ETHR_HAVE_INTERLOCKED_ACQUIRE_RELEASE_BARRIERS +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64_acq +#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64_rel +#else +#define ETHR_ILCKD_ACQ__(X) _Interlocked ## X ## 64 +#define ETHR_ILCKD_REL__(X) _Interlocked ## X ## 64 +#endif + +#define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X +#define ETHR_ATMC_T__ ethr_native_atomic64_t +#define ETHR_AINT_T__ ethr_sint64_t + +#else +#error "Unsupported integer size" +#endif typedef struct { - volatile LONG value; -} ethr_native_atomic_t; + volatile ETHR_AINT_T__ value; +} ETHR_ATMC_T__; -#define ETHR_MEMORY_BARRIER \ -do { \ - volatile LONG x___ = 0; \ - _InterlockedCompareExchange(&x___, (LONG) 1, (LONG) 0); \ -} while (0) +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) -#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) +static ETHR_INLINE ETHR_AINT_T__ * +ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__ *) &var->value; +} static ETHR_INLINE void -ethr_native_atomic_init(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(init)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - var->value = (LONG) i; +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ + var->value = i; +#else + (void) ETHR_ILCKD__(Exchange)(&var->value, i); +#endif } static ETHR_INLINE void -ethr_native_atomic_set(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ - var->value = (LONG) i; +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ + var->value = i; #else - (void) InterlockedExchange(&var->value, (LONG) i); + (void) ETHR_ILCKD__(Exchange)(&var->value, i); #endif } -static ETHR_INLINE long -ethr_native_atomic_read(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ return var->value; #else - return InterlockedExchangeAdd(&var->value, (LONG) 0); + return ETHR_ILCKD__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); #endif } static ETHR_INLINE void -ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +ETHR_NATMC_FUNC__(add)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) { - (void) InterlockedExchangeAdd(&var->value, (LONG) incr); + (void) ETHR_ILCKD__(ExchangeAdd)(&var->value, incr); } -static ETHR_INLINE long -ethr_native_atomic_add_return(ethr_native_atomic_t *var, long i) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - LONG tmp = InterlockedExchangeAdd(&var->value, (LONG) i); - return tmp + i; + return ETHR_ILCKD__(ExchangeAdd)(&var->value, i) + i; } static ETHR_INLINE void -ethr_native_atomic_inc(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(inc)(ETHR_ATMC_T__ *var) { - (void) InterlockedIncrement(&var->value); + (void) ETHR_ILCKD__(Increment)(&var->value); } static ETHR_INLINE void -ethr_native_atomic_dec(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec)(ETHR_ATMC_T__ *var) { - (void) InterlockedDecrement(&var->value); + (void) ETHR_ILCKD__(Decrement)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return)(ETHR_ATMC_T__ *var) { - return (long) InterlockedIncrement(&var->value); + return ETHR_ILCKD__(Increment)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) { - return (long) InterlockedDecrement(&var->value); + return ETHR_ILCKD__(Decrement)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(and_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - return (long) _InterlockedAnd(&var->value, mask); + return ETHR_ILCKD__(And)(&var->value, mask); } -static ETHR_INLINE long -ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(or_retold)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ mask) { - return (long) _InterlockedOr(&var->value, mask); + return ETHR_ILCKD__(Or)(&var->value, mask); } - -static ETHR_INLINE long -ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { - return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); + return ETHR_ILCKD__(CompareExchange)(&var->value, new, old); } -static ETHR_INLINE long -ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(xchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new) { - return (long) InterlockedExchange(&var->value, (LONG) new); + return ETHR_ILCKD__(Exchange)(&var->value, new); } /* * Atomic ops with at least specified barriers. */ -static ETHR_INLINE long -ethr_native_atomic_read_acqb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) { -#ifdef ETHR_HAVE_INTERLOCKEDEXCHANGEADDACQUIRE - return (long) InterlockedExchangeAddAcquire(&var->value, (LONG) 0); +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_AINT_T__ val = var->value; + ETHR_COMPILER_BARRIER; + return val; #else - return (long) InterlockedExchangeAdd(&var->value, (LONG) 0); + return ETHR_ILCKD_ACQ__(ExchangeAdd)(&var->value, (ETHR_AINT_T__) 0); #endif } -static ETHR_INLINE long -ethr_native_atomic_inc_return_acqb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) { -#ifdef ETHR_HAVE_INTERLOCKEDINCREMENTACQUIRE - return (long) InterlockedIncrementAcquire(&var->value); -#else - return (long) InterlockedIncrement(&var->value); -#endif + return ETHR_ILCKD_ACQ__(Increment)(&var->value); } static ETHR_INLINE void -ethr_native_atomic_set_relb(ethr_native_atomic_t *var, long i) +ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ i) { - (void) InterlockedExchange(&var->value, (LONG) i); +#if ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + ETHR_COMPILER_BARRIER; + var->value = i; +#else + (void) ETHR_ILCKD_REL__(Exchange)(&var->value, i); +#endif } static ETHR_INLINE void -ethr_native_atomic_dec_relb(ethr_native_atomic_t *var) +ETHR_NATMC_FUNC__(dec_relb)(ETHR_ATMC_T__ *var) { -#ifdef ETHR_HAVE_INTERLOCKEDDECREMENTRELEASE - (void) InterlockedDecrementRelease(&var->value); -#else - (void) InterlockedDecrement(&var->value); -#endif + (void) ETHR_ILCKD_REL__(Decrement)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_dec_return_relb(ethr_native_atomic_t *var) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) { -#ifdef ETHR_HAVE_INTERLOCKEDDECREMENTRELEASE - return (long) InterlockedDecrementRelease(&var->value); -#else - return (long) InterlockedDecrement(&var->value); -#endif + return ETHR_ILCKD_REL__(Decrement)(&var->value); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg_acqb(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { -#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_ACQ - return (long) _InterlockedCompareExchange_acq(&var->value, (LONG) new, (LONG) old); -#else - return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); -#endif + return ETHR_ILCKD_ACQ__(CompareExchange)(&var->value, new, old); } -static ETHR_INLINE long -ethr_native_atomic_cmpxchg_relb(ethr_native_atomic_t *var, long new, long old) +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ old) { - -#ifdef ETHR_HAVE_INTERLOCKEDCOMPAREEXCHANGE_REL - return (long) _InterlockedCompareExchange_rel(&var->value, (LONG) new, (LONG) old); -#else - return (long) _InterlockedCompareExchange(&var->value, (LONG) new, (LONG) old); -#endif + return ETHR_ILCKD_REL__(CompareExchange)(&var->value, new, old); } -#endif +#endif /* ETHR_TRY_INLINE_FUNCS */ -#endif +#undef ETHR_ILCKD__ +#undef ETHR_ILCKD_ACQ__ +#undef ETHR_ILCKD_REL__ +#undef ETHR_NATMC_FUNC__ +#undef ETHR_ATMC_T__ +#undef ETHR_AINT_T__ +#undef ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ +#undef ETHR_READ_ACQB_AND_SET_RELB_COMPILER_BARRIER_ONLY__ + +#endif /* _MSC_VER */ + +#endif /* ETHR_INCLUDE_ATOMIC_IMPL__ */ diff --git a/erts/include/internal/win/ethr_event.h b/erts/include/internal/win/ethr_event.h index af57c20f91..598816b2c6 100644 --- a/erts/include/internal/win/ethr_event.h +++ b/erts/include/internal/win/ethr_event.h @@ -21,22 +21,24 @@ * Author: Rickard Green */ -#define ETHR_EVENT_OFF_WAITER__ ((LONG) -1) -#define ETHR_EVENT_OFF__ ((LONG) 1) -#define ETHR_EVENT_ON__ ((LONG) 0) +#define ETHR_EVENT_OFF_WAITER__ ((long) -1) +#define ETHR_EVENT_OFF__ ((long) 1) +#define ETHR_EVENT_ON__ ((long) 0) typedef struct { - volatile LONG state; + volatile long state; HANDLE handle; } ethr_event; #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_EVENT_IMPL__) +#pragma intrinsic(_InterlockedExchange) + static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) { - /* InterlockedExchange() imply a full memory barrier which is important */ - LONG state = InterlockedExchange(&e->state, ETHR_EVENT_ON__); + /* _InterlockedExchange() imply a full memory barrier which is important */ + long state = _InterlockedExchange(&e->state, ETHR_EVENT_ON__); if (state == ETHR_EVENT_OFF_WAITER__) { if (!SetEvent(e->handle)) ETHR_FATAL_ERROR__(ethr_win_get_errno__()); @@ -46,7 +48,7 @@ ETHR_INLINE_FUNC_NAME_(ethr_event_set)(ethr_event *e) static ETHR_INLINE void ETHR_INLINE_FUNC_NAME_(ethr_event_reset)(ethr_event *e) { - /* InterlockedExchange() imply a full memory barrier which is important */ + /* _InterlockedExchange() imply a full memory barrier which is important */ InterlockedExchange(&e->state, ETHR_EVENT_OFF__); } diff --git a/erts/include/internal/win/ethread.h b/erts/include/internal/win/ethread.h index b52710f6a3..c01b17cf14 100644 --- a/erts/include/internal/win/ethread.h +++ b/erts/include/internal/win/ethread.h @@ -25,7 +25,11 @@ #ifndef ETHREAD_WIN_H__ #define ETHREAD_WIN_H__ +#define ETHR_ATOMIC_WANT_32BIT_IMPL__ #include "ethr_atomic.h" -#define ETHR_HAVE_NATIVE_ATOMICS 1 +#if ETHR_SIZEOF_PTR == 8 +# define ETHR_ATOMIC_WANT_64BIT_IMPL__ +# include "ethr_atomic.h" +#endif #endif diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index 0d3181cace..757b3b24e2 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -283,6 +283,7 @@ endif ETHR_THR_LIB_BASE_DIR=@ETHR_THR_LIB_BASE_DIR@ ifneq ($(strip $(ETHR_LIB_NAME)),) ETHREAD_LIB_SRC=common/ethr_aux.c \ + common/ethr_atomics.c \ common/ethr_mutex.c \ common/ethr_cbf.c \ $(ETHR_THR_LIB_BASE_DIR)/ethread.c \ @@ -381,6 +382,11 @@ $(ERTS_LIB): $(ERTS_LIB_OBJS) # Object files # +ifeq ($(TYPE)-@GCC@,debug-yes) +$(r_OBJ_DIR)/ethr_aux.o: common/ethr_aux.c + $(CC) $(THR_DEFS) $(CFLAGS) -Wno-unused-function $(INCLUDES) -c $< -o $@ +endif + $(r_OBJ_DIR)/%.o: common/%.c $(CC) $(THR_DEFS) $(CFLAGS) $(INCLUDES) -c $< -o $@ @@ -445,6 +451,7 @@ INTERNAL_RELEASE_INCLUDES= \ $(ERTS_INCL_INT)/ethread.h \ $(ERTS_INCL_INT)/ethr_mutex.h \ $(ERTS_INCL_INT)/ethr_optimized_fallbacks.h \ + $(ERTS_INCL_INT)/ethr_atomics.h \ $(ERTS_INCL_INT)/$(TARGET)/ethread.mk \ $(ERTS_INCL_INT)/$(TARGET)/erts_internal.mk \ $(ERTS_INCL_INT)/$(TARGET)/ethread_header_config.h \ diff --git a/erts/lib_src/common/erl_misc_utils.c b/erts/lib_src/common/erl_misc_utils.c index 498ce6837a..4c881993a5 100644 --- a/erts/lib_src/common/erl_misc_utils.c +++ b/erts/lib_src/common/erl_misc_utils.c @@ -71,6 +71,19 @@ (CPUSET)) != 0 ? -errno : 0) #define ERTS_MU_SET_THR_AFFINITY__(SETP) \ (sched_setaffinity(0, sizeof(cpu_set_t), (SETP)) != 0 ? -errno : 0) +#elif defined(HAVE_CPUSET_xETAFFINITY) +# include <sys/param.h> +# include <sys/cpuset.h> +# define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ +#define ERTS_MU_GET_PROC_AFFINITY__(CPUINFOP, CPUSET) \ + (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, -1, \ + sizeof(cpuset_t), \ + (CPUSET)) != 0 ? -errno : 0) +#define ERTS_MU_SET_THR_AFFINITY__(CPUSETP) \ + (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, \ + sizeof(cpuset_t), \ + (CPUSETP)) != 0 ? -errno : 0) +# define cpu_set_t cpuset_t #elif defined(__WIN32__) # define ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__ # define cpu_set_t DWORD @@ -100,6 +113,11 @@ # define ERTS_SYS_CPU_PATH "/sys/devices/system/cpu" #endif +#ifdef __FreeBSD__ +#include <sys/types.h> +#include <sys/sysctl.h> +#endif + static int read_topology(erts_cpu_info_t *cpuinfo); #if defined(ERTS_HAVE_MISC_UTIL_AFFINITY_MASK__) @@ -1380,6 +1398,245 @@ read_topology(erts_cpu_info_t *cpuinfo) return res; } +#elif defined(__FreeBSD__) + +/** + * FreeBSD topology detection is based on kern.sched.topology_spec XML as + * exposed by the ULE scheduler and described in SMP(4). It is available in + * 8.0 and higher. + * + * Threads are identified in this XML chunk with a THREAD flag. The function + * (simplistically) distinguishes cores and processors by the amount of cache + * they share (0 => processor, otherwise => core). Nodes are not identified + * (ULE doesn't handle NUMA yet, I believe). + */ + +/** + * Recursively parse a topology_spec <group> tag. + */ +static +const char* parse_topology_spec_group(erts_cpu_info_t *cpuinfo, const char* xml, int parentCacheLevel, int* processor_p, int* core_p, int* index_procs_p) { + int error = 0; + int cacheLevel = parentCacheLevel; + const char* next_group_start = strstr(xml + 1, "<group"); + int is_thread_group = 0; + const char* next_cache_level; + const char* next_thread_flag; + const char* next_group_end; + const char* next_children; + const char* next_children_end; + + /* parse the cache level */ + next_cache_level = strstr(xml, "cache-level=\""); + if (next_cache_level && (next_group_start == NULL || next_cache_level < next_group_start)) { + sscanf(next_cache_level, "cache-level=\"%i\"", &cacheLevel); + } + + /* parse the threads flag */ + next_thread_flag = strstr(xml, "THREAD"); + if (next_thread_flag && (next_group_start == NULL || next_thread_flag < next_group_start)) + is_thread_group = 1; + + /* Determine if it's a leaf with the position of the next children tag */ + next_group_end = strstr(xml, "</group>"); + next_children = strstr(xml, "<children>"); + next_children_end = strstr(xml, "</children>"); + if (next_children == NULL || next_group_end < next_children) { + do { + const char* next_cpu_start; + const char* next_cpu_cdata; + const char* next_cpu_end; + int cpu_str_size; + char* cpu_str; + char* cpu_crsr; + char* brkb; + int thread = 0; + int index_procs = *index_procs_p; + + next_cpu_start = strstr(xml, "<cpu"); + if (!next_cpu_start) { + error = 1; + break; + } + next_cpu_cdata = strstr(next_cpu_start, ">") + 1; + if (!next_cpu_cdata) { + error = 1; + break; + } + next_cpu_end = strstr(next_cpu_cdata, "</cpu>"); + if (!next_cpu_end) { + error = 1; + break; + } + cpu_str_size = next_cpu_end - next_cpu_cdata; + cpu_str = (char*) malloc(cpu_str_size + 1); + memcpy(cpu_str, (const char*) next_cpu_cdata, cpu_str_size); + cpu_str[cpu_str_size] = 0; + for (cpu_crsr = strtok_r(cpu_str, " \t,", &brkb); cpu_crsr; cpu_crsr = strtok_r(NULL, " \t,", &brkb)) { + int cpu_id; + if (index_procs >= cpuinfo->configured) { + void* t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t) * (index_procs + 1))); + if (t) { + cpuinfo->topology = t; + } else { + error = 1; + break; + } + } + cpu_id = atoi(cpu_crsr); + cpuinfo->topology[index_procs].node = -1; + cpuinfo->topology[index_procs].processor = *processor_p; + cpuinfo->topology[index_procs].processor_node = -1; + cpuinfo->topology[index_procs].core = *core_p; + cpuinfo->topology[index_procs].thread = thread; + cpuinfo->topology[index_procs].logical = cpu_id; + if (is_thread_group) { + thread++; + } else { + *core_p = (*core_p)++; + } + index_procs++; + } + *index_procs_p = index_procs; + free(cpu_str); + } while (0); + xml = next_group_end; + } else { + while (next_group_start != NULL && next_group_start < next_children_end) { + xml = parse_topology_spec_group(cpuinfo, next_group_start, cacheLevel, processor_p, core_p, index_procs_p); + if (!xml) + break; + next_group_start = strstr(xml, "<group"); + next_children_end = strstr(xml, "</children>"); + } + } + + if (cacheLevel == 0) { + *core_p = 0; + *processor_p = (*processor_p)++; + } else { + *core_p = (*core_p)++; + } + + if (error) + xml = NULL; + + return xml; +} + +/** + * Parse the topology_spec. Return the number of CPUs or 0 if parsing failed. + */ +static +int parse_topology_spec(erts_cpu_info_t *cpuinfo, const char* xml) { + int res = 1; + int index_procs = 0; + int core = 0; + int processor = 0; + xml = strstr(xml, "<groups"); + if (!xml) + return -1; + + xml += 7; + xml = strstr(xml, "<group"); + while (xml) { + xml = parse_topology_spec_group(cpuinfo, xml, 0, &processor, &core, &index_procs); + if (!xml) { + res = 0; + break; + } + xml = strstr(xml, "<group"); + } + + if (res) + res = index_procs; + + return res; +} + +static int +read_topology(erts_cpu_info_t *cpuinfo) +{ + int ix; + int res = 0; + size_t topology_spec_size = 0; + void* topology_spec = NULL; + + errno = 0; + + if (cpuinfo->configured < 1) + goto error; + + cpuinfo->topology_size = cpuinfo->configured; + cpuinfo->topology = malloc(sizeof(erts_cpu_topology_t) + * cpuinfo->configured); + if (!cpuinfo->topology) { + res = -ENOMEM; + goto error; + } + + for (ix = 0; ix < cpuinfo->configured; ix++) { + cpuinfo->topology[ix].node = -1; + cpuinfo->topology[ix].processor = -1; + cpuinfo->topology[ix].processor_node = -1; + cpuinfo->topology[ix].core = -1; + cpuinfo->topology[ix].thread = -1; + cpuinfo->topology[ix].logical = -1; + } + + if (!sysctlbyname("kern.sched.topology_spec", NULL, &topology_spec_size, NULL, 0)) { + topology_spec = malloc(topology_spec_size); + if (!topology_spec) { + res = -ENOMEM; + goto error; + } + + if (sysctlbyname("kern.sched.topology_spec", topology_spec, &topology_spec_size, NULL, 0)) { + goto error; + } + + res = parse_topology_spec(cpuinfo, topology_spec); + if (!res || res < cpuinfo->online) + res = 0; + else { + cpuinfo->topology_size = res; + + if (cpuinfo->topology_size != cpuinfo->configured) { + void *t = realloc(cpuinfo->topology, (sizeof(erts_cpu_topology_t) + * cpuinfo->topology_size)); + if (t) + cpuinfo->topology = t; + } + + adjust_processor_nodes(cpuinfo, 1); + + qsort(cpuinfo->topology, + cpuinfo->topology_size, + sizeof(erts_cpu_topology_t), + cpu_cmp); + } + } + +error: + + if (res == 0) { + cpuinfo->topology_size = 0; + if (cpuinfo->topology) { + free(cpuinfo->topology); + cpuinfo->topology = NULL; + } + if (errno) + res = -errno; + else + res = -EINVAL; + } + + if (topology_spec) + free(topology_spec); + + return res; +} + #else static int diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c new file mode 100644 index 0000000000..94557d904a --- /dev/null +++ b/erts/lib_src/common/ethr_atomics.c @@ -0,0 +1,402 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: The ethread atomic API + * Author: Rickard Green + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#define ETHR_INLINE_FUNC_NAME_(X) X ## __ +#define ETHR_ATOMIC_IMPL__ + +#include "ethread.h" +#include "ethr_internal.h" + +#ifndef ETHR_HAVE_NATIVE_ATOMICS +ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; +#endif + +int +ethr_init_atomics(void) +{ +#ifndef ETHR_HAVE_NATIVE_ATOMICS + { + int i; + for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { + int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); + if (res != 0) + return res; + } + } +#endif + return 0; +} + +/* + * --- Pointer size atomics --------------------------------------------------- + */ + +ethr_sint_t * +ethr_atomic_addr(ethr_atomic_t *var) +{ + ETHR_ASSERT(var); + return ethr_atomic_addr__(var); +} + +void +ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t i) +{ + ETHR_ASSERT(var); + ethr_atomic_init__(var, i); +} + +void +ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set__(var, i); +} + +ethr_sint_t +ethr_atomic_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read__(var); +} + +ethr_sint_t +ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_add_read__(var, incr); +} + +ethr_sint_t +ethr_atomic_inc_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read__(var); +} + +ethr_sint_t +ethr_atomic_dec_read(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read__(var); +} + +void +ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_add__(var, incr); +} + +void +ethr_atomic_inc(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_inc__(var); +} + +void +ethr_atomic_dec(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec__(var); +} + +ethr_sint_t +ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_band__(var, mask); +} + +ethr_sint_t +ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_bor__(var, mask); +} + +ethr_sint_t +ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t new) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_xchg__(var, new); +} + +ethr_sint_t +ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t expected) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg__(var, new, expected); +} + +ethr_sint_t +ethr_atomic_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_read_acqb__(var); +} + +ethr_sint_t +ethr_atomic_inc_read_acqb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_inc_read_acqb__(var); +} + +void +ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_set_relb__(var, i); +} + +void +ethr_atomic_dec_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic_dec_relb__(var); +} + +ethr_sint_t +ethr_atomic_dec_read_relb(ethr_atomic_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_dec_read_relb__(var); +} + +ethr_sint_t +ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_acqb__(var, new, exp); +} + +ethr_sint_t +ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t new, ethr_sint_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic_cmpxchg_relb__(var, new, exp); +} + + +/* + * --- 32-bit atomics --------------------------------------------------------- + */ + +ethr_sint32_t * +ethr_atomic32_addr(ethr_atomic32_t *var) +{ + ETHR_ASSERT(var); + return ethr_atomic32_addr__(var); +} + +void +ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t i) +{ + ETHR_ASSERT(var); + ethr_atomic32_init__(var, i); +} + +void +ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_set__(var, i); +} + +ethr_sint32_t +ethr_atomic32_read(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_read__(var); +} + + +ethr_sint32_t +ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_add_read__(var, incr); +} + +ethr_sint32_t +ethr_atomic32_inc_read(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_inc_read__(var); +} + +ethr_sint32_t +ethr_atomic32_dec_read(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_dec_read__(var); +} + +void +ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t incr) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_add__(var, incr); +} + +void +ethr_atomic32_inc(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_inc__(var); +} + +void +ethr_atomic32_dec(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_dec__(var); +} + +ethr_sint32_t +ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_read_band__(var, mask); +} + +ethr_sint32_t +ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t mask) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_read_bor__(var, mask); +} + +ethr_sint32_t +ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t new) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_xchg__(var, new); +} + +ethr_sint32_t +ethr_atomic32_cmpxchg(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t expected) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_cmpxchg__(var, new, expected); +} + +ethr_sint32_t +ethr_atomic32_read_acqb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_read_acqb__(var); +} + +ethr_sint32_t +ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_inc_read_acqb__(var); +} + +void +ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t i) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_set_relb__(var, i); +} + +void +ethr_atomic32_dec_relb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + ethr_atomic32_dec_relb__(var); +} + +ethr_sint32_t +ethr_atomic32_dec_read_relb(ethr_atomic32_t *var) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_dec_read_relb__(var); +} + +ethr_sint32_t +ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_cmpxchg_acqb__(var, new, exp); +} + +ethr_sint32_t +ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t exp) +{ + ETHR_ASSERT(!ethr_not_inited__); + ETHR_ASSERT(var); + return ethr_atomic32_cmpxchg_relb__(var, new, exp); +} + diff --git a/erts/lib_src/common/ethr_aux.c b/erts/lib_src/common/ethr_aux.c index 4db4cffd3a..2c3e25a805 100644 --- a/erts/lib_src/common/ethr_aux.c +++ b/erts/lib_src/common/ethr_aux.c @@ -31,7 +31,10 @@ #define ETHR_INLINE_FUNC_NAME_(X) X ## __ #define ETHR_AUX_IMPL__ - +#define ETHR_ATOMIC_IMPL__ /* Needed in order to pull in + native atomic implementations + for optimized fallbacks of + spinlocks and rwspinlocks */ #include "ethread.h" #include "ethr_internal.h" #include <string.h> @@ -51,10 +54,6 @@ int ethr_not_inited__ = 1; ethr_memory_allocators ethr_mem__ = ETHR_MEM_ALLOCS_DEF_INITER__; -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATOMIC_ADDR_BITS]; -#endif - void *(*ethr_thr_prepare_func__)(void) = NULL; void (*ethr_thr_parent_func__)(void *) = NULL; void (*ethr_thr_child_func__)(void *) = NULL; @@ -138,16 +137,9 @@ ethr_init_common__(ethr_init_data *id) #endif ethr_max_stack_size__ = ETHR_B2KW(ethr_max_stack_size__); -#ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS - { - int i; - for (i = 0; i < (1 << ETHR_ATOMIC_ADDR_BITS); i++) { - res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); - if (res != 0) - return res; - } - } -#endif + res = ethr_init_atomics(); + if (res != 0) + return res; res = ethr_mutex_lib_init(erts_get_cpu_configured(ethr_cpu_info__)); if (res != 0) @@ -279,14 +271,6 @@ typedef union { static ethr_spinlock_t ts_ev_alloc_lock; static ethr_ts_event *free_ts_ev; -#if SIZEOF_VOID_P == SIZEOF_INT -typedef unsigned int EthrPtrSzUInt; -#elif SIZEOF_VOID_P == SIZEOF_LONG -typedef unsigned long EthrPtrSzUInt; -#else -#error No pointer sized integer type -#endif - static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp) { int i; @@ -295,16 +279,16 @@ static ethr_ts_event *ts_event_pool(int size, ethr_ts_event **endpp) + ETHR_CACHE_LINE_SIZE); if (!atsev) return NULL; - if ((((EthrPtrSzUInt) atsev) & ETHR_CACHE_LINE_MASK) == 0) + if ((((ethr_uint_t) atsev) & ETHR_CACHE_LINE_MASK) == 0) atsev = ((ethr_aligned_ts_event *) - ((((EthrPtrSzUInt) atsev) & ~ETHR_CACHE_LINE_MASK) + ((((ethr_uint_t) atsev) & ~ETHR_CACHE_LINE_MASK) + ETHR_CACHE_LINE_SIZE)); for (i = 1; i < size; i++) { atsev[i-1].ts_ev.next = &atsev[i].ts_ev; - ethr_atomic_init(&atsev[i-1].ts_ev.uaflgs, 0); + ethr_atomic32_init(&atsev[i-1].ts_ev.uaflgs, 0); atsev[i-1].ts_ev.iflgs = 0; } - ethr_atomic_init(&atsev[size-1].ts_ev.uaflgs, 0); + ethr_atomic32_init(&atsev[size-1].ts_ev.uaflgs, 0); atsev[size-1].ts_ev.iflgs = 0; atsev[size-1].ts_ev.next = NULL; if (endpp) @@ -466,170 +450,6 @@ int ethr_get_main_thr_status(int *on) return 0; } - -/* Atomics */ - -void -ethr_atomic_init(ethr_atomic_t *var, long i) -{ - ETHR_ASSERT(var); - ethr_atomic_init__(var, i); -} - -void -ethr_atomic_set(ethr_atomic_t *var, long i) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_set__(var, i); -} - -long -ethr_atomic_read(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_read__(var); -} - - -long -ethr_atomic_add_read(ethr_atomic_t *var, long incr) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_add_read__(var, incr); -} - -long -ethr_atomic_inc_read(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_inc_read__(var); -} - -long -ethr_atomic_dec_read(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_dec_read__(var); -} - -void -ethr_atomic_add(ethr_atomic_t *var, long incr) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_add__(var, incr); -} - -void -ethr_atomic_inc(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_inc__(var); -} - -void -ethr_atomic_dec(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_dec__(var); -} - -long -ethr_atomic_read_band(ethr_atomic_t *var, long mask) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_read_band__(var, mask); -} - -long -ethr_atomic_read_bor(ethr_atomic_t *var, long mask) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_read_bor__(var, mask); -} - -long -ethr_atomic_xchg(ethr_atomic_t *var, long new) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_xchg__(var, new); -} - -long -ethr_atomic_cmpxchg(ethr_atomic_t *var, long new, long expected) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_cmpxchg__(var, new, expected); -} - -long -ethr_atomic_read_acqb(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_read_acqb__(var); -} - -long -ethr_atomic_inc_read_acqb(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_inc_read_acqb__(var); -} - -void -ethr_atomic_set_relb(ethr_atomic_t *var, long i) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_set_relb__(var, i); -} - -void -ethr_atomic_dec_relb(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - ethr_atomic_dec_relb__(var); -} - -long -ethr_atomic_dec_read_relb(ethr_atomic_t *var) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_dec_read_relb__(var); -} - -long -ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, long new, long exp) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_cmpxchg_acqb__(var, new, exp); -} - -long -ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, long new, long exp) -{ - ETHR_ASSERT(!ethr_not_inited__); - ETHR_ASSERT(var); - return ethr_atomic_cmpxchg_relb__(var, new, exp); -} - - /* Spinlocks and rwspinlocks */ int diff --git a/erts/lib_src/common/ethr_mutex.c b/erts/lib_src/common/ethr_mutex.c index 78323b62a3..2ddef32dfc 100644 --- a/erts/lib_src/common/ethr_mutex.c +++ b/erts/lib_src/common/ethr_mutex.c @@ -205,12 +205,17 @@ static void hard_debug_chk_q__(struct ethr_mutex_base_ *, int); #ifdef ETHR_USE_OWN_RWMTX_IMPL__ static void +rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx, + ethr_sint32_t initial, + int q_locked); +static void rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, - long initial); + ethr_sint32_t initial, + int transfer_read_lock); static int rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, - long initial, + ethr_sint32_t initial, ethr_ts_event *tse, int start_next_ix, int check_before_try, @@ -237,12 +242,12 @@ rwmutex_freqread_rdrs_add(ethr_rwmutex *rwmtx, int inc) { if (type == ETHR_RWMUTEX_TYPE_FREQUENT_READ || ix == 0) - ethr_atomic_add(&rwmtx->tdata.ra[ix].data.readers, inc); + ethr_atomic32_add(&rwmtx->tdata.ra[ix].data.readers, inc); else { ETHR_ASSERT(type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); - ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0); + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0); ETHR_ASSERT(inc == 1); - ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1); + ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1); } } @@ -253,18 +258,20 @@ rwmutex_freqread_rdrs_inc(ethr_rwmutex *rwmtx, ethr_ts_event *tse) if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { ix = tse->rgix; atomic_inc: - ethr_atomic_inc(&rwmtx->tdata.ra[ix].data.readers); + ethr_atomic32_inc(&rwmtx->tdata.ra[ix].data.readers); } else { ix = tse->mtix; if (ix == 0) goto atomic_inc; ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); - ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 0); - ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 1); + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 0); + ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 1); } } +#if 0 /* Not used */ + static ETHR_INLINE void rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse) { @@ -272,69 +279,72 @@ rwmutex_freqread_rdrs_dec(ethr_rwmutex *rwmtx, ethr_ts_event *tse) if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { ix = tse->rgix; atomic_dec: - ethr_atomic_dec(&rwmtx->tdata.ra[ix].data.readers); + ethr_atomic32_dec(&rwmtx->tdata.ra[ix].data.readers); } else { ix = tse->mtix; if (ix == 0) goto atomic_dec; ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); - ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1); - ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0); + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0); } } -static ETHR_INLINE long +#endif + +static ETHR_INLINE ethr_sint32_t rwmutex_freqread_rdrs_dec_read(ethr_rwmutex *rwmtx, ethr_ts_event *tse) { int ix; if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { ix = tse->rgix; atomic_dec_read: - return ethr_atomic_dec_read(&rwmtx->tdata.ra[ix].data.readers); + return ethr_atomic32_dec_read(&rwmtx->tdata.ra[ix].data.readers); } else { ix = tse->mtix; if (ix == 0) goto atomic_dec_read; ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); - ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1); - ethr_atomic_set(&rwmtx->tdata.ra[ix].data.readers, (long) 0); - return (long) 0; + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic32_set(&rwmtx->tdata.ra[ix].data.readers, (ethr_sint32_t) 0); + return (ethr_sint32_t) 0; } } -static ETHR_INLINE long +static ETHR_INLINE ethr_sint32_t rwmutex_freqread_rdrs_dec_read_relb(ethr_rwmutex *rwmtx, ethr_ts_event *tse) { int ix; if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { ix = tse->rgix; atomic_dec_read: - return ethr_atomic_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers); + return ethr_atomic32_dec_read_relb(&rwmtx->tdata.ra[ix].data.readers); } else { ix = tse->mtix; if (ix == 0) goto atomic_dec_read; ETHR_ASSERT(rwmtx->type == ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ); - ETHR_ASSERT(ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers) == 1); - ethr_atomic_set_relb(&rwmtx->tdata.ra[ix].data.readers, (long) 0); - return (long) 0; + ETHR_ASSERT(ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers) == 1); + ethr_atomic32_set_relb(&rwmtx->tdata.ra[ix].data.readers, + (ethr_sint32_t) 0); + return (ethr_sint32_t) 0; } } -static ETHR_INLINE long +static ETHR_INLINE ethr_sint32_t rwmutex_freqread_rdrs_read(ethr_rwmutex *rwmtx, int ix) { - long res = ethr_atomic_read(&rwmtx->tdata.ra[ix].data.readers); + ethr_sint32_t res = ethr_atomic32_read(&rwmtx->tdata.ra[ix].data.readers); #ifdef ETHR_DEBUG switch (rwmtx->type) { case ETHR_RWMUTEX_TYPE_FREQUENT_READ: ETHR_ASSERT(res >= 0); break; case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: - ETHR_ASSERT(res == 0 || res == 1); + ETHR_ASSERT(ix == 0 ? res >= 0 : (res == 0 || res == 1)); break; default: ETHR_ASSERT(0); @@ -393,18 +403,19 @@ static void event_wait(struct ethr_mutex_base_ *mtxb, ethr_ts_event *tse, int spincount, - long type, + ethr_sint32_t type, int is_rwmtx, int is_freq_read) { int locked = 0; - long act; + ethr_sint32_t act; int need_try_complete_runlock = 0; + int transfer_read_lock = 0; /* Need to enqueue and wait... */ tse->uflgs = type; - ethr_atomic_set(&tse->uaflgs, type); + ethr_atomic32_set(&tse->uaflgs, type); ETHR_MTX_Q_LOCK(&mtxb->qlck); locked = 1; @@ -413,7 +424,7 @@ event_wait(struct ethr_mutex_base_ *mtxb, hard_debug_chk_q__(mtxb, is_rwmtx); #endif - act = ethr_atomic_read(&mtxb->flgs); + act = ethr_atomic32_read(&mtxb->flgs); if (act & type) { @@ -443,9 +454,9 @@ event_wait(struct ethr_mutex_base_ *mtxb, /* Set wait bit */ while (1) { - long new, exp = act; - int freqread_tryrlock = 0; + ethr_sint32_t new, exp = act; need_try_complete_runlock = 0; + transfer_read_lock = 0; if (type == ETHR_RWMTX_W_WAIT_FLG__) { if (is_freq_read && act == ETHR_RWMTX_R_FLG__) @@ -465,19 +476,16 @@ event_wait(struct ethr_mutex_base_ *mtxb, new = act + 1; /* Try to get it */ } else { - if (act & ~ETHR_RWMTX_R_FLG__) - new = act | ETHR_RWMTX_R_WAIT_FLG__; - else { /* Try to get it */ - ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; - rwmutex_freqread_rdrs_inc(rwmtx, tse); - ETHR_MEMORY_BARRIER; - new = act | ETHR_RWMTX_R_FLG__; - freqread_tryrlock = 1; + new = act | ETHR_RWMTX_R_WAIT_FLG__; + if ((act & (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_W_WAIT_FLG__)) == 0) { + /* Transfer read lock to this thread. */ + transfer_read_lock = 1; } } } - act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, new, exp); + act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs, new, exp); if (exp == act) { if (new & type) { act = new; @@ -488,24 +496,6 @@ event_wait(struct ethr_mutex_base_ *mtxb, goto done; } } - - if (freqread_tryrlock) { - ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; - - /* We didn't set ETHR_RWMTX_R_FLG__, however someone - else might have */ - if (act == ETHR_RWMTX_R_FLG__) - goto done; /* Got it by help from someone else */ - - ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); - /* - * We know that no waiter flags have been set, i.e., - * we cannot get into a situation where we need to wake - * someone up here. Just restore the readers counter - * and do it over again... - */ - rwmutex_freqread_rdrs_dec(rwmtx, tse); - } } /* Enqueue */ @@ -535,26 +525,42 @@ event_wait(struct ethr_mutex_base_ *mtxb, /* Wait */ locked = 0; - ETHR_MTX_Q_UNLOCK(&mtxb->qlck); - if (need_try_complete_runlock) { + ETHR_ASSERT(!(transfer_read_lock && need_try_complete_runlock)); + + if (transfer_read_lock) { ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type != ETHR_RWMUTEX_TYPE_NORMAL); /* - * We were the only one in queue when we enqueued, and it - * was seemingly read locked. We need to try to complete a - * runlock otherwise we might be hanging forever. If the - * runlock could be completed we will be dequeued and - * woken by ourselves. + * We are the only one in the queue and we are not write + * locked; rwmutex_transfer_read_lock() will: + * - transfer a read lock to us (since we're first in q) + * - unlock the Q-lock */ - rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb, - act, tse, 0, 1, 0); + rwmutex_transfer_read_lock(((ethr_rwmutex *) mtxb), act, 1); + } + else { + ETHR_MTX_Q_UNLOCK(&mtxb->qlck); + + if (need_try_complete_runlock) { + ETHR_ASSERT(((ethr_rwmutex *) mtxb)->type + != ETHR_RWMUTEX_TYPE_NORMAL); + /* + * We were the only one in queue when we enqueued, and it + * was seemingly read locked. We need to try to complete a + * runlock otherwise we might be hanging forever. If the + * runlock could be completed we will be dequeued and + * woken by ourselves. + */ + rwmutex_try_complete_runlock((ethr_rwmutex *) mtxb, + act, tse, 0, 1, 0); + } } while (1) { ethr_event_reset(&tse->event); - act = ethr_atomic_read_acqb(&tse->uaflgs); + act = ethr_atomic32_read_acqb(&tse->uaflgs); if (!act) goto done; /* Got it */ @@ -562,7 +568,7 @@ event_wait(struct ethr_mutex_base_ *mtxb, ethr_event_swait(&tse->event, spincount); /* swait result: 0 || EINTR */ - act = ethr_atomic_read_acqb(&tse->uaflgs); + act = ethr_atomic32_read_acqb(&tse->uaflgs); if (!act) goto done; /* Got it */ } @@ -582,7 +588,7 @@ wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx) dequeue(&mtxb->q, tse, tse); ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__); - ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__); #ifdef ETHR_MTX_HARD_DEBUG_WSQ mtxb->ws--; #endif @@ -592,7 +598,7 @@ wake_writer(struct ethr_mutex_base_ *mtxb, int is_rwmtx) ETHR_MTX_Q_UNLOCK(&mtxb->qlck); - ethr_atomic_set(&tse->uaflgs, 0); + ethr_atomic32_set(&tse->uaflgs, 0); ethr_event_set(&tse->event); } @@ -644,17 +650,15 @@ int check_readers_array(ethr_rwmutex *rwmtx, static ETHR_INLINE void write_lock_wait(struct ethr_mutex_base_ *mtxb, - long initial, + ethr_sint32_t initial, int is_rwmtx, int is_freq_read) { - long act = initial; + ethr_sint32_t act = initial; int scnt, start_scnt; ethr_ts_event *tse = NULL; int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; int res; - int freq_read_size = -1; - int freq_read_start_ix = -1; ETHR_ASSERT(!is_freq_read || is_rwmtx); @@ -666,44 +670,23 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb, */ while (1) { - long exp; - while (act != 0) { if (is_freq_read && act == ETHR_RWMTX_R_FLG__) { ethr_rwmutex *rwmtx = (ethr_rwmutex *) mtxb; + scnt--; if (!tse) tse = ethr_get_ts_event(); - if (freq_read_size < 0) { - if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { - freq_read_size = reader_groups_array_size; - freq_read_start_ix = tse->rgix; - } - else { - freq_read_size = main_threads_array_size; - freq_read_start_ix = tse->mtix; - } - } - res = check_readers_array(rwmtx, - freq_read_start_ix, - freq_read_size); - scnt--; - if (res == 0) { - act = ethr_atomic_read(&mtxb->flgs); - if (act & ETHR_RWMTX_R_MASK__) { - res = rwmutex_try_complete_runlock(rwmtx, act, - tse, 0, 0, - 1); - if (res != EBUSY) - goto done; /* Got it */ - } - if (scnt <= 0) - goto chk_spin; - if (--until_yield == 0) { - until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; - ETHR_YIELD(); - } - continue; + res = rwmutex_try_complete_runlock(rwmtx, act, + tse, 0, 0, + 1); + if (res != EBUSY) + goto done; /* Got it */ + if (scnt <= 0) + goto chk_spin; + if (--until_yield == 0) { + until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; + ETHR_YIELD(); } } @@ -724,15 +707,13 @@ write_lock_wait(struct ethr_mutex_base_ *mtxb, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; ETHR_YIELD(); } - act = ethr_atomic_read(&mtxb->flgs); + act = ethr_atomic32_read(&mtxb->flgs); scnt--; } - exp = act; - - act = ethr_atomic_cmpxchg_acqb(&mtxb->flgs, - ETHR_RWMTX_W_FLG__, - exp); + act = ethr_atomic32_cmpxchg_acqb(&mtxb->flgs, + ETHR_RWMTX_W_FLG__, + 0); if (act == 0) goto done; /* Got it */ } @@ -753,6 +734,7 @@ mtxb_init(struct ethr_mutex_base_ *mtxb, #ifdef ETHR_MTX_HARD_DEBUG_WSQ mtxb->ws = 0; #endif + ETHR_MTX_CHK_EXCL_INIT(mtxb); if (no_spin) { mtxb->main_scnt = 0; mtxb->aux_scnt = 0; @@ -775,16 +757,16 @@ mtxb_init(struct ethr_mutex_base_ *mtxb, } mtxb->q = NULL; - ethr_atomic_init(&mtxb->flgs, 0); + ethr_atomic32_init(&mtxb->flgs, 0); return ETHR_MTX_QLOCK_INIT(&mtxb->qlck); } static int mtxb_destroy(struct ethr_mutex_base_ *mtxb) { - long act; + ethr_sint32_t act; ETHR_MTX_Q_LOCK(&mtxb->qlck); - act = ethr_atomic_read(&mtxb->flgs); + act = ethr_atomic32_read(&mtxb->flgs); ETHR_MTX_Q_UNLOCK(&mtxb->qlck); if (act != 0) return EINVAL; @@ -850,13 +832,13 @@ ethr_mutex_destroy(ethr_mutex *mtx) } void -ethr_mutex_lock_wait__(ethr_mutex *mtx, long initial) +ethr_mutex_lock_wait__(ethr_mutex *mtx, ethr_sint32_t initial) { write_lock_wait(&mtx->mtxb, initial, 0, 0); } void -ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial) +ethr_mutex_unlock_wake__(ethr_mutex *mtx, ethr_sint32_t initial) { ethr_ts_event *tse; @@ -864,7 +846,7 @@ ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial) tse = mtx->mtxb.q; ETHR_ASSERT(tse); - ETHR_ASSERT(ethr_atomic_read(&mtx->mtxb.flgs) + ETHR_ASSERT(ethr_atomic32_read(&mtx->mtxb.flgs) == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)); ETHR_ASSERT(initial & ETHR_RWMTX_W_WAIT_FLG__); ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); @@ -874,7 +856,7 @@ ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial) * mtxb->flgs; otherwise, we need to clear the write wait bit... */ if (tse->next == mtx->mtxb.q) - ethr_atomic_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__); + ethr_atomic32_set(&mtx->mtxb.flgs, ETHR_RWMTX_W_FLG__); wake_writer(&mtx->mtxb, 0); } @@ -884,7 +866,7 @@ ethr_mutex_unlock_wake__(ethr_mutex *mtx, long initial) static void enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end) { - long act; + ethr_sint32_t act; /* * `ethr_cond_signal()' and `ethr_cond_broadcast()' end up here. If `mtx' @@ -913,7 +895,7 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end) } #endif - act = ethr_atomic_read(&mtx->mtxb.flgs); + act = ethr_atomic32_read(&mtx->mtxb.flgs); ETHR_ASSERT(act == 0 || act == ETHR_RWMTX_W_FLG__ || act == (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)); @@ -921,10 +903,10 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end) /* The normal sane case */ if (!(act & ETHR_RWMTX_W_WAIT_FLG__)) { ETHR_ASSERT(!mtx->mtxb.q); - act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, - (ETHR_RWMTX_W_FLG__ - | ETHR_RWMTX_W_WAIT_FLG__), - ETHR_RWMTX_W_FLG__); + act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs, + (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_W_WAIT_FLG__), + ETHR_RWMTX_W_FLG__); if (act != ETHR_RWMTX_W_FLG__) { /* * Sigh... this wasn't so sane after all since, the mutex was @@ -956,14 +938,14 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end) multi = tse_start != tse_end; while (1) { - long new, exp = act; + ethr_sint32_t new, exp = act; if (multi || (act & ETHR_RWMTX_W_FLG__)) new = ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__; else new = ETHR_RWMTX_W_FLG__; - act = ethr_atomic_cmpxchg(&mtx->mtxb.flgs, new, exp); + act = ethr_atomic32_cmpxchg(&mtx->mtxb.flgs, new, exp); if (exp == act) { ETHR_ASSERT(!mtx->mtxb.q); if (act & ETHR_RWMTX_W_FLG__) { @@ -991,7 +973,7 @@ enqueue_mtx(ethr_mutex *mtx, ethr_ts_event *tse_start, ethr_ts_event *tse_end) ETHR_MTX_HARD_DEBUG_CHK_Q(mtx); ETHR_MTX_Q_UNLOCK(&mtx->mtxb.qlck); - ethr_atomic_set(&tse_start->uaflgs, 0); + ethr_atomic32_set(&tse_start->uaflgs, 0); ethr_event_set(&tse_start->event); } break; @@ -1082,9 +1064,9 @@ ethr_cond_signal(ethr_cond *cnd) ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_W_WAIT_FLG__); - ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__); + ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_CND_WAIT_FLG__); - ethr_atomic_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__); + ethr_atomic32_set(&tse->uaflgs, ETHR_RWMTX_W_WAIT_FLG__); dequeue(&cnd->q, tse, tse); @@ -1135,10 +1117,11 @@ ethr_cond_broadcast(ethr_cond *cnd) /* The normal case */ ETHR_ASSERT(tse_tmp->uflgs == ETHR_RWMTX_W_WAIT_FLG__); - ETHR_ASSERT(ethr_atomic_read(&tse_tmp->uaflgs) + ETHR_ASSERT(ethr_atomic32_read(&tse_tmp->uaflgs) == ETHR_CND_WAIT_FLG__); - ethr_atomic_set(&tse_tmp->uaflgs, ETHR_RWMTX_W_WAIT_FLG__); + ethr_atomic32_set(&tse_tmp->uaflgs, + ETHR_RWMTX_W_WAIT_FLG__); } else { /* Should be very unusual */ @@ -1191,7 +1174,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) tse->udata = (void *) mtx; tse->uflgs = ETHR_RWMTX_W_WAIT_FLG__; /* Prep for mutex lock op */ - ethr_atomic_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__); + ethr_atomic32_set(&tse->uaflgs, ETHR_CND_WAIT_FLG__); ETHR_MTX_Q_LOCK(&cnd->qlck); @@ -1204,11 +1187,11 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) /* Wait */ woken = 0; while (1) { - long act; + ethr_sint32_t act; ethr_event_reset(&tse->event); - act = ethr_atomic_read_acqb(&tse->uaflgs); + act = ethr_atomic32_read_acqb(&tse->uaflgs); if (!act) break; /* Mtx locked */ @@ -1224,7 +1207,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) */ if (act == ETHR_CND_WAIT_FLG__) { ETHR_MTX_Q_LOCK(&cnd->qlck); - act = ethr_atomic_read(&tse->uaflgs); + act = ethr_atomic32_read(&tse->uaflgs); ETHR_ASSERT(act == ETHR_CND_WAIT_FLG__ || act == ETHR_RWMTX_W_WAIT_FLG__); /* @@ -1254,7 +1237,7 @@ ethr_cond_wait(ethr_cond *cnd, ethr_mutex *mtx) ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&mtx->mtxb); ETHR_MTX_HARD_DEBUG_FENCE_CHK(cnd); ETHR_MTX_HARD_DEBUG_FENCE_CHK(mtx); - + ETHR_MTX_CHK_EXCL_SET_EXCL(&mtx->mtxb); tse->udata = udata; ethr_leave_ts_event(tse); return 0; @@ -1426,7 +1409,7 @@ wake_readers(ethr_rwmutex *rwmtx, int rs) rwmtx->rq_end = NULL; ETHR_ASSERT(!rwmtx->mtxb.q - || (ethr_atomic_read(&rwmtx->mtxb.q->uaflgs) + || (ethr_atomic32_read(&rwmtx->mtxb.q->uaflgs) == ETHR_RWMTX_W_WAIT_FLG__)); ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); @@ -1437,7 +1420,7 @@ wake_readers(ethr_rwmutex *rwmtx, int rs) #ifdef ETHR_DEBUG ETHR_ASSERT(tse->uflgs == ETHR_RWMTX_R_WAIT_FLG__); - ETHR_ASSERT(ethr_atomic_read(&tse->uaflgs) + ETHR_ASSERT(ethr_atomic32_read(&tse->uaflgs) == ETHR_RWMTX_R_WAIT_FLG__); drs++; #endif @@ -1445,7 +1428,7 @@ wake_readers(ethr_rwmutex *rwmtx, int rs) tse_next = tse->next; /* we aren't allowed to read tse->next after we have reset uaflgs */ - ethr_atomic_set(&tse->uaflgs, 0); + ethr_atomic32_set(&tse->uaflgs, 0); ethr_event_set(&tse->event); tse = tse_next; } @@ -1488,7 +1471,7 @@ int check_readers_array(ethr_rwmutex *rwmtx, ETHR_MEMORY_BARRIER; do { - long act = rwmutex_freqread_rdrs_read(rwmtx, ix); + ethr_sint32_t act = rwmutex_freqread_rdrs_read(rwmtx, ix); if (act != 0) return EBUSY; ix++; @@ -1499,55 +1482,101 @@ int check_readers_array(ethr_rwmutex *rwmtx, return 0; } -static ETHR_INLINE void +static void +rwmutex_freqread_rdrs_dec_chk_wakeup(ethr_rwmutex *rwmtx, + ethr_ts_event *tse, + ethr_sint32_t initial) +{ + ethr_sint32_t act = initial; + + if ((act & (ETHR_RWMTX_W_FLG__| + ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) == 0) { + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) { + /* + * We *need* to try to complete the runlock. + * A writer that just enqueued (not seen by us + * in flag field) may depend on someone else + * completing the runlock. We just took over + * that responsibilty since we modified reader + * groups. + */ + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + } + } + else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_R_WAIT_FLG__) + rwmutex_transfer_read_lock(rwmtx, act, 0); + else if ((act & ETHR_RWMTX_WAIT_FLGS__) == ETHR_RWMTX_W_WAIT_FLG__) + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + else { + /* + * Don't know if we got readers or writers + * first in queue; need to peek + */ + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + if (!rwmtx->mtxb.q) + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + else if (is_w_waiter(rwmtx->mtxb.q)) { + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + if ((act & ETHR_RWMTX_W_FLG__) == 0) + rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); + } + else { + /* + * rwmutex_transfer_read_lock() will + * unlock Q lock. + */ + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + if (act & ETHR_RWMTX_W_FLG__) + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + else + rwmutex_transfer_read_lock(rwmtx, act, 1); + } + } + } +} + +static void rwmutex_freqread_restore_failed_tryrlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse) { - long act; + ethr_sint32_t act; /* * Restore failed increment */ act = rwmutex_freqread_rdrs_dec_read(rwmtx, tse); - ETHR_WRITE_MEMORY_BARRIER; + ETHR_MEMORY_BARRIER; if (act == 0) { - -#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL - ETHR_READ_MEMORY_BARRIER; -#endif - - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - - if ((act & ETHR_RWMTX_W_FLG__) == 0 - && act & (ETHR_RWMTX_WAIT_FLGS__|ETHR_RWMTX_R_PEND_UNLCK_MASK__)) { - /* - * We either got waiters, or someone else trying - * to read unlock which we might have to help. - */ - rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 1, 0); - } + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act); } } static int rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, - long initial, + ethr_sint32_t initial, ethr_ts_event *tse, int start_next_ix, int check_before_try, int try_write_lock) { ethr_ts_event *tse_tmp; - long act = initial; + ethr_sint32_t act = initial; int six, res, length; + ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); + + if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) + return try_write_lock ? EBUSY : 0; + tse_tmp = tse; if (!tse_tmp) tse_tmp = ethr_get_ts_event(); - if ((act & ETHR_RWMTX_WAIT_FLGS__) - && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) + if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) goto check_waiters; if (rwmtx->type == ETHR_RWMUTEX_TYPE_FREQUENT_READ) { @@ -1569,24 +1598,33 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, if (check_before_try) { res = check_readers_array(rwmtx, six, length); + + ETHR_MEMORY_BARRIER; + if (res == EBUSY) return try_write_lock ? EBUSY : 0; } + restart: + while (1) { - long exp = act; - long new = act+1; + ethr_sint32_t exp = act; + ethr_sint32_t new = act+1; + + ETHR_ASSERT((act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) == 0); ETHR_ASSERT((act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) < ETHR_RWMTX_R_PEND_UNLCK_MASK__); - act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp); if (exp == act) { act = new; break; } + if (!try_write_lock) { - if (act == ETHR_RWMTX_W_FLG__ || act == 0) + if (act == 0 || (act & (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_R_ABRT_UNLCK_FLG__))) return 0; if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { if ((act & ETHR_RWMTX_R_FLG__) == 0) @@ -1601,33 +1639,50 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, else { if (act == 0) goto tryrwlock; - if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) + if (act & (ETHR_RWMTX_W_FLG__ + | ETHR_RWMTX_R_ABRT_UNLCK_FLG__)) return EBUSY; } } res = check_readers_array(rwmtx, six, length); - if (res == EBUSY) { - act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs); - if (act & ETHR_RWMTX_R_MASK__) - return try_write_lock ? EBUSY : 0; - } - else { - while (1) { - long exp = act; - long new = act; - new &= ~ETHR_RWMTX_R_FLG__; - new--; - ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__); + ETHR_MEMORY_BARRIER; - act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); - if (exp == act) { - if (new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) - return try_write_lock ? EBUSY : 0; - act = new; - break; + ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); + + while (1) { + int finished_abort = 0; + ethr_sint32_t exp = act; + ethr_sint32_t new = act; + + new--; + if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) { + if ((new & ETHR_RWMTX_R_PEND_UNLCK_MASK__) == 0) { + new &= ~ETHR_RWMTX_R_ABRT_UNLCK_FLG__; + finished_abort = 1; } + ETHR_ASSERT(act & ETHR_RWMTX_R_FLG__); + } + else if ((act & ETHR_RWMTX_R_FLG__) && res != EBUSY) { + new &= ~ETHR_RWMTX_R_FLG__; + } + + ETHR_ASSERT(act & ETHR_RWMTX_R_PEND_UNLCK_MASK__); + + act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (exp == act) { + act = new; + if (act & ETHR_RWMTX_W_FLG__) + return try_write_lock ? EBUSY : 0; + if (finished_abort && (act & ETHR_RWMTX_WAIT_FLGS__)) + goto restart; + if (act & (ETHR_RWMTX_R_FLG__ + | ETHR_RWMTX_R_ABRT_UNLCK_FLG__ + | ETHR_RWMTX_R_PEND_UNLCK_MASK__)) + return try_write_lock ? EBUSY : 0; + /* Read unlock completed */ + break; } } @@ -1637,12 +1692,9 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, * to write lock it). */ - if (act & ETHR_RWMTX_W_FLG__) - return try_write_lock ? EBUSY : 0; - if (act & ETHR_RWMTX_WAIT_FLGS__) { check_waiters: - rwmutex_unlock_wake(rwmtx, 0, act); + rwmutex_unlock_wake(rwmtx, 0, act, 0); return try_write_lock ? EBUSY : 0; } @@ -1652,9 +1704,9 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, tryrwlock: /* Try to write lock it */ - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, - ETHR_RWMTX_W_FLG__, - 0); + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, + 0); return act == 0 ? 0 : EBUSY; } @@ -1663,24 +1715,23 @@ rwmutex_try_complete_runlock(ethr_rwmutex *rwmtx, static ETHR_INLINE void rwmutex_incdec_restore_failed_tryrlock(ethr_rwmutex *rwmtx) { - long act; + ethr_sint32_t act; /* * Restore failed increment */ - act = ethr_atomic_dec_read(&rwmtx->mtxb.flgs); + act = ethr_atomic32_dec_read(&rwmtx->mtxb.flgs); if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { - rwmutex_unlock_wake(rwmtx, 0, act); + rwmutex_unlock_wake(rwmtx, 0, act, 0); } } #endif static void -rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, - long initial) +rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial) { - long act = initial, exp; + ethr_sint32_t act = initial, exp; int scnt, start_scnt; ethr_ts_event *tse = NULL; int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; @@ -1696,11 +1747,11 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, #ifdef ETHR_RLOCK_WITH_INC_DEC rwmutex_incdec_restore_failed_tryrlock(rwmtx); - act = ethr_atomic_read(&rwmtx->mtxb.flgs); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); #endif while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { - if (scnt >= 0) { + if (scnt <= 0) { tse = ethr_get_ts_event(); if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { event_wait(&rwmtx->mtxb, tse, scnt, @@ -1713,17 +1764,17 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; ETHR_YIELD(); } - act = ethr_atomic_read(&rwmtx->mtxb.flgs); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); scnt--; } exp = act; #ifdef ETHR_RLOCK_WITH_INC_DEC - act = ethr_atomic_inc_read(&rwmtx->mtxb.flgs); + act = ethr_atomic32_inc_read(&rwmtx->mtxb.flgs); if ((act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) == 0) goto done; /* Got it */ #else - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); if (act == exp) goto done; /* Got it */ #endif @@ -1736,10 +1787,83 @@ rwmutex_normal_rlock_wait(ethr_rwmutex *rwmtx, static void rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, - ethr_ts_event *tse, - long initial) + ethr_ts_event *tse); + +static int +rwmutex_freqread_rlock(ethr_rwmutex *rwmtx, ethr_ts_event *tse, int trylock) { - long act = initial; + int res = 0; + ethr_sint32_t act; + + rwmutex_freqread_rdrs_inc(rwmtx, tse); + + ETHR_MEMORY_BARRIER; + + act = ethr_atomic32_read_acqb(&rwmtx->mtxb.flgs); + + if (act != ETHR_RWMTX_R_FLG__) { + int wake_other_readers; + + while (1) { + ethr_sint32_t exp, new; + + wake_other_readers = 0; + + if (act == 0) + new = act | ETHR_RWMTX_R_FLG__; + else if (act == ETHR_RWMTX_R_FLG__) + break; /* Got it */ + else if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); + if (trylock) + res = EBUSY; + else + rwmutex_freqread_rlock_wait(rwmtx, tse); + break; + } + else if (act & ETHR_RWMTX_R_ABRT_UNLCK_FLG__) { + if ((act & ETHR_RWMTX_R_FLG__) == 0) + ETHR_FATAL_ERROR__(EFAULT); + /* + * An aborted runlock, not write locked, and no write + * waiters, i.e., we got it... + */ + if (act & ETHR_RWMTX_R_WAIT_FLG__) + wake_other_readers = 1; + break; + } + else { + new = act | ETHR_RWMTX_R_FLG__; + if (act & ETHR_RWMTX_R_PEND_UNLCK_MASK__) { + /* + * Someone is doing tryrwlock (no writer and no + * write waiters); we will try to abort that... + */ + new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__; + } + + if (act & ETHR_RWMTX_R_WAIT_FLG__) + wake_other_readers = 1; + } + + exp = act; + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + break; + } + + if (wake_other_readers) + rwmutex_transfer_read_lock(rwmtx, act, 0); + } + + return res; +} + +static void +rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, + ethr_ts_event *tse) +{ + ethr_sint32_t act; int scnt, start_scnt; int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; @@ -1752,12 +1876,10 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, while (1) { - rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - - while (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { - if (scnt >= 0) { + while (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { + if (scnt <= 0) { if (update_spincount(&rwmtx->mtxb, tse, &start_scnt, &scnt)) { event_wait(&rwmtx->mtxb, tse, scnt, ETHR_RWMTX_R_WAIT_FLG__, 1, 1); @@ -1769,74 +1891,65 @@ rwmutex_freqread_rlock_wait(ethr_rwmutex *rwmtx, until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; ETHR_YIELD(); } - act = ethr_atomic_read(&rwmtx->mtxb.flgs); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); scnt--; } - rwmutex_freqread_rdrs_inc(rwmtx, tse); - - ETHR_MEMORY_BARRIER; - - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - - if (act == ETHR_RWMTX_R_FLG__) - return; /* Got it */ - - while (1) { - long exp, new; - - if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) - break; /* Busy (need to restore inc) */ - - if (act & ETHR_RWMTX_R_FLG__) - return; /* Got it */ - - exp = act; - new = act | ETHR_RWMTX_R_FLG__; - act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); - if (act == exp) - return; /* Got it */ - } + if (rwmutex_freqread_rlock(rwmtx, tse, 1) != EBUSY) + break; /* Got it */ } } static void -rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, long initial) +rwmutex_normal_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial) { write_lock_wait(&rwmtx->mtxb, initial, 1, 0); } static void -rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, long initial) +rwmutex_freqread_rwlock_wait(ethr_rwmutex *rwmtx, ethr_sint32_t initial) { write_lock_wait(&rwmtx->mtxb, initial, 1, 1); } static ETHR_INLINE void -rwlock_wake_set_flags(ethr_rwmutex *rwmtx, long new_initial, int act_initial) +rwlock_wake_set_flags(ethr_rwmutex *rwmtx, + ethr_sint32_t new_initial, + ethr_sint32_t act_initial) { - long act, act_mask; + ethr_sint32_t act, act_mask; + int chk_abrt_flg; + + ETHR_MEMORY_BARRIER; + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) { /* r pend unlock mask may vary and must be retained */ act_mask = ETHR_RWMTX_R_PEND_UNLCK_MASK__; + if (new_initial & ETHR_RWMTX_R_FLG__) + chk_abrt_flg = 1; + else + chk_abrt_flg = 0; } else { #ifdef ETHR_RLOCK_WITH_INC_DEC /* rs mask may vary and must be retained */ act_mask = ETHR_RWMTX_RS_MASK__; + chk_abrt_flg = 0; #else /* rs mask always zero */ ETHR_ASSERT((act_initial & ETHR_RWMTX_RS_MASK__) == 0); - ethr_atomic_set(&rwmtx->mtxb.flgs, new_initial); + ethr_atomic32_set(&rwmtx->mtxb.flgs, new_initial); return; #endif } act = act_initial; while (1) { - long exp = act; - long new = new_initial + (act & act_mask); - act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + ethr_sint32_t exp = act; + ethr_sint32_t new = new_initial + (act & act_mask); + if (chk_abrt_flg && (act & act_mask)) + new |= ETHR_RWMTX_R_ABRT_UNLCK_FLG__; + act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp); if (act == exp) break; exp = act; @@ -1850,7 +1963,7 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx, int have_w, ethr_ts_event *tse) { - long exp, act, imask; + ethr_sint32_t exp, act, imask; exp = have_w ? ETHR_RWMTX_W_FLG__ : 0; @@ -1872,7 +1985,7 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx, if (rwmtx->rq_end) { exp |= ETHR_RWMTX_R_WAIT_FLG__; } - act = ethr_atomic_read(&rwmtx->mtxb.flgs); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); ETHR_ASSERT((exp & ~imask) == (act & ~imask)); ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); @@ -1883,7 +1996,15 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx, exp |= ETHR_RWMTX_R_WAIT_FLG__; if (rwmtx->rq_end->next != rwmtx->mtxb.q) exp |= ETHR_RWMTX_W_WAIT_FLG__; - act = ethr_atomic_read(&rwmtx->mtxb.flgs); + else if (exp == ETHR_RWMTX_R_WAIT_FLG__) { + if (!have_w) { + if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) + imask |= ETHR_RWMTX_R_FLG__; + else + imask |= ETHR_RWMTX_RS_MASK__; + } + } + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); ETHR_ASSERT((exp & ~imask) == (act & ~imask)); ETHR_RWMTX_HARD_DEBUG_CHK_Q(rwmtx); @@ -1894,41 +2015,85 @@ dbg_unlock_wake(ethr_rwmutex *rwmtx, #endif static void -rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) +rwmutex_transfer_read_lock(ethr_rwmutex *rwmtx, + ethr_sint32_t initial, + int q_locked) { - long new, act = initial; - ethr_ts_event *tse; + ethr_sint32_t act = initial; - if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { - if (!have_w) + if (!q_locked) { + ethr_ts_event *tse; + ETHR_ASSERT(initial & ETHR_RWMTX_R_WAIT_FLG__); + ETHR_ASSERT((initial & ETHR_RWMTX_W_FLG__) == 0); + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + tse = rwmtx->mtxb.q; + if ((act & ETHR_RWMTX_W_FLG__) || !tse || is_w_waiter(tse)) { + /* Someone else woke the readers up... */ + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); return; - else { - while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { - long exp = act; - new = exp & ~ETHR_RWMTX_W_FLG__; - act = ethr_atomic_cmpxchg(&rwmtx->mtxb.flgs, new, exp); - if (act == exp) - return; - } } } - ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); - tse = rwmtx->mtxb.q; + rwmutex_unlock_wake(rwmtx, 0, initial, 1); +} - if (!have_w) { - if (!tse) { +static void +rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, ethr_sint32_t initial, + int transfer_read_lock) +{ + ethr_sint32_t new, act = initial; + ethr_ts_event *tse; + + if (transfer_read_lock) { + /* + * - Q already locked + * - Got R waiters first in Q + * - Not W locked + */ + tse = rwmtx->mtxb.q; + + ETHR_ASSERT(act & ETHR_RWMTX_R_WAIT_FLG__); + ETHR_ASSERT((act & (ETHR_RWMTX_W_FLG__)) == 0); + ETHR_ASSERT(tse && !is_w_waiter(tse)); + } + else { + + if ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + if (!have_w) + return; + else { + while ((act & ETHR_RWMTX_WAIT_FLGS__) == 0) { + ethr_sint32_t exp = act; + new = exp & ~ETHR_RWMTX_W_FLG__; + act = ethr_atomic32_cmpxchg(&rwmtx->mtxb.flgs, new, exp); + if (act == exp) + return; + } + } + } + + ETHR_MTX_Q_LOCK(&rwmtx->mtxb.qlck); + tse = rwmtx->mtxb.q; + + if (!have_w) { + if (!tse) { #ifdef ETHR_DEBUG - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + ETHR_ASSERT((act & ETHR_RWMTX_WAIT_FLGS__) == 0); #endif - goto already_served; - } - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - if (act & ~ETHR_RWMTX_WAIT_FLGS__) { - already_served: - ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); - return; + goto already_served; + } + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + if (act == (ETHR_RWMTX_R_WAIT_FLG__|ETHR_RWMTX_R_FLG__)) { + ETHR_ASSERT(tse && !is_w_waiter(tse)); + } + else if (act & ~ETHR_RWMTX_WAIT_FLGS__) { + already_served: + ETHR_MTX_Q_UNLOCK(&rwmtx->mtxb.qlck); + return; + } } } @@ -1939,9 +2104,12 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) if (is_w_waiter(tse)) { if (!have_w) { - act = ethr_atomic_read_bor(&rwmtx->mtxb.flgs, + act = ethr_atomic32_read_bor(&rwmtx->mtxb.flgs, ETHR_RWMTX_W_FLG__); - ETHR_ASSERT((act & ~ETHR_RWMTX_WAIT_FLGS__) == 0); + ETHR_ASSERT((act & ~(ETHR_RWMTX_WAIT_FLGS__ + | (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL + ? 0 + : ETHR_RWMTX_R_PEND_UNLCK_MASK__))) == 0); ETHR_ASSERT(act & ETHR_RWMTX_W_WAIT_FLG__); act |= ETHR_RWMTX_W_FLG__; } @@ -1968,7 +2136,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) if (rwmtx->type == ETHR_RWMUTEX_TYPE_NORMAL) { rs = rwmtx->tdata.rs; - new = (long) rs; + new = (ethr_sint32_t) rs; rwmtx->tdata.rs = 0; } else { @@ -1988,6 +2156,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) rwmutex_freqread_rdrs_add(rwmtx, type, ix, wrs); } } + new = ETHR_RWMTX_R_FLG__; } @@ -1995,6 +2164,7 @@ rwmutex_unlock_wake(ethr_rwmutex *rwmtx, int have_w, long initial) new |= ETHR_RWMTX_W_WAIT_FLG__; rwlock_wake_set_flags(rwmtx, new, act); + wake_readers(rwmtx, rs); } } @@ -2022,16 +2192,16 @@ alloc_readers_array(int length, ethr_rwmutex_lived lived) if (!mem) return NULL; - if ((((unsigned long) mem) & ETHR_CACHE_LINE_MASK) == 0) { + if ((((ethr_uint_t) mem) & ETHR_CACHE_LINE_MASK) == 0) { ra = (ethr_rwmtx_readers_array__ *) mem; ra->data.byte_offset = 0; } else { ra = ((ethr_rwmtx_readers_array__ *) - ((((unsigned long) mem) & ~ETHR_CACHE_LINE_MASK) + ((((ethr_uint_t) mem) & ~ETHR_CACHE_LINE_MASK) + ETHR_CACHE_LINE_SIZE)); - ra->data.byte_offset = (int) ((unsigned long) ra - - (unsigned long) mem); + ra->data.byte_offset = (int) ((ethr_uint_t) ra + - (ethr_uint_t) mem); } ra->data.lived = lived; return ra; @@ -2105,7 +2275,7 @@ ethr_rwmutex_init_opt(ethr_rwmutex *rwmtx, ethr_rwmutex_opt *opt) rwmtx->tdata.ra = ra; for (ix = 0; ix < length; ix++) { - ethr_atomic_init(&rwmtx->tdata.ra[ix].data.readers, 0); + ethr_atomic32_init(&rwmtx->tdata.ra[ix].data.readers, 0); rwmtx->tdata.ra[ix].data.waiting_readers = 0; } break; @@ -2157,8 +2327,9 @@ ethr_rwmutex_destroy(ethr_rwmutex *rwmtx) return EINVAL; } #endif + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); if (rwmtx->type != ETHR_RWMUTEX_TYPE_NORMAL) { - long act = ethr_atomic_read(&rwmtx->mtxb.flgs); + ethr_sint32_t act = ethr_atomic32_read(&rwmtx->mtxb.flgs); if (act == ETHR_RWMTX_R_FLG__) rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 0, 0); } @@ -2179,7 +2350,7 @@ int ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) { int res = 0; - long act; + ethr_sint32_t act; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(rwmtx); @@ -2187,25 +2358,27 @@ ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + switch (rwmtx->type) { case ETHR_RWMUTEX_TYPE_NORMAL: { #ifdef ETHR_RLOCK_WITH_INC_DEC - act = ethr_atomic_read(&rwmtx->mtxb.flgs); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) res = EBUSY; else { - act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs); + act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs); if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { rwmutex_incdec_restore_failed_tryrlock(rwmtx); res = EBUSY; } } #else - long exp = 0; + ethr_sint32_t exp = 0; int tries = 0; while (1) { - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); if (act == exp) { res = 0; break; @@ -2225,49 +2398,30 @@ ethr_rwmutex_tryrlock(ethr_rwmutex *rwmtx) case ETHR_RWMUTEX_TYPE_FREQUENT_READ: case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { ethr_ts_event *tse = ethr_get_ts_event(); - - rwmutex_freqread_rdrs_inc(rwmtx, tse); - - ETHR_MEMORY_BARRIER; - - act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs); - - if (act != ETHR_RWMTX_R_FLG__) { - while (1) { - long exp, new; - - if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { - rwmutex_freqread_restore_failed_tryrlock(rwmtx, tse); - res = EBUSY; - break; - } - - if (act & ETHR_RWMTX_R_FLG__) - break; - - exp = act; - new = act | ETHR_RWMTX_R_FLG__; - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); - if (act == exp) - break; - } - } - + res = rwmutex_freqread_rlock(rwmtx, tse, 1); ethr_leave_ts_event(tse); break; } } +#ifdef ETHR_MTX_CHK_EXCL + if (res == 0) { + ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb); + } +#endif + ETHR_MTX_HARD_DEBUG_LFS_TRYRLOCK(&rwmtx->mtxb, res); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); return res; } void ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) { - long act; + ethr_sint32_t act; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(rwmtx); @@ -2275,20 +2429,21 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + switch (rwmtx->type) { case ETHR_RWMUTEX_TYPE_NORMAL: { #ifdef ETHR_RLOCK_WITH_INC_DEC - act = ethr_atomic_inc_read_acqb(&rwmtx->mtxb.flgs); + act = ethr_atomic32_inc_read_acqb(&rwmtx->mtxb.flgs); if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) rwmutex_normal_rlock_wait(rwmtx, act); #else - long exp = 0; + ethr_sint32_t exp = 0; while (1) { - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); - if (act == exp) { + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, exp+1, exp); + if (act == exp) break; - } if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_W_WAIT_FLG__)) { rwmutex_normal_rlock_wait(rwmtx, act); @@ -2303,38 +2458,15 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) case ETHR_RWMUTEX_TYPE_FREQUENT_READ: case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: { ethr_ts_event *tse = ethr_get_ts_event(); - - rwmutex_freqread_rdrs_inc(rwmtx, tse); - - ETHR_MEMORY_BARRIER; - - act = ethr_atomic_read_acqb(&rwmtx->mtxb.flgs); - - if (act != ETHR_RWMTX_R_FLG__) { - while (1) { - long exp, new; - - if (act & ~(ETHR_RWMTX_R_FLG__|ETHR_RWMTX_R_WAIT_FLG__)) { - rwmutex_freqread_rlock_wait(rwmtx, tse, act); - break; - } - - if (act & ETHR_RWMTX_R_FLG__) - break; - - exp = act; - new = act | ETHR_RWMTX_R_FLG__; - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, new, exp); - if (act == exp) - break; - } - } - + rwmutex_freqread_rlock(rwmtx, tse, 0); ethr_leave_ts_event(tse); break; } } + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + ETHR_MTX_CHK_EXCL_SET_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb); ETHR_MTX_HARD_DEBUG_LFS_RLOCK(&rwmtx->mtxb); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); } @@ -2342,8 +2474,10 @@ ethr_rwmutex_rlock(ethr_rwmutex *rwmtx) void ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) { - long act; + ethr_sint32_t act; + ETHR_MTX_CHK_EXCL_IS_NOT_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_UNSET_NON_EXCL(&rwmtx->mtxb); ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(rwmtx); ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); @@ -2351,13 +2485,15 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); ETHR_MTX_HARD_DEBUG_LFS_RUNLOCK(&rwmtx->mtxb); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + switch (rwmtx->type) { case ETHR_RWMUTEX_TYPE_NORMAL: - act = ethr_atomic_dec_read_relb(&rwmtx->mtxb.flgs); + act = ethr_atomic32_dec_read_relb(&rwmtx->mtxb.flgs); if ((act & ETHR_RWMTX_WAIT_FLGS__) && (act & ~ETHR_RWMTX_WAIT_FLGS__) == 0) { ETHR_ASSERT((act & ETHR_RWMTX_W_FLG__) == 0); - rwmutex_unlock_wake(rwmtx, 0, act); + rwmutex_unlock_wake(rwmtx, 0, act, 0); } break; @@ -2369,21 +2505,12 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) ETHR_ASSERT(act >= 0); - ETHR_WRITE_MEMORY_BARRIER; + ETHR_MEMORY_BARRIER; if (act == 0) { - -#ifndef ETHR_WRITE_MEMORY_BARRIER_IS_FULL - ETHR_READ_MEMORY_BARRIER; -#endif - act = ethr_atomic_read(&rwmtx->mtxb.flgs); - - if ((act & ETHR_RWMTX_W_FLG__) == 0 - && (act & (ETHR_RWMTX_WAIT_FLGS__ - | ETHR_RWMTX_R_PEND_UNLCK_MASK__))) { - rwmutex_try_complete_runlock(rwmtx, act, tse, 1, 0, 0); - } - + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); + if (act != ETHR_RWMTX_R_FLG__) + rwmutex_freqread_rdrs_dec_chk_wakeup(rwmtx, tse, act); } ethr_leave_ts_event(tse); @@ -2391,6 +2518,7 @@ ethr_rwmutex_runlock(ethr_rwmutex *rwmtx) } } + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); } @@ -2398,7 +2526,7 @@ int ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) { int res = 0; - long act; + ethr_sint32_t act; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(rwmtx); @@ -2406,10 +2534,12 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + switch (rwmtx->type) { case ETHR_RWMUTEX_TYPE_NORMAL: - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, - ETHR_RWMTX_W_FLG__, 0); + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); if (act != 0) res = EBUSY; break; @@ -2418,29 +2548,36 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: res = 0; - act = ethr_atomic_read(&rwmtx->mtxb.flgs); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); do { - if (act & (ETHR_RWMTX_W_FLG__|ETHR_RWMTX_WAIT_FLGS__)) { - res = EBUSY; - break; - } - - if (act & ETHR_RWMTX_R_MASK__) { + if (act == 0) + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); + else if (act == ETHR_RWMTX_R_FLG__) { res = rwmutex_try_complete_runlock(rwmtx, act, NULL, 0, 1, 1); break; } - - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, - ETHR_RWMTX_W_FLG__, 0); + else { + res = EBUSY; + break; + } } while (act != 0); break; } +#ifdef ETHR_MTX_CHK_EXCL + if (res == 0) { + ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb); + } +#endif + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); ETHR_MTX_HARD_DEBUG_LFS_TRYRWLOCK(&rwmtx->mtxb, res); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); @@ -2450,17 +2587,19 @@ ethr_rwmutex_tryrwlock(ethr_rwmutex *rwmtx) void ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) { - long act; + ethr_sint32_t act; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(rwmtx); ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + switch (rwmtx->type) { case ETHR_RWMUTEX_TYPE_NORMAL: - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, - ETHR_RWMTX_W_FLG__, 0); + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); if (act != 0) rwmutex_normal_rwlock_wait(rwmtx, act); break; @@ -2468,7 +2607,7 @@ ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) case ETHR_RWMUTEX_TYPE_FREQUENT_READ: case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: - act = ethr_atomic_read(&rwmtx->mtxb.flgs); + act = ethr_atomic32_read(&rwmtx->mtxb.flgs); do { @@ -2477,23 +2616,26 @@ ethr_rwmutex_rwlock(ethr_rwmutex *rwmtx) break; } - act = ethr_atomic_cmpxchg_acqb(&rwmtx->mtxb.flgs, - ETHR_RWMTX_W_FLG__, 0); + act = ethr_atomic32_cmpxchg_acqb(&rwmtx->mtxb.flgs, + ETHR_RWMTX_W_FLG__, 0); } while (act != 0); break; } + ETHR_MTX_CHK_EXCL_SET_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb); ETHR_MTX_HARD_DEBUG_LFS_RWLOCK(&rwmtx->mtxb); ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); } void ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) { - long act; + ethr_sint32_t act; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(rwmtx); ETHR_ASSERT(rwmtx->initialized == ETHR_RWMUTEX_INITIALIZED); @@ -2501,24 +2643,30 @@ ethr_rwmutex_rwunlock(ethr_rwmutex *rwmtx) ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); ETHR_MTX_HARD_DEBUG_LFS_RWUNLOCK(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_IS_NOT_NON_EXCL(&rwmtx->mtxb); + ETHR_MTX_CHK_EXCL_UNSET_EXCL(&rwmtx->mtxb); + + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); + switch (rwmtx->type) { case ETHR_RWMUTEX_TYPE_NORMAL: - act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, - 0, ETHR_RWMTX_W_FLG__); + act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs, + 0, ETHR_RWMTX_W_FLG__); if (act != ETHR_RWMTX_W_FLG__) - rwmutex_unlock_wake(rwmtx, 1, act); + rwmutex_unlock_wake(rwmtx, 1, act, 0); break; case ETHR_RWMUTEX_TYPE_FREQUENT_READ: case ETHR_RWMUTEX_TYPE_EXTREMELY_FREQUENT_READ: - act = ethr_atomic_cmpxchg_relb(&rwmtx->mtxb.flgs, 0, - ETHR_RWMTX_W_FLG__); + act = ethr_atomic32_cmpxchg_relb(&rwmtx->mtxb.flgs, 0, + ETHR_RWMTX_W_FLG__); if (act != ETHR_RWMTX_W_FLG__) - rwmutex_unlock_wake(rwmtx, 1, act); + rwmutex_unlock_wake(rwmtx, 1, act, 0); break; } ETHR_MTX_HARD_DEBUG_FENCE_CHK(rwmtx); + ETHR_MTX_DBG_CHK_UNUSED_FLG_BITS(rwmtx); } #else @@ -2636,7 +2784,7 @@ static void hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx) { int res; - long flgs = ethr_atomic_read(&mtxb->flgs); + ethr_sint32_t flgs = ethr_atomic32_read(&mtxb->flgs); ETHR_MTX_HARD_ASSERT(res == 0); @@ -2659,12 +2807,12 @@ hard_debug_chk_q__(struct ethr_mutex_base_ *mtxb, int is_rwmtx) tse = mtxb->q; do { - long type; + ethr_sint32_t type; ETHR_MTX_HARD_ASSERT(tse->next->prev == tse); ETHR_MTX_HARD_ASSERT(tse->prev->next == tse); - type = ethr_atomic_read(&tse->uaflgs); + type = ethr_atomic32_read(&tse->uaflgs); ETHR_MTX_HARD_ASSERT(type == tse->uflgs); switch (type) { case ETHR_RWMTX_W_WAIT_FLG__: diff --git a/erts/lib_src/pthread/ethr_event.c b/erts/lib_src/pthread/ethr_event.c index 6731c0eb46..9434d60d0a 100644 --- a/erts/lib_src/pthread/ethr_event.c +++ b/erts/lib_src/pthread/ethr_event.c @@ -24,6 +24,10 @@ #define ETHR_INLINE_FUNC_NAME_(X) X ## __ #define ETHR_EVENT_IMPL__ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + #include "ethread.h" #if defined(ETHR_LINUX_FUTEX_IMPL__) @@ -37,7 +41,7 @@ int ethr_event_init(ethr_event *e) { - ethr_atomic_init(&e->futex, ETHR_EVENT_OFF__); + ethr_atomic32_init(&e->futex, ETHR_EVENT_OFF__); return 0; } @@ -52,7 +56,7 @@ wait__(ethr_event *e, int spincount) { unsigned sc = spincount; int res; - long val; + ethr_sint32_t val; int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; if (spincount < 0) @@ -60,7 +64,7 @@ wait__(ethr_event *e, int spincount) while (1) { while (1) { - val = ethr_atomic_read(&e->futex); + val = ethr_atomic32_read(&e->futex); if (val == ETHR_EVENT_ON__) return 0; if (sc == 0) @@ -76,16 +80,18 @@ wait__(ethr_event *e, int spincount) } if (val != ETHR_EVENT_OFF_WAITER__) { - val = ethr_atomic_cmpxchg(&e->futex, - ETHR_EVENT_OFF_WAITER__, - ETHR_EVENT_OFF__); + val = ethr_atomic32_cmpxchg(&e->futex, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); if (val == ETHR_EVENT_ON__) return 0; ETHR_ASSERT(val == ETHR_EVENT_OFF__); } - res = ETHR_FUTEX__(&e->futex, ETHR_FUTEX_WAIT__, ETHR_EVENT_OFF_WAITER__); + res = ETHR_FUTEX__(&e->futex, + ETHR_FUTEX_WAIT__, + ETHR_EVENT_OFF_WAITER__); if (res == EINTR) break; if (res != 0 && res != EWOULDBLOCK) @@ -102,7 +108,7 @@ int ethr_event_init(ethr_event *e) { int res; - ethr_atomic_init(&e->state, ETHR_EVENT_OFF__); + ethr_atomic32_init(&e->state, ETHR_EVENT_OFF__); res = pthread_mutex_init(&e->mtx, NULL); if (res != 0) return res; @@ -131,7 +137,7 @@ static ETHR_INLINE int wait__(ethr_event *e, int spincount) { int sc = spincount; - long val; + ethr_sint32_t val; int res, ulres; int until_yield = ETHR_YIELD_AFTER_BUSY_LOOPS; @@ -139,7 +145,7 @@ wait__(ethr_event *e, int spincount) ETHR_FATAL_ERROR__(EINVAL); while (1) { - val = ethr_atomic_read(&e->state); + val = ethr_atomic32_read(&e->state); if (val == ETHR_EVENT_ON__) return 0; if (sc == 0) @@ -155,9 +161,9 @@ wait__(ethr_event *e, int spincount) } if (val != ETHR_EVENT_OFF_WAITER__) { - val = ethr_atomic_cmpxchg(&e->state, - ETHR_EVENT_OFF_WAITER__, - ETHR_EVENT_OFF__); + val = ethr_atomic32_cmpxchg(&e->state, + ETHR_EVENT_OFF_WAITER__, + ETHR_EVENT_OFF__); if (val == ETHR_EVENT_ON__) return 0; ETHR_ASSERT(val == ETHR_EVENT_OFF__); @@ -172,7 +178,7 @@ wait__(ethr_event *e, int spincount) while (1) { - val = ethr_atomic_read(&e->state); + val = ethr_atomic32_read(&e->state); if (val == ETHR_EVENT_ON__) break; diff --git a/erts/lib_src/pthread/ethread.c b/erts/lib_src/pthread/ethread.c index ea1d9d43f0..f047104103 100644 --- a/erts/lib_src/pthread/ethread.c +++ b/erts/lib_src/pthread/ethread.c @@ -72,7 +72,7 @@ static void thr_exit_cleanup(void) /* Argument passed to thr_wrapper() */ typedef struct { - ethr_atomic_t result; + ethr_atomic32_t result; ethr_ts_event *tse; void *(*thr_func)(void *); void *arg; @@ -81,14 +81,14 @@ typedef struct { static void *thr_wrapper(void *vtwd) { - long result; + ethr_sint32_t result; void *res; ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd; void *(*thr_func)(void *) = twd->thr_func; void *arg = twd->arg; ethr_ts_event *tsep = NULL; - result = (long) ethr_make_ts_event__(&tsep); + result = (ethr_sint32_t) ethr_make_ts_event__(&tsep); if (result == 0) { tsep->iflgs |= ETHR_TS_EV_ETHREAD; @@ -99,7 +99,7 @@ static void *thr_wrapper(void *vtwd) tsep = twd->tse; /* We aren't allowed to follow twd after result has been set! */ - ethr_atomic_set(&twd->result, result); + ethr_atomic32_set(&twd->result, result); ethr_event_set(&tsep->event); @@ -191,7 +191,7 @@ ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, } #endif - ethr_atomic_init(&twd.result, -1); + ethr_atomic32_init(&twd.result, (ethr_sint32_t) -1); twd.tse = ethr_get_ts_event(); twd.thr_func = func; twd.arg = arg; @@ -252,10 +252,10 @@ ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, /* Wait for child to initialize... */ while (1) { - long result; + ethr_sint32_t result; ethr_event_reset(&twd.tse->event); - result = ethr_atomic_read(&twd.result); + result = ethr_atomic32_read(&twd.result); if (result == 0) break; @@ -349,32 +349,6 @@ ethr_leave_ts_event(ethr_ts_event *tsep) } /* - * Current time - */ - -int -ethr_time_now(ethr_timeval *time) -{ - int res; - struct timeval tv; -#if ETHR_XCHK - if (ethr_not_inited__) { - ETHR_ASSERT(0); - return EACCES; - } - if (!time) { - ETHR_ASSERT(0); - return EINVAL; - } -#endif - - res = gettimeofday(&tv, NULL); - time->tv_sec = (long) tv.tv_sec; - time->tv_nsec = ((long) tv.tv_usec)*1000; - return res; -} - -/* * Thread specific data */ diff --git a/erts/lib_src/win/ethr_event.c b/erts/lib_src/win/ethr_event.c index ddb4780ff1..68f093f49c 100644 --- a/erts/lib_src/win/ethr_event.c +++ b/erts/lib_src/win/ethr_event.c @@ -28,6 +28,9 @@ /* --- Windows implementation of thread events ------------------------------ */ +#pragma intrinsic(_InterlockedExchangeAdd) +#pragma intrinsic(_InterlockedCompareExchange) + int ethr_event_init(ethr_event *e) { @@ -72,10 +75,10 @@ wait(ethr_event *e, int spincount) while (1) { long on; while (1) { -#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ +#if ETHR_READ_AND_SET_WITHOUT_INTERLOCKED_OP__ state = e->state; #else - state = InterlockedExchangeAdd(&e->state, (LONG) 0); + state = _InterlockedExchangeAdd(&e->state, (LONG) 0); #endif if (state == ETHR_EVENT_ON__) return 0; diff --git a/erts/lib_src/win/ethread.c b/erts/lib_src/win/ethread.c index 69523edf94..789a360b11 100644 --- a/erts/lib_src/win/ethread.c +++ b/erts/lib_src/win/ethread.c @@ -49,7 +49,7 @@ /* Argument passed to thr_wrapper() */ typedef struct { ethr_tid *tid; - ethr_atomic_t result; + ethr_atomic32_t result; ethr_ts_event *tse; void *(*thr_func)(void *); void *arg; @@ -93,20 +93,20 @@ static void thr_exit_cleanup(ethr_tid *tid, void *res) static unsigned __stdcall thr_wrapper(LPVOID vtwd) { ethr_tid my_tid; - long result; + ethr_sint32_t result; void *res; ethr_thr_wrap_data__ *twd = (ethr_thr_wrap_data__ *) vtwd; void *(*thr_func)(void *) = twd->thr_func; void *arg = twd->arg; ethr_ts_event *tsep = NULL; - result = (long) ethr_make_ts_event__(&tsep); + result = (ethr_sint32_t) ethr_make_ts_event__(&tsep); if (result == 0) { tsep->iflgs |= ETHR_TS_EV_ETHREAD; my_tid = *twd->tid; if (!TlsSetValue(own_tid_key, (LPVOID) &my_tid)) { - result = (long) ethr_win_get_errno__(); + result = (ethr_sint32_t) ethr_win_get_errno__(); ethr_free_ts_event__(tsep); } else { @@ -118,7 +118,7 @@ static unsigned __stdcall thr_wrapper(LPVOID vtwd) tsep = twd->tse; /* We aren't allowed to follow twd after result has been set! */ - ethr_atomic_set(&twd->result, result); + ethr_atomic32_set(&twd->result, result); ethr_event_set(&tsep->event); @@ -128,28 +128,6 @@ static unsigned __stdcall thr_wrapper(LPVOID vtwd) return 0; } -#ifdef __GNUC__ -#define LL_LITERAL(X) X##LL -#else -#define LL_LITERAL(X) X##i64 -#endif - -#define EPOCH_JULIAN_DIFF LL_LITERAL(11644473600) - -static ETHR_INLINE void -get_curr_time(long *sec, long *nsec) -{ - SYSTEMTIME t; - FILETIME ft; - LONGLONG lft; - - GetSystemTime(&t); - SystemTimeToFileTime(&t, &ft); - memcpy(&lft, &ft, sizeof(lft)); - *nsec = ((long) (lft % LL_LITERAL(10000000)))*100; - *sec = (long) ((lft / LL_LITERAL(10000000)) - EPOCH_JULIAN_DIFF); -} - /* internal exports */ int @@ -320,7 +298,7 @@ ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, ETHR_PAGE_ALIGN(ETHR_KW2B(suggested_stack_size)); } - ethr_atomic_init(&twd.result, -1); + ethr_atomic32_init(&twd.result, -1); twd.tid = tid; twd.thr_func = func; @@ -352,11 +330,11 @@ ethr_thr_create(ethr_tid *tid, void * (*func)(void *), void *arg, /* Wait for child to initialize... */ while (1) { - long result; + ethr_sint32_t result; int err; ethr_event_reset(&twd.tse->event); - result = ethr_atomic_read(&twd.result); + result = ethr_atomic32_read(&twd.result); if (result == 0) break; @@ -517,23 +495,6 @@ ethr_equal_tids(ethr_tid tid1, ethr_tid tid2) return tid1.id == tid2.id && tid1.id != ETHR_INVALID_TID_ID; } -int -ethr_time_now(ethr_timeval *time) -{ -#if ETHR_XCHK - if (ethr_not_inited__) { - ETHR_ASSERT(0); - return EACCES; - } - if (!time) { - ETHR_ASSERT(0); - return EINVAL; - } -#endif - get_curr_time(&time->tv_sec, &time->tv_nsec); - return 0; -} - /* * Thread specific data */ diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex 222809e662..bff3f7f9de 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 65c7369b76..5d2f187435 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex c8d3b78b35..7e492057da 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam Binary files differindex abf17bcb0e..c443866671 100644 --- a/erts/preloaded/ebin/otp_ring0.beam +++ b/erts/preloaded/ebin/otp_ring0.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex a3f300268f..4a75e43e73 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex a777971b32..30bbfb0943 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam Binary files differindex 0fe38a1fb2..6681466767 100644 --- a/erts/preloaded/ebin/prim_zip.beam +++ b/erts/preloaded/ebin/prim_zip.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex 7108bf44d0..593bb8dbed 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 935c2de253..4679a916c7 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1996-2010. All Rights Reserved. +%% Copyright Ericsson AB 1996-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -427,17 +427,11 @@ delay_trap(Result, Timeout) -> receive after Timeout -> Result end. %% have to reflect that, which we cannot forsee. %% set_cookie(Node, C) when Node =/= nonode@nohost, is_atom(Node) -> - Res = case C of - _ when is_atom(C) -> - auth:set_cookie(Node, C); - {CI,CO} when is_atom(CI), is_atom(CO) -> - auth:set_cookie(Node, {CI, CO}); - _ -> - error - end, - case Res of - error -> exit(badarg); - Other -> Other + case is_atom(C) of + true -> + auth:set_cookie(Node, C); + false -> + error(badarg) end. -spec get_cookie() -> atom(). diff --git a/erts/preloaded/src/init.erl b/erts/preloaded/src/init.erl index 3ab9a1cd6d..24430a3d40 100644 --- a/erts/preloaded/src/init.erl +++ b/erts/preloaded/src/init.erl @@ -72,6 +72,7 @@ script_id = [], loaded = [], subscribed = []}). +-type state() :: #state{}. -define(ON_LOAD_HANDLER, init__boot__on_load_handler). @@ -146,10 +147,10 @@ restart() -> init ! {stop,restart}, ok. -spec reboot() -> 'ok'. reboot() -> init ! {stop,reboot}, ok. --spec stop() -> no_return(). +-spec stop() -> 'ok'. stop() -> init ! {stop,stop}, ok. --spec stop(non_neg_integer() | string()) -> no_return(). +-spec stop(non_neg_integer() | string()) -> 'ok'. stop(Status) -> init ! {stop,{stop,Status}}, ok. -spec boot([binary()]) -> no_return(). @@ -278,7 +279,7 @@ crash(String, List) -> halt(halt_string(String, List)). %% Status is {InternalStatus,ProvidedStatus} --spec boot_loop(pid(), #state{}) -> no_return(). +-spec boot_loop(pid(), state()) -> no_return(). boot_loop(BootPid, State) -> receive {BootPid,loaded,ModLoaded} -> diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index 7f24889bb2..10be852e92 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -109,6 +109,8 @@ -define(FILE_RESP_LDATA, 6). -define(FILE_RESP_N2DATA, 7). -define(FILE_RESP_EOF, 8). +-define(FILE_RESP_FNAME, 9). +-define(FILE_RESP_ALL_DATA, 10). %% Open modes for the driver's open function. -define(EFILE_MODE_READ, 1). @@ -153,7 +155,7 @@ %% Opens a file using the driver port Port. Returns {error, Reason} %% | {ok, FileDescriptor} open(Port, File, ModeList) when is_port(Port), - is_list(File), + (is_list(File) orelse is_binary(File)), is_list(ModeList) -> case open_mode(ModeList) of {Mode, _Portopts, _Setopts} -> @@ -165,10 +167,11 @@ open(_,_,_) -> {error, badarg}. %% Opens a file. Returns {error, Reason} | {ok, FileDescriptor}. -open(File, ModeList) when is_list(File), is_list(ModeList) -> +open(File, ModeList) when (is_list(File) orelse is_binary(File)), + is_list(ModeList) -> case open_mode(ModeList) of {Mode, Portopts, Setopts} -> - open_int({?FD_DRV, Portopts}, File, Mode, Setopts); + open_int({?FD_DRV, Portopts},File, Mode, Setopts); Reason -> {error, Reason} end; @@ -196,7 +199,7 @@ open_int({Driver, Portopts}, File, Mode, Setopts) -> end; open_int(Port, File, Mode, Setopts) -> M = Mode band ?EFILE_MODE_MASK, - case drv_command(Port, [<<?FILE_OPEN, M:32>>, File, 0]) of + case drv_command(Port, [<<?FILE_OPEN, M:32>>, pathname(File)]) of {ok, Number} -> open_int_setopts(Port, Number, Setopts); Error -> @@ -489,7 +492,7 @@ ipread_s32bu_p32bu(#file_descriptor{module = ?MODULE, data = {_, _}}, %% Returns {ok, Contents} | {error, Reason} -read_file(File) -> +read_file(File) when (is_list(File) orelse is_binary(File)) -> case drv_open(?FD_DRV, [binary]) of {ok, Port} -> Result = read_file(Port, File), @@ -497,11 +500,14 @@ read_file(File) -> Result; {error, _} = Error -> Error - end. + end; +read_file(_) -> + {error, badarg}. %% Takes a Port opened with open/1. -read_file(Port, File) when is_port(Port) -> - Cmd = [?FILE_READ_FILE | File], +read_file(Port, File) when is_port(Port), + (is_list(File) orelse is_binary(File))-> + Cmd = [?FILE_READ_FILE | pathname(File)], case drv_command(Port, Cmd) of {error, enomem} -> %% It could possibly help to do a @@ -512,12 +518,14 @@ read_file(Port, File) when is_port(Port) -> drv_command(Port, Cmd); Result -> Result - end. + end; +read_file(_,_) -> + {error, badarg}. %% Returns {error, Reason} | ok. -write_file(File, Bin) -> +write_file(File, Bin) when (is_list(File) orelse is_binary(File)) -> case open(File, [binary, write]) of {ok, Handle} -> Result = write(Handle, Bin), @@ -525,8 +533,10 @@ write_file(File, Bin) -> Result; Error -> Error - end. - + end; +write_file(_, _) -> + {error, badarg}. + %%%----------------------------------------------------------------- @@ -539,7 +549,7 @@ write_file(File, Bin) -> %% Returns {ok, Port}, the Port should be used as first argument in all %% the following functions. Returns {error, Reason} upon failure. start() -> - try erlang:open_port({spawn, atom_to_list(?DRV)}, []) of + try erlang:open_port({spawn, atom_to_list(?DRV)}, [binary]) of Port -> {ok, Port} catch @@ -596,7 +606,7 @@ get_cwd(_, _) -> {error, badarg}. get_cwd_int(Drive) -> - get_cwd_int({?DRV, []}, Drive). + get_cwd_int({?DRV, [binary]}, Drive). get_cwd_int(Port, Drive) -> drv_command(Port, <<?FILE_PWD, Drive>>). @@ -606,7 +616,7 @@ get_cwd_int(Port, Drive) -> %% set_cwd/{1,2} set_cwd(Dir) -> - set_cwd_int({?DRV, []}, Dir). + set_cwd_int({?DRV, [binary]}, Dir). set_cwd(Port, Dir) when is_port(Port) -> set_cwd_int(Port, Dir). @@ -632,89 +642,88 @@ set_cwd_int(Port, Dir0) -> end), %% Dir is now either a string or an EXIT tuple. %% An EXIT tuple will fail in the following catch. - drv_command(Port, [?FILE_CHDIR, Dir, 0]). + drv_command(Port, [?FILE_CHDIR, pathname(Dir)]). %% delete/{1,2} delete(File) -> - delete_int({?DRV, []}, File). + delete_int({?DRV, [binary]}, File). delete(Port, File) when is_port(Port) -> delete_int(Port, File). delete_int(Port, File) -> - drv_command(Port, [?FILE_DELETE, File, 0]). + drv_command(Port, [?FILE_DELETE, pathname(File)]). %% rename/{2,3} rename(From, To) -> - rename_int({?DRV, []}, From, To). + rename_int({?DRV, [binary]}, From, To). rename(Port, From, To) when is_port(Port) -> rename_int(Port, From, To). rename_int(Port, From, To) -> - drv_command(Port, [?FILE_RENAME, From, 0, To, 0]). + drv_command(Port, [?FILE_RENAME, pathname(From), pathname(To)]). %% make_dir/{1,2} make_dir(Dir) -> - make_dir_int({?DRV, []}, Dir). + make_dir_int({?DRV, [binary]}, Dir). make_dir(Port, Dir) when is_port(Port) -> make_dir_int(Port, Dir). make_dir_int(Port, Dir) -> - drv_command(Port, [?FILE_MKDIR, Dir, 0]). + drv_command(Port, [?FILE_MKDIR, pathname(Dir)]). %% del_dir/{1,2} del_dir(Dir) -> - del_dir_int({?DRV, []}, Dir). + del_dir_int({?DRV, [binary]}, Dir). del_dir(Port, Dir) when is_port(Port) -> del_dir_int(Port, Dir). del_dir_int(Port, Dir) -> - drv_command(Port, [?FILE_RMDIR, Dir, 0]). + drv_command(Port, [?FILE_RMDIR, pathname(Dir)]). %% read_file_info/{1,2} read_file_info(File) -> - read_file_info_int({?DRV, []}, File). + read_file_info_int({?DRV, [binary]}, File). read_file_info(Port, File) when is_port(Port) -> read_file_info_int(Port, File). read_file_info_int(Port, File) -> - drv_command(Port, [?FILE_FSTAT, File, 0]). + drv_command(Port, [?FILE_FSTAT, pathname(File)]). %% altname/{1,2} altname(File) -> - altname_int({?DRV, []}, File). + altname_int({?DRV, [binary]}, File). altname(Port, File) when is_port(Port) -> altname_int(Port, File). altname_int(Port, File) -> - drv_command(Port, [?FILE_ALTNAME, File, 0]). - + drv_command(Port, [?FILE_ALTNAME, pathname(File)]). %% write_file_info/{2,3} write_file_info(File, Info) -> - write_file_info_int({?DRV, []}, File, Info). + write_file_info_int({?DRV, [binary]}, File, Info). write_file_info(Port, File, Info) when is_port(Port) -> write_file_info_int(Port, File, Info). @@ -740,72 +749,72 @@ write_file_info_int(Port, date_to_bytes(Atime), date_to_bytes(Mtime), date_to_bytes(Ctime), - File, 0]). + pathname(File)]). %% make_link/{2,3} make_link(Old, New) -> - make_link_int({?DRV, []}, Old, New). + make_link_int({?DRV, [binary]}, Old, New). make_link(Port, Old, New) when is_port(Port) -> make_link_int(Port, Old, New). make_link_int(Port, Old, New) -> - drv_command(Port, [?FILE_LINK, Old, 0, New, 0]). + drv_command(Port, [?FILE_LINK, pathname(Old), pathname(New)]). %% make_symlink/{2,3} make_symlink(Old, New) -> - make_symlink_int({?DRV, []}, Old, New). + make_symlink_int({?DRV, [binary]}, Old, New). make_symlink(Port, Old, New) when is_port(Port) -> make_symlink_int(Port, Old, New). make_symlink_int(Port, Old, New) -> - drv_command(Port, [?FILE_SYMLINK, Old, 0, New, 0]). + drv_command(Port, [?FILE_SYMLINK, pathname(Old), pathname(New)]). %% read_link/{2,3} read_link(Link) -> - read_link_int({?DRV, []}, Link). + read_link_int({?DRV, [binary]}, Link). read_link(Port, Link) when is_port(Port) -> read_link_int(Port, Link). read_link_int(Port, Link) -> - drv_command(Port, [?FILE_READLINK, Link, 0]). + drv_command(Port, [?FILE_READLINK, pathname(Link)]). %% read_link_info/{2,3} read_link_info(Link) -> - read_link_info_int({?DRV, []}, Link). + read_link_info_int({?DRV, [binary]}, Link). read_link_info(Port, Link) when is_port(Port) -> read_link_info_int(Port, Link). read_link_info_int(Port, Link) -> - drv_command(Port, [?FILE_LSTAT, Link, 0]). + drv_command(Port, [?FILE_LSTAT, pathname(Link)]). %% list_dir/{1,2} list_dir(Dir) -> - list_dir_int({?DRV, []}, Dir). + list_dir_int({?DRV, [binary]}, Dir). list_dir(Port, Dir) when is_port(Port) -> list_dir_int(Port, Dir). list_dir_int(Port, Dir) -> - drv_command(Port, [?FILE_READDIR, Dir, 0], []). + drv_command(Port, [?FILE_READDIR, pathname(Dir)], []). @@ -1026,8 +1035,6 @@ lseek_position(_) -> translate_response(?FILE_RESP_OK, []) -> ok; -translate_response(?FILE_RESP_OK, Data) -> - {ok, Data}; translate_response(?FILE_RESP_ERROR, List) when is_list(List) -> {error, list_to_atom(List)}; translate_response(?FILE_RESP_NUMBER, List) -> @@ -1074,6 +1081,16 @@ translate_response(?FILE_RESP_N2DATA = X, L0) when is_list(L0) -> end; translate_response(?FILE_RESP_EOF, []) -> eof; +translate_response(?FILE_RESP_FNAME, []) -> + ok; +translate_response(?FILE_RESP_FNAME, Data) when is_binary(Data) -> + {ok, prim_file:internal_native2name(Data)}; +translate_response(?FILE_RESP_FNAME, Data) -> + {ok, Data}; + +translate_response(?FILE_RESP_ALL_DATA, Data) -> + {ok, Data}; + translate_response(X, Data) -> {error, {bad_response_from_port, [X | Data]}}. @@ -1209,3 +1226,9 @@ lists_split([Hd | Tl], N, Rev) -> reverse(X) -> lists:reverse(X, []). reverse(L, T) -> lists:reverse(L, T). + +% Will add zero termination too +% The 'EXIT' tuple from a bad argument will eventually generate an error +% in list_to_binary, which is caught and generates the {error,badarg} return +pathname(File) -> + (catch prim_file:internal_name2native(File)). diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl index 91d39c6a73..8f2e845b4f 100644 --- a/erts/preloaded/src/prim_inet.erl +++ b/erts/preloaded/src/prim_inet.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2010. All Rights Reserved. +%% Copyright Ericsson AB 2000-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -37,7 +37,7 @@ -export([setopt/3, setopts/2, getopt/2, getopts/2, is_sockopt_val/2]). -export([chgopt/3, chgopts/2]). -export([getstat/2, getfd/1, getindex/1, getstatus/1, gettype/1, - getiflist/1, ifget/3, ifset/3, + getifaddrs/1, getiflist/1, ifget/3, ifset/3, gethostname/1]). -export([getservbyname/3, getservbyport/3]). -export([peername/1, setpeername/2]). @@ -216,9 +216,10 @@ bindx(S, AddFlag, Addrs) -> sctp -> %% Really multi-homed "bindx". Stringified args: %% [AddFlag, (Port, IP)+]: - Args = ?int8(AddFlag) ++ - lists:concat([?int16(Port)++ip_to_bytes(IP) || - {IP, Port} <- Addrs]), + Args = + [?int8(AddFlag)| + [[?int16(Port)|ip_to_bytes(IP)] || + {IP, Port} <- Addrs]], case ctl_cmd(S, ?SCTP_REQ_BINDX, Args) of {ok,_} -> {ok, S}; Error -> Error @@ -623,7 +624,7 @@ chgopt(S, Opt, Value) when is_port(S) -> chgopts(S, [{Opt,Value}]). chgopts(S, Opts) when is_port(S), is_list(Opts) -> - case inet:getopts(S, need_template(Opts)) of + case getopts(S, need_template(Opts)) of {ok,Templates} -> try merge_options(Opts, Templates) of NewOpts -> @@ -636,7 +637,94 @@ chgopts(S, Opts) when is_port(S), is_list(Opts) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% -%% IFLIST(insock()) -> {ok,IfNameList} | {error, Reason} +%% getifaddrs(insock()) -> {ok,IfAddrsList} | {error, Reason} +%% +%% IfAddrsList = [{Name,[Opts]}] +%% Name = string() +%% Opts = {flags,[Flag]} | {addr,Addr} | {netmask,Addr} | {broadaddr,Addr} +%% | {dstaddr,Addr} | {hwaddr,HwAddr} | {mtu,integer()} +%% Flag = up | broadcast | loopback | running | multicast +%% Addr = ipv4addr() | ipv6addr() +%% HwAddr = ethernet_addr() +%% +%% get interface name and addresses list +%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +getifaddrs(S) when is_port(S) -> + case ctl_cmd(S, ?INET_REQ_GETIFADDRS, []) of + {ok, Data} -> + {ok, comp_ifaddrs(build_ifaddrs(Data), ktree_empty())}; + {error,enotsup} -> + case getiflist(S) of + {ok, IFs} -> + {ok, getifaddrs_ifget(S, IFs)}; + Err1 -> Err1 + end; + Err2 -> Err2 + end. + +%% Restructure interface properties per interface and remove duplicates + +comp_ifaddrs([{If,Opts}|IfOpts], T) -> + case ktree_is_defined(If, T) of + true -> + OptSet = comp_ifaddrs_add(ktree_get(If, T), Opts), + comp_ifaddrs(IfOpts, ktree_update(If, OptSet, T)); + false -> + OptSet = comp_ifaddrs_add(ktree_empty(), Opts), + comp_ifaddrs(IfOpts, ktree_insert(If, OptSet, T)) + end; +comp_ifaddrs([], T) -> + [{If,ktree_keys(ktree_get(If, T))} || If <- ktree_keys(T)]. + +comp_ifaddrs_add(OptSet, [Opt|Opts]) -> + case ktree_is_defined(Opt, OptSet) of + true + when element(1, Opt) =:= flags; + element(1, Opt) =:= hwaddr -> + comp_ifaddrs_add(OptSet, Opts); + _ -> + comp_ifaddrs_add(ktree_insert(Opt, undefined, OptSet), Opts) + end; +comp_ifaddrs_add(OptSet, []) -> OptSet. + +%% Legacy emulation of getifaddrs + +getifaddrs_ifget(_, []) -> []; +getifaddrs_ifget(S, [IF|IFs]) -> + case ifget(S, IF, [flags]) of + {ok,[{flags,Flags}]=FlagsVals} -> + BroadOpts = + case member(broadcast, Flags) of + true -> + [broadaddr,hwaddr]; + false -> + [hwaddr] + end, + P2POpts = + case member(pointtopoint, Flags) of + true -> + [dstaddr|BroadOpts]; + false -> + BroadOpts + end, + getifaddrs_ifget(S, IFs, IF, FlagsVals, [addr,netmask|P2POpts]); + _ -> + getifaddrs_ifget(S, IFs, IF, [], [addr,netmask,hwaddr]) + end. + +getifaddrs_ifget(S, IFs, IF, FlagsVals, Opts) -> + OptVals = + case ifget(S, IF, Opts) of + {ok,OVs} -> OVs; + _ -> [] + end, + [{IF,FlagsVals++OptVals}|getifaddrs_ifget(S, IFs)]. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% +%% getiflist(insock()) -> {ok,IfNameList} | {error, Reason} %% %% get interface name list %% @@ -1325,6 +1413,19 @@ type_value_2({enum,List}, Enum) -> {value,_} -> true; false -> false end; +type_value_2(sockaddr, Addr) -> + case Addr of + any -> true; + loopback -> true; + {A,B,C,D} when ?ip(A,B,C,D) -> true; + {A,B,C,D,E,F,G,H} when ?ip6(A,B,C,D,E,F,G,H) -> true; + _ -> false + end; +type_value_2(linkaddr, Addr) when is_list(Addr) -> + case len(Addr, 32768) of + undefined -> false; + _ -> true + end; type_value_2({bitenumlist,List}, EnumList) -> case enum_vals(EnumList, List) of Ls when is_list(Ls) -> true; @@ -1413,14 +1514,21 @@ enc_value_2(addr, {any,Port}) -> [?INET_AF_ANY|?int16(Port)]; enc_value_2(addr, {loopback,Port}) -> [?INET_AF_LOOPBACK|?int16(Port)]; -enc_value_2(addr, {IP,Port}) -> - case tuple_size(IP) of - 4 -> - [?INET_AF_INET,?int16(Port)|ip4_to_bytes(IP)]; - 8 -> - [?INET_AF_INET6,?int16(Port)|ip6_to_bytes(IP)] - end; -enc_value_2(ether, [X1,X2,X3,X4,X5,X6]) -> [X1,X2,X3,X4,X5,X6]; +enc_value_2(addr, {IP,Port}) when tuple_size(IP) =:= 4 -> + [?INET_AF_INET,?int16(Port)|ip4_to_bytes(IP)]; +enc_value_2(addr, {IP,Port}) when tuple_size(IP) =:= 8 -> + [?INET_AF_INET6,?int16(Port)|ip6_to_bytes(IP)]; +enc_value_2(ether, [_,_,_,_,_,_]=Xs) -> Xs; +enc_value_2(sockaddr, any) -> + [?INET_AF_ANY]; +enc_value_2(sockaddr, loopback) -> + [?INET_AF_LOOPBACK]; +enc_value_2(sockaddr, IP) when tuple_size(IP) =:= 4 -> + [?INET_AF_INET|ip4_to_bytes(IP)]; +enc_value_2(sockaddr, IP) when tuple_size(IP) =:= 8 -> + [?INET_AF_INET6|ip6_to_bytes(IP)]; +enc_value_2(linkaddr, Linkaddr) -> + [?int16(length(Linkaddr)),Linkaddr]; enc_value_2(sctp_assoc_id, Val) -> ?int32(Val); %% enc_value_2(sctp_assoc_id, Bin) -> [byte_size(Bin),Bin]; enc_value_2({enum,List}, Enum) -> @@ -1464,7 +1572,11 @@ dec_value(time, [X3,X2,X1,X0|T]) -> Val -> {Val, T} end; dec_value(ip, [A,B,C,D|T]) -> {{A,B,C,D}, T}; -dec_value(ether,[X1,X2,X3,X4,X5,X6|T]) -> {[X1,X2,X3,X4,X5,X6],T}; +%% dec_value(ether, [X1,X2,X3,X4,X5,X6|T]) -> {[X1,X2,X3,X4,X5,X6],T}; +dec_value(sockaddr, [X|T]) -> + get_ip(X, T); +dec_value(linkaddr, [X1,X0|T]) -> + split(?i16(X1,X0), T); dec_value({enum,List}, [X3,X2,X1,X0|T]) -> Val = ?i32(X3,X2,X1,X0), case enum_name(Val, List) of @@ -1480,7 +1592,7 @@ dec_value({bitenumlist,List}, [X3,X2,X1,X0|T]) -> %% {enum_names(Val, List), T}; dec_value(binary,[L0,L1,L2,L3|List]) -> Len = ?i32(L0,L1,L2,L3), - {X,T}=lists:split(Len,List), + {X,T}=split(Len,List), {list_to_binary(X),T}; dec_value(Types, List) when is_tuple(Types) -> {L,T} = dec_value_tuple(Types, List, 1, []), @@ -1495,7 +1607,7 @@ dec_value_tuple(Types, List, N, Acc) {Term,Tail} = dec_value(element(N, Types), List), dec_value_tuple(Types, Tail, N+1, [Term|Acc]); dec_value_tuple(_, List, _, Acc) -> - {lists:reverse(Acc),List}. + {rev(Acc),List}. borlist([V|Vs], Value) -> borlist(Vs, V bor Value); @@ -1702,11 +1814,11 @@ merge_fields(_, _, _) -> []. %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -type_ifopt(addr) -> ip; -type_ifopt(broadaddr) -> ip; -type_ifopt(dstaddr) -> ip; +type_ifopt(addr) -> sockaddr; +type_ifopt(broadaddr) -> sockaddr; +type_ifopt(dstaddr) -> sockaddr; type_ifopt(mtu) -> int; -type_ifopt(netmask) -> ip; +type_ifopt(netmask) -> sockaddr; type_ifopt(flags) -> {bitenumlist, [{up, ?INET_IFF_UP}, @@ -1718,7 +1830,7 @@ type_ifopt(flags) -> {no_pointtopoint, ?INET_IFF_NPOINTTOPOINT}, {running, ?INET_IFF_RUNNING}, {multicast, ?INET_IFF_MULTICAST}]}; -type_ifopt(hwaddr) -> ether; +type_ifopt(hwaddr) -> linkaddr; type_ifopt(Opt) when is_atom(Opt) -> undefined. enc_ifopt(addr) -> ?INET_IFOPT_ADDR; @@ -1903,6 +2015,30 @@ encode_ifname(Name) -> if N > 255 -> {error, einval}; true -> {ok,[N | Name]} end. + +build_ifaddrs(Cs) -> + build_ifaddrs(Cs, []). +%% +build_ifaddrs([], []) -> + []; +build_ifaddrs([0|Cs], Acc) -> + Name = utf8_to_characters(rev(Acc)), + {Opts,Rest} = build_ifaddrs_opts(Cs, []), + [{Name,Opts}|build_ifaddrs(Rest)]; +build_ifaddrs([C|Cs], Acc) -> + build_ifaddrs(Cs, [C|Acc]). + +build_ifaddrs_opts([0|Cs], Acc) -> + {rev(Acc),Cs}; +build_ifaddrs_opts([C|Cs]=CCs, Acc) -> + case dec_ifopt(C) of + undefined -> + erlang:error(badarg, [CCs,Acc]); + Opt -> + Type = type_ifopt(Opt), + {Val,Rest} = dec_value(Type, Cs), + build_ifaddrs_opts(Rest, [{Opt,Val}|Acc]) + end. build_iflist(Cs) -> build_iflist(Cs, [], []). @@ -1927,6 +2063,80 @@ rev(L) -> rev(L,[]). rev([C|L],Acc) -> rev(L,[C|Acc]); rev([],Acc) -> Acc. +split(N, L) -> split(N, L, []). +split(0, L, R) when is_list(L) -> {rev(R),L}; +split(N, [H|T], R) when is_integer(N), N > 0 -> split(N-1, T, [H|R]). + +len(L, N) -> len(L, N, 0). +len([], N, C) when is_integer(N), N >= 0 -> C; +len(L, 0, _) when is_list(L) -> undefined; +len([_|L], N, C) when is_integer(N), N >= 0 -> len(L, N-1, C+1). + +member(X, [X|_]) -> true; +member(X, [_|Xs]) -> member(X, Xs); +member(_, []) -> false. + + + +%% Lookup tree that keeps key insert order + +ktree_empty() -> {[],tree()}. +ktree_is_defined(Key, {_,T}) -> tree(T, Key, is_defined). +ktree_get(Key, {_,T}) -> tree(T, Key, get). +ktree_insert(Key, V, {Keys,T}) -> {[Key|Keys],tree(T, Key, {insert,V})}. +ktree_update(Key, V, {Keys,T}) -> {Keys,tree(T, Key, {update,V})}. +ktree_keys({Keys,_}) -> rev(Keys). + +%% Simple lookup tree. Hash the key to get statistical balance. +%% Key is matched equal, not compared equal. + +tree() -> nil. +tree(T, Key, Op) -> tree(T, Key, Op, erlang:phash2(Key)). + +tree(nil, _, is_defined, _) -> false; +tree(nil, K, {insert,V}, _) -> {K,V,nil,nil}; +tree({K,_,_,_}, K, is_defined, _) -> true; +tree({K,V,_,_}, K, get, _) -> V; +tree({K,_,L,R}, K, {update,V}, _) -> {K,V,L,R}; +tree({K0,V0,L,R}, K, Op, H) -> + H0 = erlang:phash2(K0), + if H0 < H; H0 =:= H, K0 < K -> + if is_tuple(Op) -> + {K0,V0,tree(L, K, Op, H),R}; + true -> + tree(L, K, Op, H) + end; + true -> + if is_tuple(Op) -> + {K0,V0,L,tree(R, K, Op, H)}; + true -> + tree(R, K, Op, H) + end + end. + + + +utf8_to_characters([]) -> []; +utf8_to_characters([B|Bs]=Arg) when (B band 16#FF) =:= B -> + if 16#F8 =< B -> + erlang:error(badarg, [Arg]); + 16#F0 =< B -> + utf8_to_characters(Bs, B band 16#07, 3); + 16#E0 =< B -> + utf8_to_characters(Bs, B band 16#0F, 2); + 16#C0 =< B -> + utf8_to_characters(Bs, B band 16#1F, 1); + 16#80 =< B -> + erlang:error(badarg, [Arg]); + true -> + [B|utf8_to_characters(Bs)] + end. +%% +utf8_to_characters(Bs, U, 0) -> + [U|utf8_to_characters(Bs)]; +utf8_to_characters([B|Bs], U, N) when ((B band 16#3F) bor 16#80) =:= B -> + utf8_to_characters(Bs, (U bsl 6) bor (B band 16#3F), N-1). + ip_to_bytes(IP) when tuple_size(IP) =:= 4 -> ip4_to_bytes(IP); ip_to_bytes(IP) when tuple_size(IP) =:= 8 -> ip6_to_bytes(IP). diff --git a/erts/test/autoimport_SUITE.erl b/erts/test/autoimport_SUITE.erl index 2430dac78d..0e4708e046 100644 --- a/erts/test/autoimport_SUITE.erl +++ b/erts/test/autoimport_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,16 +20,38 @@ -module(autoimport_SUITE). -include_lib("test_server/include/test_server.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2,autoimports/1]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, + autoimports/1]). -define(TEST_TIMEOUT, ?t:seconds(180)). -all(suite) -> [autoimports]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [autoimports]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_testcase(_Func, Config) -> Dog = test_server:timetrap(?TEST_TIMEOUT), [{watchdog, Dog} | Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> Dog = ?config(watchdog, Config), catch test_server:timetrap_cancel(Dog), ok. diff --git a/erts/test/erl_print_SUITE.erl b/erts/test/erl_print_SUITE.erl index 3bb7d4d016..ee1a200530 100644 --- a/erts/test/erl_print_SUITE.erl +++ b/erts/test/erl_print_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -32,14 +32,35 @@ -define(DEFAULT_TIMEOUT, ?t:minutes(10)). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, fin_per_testcase/2]). --export([erlang_display/1, integer/1, float/1, string/1, character/1, snprintf/1, quote/1]). +-export([erlang_display/1, integer/1, float/1, + string/1, character/1, snprintf/1, quote/1]). -include_lib("test_server/include/test_server.hrl"). - -all(doc) -> []; -all(suite) -> test_cases(). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + test_cases(). + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + %% %% @@ -47,14 +68,9 @@ all(suite) -> test_cases(). %% %% -test_cases() -> - [erlang_display, - integer, - float, - string, - character, - snprintf, - quote]. +test_cases() -> + [erlang_display, integer, float, string, character, + snprintf, quote]. erlang_display(doc) -> []; erlang_display(suite) -> []; diff --git a/erts/test/erlc_SUITE.erl b/erts/test/erlc_SUITE.erl index 4797f78be2..62e0e6813d 100644 --- a/erts/test/erlc_SUITE.erl +++ b/erts/test/erlc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2010. All Rights Reserved. +%% Copyright Ericsson AB 1997-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -20,15 +20,34 @@ %% Tests the erlc command by compiling various types of files. --export([all/1, compile_erl/1, compile_yecc/1, compile_script/1, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, compile_erl/1, + compile_yecc/1, compile_script/1, compile_mib/1, good_citizen/1, deep_cwd/1, arg_overflow/1]). -include_lib("test_server/include/test_server.hrl"). -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [compile_erl, compile_yecc, compile_script, compile_mib, good_citizen, deep_cwd, arg_overflow]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + %% Copy from erlc_SUITE_data/include/erl_test.hrl. -record(person, {name, shoe_size}). diff --git a/erts/test/erlexec_SUITE.erl b/erts/test/erlexec_SUITE.erl index 6adb865f6d..0dfe6c2e5f 100644 --- a/erts/test/erlexec_SUITE.erl +++ b/erts/test/erlexec_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2007-2010. All Rights Reserved. +%% Copyright Ericsson AB 2007-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -31,7 +31,9 @@ -define(DEFAULT_TIMEOUT, ?t:minutes(1)). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). -export([args_file/1, evil_args_file/1, env/1, args_file_env/1, otp_7461/1, otp_7461_remote/1, otp_8209/1, zdbbl_dist_buf_busy_limit/1]). @@ -43,7 +45,7 @@ init_per_testcase(Case, Config) -> SavedEnv = save_env(), [{testcase, Case}, {watchdog, Dog}, {erl_flags_env, SavedEnv} |Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog = ?config(watchdog, Config), SavedEnv = ?config(erl_flags_env, Config), restore_env(SavedEnv), @@ -51,11 +53,26 @@ fin_per_testcase(_Case, Config) -> ?t:timetrap_cancel(Dog), ok. -all(doc) -> []; -all(suite) -> - [args_file, evil_args_file, env, args_file_env, otp_7461, otp_8209, - zdbbl_dist_buf_busy_limit]. +suite() -> [{ct_hooks,[ts_install_cth]}]. +all() -> + [args_file, evil_args_file, env, args_file_env, + otp_7461, otp_8209, zdbbl_dist_buf_busy_limit]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. otp_8209(doc) -> ["Test that plain first argument does not " diff --git a/erts/test/ethread_SUITE.erl b/erts/test/ethread_SUITE.erl index 0cc315e9be..71d8c1c679 100644 --- a/erts/test/ethread_SUITE.erl +++ b/erts/test/ethread_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2004-2010. All Rights Reserved. +%% Copyright Ericsson AB 2004-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -31,13 +31,14 @@ -define(DEFAULT_TIMEOUT, ?t:minutes(10)). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, fin_per_testcase/2]). -export([create_join_thread/1, equal_tids/1, mutex/1, try_lock_mutex/1, - time_now/1, cond_wait/1, broadcast/1, detached_thread/1, @@ -50,25 +51,30 @@ -include_lib("test_server/include/test_server.hrl"). -tests() -> - [create_join_thread, - equal_tids, - mutex, - try_lock_mutex, - time_now, - cond_wait, - broadcast, - detached_thread, - max_threads, - tsd, - spinlock, - rwspinlock, - rwmutex, - atomic]. - -all(doc) -> []; -all(suite) -> tests(). +tests() -> + [create_join_thread, equal_tids, mutex, try_lock_mutex, + cond_wait, broadcast, detached_thread, + max_threads, tsd, spinlock, rwspinlock, rwmutex, atomic]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + tests(). + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. %% %% @@ -104,17 +110,6 @@ try_lock_mutex(suite) -> try_lock_mutex(Config) -> run_case(Config, "try_lock_mutex", ""). -time_now(doc) -> - ["Tests ethr_time_now by comparing time values with Erlang."]; -time_now(suite) -> - []; -time_now(Config) -> - run_case(Config, "time_now", "", fun (P) -> - spawn_link(fun () -> - watchdog(P) - end) - end). - wd_dispatch(P) -> receive bye -> @@ -171,7 +166,15 @@ max_threads(doc) -> max_threads(suite) -> []; max_threads(Config) -> - run_case(Config, "max_threads", ""). + case {os:type(), os:version()} of + {{unix,darwin}, {9, _, _}} -> + %% For some reason pthread_create() crashes when more + %% threads cannot be created, instead of returning an + %% error code on our MacOS X Leopard machine... + {skipped, "MacOS X Leopard cannot cope with this test..."}; + _ -> + run_case(Config, "max_threads", "") + end. tsd(doc) -> ["Tests thread specific data."]; diff --git a/erts/test/ethread_SUITE_data/ethread_tests.c b/erts/test/ethread_SUITE_data/ethread_tests.c index 7fc71d8047..0b59ff5aa6 100644 --- a/erts/test/ethread_SUITE_data/ethread_tests.c +++ b/erts/test/ethread_SUITE_data/ethread_tests.c @@ -514,69 +514,6 @@ try_lock_mutex_test(void) } /* - * The time now test. - * - * Tests ethr_time_now by comparing time values with Erlang. - */ -#define TNT_MAX_TIME_DIFF 200000 -#define TNT_MAX_TIME_VALUES 52 - -static void -time_now_test(void) -{ - int scanf_res, time_now_res, i, no_values, max_abs_diff; - static ethr_timeval tv[TNT_MAX_TIME_VALUES]; - static int ms[TNT_MAX_TIME_VALUES]; - - i = 0; - do { - ASSERT(i < TNT_MAX_TIME_VALUES); - scanf_res = scanf("%d", &ms[i]); - time_now_res = ethr_time_now(&tv[i]); - ASSERT(scanf_res == 1); - ASSERT(time_now_res == 0); -#if 0 - print_line("Got %d; %ld:%ld", ms[i], tv[i].tv_sec, tv[i].tv_nsec); -#endif - i++; - } while (ms[i-1] >= 0); - - no_values = i-1; - - ASSERT(ms[0] == 0); - - print_line("TNT_MAX_TIME_DIFF = %d (us)", TNT_MAX_TIME_DIFF); - - max_abs_diff = 0; - - for (i = 1; i < no_values; i++) { - long diff; - long tn_us; - long e_us; - - tn_us = (tv[i].tv_sec - tv[0].tv_sec) * 1000000; - tn_us += (tv[i].tv_nsec - tv[0].tv_nsec)/1000; - - e_us = ms[i]*1000; - - diff = e_us - tn_us; - - print_line("Erlang time = %ld us; ethr_time_now = %ld us; diff %ld us", - e_us, tn_us, diff); - - if (max_abs_diff < abs((int) diff)) { - max_abs_diff = abs((int) diff); - } - - ASSERT(e_us - TNT_MAX_TIME_DIFF <= tn_us); - ASSERT(tn_us <= e_us + TNT_MAX_TIME_DIFF); - } - - print_line("Max absolute diff = %d us", max_abs_diff); - succeed("Max absolute diff = %d us", max_abs_diff); -} - -/* * The cond wait test case. * * Tests ethr_cond_wait with ethr_cond_signal and ethr_cond_broadcast. @@ -1538,8 +1475,6 @@ main(int argc, char *argv[]) mutex_test(); else if (strcmp(testcase, "try_lock_mutex") == 0) try_lock_mutex_test(); - else if (strcmp(testcase, "time_now") == 0) - time_now_test(); else if (strcmp(testcase, "cond_wait") == 0) cond_wait_test(); else if (strcmp(testcase, "broadcast") == 0) diff --git a/erts/test/install_SUITE.erl b/erts/test/install_SUITE.erl index e14790bc1b..214031a6fe 100644 --- a/erts/test/install_SUITE.erl +++ b/erts/test/install_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2010. All Rights Reserved. +%% Copyright Ericsson AB 2010-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -29,8 +29,9 @@ %-define(line_trace, 1). --export([all/1, init_per_suite/1, end_per_suite/1, - init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, + init_per_suite/1, end_per_suite/1, + init_per_testcase/2, end_per_testcase/2]). -export([bin_default/1, bin_default_dirty/1, @@ -64,27 +65,32 @@ erlang_bindir = "", bindir_symlinks = ""}). -need_symlink_cases() -> - [bin_unreachable_absolute, - bin_unreachable_relative, - bin_same_dir, - bin_ok_symlink, - bin_dirname_fail, +need_symlink_cases() -> + [bin_unreachable_absolute, bin_unreachable_relative, + bin_same_dir, bin_ok_symlink, bin_dirname_fail, bin_no_use_dirname_fail]. -dont_need_symlink_cases() -> - [bin_default, - bin_default_dirty, - bin_outside_eprfx, - bin_outside_eprfx_dirty, - bin_not_abs, - bin_unreasonable_path, - 'bin white space', +dont_need_symlink_cases() -> + [bin_default, bin_default_dirty, bin_outside_eprfx, + bin_outside_eprfx_dirty, bin_not_abs, + bin_unreasonable_path, 'bin white space', bin_no_srcfile]. -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> dont_need_symlink_cases() ++ need_symlink_cases(). +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + %% %% The test cases %% @@ -585,7 +591,7 @@ init_per_testcase_aux(true, _OsType, Case, Config) -> {test_dir, make_dirs(?config(priv_dir, Config), atom_to_list(Case))} | Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. diff --git a/erts/test/nt_SUITE.erl b/erts/test/nt_SUITE.erl index 530fb55270..7d6da28ad6 100644 --- a/erts/test/nt_SUITE.erl +++ b/erts/test/nt_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2010. All Rights Reserved. +%% Copyright Ericsson AB 1998-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -22,7 +22,9 @@ -include_lib("test_server/include/test_server.hrl"). -include_lib("kernel/include/file.hrl"). --export([all/1,init_per_testcase/2,fin_per_testcase/2,nt/1,handle_eventlog/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2,init_per_testcase/2, + end_per_testcase/2,nt/1,handle_eventlog/2, middleman/1,service_basic/1, service_env/1, user_env/1, synced/1, service_prio/1, logout/1, debug/1, restart/1, restart_always/1,stopaction/1, @@ -31,20 +33,38 @@ -define(TEST_SERVICES, [1,2,3,4,5,6,7,8,9,10,11]). -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> case os:type() of - {win32,nt} -> - [nt, service_basic, service_env, user_env, synced, service_prio, - logout, debug, - restart, restart_always, stopaction]; - _ -> [nt] %%% Just to give a little hint why they are skipped... + {win32, nt} -> + [nt, service_basic, service_env, user_env, synced, + service_prio, logout, debug, restart, restart_always, + stopaction]; + _ -> [nt] end. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + init_per_testcase(_Func, Config) -> Dog = test_server:timetrap(?TEST_TIMEOUT), [{watchdog, Dog} | Config]. -fin_per_testcase(_Func, Config) -> +end_per_testcase(_Func, Config) -> lists:foreach(fun(X) -> catch remove_service("test_service_" ++ integer_to_list(X)) end, diff --git a/erts/test/otp_SUITE.erl b/erts/test/otp_SUITE.erl index 425ad31782..d61fbbddcf 100644 --- a/erts/test/otp_SUITE.erl +++ b/erts/test/otp_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2010. All Rights Reserved. +%% Copyright Ericsson AB 2000-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,7 +19,8 @@ -module(otp_SUITE). --export([all/1,init_per_suite/1,end_per_suite/1]). +-export([all/0, suite/0,groups/0,init_per_group/2,end_per_group/2, + init_per_suite/1,end_per_suite/1]). -export([undefined_functions/1,deprecated_not_in_obsolete/1, obsolete_but_not_deprecated/1,call_to_deprecated/1, call_to_size_1/1,strong_components/1]). @@ -28,10 +29,22 @@ -import(lists, [filter/2,foldl/3,foreach/2]). -all(suite) -> - [undefined_functions,deprecated_not_in_obsolete, - obsolete_but_not_deprecated,call_to_deprecated, - call_to_size_1,strong_components]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [undefined_functions, deprecated_not_in_obsolete, + obsolete_but_not_deprecated, call_to_deprecated, + call_to_size_1, strong_components]. + +groups() -> + []. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + init_per_suite(Config) -> Dog = test_server:timetrap(?t:minutes(10)), diff --git a/erts/test/run_erl_SUITE.erl b/erts/test/run_erl_SUITE.erl index efeafbad8c..6350dc47dd 100644 --- a/erts/test/run_erl_SUITE.erl +++ b/erts/test/run_erl_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2010. All Rights Reserved. +%% Copyright Ericsson AB 2005-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -19,7 +19,9 @@ -module(run_erl_SUITE). --export([all/1,init_per_testcase/2,fin_per_testcase/2, +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2,end_per_testcase/2, basic/1,heavy/1,heavier/1,defunct/1]). -export([ping_me_back/1]). @@ -29,13 +31,31 @@ init_per_testcase(_Case, Config) -> Dog = ?t:timetrap(?t:minutes(2)), [{watchdog, Dog}|Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. -all(suite) -> - [basic,heavy,heavier,defunct]. +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [basic, heavy, heavier, defunct]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + basic(Config) when is_list(Config) -> case os:type() of diff --git a/erts/test/system.spec b/erts/test/system.spec index 9bfe2dbcf8..e0561ba0b2 100644 --- a/erts/test/system.spec +++ b/erts/test/system.spec @@ -1 +1 @@ -{topcase, {dir, "../system_test"}}. +{suites,"../system_test",all}. diff --git a/erts/test/z_SUITE.erl b/erts/test/z_SUITE.erl index 8faddeb0d3..8fceab32a6 100644 --- a/erts/test/z_SUITE.erl +++ b/erts/test/z_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2010. All Rights Reserved. +%% Copyright Ericsson AB 2008-2011. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -35,26 +35,45 @@ -define(DEFAULT_TIMEOUT, ?t:minutes(5)). --export([all/1, init_per_testcase/2, fin_per_testcase/2]). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + init_per_testcase/2, end_per_testcase/2]). -export([search_for_core_files/1, core_files/1]). --include_lib("test_server/include/test_server.hrl"). +-include_lib("common_test/include/ct.hrl"). init_per_testcase(Case, Config) -> Dog = ?t:timetrap(?DEFAULT_TIMEOUT), [{testcase, Case}, {watchdog, Dog} |Config]. -fin_per_testcase(_Case, Config) -> +end_per_testcase(_Case, Config) -> Dog = ?config(watchdog, Config), ?t:timetrap_cancel(Dog), ok. -all(doc) -> []; -all(suite) -> +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> [core_files]. +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + + core_files(doc) -> []; @@ -253,6 +272,8 @@ core_file_search(#core_search_conf{search_dir = Base, core_cand(Conf, Core, Cores); "core." ++ _ -> core_cand(Conf, Core, Cores); + Bin when is_binary(Bin) -> %Icky filename; ignore + Cores; BName -> case lists:suffix(".core", BName) of true -> core_cand(Conf, Core, Cores); diff --git a/erts/vsn.mk b/erts/vsn.mk index a5dd62feb2..193a914a70 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2010. All Rights Reserved. +# Copyright Ericsson AB 1997-2011. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -17,8 +17,8 @@ # %CopyrightEnd% # -VSN = 5.8.2 -SYSTEM_VSN = R14B01 +VSN = 5.8.4 +SYSTEM_VSN = R14B03 # Port number 4365 in 4.2 # Port number 4366 in 4.3 |