diff options
Diffstat (limited to 'erts')
171 files changed, 9385 insertions, 3734 deletions
diff --git a/erts/.gitignore b/erts/.gitignore index 526d5da0b9..e515dc8811 100644 --- a/erts/.gitignore +++ b/erts/.gitignore @@ -11,6 +11,7 @@ /etc/common/Install /etc/common/erl.src +/etc/unix/etp-commands /test/Emakefile /test/*.beam diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index 25f40944e7..46b30a16b3 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -954,6 +954,40 @@ dnl AC_DEFUN(ERL_FIND_ETHR_LIB, [ +AC_ARG_ENABLE(native-ethr-impls, + AS_HELP_STRING([--disable-native-ethr-impls], + [disable native ethread implementations]), +[ case "$enableval" in + no) disable_native_ethr_impls=yes ;; + *) disable_native_ethr_impls=no ;; + esac ], disable_native_ethr_impls=no) + +test "X$disable_native_ethr_impls" = "Xyes" && + AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) + +AC_ARG_ENABLE(x86-out-of-order, + AS_HELP_STRING([--enable-x86-out-of-order], + [enable x86/x84_64 out of order support (default disabled)])) + +AC_ARG_ENABLE(prefer-gcc-native-ethr-impls, + AS_HELP_STRING([--enable-prefer-gcc-native-ethr-impls], + [prefer gcc native ethread implementations]), +[ case "$enableval" in + yes) enable_prefer_gcc_native_ethr_impls=yes ;; + *) enable_prefer_gcc_native_ethr_impls=no ;; + esac ], enable_prefer_gcc_native_ethr_impls=no) + +test $enable_prefer_gcc_native_ethr_impls = yes && + AC_DEFINE(ETHR_PREFER_GCC_NATIVE_IMPLS, 1, [Define if you prefer gcc native ethread implementations]) + +AC_ARG_WITH(libatomic_ops, + AS_HELP_STRING([--with-libatomic_ops=PATH], + [specify and prefer usage of libatomic_ops in the ethread library])) + +AC_ARG_WITH(with_sparc_memory_order, + AS_HELP_STRING([--with-sparc-memory-order=TSO|PSO|RMO], + [specify sparc memory order (defaults to RMO)])) + LM_CHECK_THR_LIB ERL_INTERNAL_LIBS @@ -1003,40 +1037,44 @@ case "$THR_LIB_NAME" in AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads]) - ETHR_CHK_INTERLOCKED([_InterlockedDecrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT, 1, [Define if you have _InterlockedDecrement()])) - ETHR_CHK_INTERLOCKED([_InterlockedDecrement_rel], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT_REL, 1, [Define if you have _InterlockedDecrement_rel()])) - ETHR_CHK_INTERLOCKED([_InterlockedIncrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT, 1, [Define if you have _InterlockedIncrement()])) - ETHR_CHK_INTERLOCKED([_InterlockedIncrement_acq], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ, 1, [Define if you have _InterlockedIncrement_acq()])) - ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD, 1, [Define if you have _InterlockedExchangeAdd()])) - ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd_acq], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ, 1, [Define if you have _InterlockedExchangeAdd_acq()])) - ETHR_CHK_INTERLOCKED([_InterlockedAnd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND, 1, [Define if you have _InterlockedAnd()])) - ETHR_CHK_INTERLOCKED([_InterlockedOr], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR, 1, [Define if you have _InterlockedOr()])) - ETHR_CHK_INTERLOCKED([_InterlockedExchange], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE, 1, [Define if you have _InterlockedExchange()])) - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE, 1, [Define if you have _InterlockedCompareExchange()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_acq], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ, 1, [Define if you have _InterlockedCompareExchange_acq()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_rel], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL, 1, [Define if you have _InterlockedCompareExchange_rel()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - - ETHR_CHK_INTERLOCKED([_InterlockedDecrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()])) - ETHR_CHK_INTERLOCKED([_InterlockedDecrement64_rel], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64_REL, 1, [Define if you have _InterlockedDecrement64_rel()])) - ETHR_CHK_INTERLOCKED([_InterlockedIncrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()])) - ETHR_CHK_INTERLOCKED([_InterlockedIncrement64_acq], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ, 1, [Define if you have _InterlockedIncrement64_acq()])) - ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()])) - ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64_acq], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ, 1, [Define if you have _InterlockedExchangeAdd64_acq()])) - ETHR_CHK_INTERLOCKED([_InterlockedAnd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()])) - ETHR_CHK_INTERLOCKED([_InterlockedOr64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()])) - ETHR_CHK_INTERLOCKED([_InterlockedExchange64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()])) - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_acq], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ, 1, [Define if you have _InterlockedCompareExchange64_acq()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_rel], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL, 1, [Define if you have _InterlockedCompareExchange64_rel()])) - test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes - - ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange128], [4], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128, 1, [Define if you have _InterlockedCompareExchange128()])) - + if test "X$disable_native_ethr_impls" = "Xyes"; then + have_interlocked_op=no + ethr_have_native_atomics=no + else + ETHR_CHK_INTERLOCKED([_InterlockedDecrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT, 1, [Define if you have _InterlockedDecrement()])) + ETHR_CHK_INTERLOCKED([_InterlockedDecrement_rel], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT_REL, 1, [Define if you have _InterlockedDecrement_rel()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT, 1, [Define if you have _InterlockedIncrement()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement_acq], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ, 1, [Define if you have _InterlockedIncrement_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD, 1, [Define if you have _InterlockedExchangeAdd()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd_acq], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ, 1, [Define if you have _InterlockedExchangeAdd_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedAnd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND, 1, [Define if you have _InterlockedAnd()])) + ETHR_CHK_INTERLOCKED([_InterlockedOr], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR, 1, [Define if you have _InterlockedOr()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchange], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE, 1, [Define if you have _InterlockedExchange()])) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE, 1, [Define if you have _InterlockedCompareExchange()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_acq], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ, 1, [Define if you have _InterlockedCompareExchange_acq()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_rel], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL, 1, [Define if you have _InterlockedCompareExchange_rel()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + + ETHR_CHK_INTERLOCKED([_InterlockedDecrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()])) + ETHR_CHK_INTERLOCKED([_InterlockedDecrement64_rel], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64_REL, 1, [Define if you have _InterlockedDecrement64_rel()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()])) + ETHR_CHK_INTERLOCKED([_InterlockedIncrement64_acq], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ, 1, [Define if you have _InterlockedIncrement64_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64_acq], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ, 1, [Define if you have _InterlockedExchangeAdd64_acq()])) + ETHR_CHK_INTERLOCKED([_InterlockedAnd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()])) + ETHR_CHK_INTERLOCKED([_InterlockedOr64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()])) + ETHR_CHK_INTERLOCKED([_InterlockedExchange64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()])) + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_acq], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ, 1, [Define if you have _InterlockedCompareExchange64_acq()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_rel], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL, 1, [Define if you have _InterlockedCompareExchange64_rel()])) + test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes + + ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange128], [4], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128, 1, [Define if you have _InterlockedCompareExchange128()])) + fi test "$ethr_have_native_atomics" = "yes" && ethr_have_native_spinlock=yes ;; @@ -1303,93 +1341,98 @@ case "$THR_LIB_NAME" in int128="__int128_t" fi - ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32, 1, [Define if you have __sync_val_compare_and_swap() for 32-bit integers])) - test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH32, 1, [Define if you have __sync_add_and_fetch() for 32-bit integers])) - ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND32, 1, [Define if you have __sync_fetch_and_and() for 32-bit integers])) - ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR32, 1, [Define if you have __sync_fetch_and_or() for 32-bit integers])) + if test "X$disable_native_ethr_impls" = "Xyes"; then + ethr_have_native_atomics=no + else + ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32, 1, [Define if you have __sync_val_compare_and_swap() for 32-bit integers])) + test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH32, 1, [Define if you have __sync_add_and_fetch() for 32-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND32, 1, [Define if you have __sync_fetch_and_and() for 32-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR32, 1, [Define if you have __sync_fetch_and_or() for 32-bit integers])) + + ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64, 1, [Define if you have __sync_val_compare_and_swap() for 64-bit integers])) + test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes + ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH64, 1, [Define if you have __sync_add_and_fetch() for 64-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND64, 1, [Define if you have __sync_fetch_and_and() for 64-bit integers])) + ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR64, 1, [Define if you have __sync_fetch_and_or() for 64-bit integers])) + + if test $int128 != no; then + ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [128], [$int128], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128, 1, [Define if you have __sync_val_compare_and_swap() for 128-bit integers])) + fi - ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64, 1, [Define if you have __sync_val_compare_and_swap() for 64-bit integers])) - test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes - ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH64, 1, [Define if you have __sync_add_and_fetch() for 64-bit integers])) - ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND64, 1, [Define if you have __sync_fetch_and_and() for 64-bit integers])) - ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR64, 1, [Define if you have __sync_fetch_and_or() for 64-bit integers])) + AC_MSG_CHECKING([for a usable libatomic_ops implementation]) + case "x$with_libatomic_ops" in + xno | xyes | x) + libatomic_ops_include= + ;; + *) + if test -d "${with_libatomic_ops}/include"; then + libatomic_ops_include="-I$with_libatomic_ops/include" + CPPFLAGS="$CPPFLAGS $libatomic_ops_include" + else + AC_MSG_ERROR([libatomic_ops include directory $with_libatomic_ops/include not found]) + fi;; + esac + ethr_have_libatomic_ops=no + AC_TRY_LINK([#include "atomic_ops.h"], + [ + volatile AO_t x; + AO_t y; + int z; + + AO_nop_full(); + AO_store(&x, (AO_t) 0); + z = AO_load(&x); + z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); + ], + [ethr_have_native_atomics=yes + ethr_have_libatomic_ops=yes]) + AC_MSG_RESULT([$ethr_have_libatomic_ops]) + if test $ethr_have_libatomic_ops = yes; then + AC_CHECK_SIZEOF(AO_t, , + [ + #include <stdio.h> + #include "atomic_ops.h" + ]) + AC_DEFINE_UNQUOTED(ETHR_SIZEOF_AO_T, $ac_cv_sizeof_AO_t, [Define to the size of AO_t if libatomic_ops is used]) + + AC_DEFINE(ETHR_HAVE_LIBATOMIC_OPS, 1, [Define if you have libatomic_ops atomic operations]) + if test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then + AC_DEFINE(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS, 1, [Define if you prefer libatomic_ops native ethread implementations]) + fi + ETHR_DEFS="$ETHR_DEFS $libatomic_ops_include" + elif test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then + AC_MSG_ERROR([No usable libatomic_ops implementation found]) + fi - if test $int128 != no; then - ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [128], [$int128], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128, 1, [Define if you have __sync_val_compare_and_swap() for 128-bit integers])) - fi + case "$host_cpu" in + sparc | sun4u | sparc64 | sun4v) + case "$with_sparc_memory_order" in + "TSO") + AC_DEFINE(ETHR_SPARC_TSO, 1, [Define if only run in Sparc TSO mode]);; + "PSO") + AC_DEFINE(ETHR_SPARC_PSO, 1, [Define if only run in Sparc PSO, or TSO mode]);; + "RMO"|"") + AC_DEFINE(ETHR_SPARC_RMO, 1, [Define if run in Sparc RMO, PSO, or TSO mode]);; + *) + AC_MSG_ERROR([Unsupported Sparc memory order: $with_sparc_memory_order]);; + esac + ethr_have_native_atomics=yes;; + i86pc | i*86 | x86_64 | amd64) + if test "$enable_x86_out_of_order" = "yes"; then + AC_DEFINE(ETHR_X86_OUT_OF_ORDER, 1, [Define if x86/x86_64 out of order instructions should be synchronized]) + fi + ethr_have_native_atomics=yes;; + macppc | ppc | powerpc | "Power Macintosh") + ethr_have_native_atomics=yes;; + tile) + ethr_have_native_atomics=yes;; + *) + ;; + esac - AC_MSG_CHECKING([for a usable libatomic_ops implementation]) - case "x$with_libatomic_ops" in - xno | xyes | x) - libatomic_ops_include= - ;; - *) - if test -d "${with_libatomic_ops}/include"; then - libatomic_ops_include="-I$with_libatomic_ops/include" - CPPFLAGS="$CPPFLAGS $libatomic_ops_include" - else - AC_MSG_ERROR([libatomic_ops include directory $with_libatomic_ops/include not found]) - fi;; - esac - ethr_have_libatomic_ops=no - AC_TRY_LINK([#include "atomic_ops.h"], - [ - volatile AO_t x; - AO_t y; - int z; - - AO_nop_full(); - AO_store(&x, (AO_t) 0); - z = AO_load(&x); - z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); - ], - [ethr_have_native_atomics=yes - ethr_have_libatomic_ops=yes]) - AC_MSG_RESULT([$ethr_have_libatomic_ops]) - if test $ethr_have_libatomic_ops = yes; then - AC_CHECK_SIZEOF(AO_t, , - [ - #include <stdio.h> - #include "atomic_ops.h" - ]) - AC_DEFINE_UNQUOTED(ETHR_SIZEOF_AO_T, $ac_cv_sizeof_AO_t, [Define to the size of AO_t if libatomic_ops is used]) - - AC_DEFINE(ETHR_HAVE_LIBATOMIC_OPS, 1, [Define if you have libatomic_ops atomic operations]) - if test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then - AC_DEFINE(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS, 1, [Define if you prefer libatomic_ops native ethread implementations]) - fi - ETHR_DEFS="$ETHR_DEFS $libatomic_ops_include" - elif test "x$with_libatomic_ops" != "xno" && test "x$with_libatomic_ops" != "x"; then - AC_MSG_ERROR([No usable libatomic_ops implementation found]) fi - case "$host_cpu" in - sparc | sun4u | sparc64 | sun4v) - case "$with_sparc_memory_order" in - "TSO") - AC_DEFINE(ETHR_SPARC_TSO, 1, [Define if only run in Sparc TSO mode]);; - "PSO") - AC_DEFINE(ETHR_SPARC_PSO, 1, [Define if only run in Sparc PSO, or TSO mode]);; - "RMO"|"") - AC_DEFINE(ETHR_SPARC_RMO, 1, [Define if run in Sparc RMO, PSO, or TSO mode]);; - *) - AC_MSG_ERROR([Unsupported Sparc memory order: $with_sparc_memory_order]);; - esac - ethr_have_native_atomics=yes;; - i86pc | i*86 | x86_64 | amd64) - if test "$enable_x86_out_of_order" = "yes"; then - AC_DEFINE(ETHR_X86_OUT_OF_ORDER, 1, [Define if x86/x86_64 out of order instructions should be synchronized]) - fi - ethr_have_native_atomics=yes;; - macppc | ppc | "Power Macintosh") - ethr_have_native_atomics=yes;; - tile) - ethr_have_native_atomics=yes;; - *) - ;; - esac - test ethr_have_native_atomics = "yes" && ethr_have_native_spinlock=yes dnl Restore LIBS @@ -1451,40 +1494,6 @@ esac AC_C_DOUBLE_MIDDLE_ENDIAN -AC_ARG_ENABLE(native-ethr-impls, - AS_HELP_STRING([--disable-native-ethr-impls], - [disable native ethread implementations]), -[ case "$enableval" in - no) disable_native_ethr_impls=yes ;; - *) disable_native_ethr_impls=no ;; - esac ], disable_native_ethr_impls=no) - -AC_ARG_ENABLE(x86-out-of-order, - AS_HELP_STRING([--enable-x86-out-of-order], - [enable x86/x84_64 out of order support (default disabled)])) - -test "X$disable_native_ethr_impls" = "Xyes" && - AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations]) - -AC_ARG_ENABLE(prefer-gcc-native-ethr-impls, - AS_HELP_STRING([--enable-prefer-gcc-native-ethr-impls], - [prefer gcc native ethread implementations]), -[ case "$enableval" in - yes) enable_prefer_gcc_native_ethr_impls=yes ;; - *) enable_prefer_gcc_native_ethr_impls=no ;; - esac ], enable_prefer_gcc_native_ethr_impls=no) - -test $enable_prefer_gcc_native_ethr_impls = yes && - AC_DEFINE(ETHR_PREFER_GCC_NATIVE_IMPLS, 1, [Define if you prefer gcc native ethread implementations]) - -AC_ARG_WITH(libatomic_ops, - AS_HELP_STRING([--with-libatomic_ops=PATH], - [specify and prefer usage of libatomic_ops in the ethread library])) - -AC_ARG_WITH(with_sparc_memory_order, - AS_HELP_STRING([--with-sparc-memory-order=TSO|PSO|RMO], - [specify sparc memory order (defaults to RMO)])) - ETHR_X86_SSE2_ASM=no case "$GCC-$ac_cv_sizeof_void_p-$host_cpu" in yes-4-i86pc | yes-4-i*86 | yes-4-x86_64 | yes-4-amd64) diff --git a/erts/autoconf/vxworks/sed.general b/erts/autoconf/vxworks/sed.general index e2d2f415f1..dbb9420b67 100644 --- a/erts/autoconf/vxworks/sed.general +++ b/erts/autoconf/vxworks/sed.general @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 1997-2012. All Rights Reserved. +# Copyright Ericsson AB 1997-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/autoconf/vxworks/sed.vxworks_ppc32 b/erts/autoconf/vxworks/sed.vxworks_ppc32 index 48ec912b4f..f00daef74c 100644 --- a/erts/autoconf/vxworks/sed.vxworks_ppc32 +++ b/erts/autoconf/vxworks/sed.vxworks_ppc32 @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2006-2010. All Rights Reserved. +# Copyright Ericsson AB 2006-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/autoconf/vxworks/sed.vxworks_simlinux b/erts/autoconf/vxworks/sed.vxworks_simlinux index fbf6d7bc9d..67ddbf1575 100644 --- a/erts/autoconf/vxworks/sed.vxworks_simlinux +++ b/erts/autoconf/vxworks/sed.vxworks_simlinux @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2010. All Rights Reserved. +# Copyright Ericsson AB 2008-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/autoconf/vxworks/sed.vxworks_simso b/erts/autoconf/vxworks/sed.vxworks_simso index 17ec092dee..1af88cef31 100644 --- a/erts/autoconf/vxworks/sed.vxworks_simso +++ b/erts/autoconf/vxworks/sed.vxworks_simso @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2005-2010. All Rights Reserved. +# Copyright Ericsson AB 2005-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/configure.in b/erts/configure.in index c391aece14..f17f4cb5c8 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -150,6 +150,14 @@ AS_HELP_STRING([--disable-smp-support], [disable smp support]), *) enable_smp_support=yes ;; esac ], enable_smp_support=unknown) +AC_ARG_ENABLE(smp-require-native-atomics, + AS_HELP_STRING([--disable-smp-require-native-atomics], + [disable the SMP requirement of a native atomic implementation]), +[ case "$enableval" in + no) smp_require_native_atomics=no ;; + *) smp_require_native_atomics=yes ;; + esac ], smp_require_native_atomics=yes) + AC_ARG_WITH(termcap, AS_HELP_STRING([--with-termcap], [use termcap (default)]) AS_HELP_STRING([--without-termcap], @@ -429,7 +437,7 @@ case $host_os in win32) # The ethread library requires _WIN32_WINNT of at least 0x0403. # -D_WIN32_WINNT=* from CPPFLAGS is saved in ETHR_DEFS. - CPPFLAGS="$CPPFLAGS -D_WIN32_WINNT=0x0500 -DWINVER=0x0500" + CPPFLAGS="$CPPFLAGS -D_WIN32_WINNT=0x0501 -DWINVER=0x0501" ;; darwin*) CPPFLAGS="$CPPFLAGS -D_XOPEN_SOURCE" @@ -624,6 +632,7 @@ case $chk_arch_ in armv5teb) ARCH=arm;; armv5tel) ARCH=arm;; armv5tejl) ARCH=arm;; + armv6l) ARCH=arm;; armv7l) ARCH=arm;; tile) ARCH=tile;; *) ARCH=noarch;; @@ -1042,11 +1051,18 @@ if test $ERTS_BUILD_SMP_EMU = yes; then AC_DEFINE(ERTS_HAVE_SMP_EMU, 1, [Define if the smp emulator is built]) - case "$ethr_have_native_atomics-$ethr_have_native_spinlock" in + test "X$smp_require_native_atomics" = "Xyes" && + AC_DEFINE(ETHR_SMP_REQUIRE_NATIVE_IMPLS, 1, [Define if you want to enable check for native ethread implementations]) + + case "$ethr_have_native_atomics-$smp_require_native_atomics-$ethr_have_native_spinlock" in yes-*) ;; - no-yes) + no-yes-*) + AC_MSG_ERROR([No native atomic implementation found. See Configuring section in INSTALL.md for more information.]) + ;; + + no-no-yes) test -f "$ERL_TOP/erts/CONF_INFO" || echo "" > "$ERL_TOP/erts/CONF_INFO" @@ -1055,12 +1071,13 @@ if test $ERTS_BUILD_SMP_EMU = yes; then No native atomic implementation available. Fallbacks implemented using spinlocks will be used. Note that the performance of the SMP - runtime system will suffer due to this. + runtime system will suffer immensely due to + this. EOF ;; - no-no) + no-no-no) test -f "$ERL_TOP/erts/CONF_INFO" || echo "" > "$ERL_TOP/erts/CONF_INFO" @@ -1070,7 +1087,7 @@ EOF spinlock implementation available. Fallbacks implemented using mutexes will be used. Note that the performance of the SMP runtime system - will suffer much due to this. + will suffer immensely due to this. EOF ;; @@ -1082,8 +1099,26 @@ fi AC_SUBST(ERTS_BUILD_SMP_EMU) -AC_CHECK_FUNCS([posix_fadvise, fallocate]) +AC_CHECK_FUNCS([posix_fadvise]) AC_CHECK_HEADERS([linux/falloc.h]) +dnl * Old glibcs have broken fallocate64(). Make sure not to use it. +AC_CACHE_CHECK([whether fallocate() works],i_cv_fallocate_works,[ + AC_TRY_LINK([ + #include <stdio.h> + #include <stdlib.h> + #include <fcntl.h> + #include <unistd.h> + #include <fcntl.h> + #include <linux/falloc.h> + ], + [ + int fd = creat("conftest.temp", 0600); + fallocate(fd, FALLOC_FL_KEEP_SIZE,(off_t) 1024,(off_t) 1024); + ], i_cv_fallocate_works=yes, i_cv_fallocate_works=no) +]) +if test $i_cv_fallocate_works = yes; then + AC_DEFINE(HAVE_FALLOCATE, 1, Define if you have a working fallocate()) +fi dnl * Old glibcs have broken posix_fallocate(). Make sure not to use it. dnl * It may also be broken in AIX. @@ -1324,6 +1359,17 @@ if test "x$TERMCAP_LIB" != "x"; then AC_DEFINE(HAVE_TERMCAP, 1, [Define if termcap functions exists]) fi +if test "X$host" != "Xwin32"; then + AC_MSG_CHECKING(for wcwidth) + AC_TRY_LINK([#include <wchar.h>], [wcwidth(0);], + have_wcwidth=yes, have_wcwidth=no) + if test $have_wcwidth = yes; then + AC_MSG_RESULT([yes]) + AC_DEFINE(HAVE_WCWIDTH, [1], + [Define to 1 if you have a `wcwidth' function.]) + fi +fi + dnl ------------- dnl zlib dnl ------------- @@ -1633,6 +1679,10 @@ if test x"$ac_cv_header_netinet_sctp_h" = x"yes"; then ]) fi +dnl Check for setns +AC_CHECK_HEADERS(sched.h setns.h) +AC_CHECK_FUNCS([setns]) + HAVE_VALGRIND=no AC_CHECK_HEADER(valgrind/valgrind.h, HAVE_VALGRIND=yes) AC_SUBST(HAVE_VALGRIND) @@ -3558,10 +3608,7 @@ DED_EXT=so case $host_os in win32) DED_EXT=dll;; darwin*) - DED_CFLAGS="$DED_CFLAGS -fno-common" - if test "X$STATIC_CFLAGS" = "X"; then - STATIC_CFLAGS="-mdynamic-no-pic" - fi;; + DED_CFLAGS="$DED_CFLAGS -fno-common";; *) ;; esac diff --git a/erts/doc/src/crash_dump.xml b/erts/doc/src/crash_dump.xml index b3c4671c3d..8f5515baca 100644 --- a/erts/doc/src/crash_dump.xml +++ b/erts/doc/src/crash_dump.xml @@ -4,7 +4,7 @@ <chapter> <header> <copyright> - <year>1999</year><year>2010</year> + <year>1999</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -290,6 +290,10 @@ <em>Stack+heap</em>, <em>OldHeap</em>, <em>Heap unused</em> and <em>OldHeap unused</em> do not exist. Instead this field presents the size of the process' stack.</item> + <tag><em>Memory</em></tag> + <item>The total memory used by this process. This includes call stack, + heap and internal structures. Same as <seealso marker="erlang#process_info-2">erlang:process_info(Pid,memory)</seealso>. + </item> <tag><em>Program counter</em></tag> <item>The current instruction pointer. This is only interesting for runtime system developers. The function into which the program diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 9f5de8a9ae..c16b45856d 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -536,15 +536,35 @@ <tag><marker id="file_name_encoding"></marker><c><![CDATA[+fnl]]></c></tag> <item> <p>The VM works with file names as if they are encoded using the ISO-latin-1 encoding, disallowing Unicode characters with codepoints beyond 255. This is default on operating systems that have transparent file naming, i.e. all Unixes except MacOSX.</p> + <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names.</p> </item> - <tag><c><![CDATA[+fnu]]></c></tag> + <tag><c><![CDATA[+fnu[{w|i|e}]]]></c></tag> <item> <p>The VM works with file names as if they are encoded using UTF-8 (or some other system specific Unicode encoding). This is the default on operating systems that enforce Unicode encoding, i.e. Windows and MacOSX.</p> - <p>By enabling Unicode file name translation on systems where this is not default, you open up to the possibility that some file names can not be interpreted by the VM and therefore will be returned to the program as raw binaries. The option is therefore considered experimental.</p> - </item> - <tag><c><![CDATA[+fna]]></c></tag> - <item> - <p>Selection between <c>+fnl</c> and <c>+fnu</c> is done based on the current locale settings in the OS, meaning that if you have set your terminal for UTF-8 encoding, the filesystem is expected to use the same encoding for filenames (use with care).</p> + <p>The <c>+fnu</c> switch can be followed by <c>w</c>, + <c>i</c>, or <c>e</c> to control the way wrongly encoded file + names are to be reported. <c>w</c> means that a warning is + sent to the <c>error_logger</c> whenever a wrongly encoded + file name is "skipped" in directory listings, <c>i</c> means + that those wrongly encoded file names are silently ignored and + <c>e</c> means that the API function will return an error + whenever a wrongly encoded file (or directory) name is + encountered. <c>w</c> is the default. Note that + <c>file:read_link/1</c> will always return an error if the + link points to an invalid file name.</p> + <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names.</p> + </item> + <tag><c><![CDATA[+fna[{w|i|e}]]]></c></tag> + <item> + <p>Selection between <c>+fnl</c> and <c>+fnu</c> is done based on the current locale settings in the OS, meaning that if you have set your terminal for UTF-8 encoding, the filesystem is expected to use the same encoding for file names (use with care).</p> + <p>The <c>+fna</c> switch can be followed by <c>w</c>, + <c>i</c>, or <c>e</c>. This will have effect if the locale + settings cause the behavior of <c>+fnu</c> to be selected. + See the description of <c>+fnu</c> above. If the locale + settings cause the behavior of <c>+fnl</c> to be selected, + then <c>w</c>, <c>i</c>, or <c>e</c> will not have any + effect.</p> + <p>See <seealso marker="stdlib:unicode_usage#unicode_file_names">STDLIB User's Guide</seealso> for more infomation about unicode file names.</p> </item> <tag><c><![CDATA[+hms Size]]></c></tag> <item> @@ -571,9 +591,9 @@ </item> <tag><c><![CDATA[+L]]></c></tag> <item> - <p>Don't load information about source filenames and line numbers. + <p>Don't load information about source file names and line numbers. This will save some memory, but exceptions will not contain - information about the filenames and line numbers. + information about the file names and line numbers. </p> </item> <tag><marker id="erts_alloc"><c><![CDATA[+MFlag Value]]></c></marker></tag> @@ -615,7 +635,7 @@ debugging.</item> </taglist> </item> - <tag><marker id="+pc"/><marker id="max_processes"><c><![CDATA[+pc Range]]></c></marker></tag> + <tag><marker id="+pc"/><marker id="printable_character_range"><c><![CDATA[+pc Range]]></c></marker></tag> <item> <p>Sets the range of characters that the system will consider printable in heuristic detection of strings. This typically affects the shell, debugger and io:format functions (when ~tp is used in the format string).</p> <p>Currently two values for the <c>Range</c> are supported: @@ -637,10 +657,11 @@ <p>Se also <seealso marker="stdlib:io#printable_range/0"> io:printable_range/0</seealso>.</p> </item> - <tag><marker id="+P"/><marker id="max_processes"><c><![CDATA[+P Number]]></c></marker></tag> + <tag><marker id="+P"/><marker id="max_processes"><c><![CDATA[+P Number|legacy]]></c></marker></tag> <item> <p>Sets the maximum number of simultaneously existing processes for this - system. Valid range for <c>Number</c> is <c>[1024-134217727]</c></p> + system if a <c>Number</c> is passed as value. Valid range for + <c>Number</c> is <c>[1024-134217727]</c></p> <p><em>NOTE</em>: The actual maximum chosen may be much larger than the <c>Number</c> passed. Currently the runtime system often, but not always, chooses a value that is a power of 2. This might, @@ -648,11 +669,19 @@ checked by calling <seealso marker="erlang#system_info_process_limit">erlang:system_info(process_limit)</seealso>.</p> <p>The default value is <c>262144</c></p> + <p>If <c>legacy</c> is passed as value, the legacy algorithm for + allocation of process identifiers will be used. Using the legacy + algorithm, identifiers will be allocated in a strictly increasing + fashion until largest possible identifier has been reached. Note that + this algorithm suffers from performance issues and can under certain + circumstances be extremely expensive. The legacy algoritm is deprecated, + and the <c>legacy</c> option is scheduled for removal in OTP-R18.</p> </item> - <tag><marker id="+Q"/><marker id="max_ports"><c><![CDATA[+Q Number]]></c></marker></tag> + <tag><marker id="+Q"/><marker id="max_ports"><c><![CDATA[+Q Number|legacy]]></c></marker></tag> <item> <p>Sets the maximum number of simultaneously existing ports for this - system. Valid range for <c>Number</c> is <c>[1024-134217727]</c></p> + system if a Number is passed as value. Valid range for <c>Number</c> + is <c>[1024-134217727]</c></p> <p><em>NOTE</em>: The actual maximum chosen may be much larger than the actual <c>Number</c> passed. Currently the runtime system often, but not always, chooses a value that is a power of 2. This might, @@ -671,6 +700,13 @@ for setting the maximum number of simultaneously existing ports. This environment variable is deprecated, and scheduled for removal in OTP-R17, but can still be used.</p> + <p>If <c>legacy</c> is passed as value, the legacy algorithm for + allocation of port identifiers will be used. Using the legacy + algorithm, identifiers will be allocated in a strictly increasing + fashion until largest possible identifier has been reached. Note that + this algorithm suffers from performance issues and can under certain + circumstances be extremely expensive. The legacy algoritm is deprecated, + and the <c>legacy</c> option is scheduled for removal in OTP-R18.</p> </item> <tag><marker id="compat_rel"><c><![CDATA[+R ReleaseNumber]]></c></marker></tag> <item> @@ -712,19 +748,47 @@ </item> <tag><marker id="+S"><c><![CDATA[+S Schedulers:SchedulerOnline]]></c></marker></tag> <item> - <p>Sets the amount of scheduler threads to create and scheduler - threads to set online when SMP support has been enabled. - Valid range for both values are 1-1024. If the - Erlang runtime system is able to determine the amount - of logical processors configured and logical processors available, - <c>Schedulers</c> will default to logical processors configured, - and <c>SchedulersOnline</c> will default to logical processors - available; otherwise, the default values will be 1. <c>Schedulers</c> - may be omitted if <c>:SchedulerOnline</c> is not and vice versa. The - amount of schedulers online can be changed at run time via + <p>Sets the number of scheduler threads to create and scheduler + threads to set online when SMP support has been enabled. The maximum for + both values is 1024. If the Erlang runtime system is able to determine the + amount of logical processors configured and logical processors available, + <c>Schedulers</c> will default to logical processors configured, and + <c>SchedulersOnline</c> will default to logical processors available; + otherwise, the default values will be 1. <c>Schedulers</c> may be omitted + if <c>:SchedulerOnline</c> is not and vice versa. The number of schedulers + online can be changed at run time via <seealso marker="erlang#system_flag_schedulers_online">erlang:system_flag(schedulers_online, SchedulersOnline)</seealso>. </p> - <p>This flag will be ignored if the emulator doesn't have + <p>If <c>Schedulers</c> or <c>SchedulersOnline</c> is specified as a + negative number, the value is subtracted from the default number of + logical processors configured or logical processors available, respectively. + </p> + <p>Specifying the value 0 for <c>Schedulers</c> or <c>SchedulersOnline</c> + resets the number of scheduler threads or scheduler threads online respectively + to its default value. + </p> + <p>This option is ignored if the emulator doesn't have + SMP support enabled (see the <seealso marker="#smp">-smp</seealso> + flag).</p> + </item> + <tag><marker id="+SP"><c><![CDATA[+SP SchedulersPercentage:SchedulersOnlinePercentage]]></c></marker></tag> + <item> + <p>Similar to <seealso marker="#+S">+S</seealso> but uses percentages to set the + number of scheduler threads to create, based on logical processors configured, + and scheduler threads to set online, based on logical processors available, when + SMP support has been enabled. Specified values must be greater than 0. For example, + <c>+SP 50:25</c> sets the number of scheduler threads to 50% of the logical processors + configured and the number of scheduler threads online to 25% of the logical processors available. + <c>SchedulersPercentage</c> may be omitted if <c>:SchedulersOnlinePercentage</c> is + not and vice versa. The number of schedulers online can be changed at run time via + <seealso marker="erlang#system_flag_schedulers_online">erlang:system_flag(schedulers_online, SchedulersOnline)</seealso>. + </p> + <p>This option interacts with <seealso marker="#+S">+S</seealso> settings. + For example, on a system with 8 logical cores configured and 8 logical cores + available, the combination of the options <c>+S 4:4 +SP 50:25</c> (in either order) + results in 2 scheduler threads (50% of 4) and 1 scheduler thread online (25% of 4). + </p> + <p>This option is ignored if the emulator doesn't have SMP support enabled (see the <seealso marker="#smp">-smp</seealso> flag).</p> </item> @@ -1001,6 +1065,20 @@ <p>For more information, see <seealso marker="erlang#system_info_cpu_topology">erlang:system_info(cpu_topology)</seealso>.</p> </item> + <tag><marker id="+sfwi"><c>+sfwi Interval</c></marker></tag> + <item> + <p>Set scheduler forced wakeup interval. All run queues will + be scanned each <c>Interval</c> milliseconds. While there are + sleeping schedulers in the system, one scheduler will be woken + for each non-empty run queue found. An <c>Interval</c> of zero + disables this feature, which also is the default. + </p> + <p>This feature has been introduced as a temporary workaround + for lengthy executing native code, and native code that do not + bump reductions properly in OTP. When these bugs have be fixed + the <c>+sfwi</c> flag will be removed. + </p> + </item> <tag><marker id="+stbt"><c>+stbt BindType</c></marker></tag> <item> <p>Try to set scheduler bind type. The same as the diff --git a/erts/doc/src/erl_driver.xml b/erts/doc/src/erl_driver.xml index efe0483b31..c055d1ca9e 100644 --- a/erts/doc/src/erl_driver.xml +++ b/erts/doc/src/erl_driver.xml @@ -666,7 +666,7 @@ typedef struct ErlDrvBinary { <item> <p>The <c>ErlDrvData</c> is a handle to driver-specific data, passed to the driver call-backs. It is a pointer, and is - most often type casted to a specific pointer in the driver.</p> + most often type cast to a specific pointer in the driver.</p> </item> <tag>SysIOVec</tag> <item> @@ -1014,7 +1014,7 @@ typedef struct ErlIOVec { <fsummary>Read a system timestamp</fsummary> <desc> <marker id="driver_get_now"></marker> - <p>This function reads a timestamp into the memory pointed to by + <p>This function reads a timestamp into the memory pointed to by the parameter <c>now</c>. See the description of <seealso marker="#ErlDrvNowData">ErlDrvNowData</seealso> for specification of its fields. </p> <p>The return value is 0 unless the <c>now</c> pointer is not @@ -1056,7 +1056,7 @@ typedef struct ErlIOVec { returned. Another thread may still be using the event object internally. To safely close an event object call <c>driver_select</c> with <c>ERL_DRV_USE</c> and <c>on==0</c>. That - will clear all events and then call + will clear all events and then call <seealso marker="driver_entry#stop_select">stop_select</seealso> when it is safe to close the event object. <c>ERL_DRV_USE</c> should be set together with the first event @@ -1068,7 +1068,7 @@ typedef struct ErlIOVec { <p>ERL_DRV_USE was added in OTP release R13. Old drivers will still work as before. But it is recommended to update them to use <c>ERL_DRV_USE</c> and <c>stop_select</c> to make sure that event objects are closed in a safe way.</p> - </note> + </note> <p>The return value is 0 (failure, -1, only if the <c>ready_input</c>/<c>ready_output</c> is <c>NULL</c>).</p> @@ -1524,7 +1524,7 @@ typedef struct ErlIOVec { <marker id="remove_driver_entry"></marker> <p>This function removes a driver entry <c>de</c> previously added with <c>add_driver_entry</c>.</p> - <p>Driver entries added by the <c>erl_ddll</c> erlang interface can + <p>Driver entries added by the <c>erl_ddll</c> erlang interface can not be removed by using this interface.</p> </desc> </func> @@ -1758,7 +1758,7 @@ typedef struct ErlIOVec { <pre> Term type Argument(s) =========================================== -ERL_DRV_NIL +ERL_DRV_NIL ERL_DRV_ATOM ErlDrvTermData atom (from driver_mk_atom(char *string)) ERL_DRV_INT ErlDrvSInt integer ERL_DRV_UINT ErlDrvUInt integer @@ -1779,11 +1779,11 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len signed integer data type <c>ErlDrvSInt</c> are 64 bits wide on a 64 bit runtime system and 32 bits wide on a 32 bit runtime system. They were introduced in erts version 5.6, - and replaced some of the <c>int</c> arguments in the list above. + and replaced some of the <c>int</c> arguments in the list above. </p> <p>The unsigned integer data type <c>ErlDrvUInt64</c> and the signed integer data type <c>ErlDrvSInt64</c> are always 64 bits - wide. They were introduced in erts version 5.7.4. + wide. They were introduced in erts version 5.7.4. </p> <p>To build the tuple <c>{tcp, Port, [100 | Binary]}</c>, the @@ -1879,7 +1879,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len <fsummary>Send term data from driver to port owner</fsummary> <desc> <marker id="driver_output_term"></marker> - <warning><p><c>driver_output_term()</c> is deferred and will + <warning><p><c>driver_output_term()</c> is deprecated and will be removed in the OTP-R17 release. Use <seealso marker="#erl_drv_send_term">erl_drv_output_term()</seealso> instead.</p> @@ -1937,7 +1937,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len <fsummary>Send term data to other process than port owner process</fsummary> <desc> <marker id="driver_send_term"></marker> - <warning><p><c>driver_send_term()</c> is deferred and will + <warning><p><c>driver_send_term()</c> is deprecated and will be removed in the OTP-R17 release. Use <seealso marker="#erl_drv_send_term">erl_drv_send_term()</seealso> instead.</p> @@ -1981,7 +1981,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len thread, the following call can be used:</p> <p></p> <code type="none"><![CDATA[ - unsigned int myKey = (unsigned int) myPort; + unsigned int myKey = driver_async_port_key(myPort); r = driver_async(myPort, &myKey, myData, myFunc); ]]></code> @@ -1998,7 +1998,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len The data should be freed in <c>async_free</c>, because it's called if <c>driver_async_cancel</c> is called.</p> <p>When the async operation is done, <seealso marker="driver_entry#ready_async">ready_async</seealso> driver - entry function is called. If <c>async_ready</c> is null in + entry function is called. If <c>ready_async</c> is null in the driver entry, the <c>async_free</c> function is called instead.</p> <p>The return value is a handle to the asynchronous task, which @@ -2022,6 +2022,24 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len </desc> </func> <func> + <name><ret>unsigned int</ret><nametext>driver_async_port_key (ErlDrvPort port)</nametext></name> + <fsummary>Calculate an async key from an ErlDrvPort</fsummary> + <desc> + <marker id="driver_async_port_key"></marker> + <p>This function calculates a key for later use in <seealso + marker="#driver_async">driver_async()</seealso>. The keys are + evenly distributed so that a fair mapping between port id's + and async thread id's is achieved.</p> + <note> + <p>Before OTP-R16, the actual port id could be used as a key + with proper casting, but after the rewrite of the port + subsystem, this is no longer the case. With this function, you + can achieve the same distribution based on port id's as before + OTP-R16.</p> + </note> + </desc> + </func> + <func> <name><ret>int</ret><nametext>driver_async_cancel(long id)</nametext></name> <fsummary>Cancel an asynchronous call</fsummary> <desc> @@ -2033,10 +2051,10 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len The user had to implement synchronization of cancellation anyway. It also unnecessarily complicated the implementation. Therefore, as of OTP-R15B <c>driver_async_cancel()</c> is deprecated, and - scheduled for removal in OTP-R16. It will currently always fail, + scheduled for removal in OTP-R17. It will currently always fail, and return 0.</p> - <warning><p><c>driver_async_cancel()</c> is deferred and will - be removed in the OTP-R16 release.</p> + <warning><p><c>driver_async_cancel()</c> is deprecated and will + be removed in the OTP-R17 release.</p> </warning> </desc> @@ -2048,7 +2066,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len <marker id="driver_lock_driver"></marker> <p>This function locks the driver used by the port <c>port</c> in memory for the rest of the emulator process' - lifetime. After this call, the driver behaves as one of Erlang's + lifetime. After this call, the driver behaves as one of Erlang's statically linked in drivers.</p> </desc> </func> @@ -2076,7 +2094,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len <seealso marker="driver_entry">driver_entry</seealso>).</item> <tag><c>drv_data</c></tag> <item>The driver defined handle that will be passed in subsequent - calls to driver call-backs. Note, that the + calls to driver call-backs. Note, that the <seealso marker="driver_entry#start">driver start call-back</seealso> will not be called for this new driver instance. The driver defined handle is normally created in the @@ -2284,7 +2302,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len <item>A thread identifier.</item> </taglist> <p>This function compares two thread identifiers for equality, - and returns <c>0</c> it they aren't equal, and + and returns <c>0</c> it they aren't equal, and a value not equal to <c>0</c> if they are equal.</p> <note><p>A Thread identifier may be reused very quickly after a thread has terminated. Therefore, if a thread @@ -2469,7 +2487,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len </taglist> <p>This function broadcasts on a condition variable. That is, if other threads are waiting on the condition variable being - broadcasted on, <em>all</em> of them will be woken. + broadcast on, <em>all</em> of them will be woken. </p> <p>This function is thread-safe.</p> </desc> @@ -2498,7 +2516,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len the calling thread when calling this function. </p> <note><p><c>erl_drv_cond_wait()</c> might return even though - no-one has signaled or broadcasted on the condition + no-one has signaled or broadcast on the condition variable. Code calling <c>erl_drv_cond_wait()</c> should always be prepared for <c>erl_drv_cond_wait()</c> returning even though the condition that the thread was @@ -2822,7 +2840,7 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len <item>A pointer to an output buffer.</item> <tag><c>value_size</c></tag> <item>A pointer to an integer. The integer is both used for - passing input and output sizes (see below). + passing input and output sizes (see below). </item> </taglist> <p>This function retrieves the value of an environment variable. @@ -2889,8 +2907,84 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len beginning of this document.</p> </desc> </func> - </funcs> + <func> + <name><ret>char *</ret><nametext>erl_drv_cond_name(ErlDrvCond *cnd)</nametext></name> + <fsummary>Get name of driver mutex.</fsummary> + <desc> + <marker id="erl_drv_cnd_name"></marker> + <p>Arguments:</p> + <taglist> + <tag><c>cnd</c></tag> + <item>A pointer to an initialized condition.</item> + </taglist> + <p> + Returns a pointer to the name of the condition. + </p> + <note> + <p>This function is intended for debugging purposes only.</p> + </note> + </desc> + </func> + + <func> + <name><ret>char *</ret><nametext>erl_drv_mutex_name(ErlDrvMutex *mtx)</nametext></name> + <fsummary>Get name of driver mutex.</fsummary> + <desc> + <marker id="erl_drv_mutex_name"></marker> + <p>Arguments:</p> + <taglist> + <tag><c>mtx</c></tag> + <item>A pointer to an initialized mutex.</item> + </taglist> + <p> + Returns a pointer to the name of the mutex. + </p> + <note> + <p>This function is intended for debugging purposes only.</p> + </note> + </desc> + </func> + + <func> + <name><ret>char *</ret><nametext>erl_drv_rwlock_name(ErlDrvRWLock *rwlck)</nametext></name> + <fsummary>Get name of driver mutex.</fsummary> + <desc> + <marker id="erl_drv_rwlock_name"></marker> + <p>Arguments:</p> + <taglist> + <tag><c>rwlck</c></tag> + <item>A pointer to an initialized r/w-lock.</item> + </taglist> + <p> + Returns a pointer to the name of the r/w-lock. + </p> + <note> + <p>This function is intended for debugging purposes only.</p> + </note> + </desc> + </func> + + <func> + <name><ret>char *</ret><nametext>erl_drv_thread_name(ErlDrvTid tid)</nametext></name> + <fsummary>Get name of driver mutex.</fsummary> + <desc> + <marker id="erl_drv_rwlock_name"></marker> + <p>Arguments:</p> + <taglist> + <tag><c>tid</c></tag> + <item>A thread identifier.</item> + </taglist> + <p> + Returns a pointer to the name of the thread. + </p> + <note> + <p>This function is intended for debugging purposes only.</p> + </note> + </desc> + </func> + + </funcs> <section> <title>SEE ALSO</title> <p><seealso marker="driver_entry">driver_entry(3)</seealso>, @@ -2900,4 +2994,3 @@ ERL_DRV_EXT2TERM char *buf, ErlDrvUInt len Guide Ch. 3)</p> </section> </cref> - diff --git a/erts/doc/src/erl_ext_fig.ps b/erts/doc/src/erl_ext_fig.ps deleted file mode 100644 index 2501dc3c05..0000000000 --- a/erts/doc/src/erl_ext_fig.ps +++ /dev/null @@ -1,153 +0,0 @@ -%!PS-Adobe-3.0 EPSF-2.0 -%%BoundingBox: 0 0 600 520 -%%Creator: mscgen 1 -%%EndComments -0.70 0.70 scale -0 0 moveto -0 520 lineto -600 520 lineto -600 0 lineto -closepath -clip -%PageTrailer -%Page: 1 1 -/Helvetica findfont -10 scalefont -setfont -0 520 translate -/mtrx matrix def -/ellipse - { /endangle exch def - /startangle exch def - /ydia exch def - /xdia exch def - /y exch def - /x exch def - /savematrix mtrx currentmatrix def - x y translate - xdia 2 div ydia 2 div scale - 0 0 1 startangle endangle arc - savematrix setmatrix -} def -150 -12 moveto (Client (or Node)) dup stringwidth pop 2 div neg 0 rmoveto show -450 -12 moveto (EPMD) dup stringwidth pop 2 div neg 0 rmoveto show -newpath 150 -20 moveto 150 -45 lineto stroke -newpath 450 -20 moveto 450 -45 lineto stroke -newpath 150 -32 moveto 450 -32 lineto stroke -newpath 450 -32 moveto 440 -38 lineto 440 -26 lineto closepath fill -270 -30 moveto (ALIVE2_REQ) show -newpath 150 -45 moveto 150 -70 lineto stroke -newpath 450 -45 moveto 450 -70 lineto stroke -[2] 0 setdash -newpath 450 -57 moveto 150 -57 lineto stroke -[] 0 setdash -newpath 150 -57 moveto 160 -63 lineto 160 -51 lineto closepath fill -267 -55 moveto (ALIVE2_RESP) show -[2] 0 setdash -newpath 150 -70 moveto 150 -95 lineto stroke -[] 0 setdash -[2] 0 setdash -newpath 450 -70 moveto 450 -95 lineto stroke -[] 0 setdash -newpath 150 -95 moveto 150 -120 lineto stroke -newpath 450 -95 moveto 450 -120 lineto stroke -newpath 150 -107 moveto 450 -107 lineto stroke -newpath 450 -107 moveto 440 -113 lineto 440 -101 lineto closepath fill -253 -105 moveto (ALIVE_CLOSE_REQ) show -[2] 0 setdash -newpath 150 -120 moveto 150 -145 lineto stroke -[] 0 setdash -[2] 0 setdash -newpath 450 -120 moveto 450 -145 lineto stroke -[] 0 setdash -newpath 150 -145 moveto 150 -170 lineto stroke -newpath 450 -145 moveto 450 -170 lineto stroke -newpath 150 -157 moveto 450 -157 lineto stroke -newpath 450 -157 moveto 440 -163 lineto 440 -151 lineto closepath fill -248 -155 moveto (PORT_PLEASE2_REQ) show -newpath 150 -170 moveto 150 -195 lineto stroke -newpath 450 -170 moveto 450 -195 lineto stroke -[2] 0 setdash -newpath 450 -182 moveto 150 -182 lineto stroke -[] 0 setdash -newpath 150 -182 moveto 160 -188 lineto 160 -176 lineto closepath fill -267 -180 moveto (PORT2_RESP) show -[2] 0 setdash -newpath 150 -195 moveto 150 -220 lineto stroke -[] 0 setdash -[2] 0 setdash -newpath 450 -195 moveto 450 -220 lineto stroke -[] 0 setdash -newpath 150 -220 moveto 150 -245 lineto stroke -newpath 450 -220 moveto 450 -245 lineto stroke -newpath 150 -232 moveto 450 -232 lineto stroke -newpath 450 -232 moveto 440 -238 lineto 440 -226 lineto closepath fill -269 -230 moveto (NAMES_REQ) show -newpath 150 -245 moveto 150 -270 lineto stroke -newpath 450 -245 moveto 450 -270 lineto stroke -[2] 0 setdash -newpath 450 -257 moveto 150 -257 lineto stroke -[] 0 setdash -newpath 150 -257 moveto 160 -263 lineto 160 -251 lineto closepath fill -266 -255 moveto (NAMES_RESP) show -[2] 0 setdash -newpath 150 -270 moveto 150 -295 lineto stroke -[] 0 setdash -[2] 0 setdash -newpath 450 -270 moveto 450 -295 lineto stroke -[] 0 setdash -newpath 150 -295 moveto 150 -320 lineto stroke -newpath 450 -295 moveto 450 -320 lineto stroke -newpath 150 -307 moveto 450 -307 lineto stroke -newpath 450 -307 moveto 440 -313 lineto 440 -301 lineto closepath fill -272 -305 moveto (DUMP_REQ) show -newpath 150 -320 moveto 150 -345 lineto stroke -newpath 450 -320 moveto 450 -345 lineto stroke -[2] 0 setdash -newpath 450 -332 moveto 150 -332 lineto stroke -[] 0 setdash -newpath 150 -332 moveto 160 -338 lineto 160 -326 lineto closepath fill -269 -330 moveto (DUMP_RESP) show -[2] 0 setdash -newpath 150 -345 moveto 150 -370 lineto stroke -[] 0 setdash -[2] 0 setdash -newpath 450 -345 moveto 450 -370 lineto stroke -[] 0 setdash -newpath 150 -370 moveto 150 -395 lineto stroke -newpath 450 -370 moveto 450 -395 lineto stroke -newpath 150 -382 moveto 450 -382 lineto stroke -newpath 450 -382 moveto 440 -388 lineto 440 -376 lineto closepath fill -277 -380 moveto (KILL_REQ) show -newpath 150 -395 moveto 150 -420 lineto stroke -newpath 450 -395 moveto 450 -420 lineto stroke -[2] 0 setdash -newpath 450 -407 moveto 150 -407 lineto stroke -[] 0 setdash -newpath 150 -407 moveto 160 -413 lineto 160 -401 lineto closepath fill -274 -405 moveto (KILL_RESP) show -[2] 0 setdash -newpath 150 -420 moveto 150 -445 lineto stroke -[] 0 setdash -[2] 0 setdash -newpath 450 -420 moveto 450 -445 lineto stroke -[] 0 setdash -newpath 150 -445 moveto 150 -470 lineto stroke -newpath 450 -445 moveto 450 -470 lineto stroke -newpath 150 -457 moveto 450 -457 lineto stroke -newpath 450 -457 moveto 440 -463 lineto 440 -451 lineto closepath fill -273 -455 moveto (STOP_REQ) show -newpath 150 -470 moveto 150 -495 lineto stroke -newpath 450 -470 moveto 450 -495 lineto stroke -[2] 0 setdash -newpath 450 -482 moveto 150 -482 lineto stroke -[] 0 setdash -newpath 150 -482 moveto 160 -488 lineto 160 -476 lineto closepath fill -260 -480 moveto (STOP_OK_RESP) show -newpath 150 -495 moveto 150 -520 lineto stroke -newpath 450 -495 moveto 450 -520 lineto stroke -[2] 0 setdash -newpath 450 -507 moveto 150 -507 lineto stroke -[] 0 setdash -newpath 150 -507 moveto 160 -513 lineto 160 -501 lineto closepath fill -250 -505 moveto (STOP_NOTOK_RESP) show diff --git a/erts/doc/src/erl_fix_alloc.ps b/erts/doc/src/erl_fix_alloc.ps deleted file mode 100644 index bf65d1556c..0000000000 --- a/erts/doc/src/erl_fix_alloc.ps +++ /dev/null @@ -1,646 +0,0 @@ -%!PS-Adobe-2.0 EPSF-2.0 -%%Title: erl_fix_alloc.fig -%%Creator: fig2dev Version 3.1 Patchlevel 2 -%%CreationDate: Tue May 20 11:10:33 1997 -%%For: jocke@akvavit (Joakim Greben|,ETX/B/DUP) -%Magnification: 1.00 -%%Orientation: Portrait -%%BoundingBox: 0 0 506 462 -%%Pages: 0 -%%BeginSetup -%%IncludeFeature: *PageSize A4 -%%EndSetup -%%EndComments -/MyAppDict 100 dict dup begin def -/$F2psDict 200 dict def -$F2psDict begin -$F2psDict /mtrx matrix put -/col-1 {0 setgray} bind def -/col0 {0.000 0.000 0.000 srgb} bind def -/col1 {0.000 0.000 1.000 srgb} bind def -/col2 {0.000 1.000 0.000 srgb} bind def -/col3 {0.000 1.000 1.000 srgb} bind def -/col4 {1.000 0.000 0.000 srgb} bind def -/col5 {1.000 0.000 1.000 srgb} bind def -/col6 {1.000 1.000 0.000 srgb} bind def -/col7 {1.000 1.000 1.000 srgb} bind def -/col8 {0.000 0.000 0.560 srgb} bind def -/col9 {0.000 0.000 0.690 srgb} bind def -/col10 {0.000 0.000 0.820 srgb} bind def -/col11 {0.530 0.810 1.000 srgb} bind def -/col12 {0.000 0.560 0.000 srgb} bind def -/col13 {0.000 0.690 0.000 srgb} bind def -/col14 {0.000 0.820 0.000 srgb} bind def -/col15 {0.000 0.560 0.560 srgb} bind def -/col16 {0.000 0.690 0.690 srgb} bind def -/col17 {0.000 0.820 0.820 srgb} bind def -/col18 {0.560 0.000 0.000 srgb} bind def -/col19 {0.690 0.000 0.000 srgb} bind def -/col20 {0.820 0.000 0.000 srgb} bind def -/col21 {0.560 0.000 0.560 srgb} bind def -/col22 {0.690 0.000 0.690 srgb} bind def -/col23 {0.820 0.000 0.820 srgb} bind def -/col24 {0.500 0.190 0.000 srgb} bind def -/col25 {0.630 0.250 0.000 srgb} bind def -/col26 {0.750 0.380 0.000 srgb} bind def -/col27 {1.000 0.500 0.500 srgb} bind def -/col28 {1.000 0.630 0.630 srgb} bind def -/col29 {1.000 0.750 0.750 srgb} bind def -/col30 {1.000 0.880 0.880 srgb} bind def -/col31 {1.000 0.840 0.000 srgb} bind def - -end -save --18.0 481.0 translate -1 -1 scale -.9 .9 scale % to make patterns same scale as in xfig - -% This junk string is used by the show operators -/PATsstr 1 string def -/PATawidthshow { % cx cy cchar rx ry string - % Loop over each character in the string - { % cx cy cchar rx ry char - % Show the character - dup % cx cy cchar rx ry char char - PATsstr dup 0 4 -1 roll put % cx cy cchar rx ry char (char) - false charpath % cx cy cchar rx ry char - /clip load PATdraw - % Move past the character (charpath modified the - % current point) - currentpoint % cx cy cchar rx ry char x y - newpath - moveto % cx cy cchar rx ry char - % Reposition by cx,cy if the character in the string is cchar - 3 index eq { % cx cy cchar rx ry - 4 index 4 index rmoveto - } if - % Reposition all characters by rx ry - 2 copy rmoveto % cx cy cchar rx ry - } forall - pop pop pop pop pop % - - currentpoint - newpath - moveto -} bind def -/PATcg { - 7 dict dup begin - /lw currentlinewidth def - /lc currentlinecap def - /lj currentlinejoin def - /ml currentmiterlimit def - /ds [ currentdash ] def - /cc [ currentrgbcolor ] def - /cm matrix currentmatrix def - end -} bind def -% PATdraw - calculates the boundaries of the object and -% fills it with the current pattern -/PATdraw { % proc - save exch - PATpcalc % proc nw nh px py - 5 -1 roll exec % nw nh px py - newpath - PATfill % - - restore -} bind def -% PATfill - performs the tiling for the shape -/PATfill { % nw nh px py PATfill - - PATDict /CurrentPattern get dup begin - setfont - % Set the coordinate system to Pattern Space - PatternGState PATsg - % Set the color for uncolored pattezns - PaintType 2 eq { PATDict /PColor get PATsc } if - % Create the string for showing - 3 index string % nw nh px py str - % Loop for each of the pattern sources - 0 1 Multi 1 sub { % nw nh px py str source - % Move to the starting location - 3 index 3 index % nw nh px py str source px py - moveto % nw nh px py str source - % For multiple sources, set the appropriate color - Multi 1 ne { dup PC exch get PATsc } if - % Set the appropriate string for the source - 0 1 7 index 1 sub { 2 index exch 2 index put } for pop - % Loop over the number of vertical cells - 3 index % nw nh px py str nh - { % nw nh px py str - currentpoint % nw nh px py str cx cy - 2 index show % nw nh px py str cx cy - YStep add moveto % nw nh px py str - } repeat % nw nh px py str - } for - 5 { pop } repeat - end -} bind def - -% PATkshow - kshow with the current pattezn -/PATkshow { % proc string - exch bind % string proc - 1 index 0 get % string proc char - % Loop over all but the last character in the string - 0 1 4 index length 2 sub { - % string proc char idx - % Find the n+1th character in the string - 3 index exch 1 add get % string proe char char+1 - exch 2 copy % strinq proc char+1 char char+1 char - % Now show the nth character - PATsstr dup 0 4 -1 roll put % string proc chr+1 chr chr+1 (chr) - false charpath % string proc char+1 char char+1 - /clip load PATdraw - % Move past the character (charpath modified the current point) - currentpoint newpath moveto - % Execute the user proc (should consume char and char+1) - mark 3 1 roll % string proc char+1 mark char char+1 - 4 index exec % string proc char+1 mark... - cleartomark % string proc char+1 - } for - % Now display the last character - PATsstr dup 0 4 -1 roll put % string proc (char+1) - false charpath % string proc - /clip load PATdraw - neewath - pop pop % - -} bind def -% PATmp - the makepattern equivalent -/PATmp { % patdict patmtx PATmp patinstance - exch dup length 7 add % We will add 6 new entries plus 1 FID - dict copy % Create a new dictionary - begin - % Matrix to install when painting the pattern - TilingType PATtcalc - /PatternGState PATcg def - PatternGState /cm 3 -1 roll put - % Check for multi pattern sources (Level 1 fast color patterns) - currentdict /Multi known not { /Multi 1 def } if - % Font dictionary definitions - /FontType 3 def - % Create a dummy encoding vector - /Encoding 256 array def - 3 string 0 1 255 { - Encoding exch dup 3 index cvs cvn put } for pop - /FontMatrix matrix def - /FontBBox BBox def - /BuildChar { - mark 3 1 roll % mark dict char - exch begin - Multi 1 ne {PaintData exch get}{pop} ifelse % mark [paintdata] - PaintType 2 eq Multi 1 ne or - { XStep 0 FontBBox aload pop setcachedevice } - { XStep 0 setcharwidth } ifelse - currentdict % mark [paintdata] dict - /PaintProc load % mark [paintdata] dict paintproc - end - gsave - false PATredef exec true PATredef - grestore - cleartomark % - - } bind def - currentdict - end % newdict - /foo exch % /foo newlict - definefont % newfont -} bind def -% PATpcalc - calculates the starting point and width/height -% of the tile fill for the shape -/PATpcalc { % - PATpcalc nw nh px py - PATDict /CurrentPattern get begin - gsave - % Set up the coordinate system to Pattern Space - % and lock down pattern - PatternGState /cm get setmatrix - BBox aload pop pop pop translate - % Determine the bounding box of the shape - pathbbox % llx lly urx ury - grestore - % Determine (nw, nh) the # of cells to paint width and height - PatHeight div ceiling % llx lly urx qh - 4 1 roll % qh llx lly urx - PatWidth div ceiling % qh llx lly qw - 4 1 roll % qw qh llx lly - PatHeight div floor % qw qh llx ph - 4 1 roll % ph qw qh llx - PatWidth div floor % ph qw qh pw - 4 1 roll % pw ph qw qh - 2 index sub cvi abs % pw ph qs qh-ph - exch 3 index sub cvi abs exch % pw ph nw=qw-pw nh=qh-ph - % Determine the starting point of the pattern fill - %(px, py) - 4 2 roll % nw nh pw ph - PatHeight mul % nw nh pw py - exch % nw nh py pw - PatWidth mul exch % nw nh px py - end -} bind def - -% Save the original routines so that we can use them later on -/oldfill /fill load def -/oldeofill /eofill load def -/oldstroke /stroke load def -/oldshow /show load def -/oldashow /ashow load def -/oldwidthshow /widthshow load def -/oldawidthshow /awidthshow load def -/oldkshow /kshow load def - -% These defs are necessary so that subsequent procs don't bind in -% the originals -/fill { oldfill } bind def -/eofill { oldeofill } bind def -/stroke { oldstroke } bind def -/show { oldshow } bind def -/ashow { oldashow } bind def -/widthshow { oldwidthshow } bind def -/awidthshow { oldawidthshow } bind def -/kshow { oldkshow } bind def -/PATredef { - MyAppDict begin - { - /fill { /clip load PATdraw newpath } bind def - /eofill { /eoclip load PATdraw newpath } bind def - /stroke { PATstroke } bind def - /show { 0 0 null 0 0 6 -1 roll PATawidthshow } bind def - /ashow { 0 0 null 6 3 roll PATawidthshow } - bind def - /widthshow { 0 0 3 -1 roll PATawidthshow } - bind def - /awidthshow { PATawidthshow } bind def - /kshow { PATkshow } bind def - } { - /fill { oldfill } bind def - /eofill { oldeofill } bind def - /stroke { oldstroke } bind def - /show { oldshow } bind def - /ashow { oldashow } bind def - /widthshow { oldwidthshow } bind def - /awidthshow { oldawidthshow } bind def - /kshow { oldkshow } bind def - } ifelse - end -} bind def -false PATredef -% Conditionally define setcmykcolor if not available -/setcmykcolor where { pop } { - /setcmykcolor { - 1 sub 4 1 roll - 3 { - 3 index add neg dup 0 lt { pop 0 } if 3 1 roll - } repeat - setrgbcolor - pop - } bind def -} ifelse -/PATsc { % colorarray - aload length % c1 ... cn length - dup 1 eq { pop setgray } { 3 eq { setrgbcolor } { setcmykcolor - } ifelse } ifelse -} bind def -/PATsg { % dict - begin - lw setlinewidth - lc setlinecap - lj setlinejoin - ml setmiterlimit - ds aload pop setdash - cc aload pop setrgbcolor - cm setmatrix - end -} bind def - -/PATDict 3 dict def -/PATsp { - true PATredef - PATDict begin - /CurrentPattern exch def - % If it's an uncolored pattern, save the color - CurrentPattern /PaintType get 2 eq { - /PColor exch def - } if - /CColor [ currentrgbcolor ] def - end -} bind def -% PATstroke - stroke with the current pattern -/PATstroke { - countdictstack - save - mark - { - currentpoint strokepath moveto - PATpcalc % proc nw nh px py - clip newpath PATfill - } stopped { - (*** PATstroke Warning: Path is too complex, stroking - with gray) = - cleartomark - restore - countdictstack exch sub dup 0 gt - { { end } repeat } { pop } ifelse - gsave 0.5 setgray oldstroke grestore - } { pop restore pop } ifelse - newpath -} bind def -/PATtcalc { % modmtx tilingtype PATtcalc tilematrix - % Note: tiling types 2 and 3 are not supported - gsave - exch concat % tilingtype - matrix currentmatrix exch % cmtx tilingtype - % Tiling type 1 and 3: constant spacing - 2 ne { - % Distort the pattern so that it occupies - % an integral number of device pixels - dup 4 get exch dup 5 get exch % tx ty cmtx - XStep 0 dtransform - round exch round exch % tx ty cmtx dx.x dx.y - XStep div exch XStep div exch % tx ty cmtx a b - 0 YStep dtransform - round exch round exch % tx ty cmtx a b dy.x dy.y - YStep div exch YStep div exch % tx ty cmtx a b c d - 7 -3 roll astore % { a b c d tx ty } - } if - grestore -} bind def -/PATusp { - false PATredef - PATDict begin - CColor PATsc - end -} bind def - -% right30 -11 dict begin -/PaintType 1 def -/PatternType 1 def -/TilingType 1 def -/BBox [0 0 1 1] def -/XStep 1 def -/YStep 1 def -/PatWidth 1 def -/PatHeight 1 def -/Multi 2 def -/PaintData [ - { clippath } bind - { 32 16 true [ 32 0 0 -16 0 16 ] - {<00030003000c000c0030003000c000c0030003000c000c00 - 30003000c000c00000030003000c000c0030003000c000c0 - 030003000c000c0030003000c000c000>} - imagemask } bind -] def -/PaintProc { - pop - exec fill -} def -currentdict -end -/P2 exch def -1.1111 1.1111 scale %restore scale - -/cp {closepath} bind def -/ef {eofill} bind def -/gr {grestore} bind def -/gs {gsave} bind def -/sa {save} bind def -/rs {restore} bind def -/l {lineto} bind def -/m {moveto} bind def -/rm {rmoveto} bind def -/n {newpath} bind def -/s {stroke} bind def -/sh {show} bind def -/slc {setlinecap} bind def -/slj {setlinejoin} bind def -/slw {setlinewidth} bind def -/srgb {setrgbcolor} bind def -/rot {rotate} bind def -/sc {scale} bind def -/sd {setdash} bind def -/ff {findfont} bind def -/sf {setfont} bind def -/scf {scalefont} bind def -/sw {stringwidth} bind def -/tr {translate} bind def -/tnt {dup dup currentrgbcolor - 4 -2 roll dup 1 exch sub 3 -1 roll mul add - 4 -2 roll dup 1 exch sub 3 -1 roll mul add - 4 -2 roll dup 1 exch sub 3 -1 roll mul add srgb} - bind def -/shd {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul - 4 -2 roll mul srgb} bind def - /DrawSplineSection { - /y3 exch def - /x3 exch def - /y2 exch def - /x2 exch def - /y1 exch def - /x1 exch def - /xa x1 x2 x1 sub 0.666667 mul add def - /ya y1 y2 y1 sub 0.666667 mul add def - /xb x3 x2 x3 sub 0.666667 mul add def - /yb y3 y2 y3 sub 0.666667 mul add def - x1 y1 lineto - xa ya xb yb x3 y3 curveto - } def - -/$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def -/$F2psEnd {$F2psEnteredState restore end} def -%%EndProlog - -$F2psBegin -10 setmiterlimit -n 0 842 m 0 0 l 595 0 l 595 842 l cp clip - 0.06000 0.06000 sc -7.500 slw -% Polyline -n 750 3600 m 750 1200 l 2325 1200 l 2325 3600 l gs col-1 s gr -% Polyline -n 750 1500 m 2325 1500 l gs col-1 s gr -% Polyline -n 750 1800 m 2325 1800 l gs col-1 s gr -15.000 slw -% Polyline -n 375 2100 m 2775 2100 l gs col-1 s gr -/Times-Roman ff 180.00 scf sf -900 2025 m -gs 1 -1 sc (allocated_blocks) col-1 sh gr -/Times-Roman ff 180.00 scf sf -1200 1725 m -gs 1 -1 sc (free_list) col-1 sh gr -/Times-Roman ff 180.00 scf sf -1200 1425 m -gs 1 -1 sc (item_size) col-1 sh gr -7.500 slw -% Polyline -n 3525 1200 m 5025 1200 l 5025 1800 l 3525 1800 l cp gs col-1 s gr -% Polyline -n 3525 1500 m 5025 1500 l gs col-1 s gr -/Times-Roman ff 180.00 scf sf -4050 1425 m -gs 1 -1 sc (next) col-1 sh gr -/Times-Roman ff 180.00 scf sf -4050 1725 m -gs 1 -1 sc (block) col-1 sh gr -% Polyline -n 5850 1200 m 7350 1200 l 7350 1800 l 5850 1800 l cp gs col-1 s gr -% Polyline -n 5850 1500 m 7350 1500 l gs col-1 s gr -/Times-Roman ff 180.00 scf sf -6375 1425 m -gs 1 -1 sc (next) col-1 sh gr -/Times-Roman ff 180.00 scf sf -6375 1725 m -gs 1 -1 sc (block) col-1 sh gr -15.000 slw -% Polyline - [100.0] 0 sd -n 3450 5700 m 5400 5700 l gs col-1 s gr [] 0 sd -7.500 slw -% Polyline -n 3600 8000 m 3600 3450 l 5175 3450 l 5175 8000 l gs col-1 s gr -15.000 slw -% Polyline - [100.0] 0 sd -n 3525 6900 m 5325 6900 l gs col-1 s gr [] 0 sd -0.000 slw -% Polyline - [33.3] 0 sd -n 3675 3525 m 5100 3525 l 5100 4425 l 3675 4425 l cp gs /PC [[1.00 1.00 1.00] [0.00 0.00 0.00]] def -15.00 15.00 sc P2 [16 0 0 -8 245.00 235.00] PATmp PATsp ef gr PATusp [] 0 sd -% Polyline - [33.3] 0 sd -n 3675 5775 m 5100 5775 l 5100 6825 l 3675 6825 l cp gs /PC [[1.00 1.00 1.00] [0.00 0.00 0.00]] def -15.00 15.00 sc P2 [16 0 0 -8 245.00 385.00] PATmp PATsp ef gr PATusp [] 0 sd -7.500 slw -% Polyline - [15 50.0] 50.0 sd -n 3600 4725 m 5250 4725 l gs col-1 s gr [] 0 sd -% Polyline - [15 50.0] 50.0 sd -n 3600 4950 m 5175 4950 l gs col-1 s gr [] 0 sd -0.000 slw -% Polyline - [15 25.0] 25.0 sd -n 6375 3750 m 6675 3750 l 6675 4050 l 6375 4050 l cp gs /PC [[1.00 1.00 1.00] [0.00 0.00 0.00]] def -15.00 15.00 sc P2 [16 0 0 -8 425.00 250.00] PATmp PATsp ef gr PATusp [] 0 sd -7.500 slw -% Polyline -gs clippath -4305 3253 m 4275 3373 l 4245 3253 l 4245 3415 l 4305 3415 l cp clip -n 4275 1800 m 4275 3400 l gs col-1 s gr gr - -% arrowhead -n 4305 3253 m 4275 3373 l 4245 3253 l 4275 3253 l 4305 3253 l cp gs 0.00 setgray ef gr col-1 s -15.000 slw -% Polyline - [100.0] 0 sd -n 3450 4500 m 5325 4500 l gs col-1 s gr [] 0 sd -7.500 slw -% Polyline -gs clippath -6630 2778 m 6600 2898 l 6570 2778 l 6570 2940 l 6630 2940 l cp clip -n 6600 1800 m 6600 2925 l gs col-1 s gr gr - -% arrowhead -n 6630 2778 m 6600 2898 l 6570 2778 l 6600 2778 l 6630 2778 l cp gs 0.00 setgray ef gr col-1 s -15.000 slw -% Polyline -gs clippath -656 840 m 723 1078 l 552 900 l 713 1181 l 817 1121 l cp clip -n 450 600 m 750 1125 l gs col-1 s gr gr - -% arrowhead -n 656 840 m 723 1078 l 552 900 l 604 870 l 656 840 l cp gs 0.00 setgray ef gr col-1 s -% Open spline -gs clippath -5627 1227 m 5749 1210 l 5650 1282 l 5800 1222 l 5778 1167 l cp clip -7.500 slw -n 4875.0 1350.0 m 5137.5 1350.0 l - 5137.5 1350.0 5400.0 1350.0 5587.5 1275.0 DrawSplineSection - 5775.0 1200.0 l gs col-1 s gr - gr - -% arrowhead -n 5627 1227 m 5749 1210 l 5650 1282 l 5639 1255 l 5627 1227 l cp gs 0.00 setgray ef gr col-1 s -% Open spline -gs clippath -7877 1227 m 7999 1210 l 7900 1282 l 8050 1222 l 8028 1167 l cp clip -n 7125.0 1350.0 m 7387.5 1350.0 l - 7387.5 1350.0 7650.0 1350.0 7837.5 1275.0 DrawSplineSection - 8025.0 1200.0 l gs col-1 s gr - gr - -% arrowhead -n 7877 1227 m 7999 1210 l 7900 1282 l 7889 1255 l 7877 1227 l cp gs 0.00 setgray ef gr col-1 s -% Interp Spline -gs clippath -3308 4453 m 3423 4496 l 3300 4512 l 3461 4532 l 3468 4472 l cp clip -n 2175 1650 m - 2346.9 1637.5 2421.9 1637.5 2475 1650 curveto - 2514.6 1659.3 2593.0 1702.0 2625 1725 curveto - 2665.4 1754.1 2753.1 1822.2 2775 1875 curveto - 2790.5 1912.4 2775.0 1990.8 2775 2025 curveto - 2775.0 2537.6 2542.8 3714.4 2775 4275 curveto - 2785.9 4301.4 2829.8 4335.4 2850 4350 curveto - 2882.0 4373.0 2962.7 4412.9 3000 4425 curveto - 3076.1 4449.8 3188.6 4468.5 3450 4500 curveto - gs col-1 s gr - gr - -% arrowhead -n 3308 4453 m 3423 4496 l 3300 4512 l 3304 4483 l 3308 4453 l cp gs 0.00 setgray ef gr col-1 s -% Interp Spline -gs clippath -5325 6973 m 5201 6969 l 5312 6914 l 5154 6949 l 5167 7007 l cp clip -n 5100 4575 m - 5270.9 4558.6 5345.9 4558.6 5400 4575 curveto - 5465.1 4594.7 5582.7 4669.6 5625 4725 curveto - 5662.1 4773.6 5691.1 4895.0 5700 4950 curveto - 5764.9 5350.0 5796.1 6267.8 5700 6675 curveto - 5690.7 6714.6 5655.4 6794.6 5625 6825 curveto - 5594.6 6855.4 5511.4 6886.6 5475 6900 curveto - 5424.7 6918.6 5349.7 6937.4 5175 6975 curveto - gs col-1 s gr - gr - -% arrowhead -n 5325 6973 m 5201 6969 l 5312 6914 l 5319 6944 l 5325 6973 l cp gs 0.00 setgray ef gr col-1 s -% Interp Spline -gs clippath -3300 1202 m 3423 1205 l 3313 1261 l 3471 1226 l 3458 1168 l cp clip -n 2250 1950 m - 2378.9 1959.4 2435.2 1959.4 2475 1950 curveto - 2514.6 1940.7 2593.0 1898.0 2625 1875 curveto - 2665.4 1845.9 2745.9 1765.4 2775 1725 curveto - 2821.1 1661.1 2878.9 1488.9 2925 1425 curveto - 2939.6 1404.8 2979.8 1364.6 3000 1350 curveto - 3032.0 1327.0 3113.7 1288.5 3150 1275 curveto - 3200.3 1256.4 3275.3 1237.7 3450 1200 curveto - gs col-1 s gr - gr - -% arrowhead -n 3300 1202 m 3423 1205 l 3313 1261 l 3306 1231 l 3300 1202 l cp gs 0.00 setgray ef gr col-1 s -/Times-Roman ff 180.00 scf sf -3975 4650 m -gs 1 -1 sc (next_free) col-1 sh gr -/Times-Roman ff 180.00 scf sf -4050 4875 m -gs 1 -1 sc (magic) col-1 sh gr -/Times-Bold ff 210.00 scf sf -6825 3975 m -gs 1 -1 sc (= allocated memory) col-1 sh gr -/Helvetica-Bold ff 180.00 scf sf -3675 1125 m -gs 1 -1 sc (AllocatedBlock) col-1 sh gr -/Helvetica-Bold ff 180.00 scf sf -1575 1125 m -gs 1 -1 sc (FixItem) col-1 sh gr -/Helvetica-Bold ff 180.00 scf sf -2400 4650 m -gs 1 -1 sc (FreeHeader) col-1 sh gr -/Times-Bold ff 180.00 scf sf -5250 3600 m -gs 1 -1 sc 270.0 rot (item_size) col-1 sh gr -/Times-Bold ff 270.00 scf sf -300 525 m -gs 1 -1 sc (fix_array) col-1 sh gr -$F2psEnd -rs -end diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 06fefa8efb..5ee40823bc 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -235,7 +235,7 @@ <code> 1> Bin = <<1,2,3,4,5,6,7,8,9,10>>. -2> binary_part(Bin,{byte_size(Bin), -5)). +2> binary_part(Bin,{byte_size(Bin), -5}). <<6,7,8,9,10>> </code> @@ -4477,6 +4477,7 @@ true</pre> </func> <func> <name name="spawn_opt" arity="2"/> + <type name="priority_level" /> <fsummary>Create a new process with a fun as entry point</fsummary> <desc> <p>Returns the pid of a new process started by the application @@ -4490,6 +4491,7 @@ true</pre> </func> <func> <name name="spawn_opt" arity="3"/> + <type name="priority_level" /> <fsummary>Create a new process with a fun as entry point on a given node</fsummary> <desc> <p>Returns the pid of a new process started by the application @@ -4501,6 +4503,7 @@ true</pre> </func> <func> <name name="spawn_opt" arity="4"/> + <type name="priority_level" /> <fsummary>Create a new process with a function as entry point</fsummary> <desc> <p>Works exactly like @@ -4602,6 +4605,7 @@ true</pre> </func> <func> <name name="spawn_opt" arity="5"/> + <type name="priority_level" /> <fsummary>Create a new process with a function as entry point on a given node</fsummary> <desc> <p>Returns the pid of a new process started by the application @@ -6054,6 +6058,49 @@ ok notice. </p> </item> + <tag><c>{long_schedule, Time}</c></tag> + <item> + <p>If a process or port in the system runs uninterrupted + for at least <c>Time</c> wall clock milliseconds, a + message <c>{monitor, PidOrPort, long_schedule, Info}</c> + is sent to <c>MonitorPid</c>. <c>PidOrPort</c> is the + process or port that was running and <c>Info</c> is a + list of two-element tuples describing the event. In case + of a <c>pid()</c>, the tuples <c>{timeout, Millis}</c>, + <c>{in, Location}</c> and <c>{out, Location}</c> will be + present, where <c>Location</c> is either an MFA + (<c>{Module, Function, Arity}</c>) describing the + function where the process was scheduled in/out, or the + atom <c>undefined</c>. In case of a <c>port()</c>, the + tuples <c>{timeout, Millis}</c> and <c>{port_op,Op}</c> + will be present. <c>Op</c> will be one of <c>proc_sig</c>, + <c>timeout</c>, <c>input</c>, <c>output</c>, + <c>event</c> or <c>dist_cmd</c>, depending on which + driver callback was executing. <c>proc_sig</c> is an + internal operation and should never appear, while the + others represent the corresponding driver callbacks + <c>timeout</c>, <c>ready_input</c>, <c>ready_output</c>, + <c>event</c> and finally <c>outputv</c> (when the port + is used by distribution). The <c>Millis</c> value in + the <c>timeout</c> tuple will tell you the actual + uninterrupted execution time of the process or port, + which will always be <c>>=</c> the <c>Time</c> value + supplied when starting the trace. New tuples may be + added to the <c>Info</c> list in the future, and the + order of the tuples in the list may be changed at any + time without prior notice. + </p> + <p>This can be used to detect problems with NIF's or + drivers that take too long to execute. Generally, 1 ms + is considered a good maximum time for a driver callback + or a NIF. However, a time sharing system should usually + consider everything below 100 ms as "possible" and + fairly "normal". Schedule times above that might however + indicate swapping or a NIF/driver that is + misbehaving. Misbehaving NIF's and drivers could cause + bad resource utilization and bad overall performance of + the system.</p> + </item> <tag><c>{large_heap, Size}</c></tag> <item> <p>If a garbage collection in the system results in diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index c73cdfd290..6ce2261430 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -75,10 +75,6 @@ segments are allocated, cached segments are used if possible instead of creating new segments. This in order to reduce the number of system calls made.</item> - <tag><c>sbmbc_alloc</c></tag> - <item>Allocator used by other allocators for allocation of carriers - where only small blocks are placed. Currently this allocator is - disabled by default.</item> </taglist> <p><c>sys_alloc</c> is always enabled and cannot be disabled. <c>mseg_alloc</c> is always enabled if it is @@ -86,9 +82,7 @@ allocators can be <seealso marker="#M_e">enabled or disabled</seealso>. By default all allocators are enabled. When an allocator is disabled, <c>sys_alloc</c> is used instead of - the disabled allocator. <c>sbmbc_alloc</c> is an exception. If - <c>sbmbc_alloc</c> is disabled, other allocators will not handle - small blocks in separate carriers.</p> + the disabled allocator.</p> <p>The main idea with the <c>erts_alloc</c> library is to separate memory blocks that are used differently into different memory areas, and by this achieving less memory fragmentation. By @@ -106,20 +100,15 @@ following does <em>not</em> apply to them.</p> <p>An allocator manages multiple areas, called carriers, in which memory blocks are placed. A carrier is either placed in a - separate memory segment (allocated via <c>mseg_alloc</c>), in - the heap segment (allocated via <c>sys_alloc</c>), or inside - another carrier (in case it is a carrier created by - <c>sbmbc_alloc</c>). Multiblock + separate memory segment (allocated via <c>mseg_alloc</c>), or in + the heap segment (allocated via <c>sys_alloc</c>). Multiblock carriers are used for storage of several blocks. Singleblock carriers are used for storage of one block. Blocks that are larger than the value of the singleblock carrier threshold (<seealso marker="#M_sbct">sbct</seealso>) parameter are placed in singleblock carriers. Blocks that are smaller than the value of the <c>sbct</c> parameter are placed in multiblock - carriers. Blocks that are smaller than the small block multiblock - carrier threshold (<seealso marker="#M_sbmbct">sbmbct</seealso>) - will be placed in multiblock carriers only used for small blocks. - Normally an allocator creates a "main multiblock + carriers. Normally an allocator creates a "main multiblock carrier". Main multiblock carriers are never deallocated. The size of the main multiblock carrier is determined by the value of the <seealso marker="#M_mmbcs">mmbcs</seealso> parameter.</p> @@ -140,9 +129,7 @@ <c>sbct</c> parameter should be larger than the value of the <c>lmbcs</c> parameter, the allocator may have to create multiblock carriers that are larger than the value of the - <c>lmbcs</c> parameter, though. The size of multiblock carriers - for small blocks is determined by the small block multiblock - carrier size (<seealso marker="#M_sbmbcs">sbmbcs</seealso>). + <c>lmbcs</c> parameter, though. Singleblock carriers allocated via <c>mseg_alloc</c> are sized to whole pages.</p> <p>Sizes of carriers allocated via <c>sys_alloc</c> are @@ -183,6 +170,24 @@ used. The time complexity is proportional to log N, where N is the number of free blocks.</p> </item> + <tag>Address order first fit carrier best fit</tag> + <item> + <p>Strategy: Find the <em>carrier</em> with the lowest address that + can satisfy the requested block size, then find a block within + that carrier using the "best fit" strategy.</p> + <p>Implementation: Balanced binary search trees are + used. The time complexity is proportional to log N, where + N is the number of free blocks.</p> + </item> + <tag>Address order first fit carrier address order best fit</tag> + <item> + <p>Strategy: Find the <em>carrier</em> with the lowest address that + can satisfy the requested block size, then find a block within + that carrier using the "adress order best fit" strategy.</p> + <p>Implementation: Balanced binary search trees are + used. The time complexity is proportional to log N, where + N is the number of free blocks.</p> + </item> <tag>Good fit</tag> <item> <p>Strategy: Try to find the best fit, but settle for the best fit @@ -219,11 +224,6 @@ but can only satisfy a limited amount of requests.</p> </section> - <note><p> - Currently only allocators using the best fit and the address order - best fit strategies are able to use "small block multi block carriers". - </p></note> - <section> <marker id="flags"></marker> <title>System Flags Effecting erts_alloc</title> @@ -245,7 +245,6 @@ the currently present allocators:</p> <list type="bulleted"> <item><c>B: binary_alloc</c></item> - <item><c>C: sbmbc_alloc</c></item> <item><c>D: std_alloc</c></item> <item><c>E: ets_alloc</c></item> <item><c>F: fix_alloc</c></item> @@ -319,10 +318,43 @@ subsystem identifier, only the specific allocator identified will be effected:</p> <taglist> - <tag><marker id="M_as"><c><![CDATA[+M<S>as bf|aobf|aoff|gf|af]]></c></marker></tag> + <tag><marker id="M_acul"><c><![CDATA[+M<S>acul <utilization>|de]]></c></marker></tag> + <item> + Abandon carrier utilization limit. A valid + <c><![CDATA[<utilization>]]></c> is an integer in the range + <c>[0, 100]</c> representing utilization in percent. When a + utilization value larger than zero is used, allocator instances + are allowed to abandon multiblock carriers. Currently the default + is zero. If <c>de</c> (default enabled) is passed instead of a + <c><![CDATA[<utilization>]]></c>, a recomended non zero utilization + value will be used. The actual value chosen depend on allocator + type and may be changed between ERTS versions. Carriers will be + abandoned when memory utilization in the allocator instance falls + below the utilization value used. Once a carrier has been abandoned, + no new allocations will be made in it. When an allocator instance + gets an increased multiblock carrier need, it will first try to + fetch an abandoned carrier from an allocator instances of the same + allocator type. If no abandoned carrier could be fetched, it will + create a new empty carrier. When an abandoned carrier has been + fetched it will function as an ordinary carrier. This feature has + special requirements on the + <seealso marker="#M_as">allocation strategy</seealso> used. Currently + only the strategies <c>aoff</c>, <c>aoffcbf</c> and <c>aoffcaobf</c> support + abandoned carriers. This feature also requires + <seealso marker="#M_t">multiple thread specific instances</seealso> + to be enabled. When enabling this feature, multiple thread specific + instances will be enabled if not already enabled, and the + <c>aoffcbf</c> strategy will be enabled if current strategy does not + support abandoned carriers. This feature can be enabled on all + allocators based on the <c>alloc_util</c> framework with the + exception of <c>temp_alloc</c> (which would be pointless). + </item> + <tag><marker id="M_as"><c><![CDATA[+M<S>as bf|aobf|aoff|aoffcbf|aoffcaobf|gf|af]]></c></marker></tag> <item> Allocation strategy. Valid strategies are <c>bf</c> (best fit), <c>aobf</c> (address order best fit), <c>aoff</c> (address order first fit), + <c>aoffcbf</c> (address order first fit carrier best fit), + <c>aoffcaobf</c> (address order first fit carrier address order best fit), <c>gf</c> (good fit), and <c>af</c> (a fit). See <seealso marker="#strategy">the description of allocation strategies</seealso> in "the <c>alloc_util</c> framework" section.</item> <tag><marker id="M_asbcst"><c><![CDATA[+M<S>asbcst <size>]]></c></marker></tag> @@ -416,20 +448,6 @@ smaller than this threshold will be placed in multiblock carriers. On 32-bit Unix style OS this threshold can not be set higher than 8 megabytes.</item> - <tag><marker id="M_sbmbcs"><c><![CDATA[+M<S>sbmbcs <size>]]></c></marker></tag> - <item> - Small block multiblock carrier size (in bytes). Memory blocks smaller - than the small block multiblock carrier threshold - (<seealso marker="#M_sbmbct">sbmbct</seealso>) will be placed in - multiblock carriers used for small blocks only. This parameter - determines the size of such carriers. - </item> - <tag><marker id="M_sbmbct"><c><![CDATA[+M<S>sbmbct <size>]]></c></marker></tag> - <item> - Small block multiblock carrier threshold (in bytes). Memory blocks - smaller than this threshold will be placed in multiblock carriers - used for small blocks only. - </item> <tag><marker id="M_smbcs"><c><![CDATA[+M<S>smbcs <size>]]></c></marker></tag> <item> Smallest (<c>mseg_alloc</c>) multiblock carrier size (in diff --git a/erts/doc/src/escript.xml b/erts/doc/src/escript.xml index 66e904f64f..9e2a87dde6 100644 --- a/erts/doc/src/escript.xml +++ b/erts/doc/src/escript.xml @@ -4,7 +4,7 @@ <comref> <header> <copyright> - <year>2007</year><year>2011</year> + <year>2007</year><year>2013</year> <holder>Ericsson AB. All Rights Reserved.</holder> </copyright> <legalnotice> @@ -85,6 +85,11 @@ $ <input>escript factorial 5</input> </pre> enter the major mode for editing Erlang source files. If the directive is present it must be located on the second line.</p> + + <p>If there is a comment selecting the <seealso + marker="stdlib:epp#encoding">encoding</seealso> it can be + located on the second line.</p> + <p>On the third line (or second line depending on the presence of the Emacs directive), it is possible to give arguments to the emulator, such as </p> @@ -133,7 +138,9 @@ halt(1).</pre> <pre> -include_lib("kernel/include/file.hrl").</pre> <p>to include the record definitions for the records used by the - <c>file:read_link_info/1</c> function.</p> + <c>file:read_link_info/1</c> function. You can also select + encoding here, but if there is a valid encoding comment on + the second line it takes precedence.</p> <p>The script will be checked for syntactic and semantic correctness before being run. If there are warnings (such as diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index 3a2c644ff7..77ffeefd04 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -1,4 +1,4 @@ -<?xml version="1.0" encoding="latin1" ?> +<?xml version="1.0" encoding="utf-8" ?> <!DOCTYPE chapter SYSTEM "chapter.dtd"> <chapter> @@ -30,6 +30,595 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 5.10.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> The documentation of predefined types and built-in + types has been corrected. </p> + <p> + Own Id: OTP-11090</p> + </item> + <item> + <p> + Fix changing terminal parameters in to_erl</p> + <p> + Change the behaviour of to_erl to use TCSADRAIN instead + of TCSANOW when changing terminal parameters. This makes + the serial driver wait for the output queues to be empty + before applying the terminal parameter change. Thanks to + Stefan Zegenhagen.</p> + <p> + Own Id: OTP-11206</p> + </item> + <item> + <p> + The default value of {flush, boolean()} in erlang:halt/2 + is documented to be 'true' if the status is an integer. + The implementation behaviour was reversed. The + Implementation is now corrected to adhere to the + documentation. Thanks to Jose Valim for reporting the + error.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-11218</p> + </item> + <item> + <p> + Fix serious race bug in R16B01 that could cause PID + mix-ups when a lot of processes were spawned and + terminated in a very rapid pace on an SMP emulator with + at least two scheduler threads.</p> + <p> + Own Id: OTP-11225</p> + </item> + <item> + <p> + Validating a trace pattern with the option silent no + longer incorrectly enables/disables the silent option of + the calling process.</p> + <p> + Own Id: OTP-11232</p> + </item> + <item> + <p> + Fixed a bug where GCC 4.8 and later use a more aggressive + loop optimization algorithm that broke some previously + working code in the efile driver. Thanks to Tomas + Abrahamsson for reporting this issue.</p> + <p> + Own Id: OTP-11246</p> + </item> + <item> + <p> + Fixed bug when printing memory allocator acul option in + crash dump.</p> + <p> + Own Id: OTP-11264</p> + </item> + <item> + <p> + Opening a new compressed file on Windows could in rare + (random) cases result in {error,eisdir} or other error + codes although it should have succeeded. This is now + corrected.</p> + <p> + Own Id: OTP-11265</p> + </item> + <item> + <p> + Fixed a race condition when closing a trace port that + would cause the emulator to crash.</p> + <p> + Own Id: OTP-11290</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + There is a new somewhat experimental socket option + 'netns' that can set the network namespace for a socket + on Linux:es where it is supported. See the documentation.</p> + <p> + Own Id: OTP-11157</p> + </item> + <item> + <p> + New allocator strategy <c>aoffcbf</c> (address order + first fit carrier best fit). Supports carrier migration + but with better CPU performance than <c>aoffcaobf</c>.</p> + <p> + Own Id: OTP-11174</p> + </item> + <item> + <p> + Introduced functionality for inspection of system and + build configuration.</p> + <p> + Own Id: OTP-11196</p> + </item> + <item> + <p> + Fix matching of floating point middle-endian machines. + Thanks to Johannes Weissl.</p> + <p> + Own Id: OTP-11201</p> + </item> + <item> + <p> + Fix compile error on ARM and GCC versions greater than + 4.1.0. Thanks to Johannes Weissl.</p> + <p> + Own Id: OTP-11214</p> + </item> + <item> + <p> + run_erl: Redirect standard streams to /dev/null. Thanks + to Johannes Weissl.</p> + <p> + Own Id: OTP-11215</p> + </item> + <item> + <p> + Misc. corrections in documentation for erl_driver. Thanks + to Giacomo Olgeni.</p> + <p> + Own Id: OTP-11227</p> + </item> + <item> + <p> + Fix documentation regarding binary_part.</p> + <p> + Own Id: OTP-11239</p> + </item> + <item> + <p> + Make edlin understand a few important control keys. + Thanks to Stefan Zegenhagen.</p> + <p> + Own Id: OTP-11251</p> + </item> + <item> + <p> + Export type zlib:zstream/0. Thanks to Loic Hoguin.</p> + <p> + Own Id: OTP-11278</p> + </item> + <item> + <p> + Add erl option to set schedulers by percentages. </p> + <p> + For applications where measurements show enhanced + performance from the use of a non-default number of + emulator scheduler threads, having to accurately set the + right number of scheduler threads across multiple hosts + each with different numbers of logical processors is + difficult because the erl +S option requires absolute + numbers of scheduler threads and scheduler threads online + to be specified.</p> + <p> + To address this issue, add a +SP option to erl, similar + to the existing +S option but allowing the number of + scheduler threads and scheduler threads online to be set + as percentages of logical processors configured and + logical processors available, respectively. For example, + "+SP 50:25" sets the number of scheduler threads to 50% + of the logical processors configured, and the number of + scheduler threads online to 25% of the logical processors + available. The +SP option also interacts with any + settings specified with the +S option, such that the + combination of options "+S 4:4 +SP 50:50" (in either + order) results in 2 scheduler threads and 2 scheduler + threads online.</p> + <p> + Thanks to Steve Vinoski</p> + <p> + Own Id: OTP-11282</p> + </item> + <item> + <p> + Extend erl_driver interface with lock names</p> + <p> + Lock and thread names are already a feature in the driver + interface. This extension will let developers read these + names which eases debugging.</p> + <p> + Own Id: OTP-11303</p> + </item> + <item> + <p> + Fix incorrect values returned by integer_to_binary/2. + Thanks to Juan Jose Comellas.</p> + <p> + Own Id: OTP-11311</p> + </item> + <item> + <p> + Fix system_flag scheduling_statistics - disable . Thanks + to Steve Vinoski.</p> + <p> + Own Id: OTP-11317</p> + </item> + <item> + <p> The documentation of predefined types has been + corrected Thanks to Kostis Sagonas. </p> + <p> + Own Id: OTP-11321</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 5.10.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + A bug in prim_inet has been corrected. If the port owner + was killed at a bad time while closing the socket port + the port could become orphaned hence causing port and + socket leaking. Reported by Fred Herbert, Dmitry Belyaev + and others.</p> + <p> + Own Id: OTP-10497 Aux Id: OTP-10562 </p> + </item> + <item> + <p> + Compilation fixes for NetBSD. Thanks to YAMAMOTO Takashi.</p> + <p> + Own Id: OTP-10941</p> + </item> + <item> + <p> + Fixed a race condition when using delayed_write when + writing to a file which would cause the same data to be + written multiple times.</p> + <p> + Own Id: OTP-10984</p> + </item> + <item> + <p> + Fix small memory leak from tracing with option + <c>meta</c>.</p> + <p> + Own Id: OTP-10997</p> + </item> + <item> + <p> + Correct typo in erlsrv usage. Thanks to Bryan Hunter</p> + <p> + Own Id: OTP-11002</p> + </item> + <item> + <p> + ct_run: delete unused function. Thanks to Tuncer Ayaz.</p> + <p> + Own Id: OTP-11003</p> + </item> + <item> + <p> + Corrections to run_erl/to_erl handshake behaviour.</p> + <p> + Own Id: OTP-11012</p> + </item> + <item> + <p> + Fix typo in type: erlang:process_info_item(). Thanks to + Andrew Tunnell-Jones.</p> + <p> + Own Id: OTP-11024</p> + </item> + <item> + <p> + Fix src/dest overlap issue in ttsl driver. Thanks to + Steve Vinoski.</p> + <p> + Own Id: OTP-11064</p> + </item> + <item> + <p> + When sending to a port using <c>erlang:send(Port, Msg, + [nosuspend])</c>, the send operation was performed + synchronously. This bug has now been fixed.</p> + <p> + Own Id: OTP-11076 Aux Id: OTP-10336 </p> + </item> + <item> + <p> + When converting a faulty binary to a list with + unicode:characters_to_list, the error return value could + contain a faulty "rest", i.e. the io_list of characters + that could not be converted was wrong. This happened only + if input was a sub binary and conversion was from utf8. + This is now corrected.</p> + <p> + Own Id: OTP-11080</p> + </item> + <item> + <p> + Runtime system could crash when reporting stale + <c>driver_select()</c>.</p> + <p> + Own Id: OTP-11084</p> + </item> + <item> + <p> + Fix lock order violation for memory instrumentation + (+Mim, +Mis, +Mit).</p> + <p> + Own Id: OTP-11085</p> + </item> + <item> + <p> + Fixed some compilation warnings on miscellaneous + platforms. Thanks to Anthony Ramine.</p> + <p> + Own Id: OTP-11086</p> + </item> + <item> + <p> + Fixed issue when flushing i/o during shutdown on windows + where the Erlang VM would sometime hang due to a race + condition.</p> + <p> + Own Id: OTP-11096</p> + </item> + <item> + <p> + Fixed issue where repeated calls to erlang:nodes() could + cause unnecessary contention in the dist_table lock.</p> + <p> + Own Id: OTP-11097</p> + </item> + <item> + <p> + Properly guard WIDE_TAG use with HAVE_WCWIDTH in + ttsl_drv. Thanks to Anthony Ramine</p> + <p> + Own Id: OTP-11106</p> + </item> + <item> + <p> + Fix some Makefile rules that didn't support silent rules. + Thanks to Anthony Ramine.</p> + <p> + Own Id: OTP-11111</p> + </item> + <item> + <p> + Fix receive support in erl_eval with a BEAM module. + Thanks to Anthony Ramine.</p> + <p> + Own Id: OTP-11137</p> + </item> + <item> + <p> + erlang:now() could suddenly jump ~24 days into the future + on Windows. This is now corrected. Thanks to Garret Smith + for reporting and testing fixes.</p> + <p> + Own Id: OTP-11146</p> + </item> + <item> + <p> + erlang:term_to_binary will now cost an appropriate amount + of reductions and will interrupt (yield) for reschedule + if the term is big. This avoids too long schedules when + term_to_binary is used. </p> + <p> + Impact: Programs running term_to_binary on large terms + will run more smothly, but rescheduling will impact the + single process performance of the BIF. Single threaded + benchmarks will show degraded performance of the BIF when + called with very large terms, while general system + behaviour will be improved. The overhead for allowing + restart and reduction counting also degrades local + performance of the BIF with between 5% and 10% even for + small terms.</p> + <p> + Own Id: OTP-11163</p> + </item> + </list> + </section> + + + <section><title>Improvements and New Features</title> + <list> + <item> + <p> + Replaced the lock protecting gathering of garbage + collection statistics with a lock-free solution.</p> + <p> + Own Id: OTP-10271 Aux Id: kunagi-108 + [04c5410f-9cc4-4696-8639-36bf98686c7a-7] </p> + </item> + <item> + <p>Support for migration of memory carriers between + memory allocator instances has been introduced.</p> + <p>By default this feature is not enabled and do not + effect the characteristics of the system. When enabled it + has the following impact on the characteristics of the + system:</p> <list> <item>Reduced memory footprint when + the memory load is unevenly distributed between scheduler + specific allocator instances.</item> <item>Depending on + the default allocaton strategy used on a specific + allocator there might or might not be a slight + performance loss.</item> <item>When enabled on the + <c>fix_alloc</c> allocator, a different strategy for + management of fix blocks will be used.</item> <item>The + information returned from <seealso + marker="erlang:system_info_allocator_tuple"><c>erlang:system_info({allocator, + A})</c></seealso>, and <seealso + marker="erlang:system_info_allocator_sizes"><c>erlang:system_info({allocator_sizes, + A})</c></seealso> will be slightly different when this + feature has been enabled. An <c>mbcs_pool</c> tuple will + be present giving information about abandoned carriers, + and in the <c>fix_alloc</c> case no <c>fix_types</c> + tuple will be present. </item></list> + <p>For more information, see the documentation of the + <seealso + marker="erts_alloc#M_acul"><c>+M<S>acul</c></seealso> + command line argument.</p> + <p> + Own Id: OTP-10279</p> + </item> + <item> + <p> + Change specs for spawn_opt to use the process_level() + type declaration instead of re-defining it in various + places. Thanks to Kostis Sagonas.</p> + <p> + Own Id: OTP-11008</p> + </item> + <item> + <p> Postscript files no longer needed for the generation + of PDF files have been removed. </p> + <p> + Own Id: OTP-11016</p> + </item> + <item> + <p>Erlang source files with non-ASCII characters are now + encoded in UTF-8 (instead of latin1).</p> + <p> + Own Id: OTP-11041 Aux Id: OTP-10907 </p> + </item> + <item> + <p> + Optimization of simultaneous <c>inet_db</c> operations on + the same socket by using a lock free implementation.</p> + <p> + Impact on the characteristics of the system: Improved + performance.</p> + <p> + Own Id: OTP-11074</p> + </item> + <item> + <p> + The <c>high_msgq_watermark</c> and + <c>low_msgq_watermark</c> <c>inet</c> socket options + introduced in OTP-R16A could only be set on TCP sockets. + These options are now generic and can be set on all types + of sockets.</p> + <p> + Own Id: OTP-11075 Aux Id: OTP-10336 </p> + </item> + <item> + <p>A new better algorithm for management of the process, + and port tables has been introduced.</p> + <p>Impact on the characteristics of the system:</p> + <list> <item>The new algorithm ensures that both insert + and delete operations can be made in O(1) time + complexity. Previously used algorithm either caused + insert or delete to be O(N).</item> <item>The new + algorithm will also ensure that reuse of identifiers will + be less frequent than when the old algorithm was + used.</item> <item>Previously used algorithm ensured that + the latest created identifier compared as the largest + when comparing two identifiers of the same type that had + been created on the same node as long as no identifiers + had been reused. Since identifiers can be reused quite + fast, one has never been able to rely on this property. + Due to the introduction of this new algorithm this + property will not hold even if no identifiers has been + reused yet. This could be considered as an + incompatibility.</item> </list> + <p>Due to the above mensioned potential incompatibility, + it will still be possible to enable the old algorithm for + some time. The command line argument <seealso + marker="erl#+P"><c>+P legacy</c></seealso> will enable + the old algorithm on the process table, and <seealso + marker="erl#+Q"><c>+Q legacy</c></seealso> will do the + same for the port table. These command line arguments are + however deprecated as of their introduction and have been + scheduled for removal in OTP-R18.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-11077</p> + </item> + <item> + <p> + Support wide characters in the shell through wcwidth(). + Thanks to Anthony Ramine. Reported by Loïc Hoguin.</p> + <p> + Own Id: OTP-11088</p> + </item> + <item> + <p> + Added total used memory for each process in erlang crash + dumps.</p> + <p> + Own Id: OTP-11098</p> + </item> + <item> + <p> + Added support for hipe on Raspberry Pi (armv6l). Thanks + to Klaus Alfert.</p> + <p> + Own Id: OTP-11125</p> + </item> + <item> + <p> + Remove 'query' from the list of reserved words in docs. + Thanks to Matthias Endler and Loïc Hoguin.</p> + <p> + Own Id: OTP-11158</p> + </item> + <item> + <p> + Lift static limitation (FD_SETSIZE) for file descriptors + on Mac OS X. (Thanks to Anthony Ramine)</p> + <p> + Own Id: OTP-11159</p> + </item> + </list> + </section> + + + <section><title>Known Bugs and Problems</title> + <list> + <item> + <p>Miscellaneous native code in OTP misbehave either due + to lengthy execution, or due to not bumping reductions + properly. Problems typically occur when passing huge sets + of data to a misbehaving BIF. Fixing this has high + priority and is being worked on, but there will remain + issues like this for some time.</p> + <p>In order to alleviate problems with scheduling which + might occur when executing misbehaving native code, the + command line argument <seealso + marker="erl#+sfwi">+sfwi</seealso> has been + introduced.</p> + <p>By default this feature is disabled and you are + advised not to enable it if you do not encounter problems + with misbehaving native code.</p> + <p>When enabled it has the following impact on the + characteristics of the system:</p> <list> <item>Work will + always be distributed between schedulers even when + executing misbehaving native code.</item> <item>It may + cause an increased amount of processes and/or ports + bouncing between schedulers which in turn will cause a + performance loss.</item> <item>It may cause reduced + performance due to reduced or lost work compaction when + all schedulers do not execute under full load.</item> + <item>An increased contention on run queue locks.</item> + </list> + <p> + Own Id: OTP-11164</p> + </item> + </list> + </section> + +</section> + <section><title>Erts 5.10.1.2</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -1598,7 +2187,7 @@ <item> <p> Fix typo in supervisor behaviour doc (Thanks to Ricardo - Catalinas Jim�nez)</p> + Catalinas Jiménez)</p> <p> Own Id: OTP-9924</p> </item> @@ -1862,7 +2451,7 @@ <item> <p> Fixes module erlang doc style: option description (Thanks - to Ricardo Catalinas Jim�nez)</p> + to Ricardo Catalinas Jiménez)</p> <p> Own Id: OTP-9697</p> </item> @@ -2311,7 +2900,7 @@ <item> <p> Fix typos in the epmd documentation (Thanks to Holger - Wei� )</p> + Weiß )</p> <p> Own Id: OTP-9387</p> </item> @@ -2416,7 +3005,7 @@ <item> <p> Fix non-existing function (erlang:disconnect/1) in - distributed reference manual (Thanks to Fabian Kr�l)</p> + distributed reference manual (Thanks to Fabian Król)</p> <p> Own Id: OTP-9504</p> </item> @@ -2444,7 +3033,7 @@ only separator characters (comma and space).</p> <p> The same applies to epmd's -address option.(Thanks to - Holger Wei�)</p> + Holger Weiß)</p> <p> Own Id: OTP-9525</p> </item> @@ -2588,7 +3177,7 @@ <p> Add support for querying the number of configured and online processors on SGI systems running IRIX.(Thanks to - Holger Wei�)</p> + Holger Weiß)</p> <p> Own Id: OTP-9531</p> </item> @@ -2698,7 +3287,7 @@ using a comma-separated list. If the loopback address is not in this list, it will be added implicitly, so that the daemon can be queried by an interactive epmd - process.(Thanks to Holger Wei�)</p> + process.(Thanks to Holger Weiß)</p> <p> Own Id: OTP-9213</p> </item> @@ -2733,7 +3322,7 @@ value over to dbg_gen_printf(). This fixes the problem that errno had been reset to zero by the time it was used (to print the corresponding error message) in the - dbg_gen_printf() function. (Thanks to Holger Wei�)</p> + dbg_gen_printf() function. (Thanks to Holger Weiß)</p> <p> Own Id: OTP-9223</p> </item> @@ -3119,7 +3708,7 @@ Mention that "-detached" implies "-noinput"</p> <p> Clarify that specifying "-noinput" is unnecessary if the - "-detached" flag is given. (thanks to Holger Wei�)</p> + "-detached" flag is given. (thanks to Holger Weiß)</p> <p> Own Id: OTP-9086</p> </item> @@ -4625,7 +5214,7 @@ failed to detect gcc C compilers with other command line names than gcc. `configure' now substitute GCC into the makefiles. If CC is a gcc C compiler, GCC will have the - value yes. (Thanks to Jean-S�bastien P�dron)</p> + value yes. (Thanks to Jean-Sébastien Pédron)</p> <p> Own Id: OTP-8373</p> </item> @@ -6995,7 +7584,7 @@ <p> IPv6 name resolving has now been fixed to use getaddrinfo() patch (thoroughly reworked) courtesy of Love - H�rnquist-�strand submitted by Fredrik Thulin. It also + Hörnquist-Åstrand submitted by Fredrik Thulin. It also can use gethostname2() patch (also reworked) courtesy of Mikael Magnusson for debian submitted by Sergei Golovan.</p> <p> diff --git a/erts/doc/src/tty.xml b/erts/doc/src/tty.xml index 7d662a2849..b16523e085 100644 --- a/erts/doc/src/tty.xml +++ b/erts/doc/src/tty.xml @@ -47,7 +47,7 @@ <section> <title>Normal Mode</title> <p>In normal mode keystrokes from the user are collected and interpreted by <c><![CDATA[tty]]></c>. Most of the <em>emacs</em> line editing commands are supported. The following is a complete list of the supported line editing commands.<br></br></p> - <p><em>Note:</em> The notation <c><![CDATA[C-a]]></c> means pressing the control key and the letter <c><![CDATA[a]]></c> simultaneously. <c><![CDATA[M-f]]></c> means pressing the <c><![CDATA[ESC]]></c> key followed by the letter <c><![CDATA[f]]></c>. + <p><em>Note:</em> The notation <c><![CDATA[C-a]]></c> means pressing the control key and the letter <c><![CDATA[a]]></c> simultaneously. <c><![CDATA[M-f]]></c> means pressing the <c><![CDATA[ESC]]></c> key followed by the letter <c><![CDATA[f]]></c>. <c><![CDATA[Home]]></c> and <c><![CDATA[End]]></c> represent the keys with the same name on the keyboard, whereas <c><![CDATA[Left]]></c> and <c><![CDATA[Right]]></c> represent the corresponding arrow keys. </p> <table> <row> @@ -55,6 +55,10 @@ <cell align="left" valign="middle"><em>Function</em></cell> </row> <row> + <cell align="left" valign="middle">Home</cell> + <cell align="left" valign="middle">Beginning of line</cell> + </row> + <row> <cell align="left" valign="middle">C-a</cell> <cell align="left" valign="middle">Beginning of line</cell> </row> @@ -63,6 +67,10 @@ <cell align="left" valign="middle">Backward character</cell> </row> <row> + <cell align="left" valign="middle">C-Left</cell> + <cell align="left" valign="middle">Backward word</cell> + </row> + <row> <cell align="left" valign="middle">M-b</cell> <cell align="left" valign="middle">Backward word</cell> </row> @@ -75,6 +83,10 @@ <cell align="left" valign="middle">Delete word</cell> </row> <row> + <cell align="left" valign="middle">End</cell> + <cell align="left" valign="middle">End of line</cell> + </row> + <row> <cell align="left" valign="middle">C-e</cell> <cell align="left" valign="middle">End of line</cell> </row> @@ -83,6 +95,10 @@ <cell align="left" valign="middle">Forward character</cell> </row> <row> + <cell align="left" valign="middle">C-Right</cell> + <cell align="left" valign="middle">Forward word</cell> + </row> + <row> <cell align="left" valign="middle">M-f</cell> <cell align="left" valign="middle">Forward word</cell> </row> @@ -95,6 +111,10 @@ <cell align="left" valign="middle">Kill line</cell> </row> <row> + <cell align="left" valign="middle">C-u</cell> + <cell align="left" valign="middle">Backward kill line</cell> + </row> + <row> <cell align="left" valign="middle">C-l</cell> <cell align="left" valign="middle">Redraw line</cell> </row> @@ -111,6 +131,10 @@ <cell align="left" valign="middle">Transpose characters</cell> </row> <row> + <cell align="left" valign="middle">C-w</cell> + <cell align="left" valign="middle">Backward kill word</cell> + </row> + <row> <cell align="left" valign="middle">C-y</cell> <cell align="left" valign="middle">Insert previously killed text</cell> </row> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 133ecd69e4..9751982103 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -48,6 +48,7 @@ CREATE_DIRS= LDFLAGS=@LDFLAGS@ ARFLAGS=rc +OMIT_OMIT_FP=no ifeq ($(TYPE),debug) PURIFY = @@ -113,6 +114,13 @@ TYPEMARKER = .lcnt TYPE_FLAGS = @CFLAGS@ -DERTS_ENABLE_LOCK_COUNT else +ifeq ($(TYPE),frmptr) +PURIFY = +OMIT_OMIT_FP=yes +TYPEMARKER = .frmptr +TYPE_FLAGS = @CFLAGS@ -DERTS_FRMPTR +else + # If type isn't one of the above, it *is* opt type... override TYPE=opt PURIFY = @@ -126,6 +134,7 @@ endif endif endif endif +endif # # NOTE: When adding a new type update ERL_BUILD_TYPE_MARKER in sys/unix/sys.c @@ -219,8 +228,6 @@ LIB_PREFIX=lib LIB_SUFFIX=.a endif -OMIT_OMIT_FP=no - ifeq (@EMU_LOCK_CHECKING@,yes) NO_INLINE_FUNCTIONS=true endif @@ -238,7 +245,7 @@ ifeq ($(NO_INLINE_FUNCTIONS),true) GEN_OPT_FLGS = $(OPT_LEVEL) -fno-inline-functions else ifeq ($(OMIT_OMIT_FP),yes) -GEN_OPT_FLGS = $(OPT_LEVEL) +GEN_OPT_FLGS = $(OPT_LEVEL) -fno-omit-frame-pointer else GEN_OPT_FLGS = $(OPT_LEVEL) -fomit-frame-pointer endif @@ -380,6 +387,13 @@ else UNIX_ONLY_BUILDS = endif +ifeq ($(TARGET), win32) +TMPVAR := $(shell LANG=C $(PERL) utils/make_compiler_flags -o $(TTF_DIR)/erl_compile_flags.h -v CONFIG_H "N/A" -v CFLAGS "$(CFLAGS)" -v LDFLAGS "$(LDFLAGS)") +else +# We force this to be run every time this makefile is executed +TMPVAR := $(shell LANG=C $(PERL) utils/make_compiler_flags -o $(TTF_DIR)/erl_compile_flags.h -f CONFIG_H "$(ERL_TOP)/erts/$(TARGET)/config.h" -v CFLAGS "$(CFLAGS)" -v LDFLAGS "$(LDFLAGS)") +endif + .PHONY: all ifdef VOID_EMULATOR all: @@ -476,8 +490,10 @@ endif ifdef DTRACE_ENABLED # global.h causes problems by including dtrace-wrapper.h which includes # the autogenerated erlang_dtrace.h ... so make erlang_dtrace.h very early. -generate: $(TARGET)/erlang_dtrace.h $(GENERATE) +DTRACE_HEADERS = $(TARGET)/erlang_dtrace.h +generate: $(DTRACE_HEADERS) $(GENERATE) else +DTRACE_HEADERS = generate: $(GENERATE) endif @@ -550,13 +566,14 @@ GENERATE += $(TTF_DIR)/driver_tab.c # Preloaded code. # # This list must be consistent with PRE_LOADED_MODULES in -# lib/kernel/src/Makefile. +# erts/preloaded/src/Makefile. ifeq ($(TARGET),win32) # On windows the preloaded objects are in a resource object. PRELOAD_OBJ = $(OBJDIR)/beams.$(RES_EXT) PRELOAD_SRC = $(TARGET)/beams.rc $(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ @@ -570,6 +587,7 @@ PRELOAD_OBJ = $(OBJDIR)/preload.o PRELOAD_SRC = $(TARGET)/preload.c $(PRELOAD_SRC): $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/init.beam \ + $(ERL_TOP)/erts/preloaded/ebin/prim_eval.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_inet.beam \ $(ERL_TOP)/erts/preloaded/ebin/prim_file.beam \ $(ERL_TOP)/erts/preloaded/ebin/zlib.beam \ @@ -863,7 +881,7 @@ $(OBJDIR)/%.o: hipe/%.c $(BINDIR)/hipe_mkliterals$(TF_MARKER): $(OBJDIR)/hipe_mkliterals.o $(ld_verbose)$(CC) $(CFLAGS) $(INCLUDES) -o $@ $< -$(OBJDIR)/hipe_mkliterals.o: $(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h \ +$(OBJDIR)/hipe_mkliterals.o: $(HIPE_ASM) $(TTF_DIR)/erl_alloc_types.h $(DTRACE_HEADERS) \ $(TTF_DIR)/OPCODES-GENERATED $(TARGET)/TABLES-GENERATED $(TTF_DIR)/hipe_literals.h: $(BINDIR)/hipe_mkliterals$(TF_MARKER) @@ -948,6 +966,7 @@ SED_REPL_TTF_DIR=s|$(TTF_DIR)/|$$(TTF_DIR)/|g SED_REPL_ERL_TOP=s|\([ ]\)$(ERL_TOP)/|\1$$(ERL_TOP)/|g;s|^$(ERL_TOP)/|$$(ERL_TOP)/|g SED_REPL_POLL=s|$$(OBJDIR)/erl_poll.o|$$(OBJDIR)/erl_poll.kp.o $$(OBJDIR)/erl_poll.nkp.o|g SED_REPL_CHK_IO=s|$$(OBJDIR)/erl_check_io.o|$$(OBJDIR)/erl_check_io.kp.o $$(OBJDIR)/erl_check_io.nkp.o|g +SED_REPL_TTF_COMP_FLAGS=s|\([^/]\)erl_compile_flags\.h|\1$$(TTF_DIR)/erl_compile_flags\.h|g ifeq ($(TARGET),win32) #SED_PREFIX=$(SED_REPL_WIN_DRIVE); @@ -962,7 +981,7 @@ else SED_SUFFIX= endif -SED_DEPEND=sed '$(SED_PREFIX)$(SED_REPL_O);$(SED_REPL_TTF_DIR);$(SED_REPL_ERL_TOP)$(SED_SUFFIX)' +SED_DEPEND=sed '$(SED_PREFIX)$(SED_REPL_O);$(SED_REPL_TTF_DIR);$(SED_REPL_ERL_TOP)$(SED_SUFFIX);$(SED_REPL_TTF_COMP_FLAGS)' SED_DEPEND_ZLIB=sed '$(SED_PREFIX)$(SED_REPL_O_ZLIB)' ifdef HIPE_ENABLED @@ -1012,7 +1031,8 @@ endif $(TARGET)/gen_git_version.mk: # We touch beam/erl_bif.info.c if we regenerated the git version to force a # rebuild. - if $(gen_verbose)utils/gen_git_version $@; then touch beam/erl_bif_info.c; fi + $(gen_verbose) + $(V_at)if utils/gen_git_version $@; then touch beam/erl_bif_info.c; fi .PHONY: depend ifdef VOID_EMULATOR diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index ce60bb9bbc..c2f32ba089 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -139,6 +139,7 @@ atom caseless atom catchlevel atom cd atom cdr +atom cflags atom characters_to_binary_int atom characters_to_list_int atom clear @@ -150,6 +151,7 @@ atom compact atom compat_rel atom compile atom compressed +atom config_h atom connect atom connected atom connection_closed @@ -178,6 +180,7 @@ atom disable_trace atom disabled atom display_items atom dist +atom dist_cmd atom Div='/' atom div atom dlink @@ -248,6 +251,7 @@ atom get_data atom get_seq_token atom get_tcw atom getenv +atom gather_gc_info_result atom gather_sched_wall_time_result atom getting_linked atom getting_unlinked @@ -299,6 +303,7 @@ atom label atom large_heap atom last_calls atom latin1 +atom ldflags atom Le='=<' atom lf atom line @@ -312,6 +317,7 @@ atom load_cancelled atom load_failure atom local atom long_gc +atom long_schedule atom low atom Lt='<' atom machine @@ -322,6 +328,7 @@ atom maximum atom max_tables max_processes atom mbuf_size atom memory +atom memory_internal atom memory_types atom message atom message_binary @@ -430,6 +437,7 @@ atom port atom ports atom port_count atom port_limit +atom port_op atom print atom priority atom private @@ -441,6 +449,7 @@ atom process_display atom process_limit atom process_dump atom procs +atom proc_sig atom profile atom protected atom protection @@ -528,6 +537,7 @@ atom system_version atom system_architecture atom SYSTEM='SYSTEM' atom table +atom term_to_binary_trap atom this atom thread_pool_size atom threads diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 9b17de34ec..68907a771a 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2012. All Rights Reserved. + * Copyright Ericsson AB 2000-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1580,6 +1580,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags) } if (common & ERTS_BPF_META_TRACE) { MatchSetUnref(bp->meta_ms); + bp_meta_unref(bp->meta_pid); } if (common & ERTS_BPF_COUNT) { ASSERT((bp->flags & ERTS_BPF_COUNT_ACTIVE) == 0); diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 944ed6da81..da36c4437e 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -924,6 +924,7 @@ extern int count_instructions; # define NOINLINE #endif + /* * The following functions are called directly by process_main(). * Don't inline them. @@ -1153,6 +1154,9 @@ void process_main(void) Eterm pt_arity; /* Used by do_put_tuple */ + Uint64 start_time = 0; /* Monitor long schedule */ + BeamInstr* start_time_i = NULL; + ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */ @@ -1175,6 +1179,16 @@ void process_main(void) do_schedule: reds_used = REDS_IN(c_p) - FCALLS; do_schedule1: + + if (start_time != 0) { + Sint64 diff = erts_timestamp_millis() - start_time; + if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { + BeamInstr *inptr = find_function_from_pc(start_time_i); + BeamInstr *outptr = find_function_from_pc(c_p->i); + monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); + } + } + PROCESS_MAIN_CHK_LOCKS(c_p); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); #if HALFWORD_HEAP @@ -1183,11 +1197,18 @@ void process_main(void) ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); c_p = schedule(c_p, reds_used); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); + start_time = 0; #ifdef DEBUG pid = c_p->common.id; /* Save for debugging purpouses */ #endif ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); PROCESS_MAIN_CHK_LOCKS(c_p); + + if (erts_system_monitor_long_schedule != 0) { + start_time = erts_timestamp_millis(); + start_time_i = c_p->i; + } + reg = ERTS_PROC_GET_SCHDATA(c_p)->x_reg_array; freg = ERTS_PROC_GET_SCHDATA(c_p)->f_reg_array; #if !HEAP_ON_C_STACK @@ -6151,6 +6172,7 @@ apply_fun(Process* p, Eterm fun, Eterm args, Eterm* reg) } + static Eterm new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) { @@ -6234,3 +6256,12 @@ erts_current_reductions(Process *current, Process *p) } } +int +erts_beam_jump_table(void) +{ +#if defined(NO_JUMP_TABLE) + return 0; +#else + return 1; +#endif +} diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index bd4e5a52d0..4193eb4f3f 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -2964,19 +2964,19 @@ gen_put_integer(LoaderState* stp, GenOpArg Fail, GenOpArg Size, NEW_GENOP(stp, op); NATIVE_ENDIAN(Flags); - if (Size.type == TAG_i && Size.val < 0) { - error: /* Negative size must fail */ - op->op = genop_badarg_1; - op->arity = 1; - op->a[0] = Fail; - } else if (Size.type == TAG_i) { + if (Size.type == TAG_i) { op->op = genop_i_new_bs_put_integer_imm_4; op->arity = 4; op->a[0] = Fail; op->a[1].type = TAG_u; if (!safe_mul(Size.val, Unit.val, &op->a[1].val)) { - goto error; + error: + op->op = genop_badarg_1; + op->arity = 1; + op->a[0] = Fail; + op->next = NULL; + return op; } op->a[1].val = Size.val * Unit.val; op->a[2].type = Flags.type; diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index ff237b6a78..755c5e6882 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3934,7 +3934,7 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) { Sint code; Eterm optlist = BIF_ARG_2; - int flush = 0; + int flush = 1; for (optlist = BIF_ARG_2; is_list(optlist); diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 8bc994c8c3..dc8e9101de 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -154,8 +154,10 @@ bif erts_internal:port_command/3 bif erts_internal:port_control/3 bif erts_internal:port_close/1 bif erts_internal:port_connect/2 -bif erts_internal:port_set_data/2 -bif erts_internal:port_get_data/1 + +# inet_db support +bif erlang:port_set_data/2 +bif erlang:port_get_data/1 # Tracing & debugging. bif erlang:trace_pattern/2 diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index acfcc845e4..6b43c53985 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -150,14 +150,14 @@ #define D2GTE(a1,a0,b1,b0) (!D2LT(a1,a0,b1,b0)) #define D2LTE(a1,a0,b1,b0) (!D2GT(a1,a0,b1,b0)) -// Add (A+B), A=(a1B+a0) B=(b1B+b0) +/* Add (A+B), A=(a1B+a0) B=(b1B+b0) */ #define D2ADD(a1,a0,b1,b0,c1,c0) do { \ ErtsDigit __ci = 0; \ DSUM(a0,b0,__ci,c0); \ DSUMc(a1,b1,__ci,c1); \ } while(0) -// Subtract (A-B), A=(a1B+a0), B=(b1B+b0) (A>=B) +/* Subtract (A-B), A=(a1B+a0), B=(b1B+b0) (A>=B) */ #define D2SUB(a1,a0,b1,b0,c1,c0) do { \ ErtsDigit __bi; \ DSUB(a0,b0,__bi,c0); \ @@ -1337,7 +1337,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y, while(bw--) *r++ = 0; - if (sw) { // NOTE! x >> 32 is not = 0! + if (sw) { /* NOTE! x >> 32 is not = 0! */ while(xl--) { a0 = (*x << sw) | a1; a1 = (*x >> (D_EXP - sw)); @@ -1384,7 +1384,7 @@ static dsize_t I_lshift(ErtsDigit* x, dsize_t xl, Sint y, x += (xl-1); r += (rl-1); xl -= bw; - if (sw) { // NOTE! x >> 32 is not = 0! + if (sw) { /* NOTE! x >> 32 is not = 0! */ while(xl--) { a1 = (*x >> sw) | a0; a0 = (*x << (D_EXP-sw)); @@ -2468,7 +2468,7 @@ int term_equals_2pow32(Eterm x) if (!is_big(x)) return 0; bp = big_val(x); -#if D_EXP == 16 // 16 bit platfrom not really supported!!! +#if D_EXP == 16 /* 16 bit platfrom not really supported!!! */ return (BIG_SIZE(bp) == 3) && !BIG_DIGIT(bp,0) && !BIG_DIGIT(bp,1) && BIG_DIGIT(bp,2) == 1; #elif D_EXP == 32 @@ -2528,17 +2528,18 @@ const byte d_base_exp_lookup[] = { 31, 19, 15, 13, 11, 11, 10, 9, 9, 8, 8, 8, 8, * end * How much can the characters which fit in 31 bit represent */ -const Uint d_base_base_lookup[] = { 2147483648, 1162261467, 1073741824, - 1220703125, 362797056, 1977326743, 1073741824, 387420489, 1000000000, - 214358881, 429981696, 815730721, 1475789056, 170859375, 268435456, - 410338673, 612220032, 893871739, 1280000000, 1801088541, 113379904, - 148035889, 191102976, 244140625, 308915776, 387420489, 481890304, - 594823321, 729000000, 887503681, 1073741824, 1291467969, 1544804416, - 1838265625, 60466176, 69343957, 79235168, 90224199, 102400000, - 115856201, 130691232, 147008443, 164916224, 184528125, 205962976, - 229345007, 254803968, 282475249, 312500000, 345025251, 380204032, - 418195493, 459165024, 503284375, 550731776, 601692057, 656356768, - 714924299, 777600000, 844596301, 916132832, 992436543, 1073741824 }; +const Uint d_base_base_lookup[] = { 2147483648u, 1162261467u, 1073741824u, + 1220703125u, 362797056u, 1977326743u, 1073741824u, 387420489u, + 1000000000u, 214358881u, 429981696u, 815730721u, 1475789056u, + 170859375u, 268435456u, 410338673u, 612220032u, 893871739u, 1280000000u, + 1801088541u, 113379904u, 148035889u, 191102976u, 244140625u, 308915776u, + 387420489u, 481890304u, 594823321u, 729000000u, 887503681u, 1073741824u, + 1291467969u, 1544804416u, 1838265625u, 60466176u, 69343957u, 79235168u, + 90224199u, 102400000u, 115856201u, 130691232u, 147008443u, 164916224u, + 184528125u, 205962976u, 229345007u, 254803968u, 282475249u, 312500000u, + 345025251u, 380204032u, 418195493u, 459165024u, 503284375u, 550731776u, + 601692057u, 656356768u, 714924299u, 777600000u, 844596301u, 916132832u, + 992436543u, 1073741824u }; Eterm erts_chars_to_integer(Process *BIF_P, char *bytes, Uint size, const int base) { diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 9aa1e5f30d..ad9a89b642 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -331,6 +331,7 @@ print_process_info(int to, void *to_arg, Process *p) erts_print(to, to_arg, "Heap unused: %bpu\n", (p->hend - p->htop)); erts_print(to, to_arg, "OldHeap unused: %bpu\n", (OLD_HEAP(p) == NULL) ? 0 : (OLD_HEND(p) - OLD_HTOP(p)) ); + erts_print(to, to_arg, "Memory: %beu\n", erts_process_memory(p)); if (garbing) { print_garb_info(to, to_arg, p); @@ -753,7 +754,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) return; /* Can't create the crash dump, skip it */ time(&now); - erts_fdprintf(fd, "=erl_crash_dump:0.1\n%s", ctime(&now)); + erts_fdprintf(fd, "=erl_crash_dump:0.2\n%s", ctime(&now)); if (file != NULL) erts_fdprintf(fd, "The error occurred in file %s, line %d\n", file, line); diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 0781665f05..44f4eb9d43 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -455,7 +455,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) Eterm nd_reason = (reason == am_no_network ? am_no_network : am_net_kernel_terminated); - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); for (tdep = erts_hidden_dist_entries; tdep; tdep = tdep->next) no_dist_port++; @@ -464,7 +464,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) /* KILL all port controllers */ if (no_dist_port == 0) - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); else { Eterm def_buf[128]; int i = 0; @@ -483,7 +483,7 @@ int erts_do_net_exits(DistEntry *dep, Eterm reason) ASSERT(is_internal_port(tdep->cid)); dist_port[i++] = tdep->cid; } - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); for (i = 0; i < no_dist_port; i++) { Port *prt = erts_port_lookup(dist_port[i], @@ -2560,9 +2560,9 @@ BIF_RETTYPE setnode_2(BIF_ALIST_2) erts_smp_proc_unlock(net_kernel, ERTS_PROC_LOCK_MAIN); #ifdef DEBUG - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); ASSERT(!erts_visible_dist_entries && !erts_hidden_dist_entries); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); #endif erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); @@ -2912,7 +2912,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) length = 0; - erts_smp_rwmtx_rwlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_rlock(&erts_dist_table_rwmtx); ASSERT(erts_no_of_not_connected_dist_entries >= 0); ASSERT(erts_no_of_hidden_dist_entries >= 0); @@ -2929,7 +2929,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) result = NIL; if (length == 0) { - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); goto done; } @@ -2958,7 +2958,7 @@ BIF_RETTYPE nodes_1(BIF_ALIST_1) hp += 2; } ASSERT(endp == hp); - erts_smp_rwmtx_rwunlock(&erts_dist_table_rwmtx); + erts_smp_rwmtx_runlock(&erts_dist_table_rwmtx); done: UnUseTmpHeap(2,BIF_P); diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 5850f80843..eca4e3b3bb 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -48,10 +48,9 @@ struct AFFreeBlock_t_ { #define MIN_MBC_FIRST_FREE_SZ (4*1024) /* Prototypes of callback functions */ -static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void link_free_block (Allctr_t *, Block_t *, Uint32); -static void unlink_free_block (Allctr_t *, Block_t *, Uint32); +static Block_t * get_free_block (Allctr_t *, Uint, Block_t *, Uint); +static void link_free_block (Allctr_t *, Block_t *); +static void unlink_free_block (Allctr_t *, Block_t *); static Eterm info_options (Allctr_t *, char *, int *, @@ -84,8 +83,7 @@ erts_afalc_start(AFAllctr_t *afallctr, sys_memcpy((void *) afallctr, (void *) &zero.allctr, sizeof(AFAllctr_t)); - init->sbmbct = 0; /* Small mbc not supported by afit */ - + allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = sizeof(AFFreeBlock_t); @@ -100,6 +98,9 @@ erts_afalc_start(AFAllctr_t *afallctr, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = NULL; allctr->destroying_mbc = NULL; + allctr->add_mbc = NULL; + allctr->remove_mbc = NULL; + allctr->largest_fblk_in_mbc = NULL; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -116,8 +117,7 @@ erts_afalc_start(AFAllctr_t *afallctr, } static Block_t * -get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size, - Uint32 flags) +get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size) { AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -135,7 +135,7 @@ get_free_block(Allctr_t *allctr, Uint size, Block_t *cand_blk, Uint cand_size, } static void -link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +link_free_block(Allctr_t *allctr, Block_t *block) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -156,7 +156,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +unlink_free_block(Allctr_t *allctr, Block_t *block) { AFFreeBlock_t *blk = (AFFreeBlock_t *) block; AFAllctr_t *afallctr = (AFAllctr_t *) allctr; @@ -259,10 +259,10 @@ info_options(Allctr_t *allctr, * to erts_afalc_test() * \* */ -unsigned long -erts_afalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_afalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - default: ASSERT(0); return ~((unsigned long) 0); + default: ASSERT(0); return ~((UWord) 0); } } diff --git a/erts/emulator/beam/erl_afit_alloc.h b/erts/emulator/beam/erl_afit_alloc.h index cf7b99c463..b90ac8f7c5 100644 --- a/erts/emulator/beam/erl_afit_alloc.h +++ b/erts/emulator/beam/erl_afit_alloc.h @@ -56,7 +56,7 @@ struct AFAllctr_t_ { AFFreeBlock_t * free_list; }; -unsigned long erts_afalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_afalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_AF_ALLOC_IMPL) && !defined(ERL_AF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 13692edbfc..6dec383cee 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -71,6 +71,23 @@ #define AU_ALLOC_DEFAULT_ENABLE(X) (X) #endif +#define ERTS_ALC_DEFAULT_ENABLED_ACUL 60 +#define ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC 45 +#define ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC 85 + +#define ERTS_ALC_DEFAULT_ACUL 0 +#define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0 +#define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0 + +#ifndef ERTS_SMP +# undef ERTS_ALC_DEFAULT_ACUL +# define ERTS_ALC_DEFAULT_ACUL 0 +# undef ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC +# define ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC 0 +# undef ERTS_ALC_DEFAULT_ACUL_LL_ALLOC +# define ERTS_ALC_DEFAULT_ACUL_LL_ALLOC 0 +#endif + #ifdef DEBUG static Uint install_debug_functions(void); #if 0 @@ -101,11 +118,9 @@ typedef union { char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))]; } ErtsAllocatorState_t; -static ErtsAllocatorState_t sbmbc_alloc_state; static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; #if HALFWORD_HEAP -static ErtsAllocatorState_t sbmbc_low_alloc_state; static ErtsAllocatorState_t std_low_alloc_state; static ErtsAllocatorState_t ll_low_alloc_state; #endif @@ -120,6 +135,7 @@ static ErtsAllocatorState_t fix_alloc_state; typedef struct { erts_smp_atomic32_t refc; int only_sz; + int internal; Uint req_sched; Process *proc; Eterm ref; @@ -162,6 +178,7 @@ enum allctr_type { struct au_init { int enable; int thr_spec; + int carrier_migration_allowed; enum allctr_type atype; struct { AllctrInit_t util; @@ -200,7 +217,6 @@ typedef struct { char *mtrace; char *nodename; } instr; - struct au_init sbmbc_alloc; struct au_init sl_alloc; struct au_init std_alloc; struct au_init ll_alloc; @@ -211,13 +227,12 @@ typedef struct { struct au_init driver_alloc; struct au_init fix_alloc; #if HALFWORD_HEAP - struct au_init sbmbc_low_alloc; struct au_init std_low_alloc; struct au_init ll_low_alloc; #endif } erts_alc_hndl_args_init_t; -#define ERTS_AU_INIT__ {0, 0, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} +#define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} #define SET_DEFAULT_ALLOC_OPTS(IP) \ do { \ @@ -225,40 +240,32 @@ do { \ sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init)); \ } while (0) -static void -set_default_sbmbc_alloc_opts(struct au_init *ip) +#if ERTS_ALC_DEFAULT_ACUL \ + || ERTS_ALC_DEFAULT_ACUL_LL_ALLOC \ + || ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC + +static ERTS_INLINE void +set_default_acul(struct au_init *ip, int acul) { - SET_DEFAULT_ALLOC_OPTS(ip); - ip->enable = 0; - ip->thr_spec = 0; - ip->atype = BESTFIT; - ip->init.bf.ao = 1; - ip->init.util.ramv = 0; - ip->init.util.mmsbc = 0; - ip->init.util.sbct = ~((UWord) 0); - ip->init.util.name_prefix = "sbmbc_"; - ip->init.util.alloc_no = ERTS_ALC_A_SBMBC; -#ifndef SMALL_MEMORY - ip->init.util.mmbcs = 2*1024*1024; /* Main carrier size */ -#else - ip->init.util.mmbcs = 1*1024*1024; /* Main carrier size */ -#endif - ip->init.util.ts = ERTS_ALC_MTA_SBMBC; - ip->init.util.asbcst = 0; - ip->init.util.rsbcst = 0; - ip->init.util.rsbcmt = 0; - ip->init.util.rmbcmt = 0; - ip->init.util.sbmbct = 0; - ip->init.util.sbmbcs = 0; + ip->thr_spec = 1; + ip->atype = AOFIRSTFIT; + ip->init.aoff.flavor = AOFF_BF; + ip->init.util.acul = acul; } +#endif + static void set_default_sl_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = GOODFIT; +#endif ip->init.util.name_prefix = "sl_"; ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED; #ifndef SMALL_MEMORY @@ -280,8 +287,12 @@ set_default_std_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; +#endif ip->init.util.name_prefix = "std_"; ip->init.util.alloc_no = ERTS_ALC_A_STANDARD; #ifndef SMALL_MEMORY @@ -297,9 +308,13 @@ set_default_ll_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL_LL_ALLOC + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_LL_ALLOC); +#else ip->thr_spec = 0; ip->atype = BESTFIT; ip->init.bf.ao = 1; +#endif ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; ip->init.util.sbct = ~((UWord) 0); @@ -315,8 +330,6 @@ set_default_ll_alloc_opts(struct au_init *ip) ip->init.util.rsbcst = 0; ip->init.util.rsbcmt = 0; ip->init.util.rmbcmt = 0; - ip->init.util.sbmbct = 0; - ip->init.util.sbmbcs = 0; } static void @@ -325,6 +338,7 @@ set_default_temp_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; + ip->carrier_migration_allowed = 0; ip->atype = AFIT; ip->init.util.name_prefix = "temp_"; ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY; @@ -347,8 +361,12 @@ set_default_eheap_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC); +#else ip->thr_spec = 1; ip->atype = GOODFIT; +#endif ip->init.util.name_prefix = "eheap_"; ip->init.util.alloc_no = ERTS_ALC_A_EHEAP; #ifndef SMALL_MEMORY @@ -369,8 +387,12 @@ set_default_binary_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; +#endif ip->init.util.name_prefix = "binary_"; ip->init.util.alloc_no = ERTS_ALC_A_BINARY; #ifndef SMALL_MEMORY @@ -386,8 +408,12 @@ set_default_ets_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; +#endif ip->init.util.name_prefix = "ets_"; ip->init.util.alloc_no = ERTS_ALC_A_ETS; #ifndef SMALL_MEMORY @@ -403,8 +429,12 @@ set_default_driver_alloc_opts(struct au_init *ip) { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; +#endif ip->init.util.name_prefix = "driver_"; ip->init.util.alloc_no = ERTS_ALC_A_DRIVER; #ifndef SMALL_MEMORY @@ -421,8 +451,12 @@ set_default_fix_alloc_opts(struct au_init *ip, { SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); +#if ERTS_ALC_DEFAULT_ACUL + set_default_acul(ip, ERTS_ALC_DEFAULT_ACUL); +#else ip->thr_spec = 1; ip->atype = BESTFIT; +#endif ip->init.bf.ao = 1; ip->init.util.name_prefix = "fix_"; ip->init.util.fix_type_size = fix_type_sizes; @@ -481,6 +515,71 @@ refuse_af_strategy(struct au_init *init) static void hdbg_init(void); #endif +static void adjust_fix_alloc_sizes(UWord extra_block_size) +{ + + if (extra_block_size && erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].enabled) { + int j; + +#ifdef ERTS_SMP + if (erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].thr_spec) { + int i; + ErtsAllocatorThrSpec_t* tspec; + + tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE]; + ASSERT(tspec->enabled); + + for (i=0; i < tspec->size; i++) { + Allctr_t* allctr = tspec->allctr[i]; + for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) { + allctr->fix[j].type_size += extra_block_size; + } + } + } + else +#endif + { + Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra; + for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) { + allctr->fix[j].type_size += extra_block_size; + } + } + } +} + +static ERTS_INLINE int +strategy_support_carrier_migration(struct au_init *auip) +{ + /* + * Currently only aoff, aoffcbf and aoffcaobf support carrier + * migration, i.e, type AOFIRSTFIT. + */ + return auip->atype == AOFIRSTFIT; +} + +static ERTS_INLINE void +check_disable_carrier_migration(struct au_init *auip) +{ + if (!strategy_support_carrier_migration(auip) || !auip->thr_spec) + auip->init.util.acul = 0; +} + +static ERTS_INLINE void +ensure_carrier_migration_support(struct au_init *auip) +{ + auip->thr_spec = 1; /* Need thread preferred */ + + /* + * If strategy cannot handle carrier migration, + * default to a strategy that can... + */ + if (!strategy_support_carrier_migration(auip)) { + /* Default to aoffcbf */ + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_BF; + } +} + void erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) { @@ -501,9 +600,9 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) = sizeof(Process); #if !HALFWORD_HEAP fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_MONITOR_SH)] - = ERTS_MONITOR_SH_SIZE; + = ERTS_MONITOR_SH_SIZE * sizeof(Uint); fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_NLINK_SH)] - = ERTS_LINK_SH_SIZE; + = ERTS_LINK_SH_SIZE * sizeof(Uint); #endif fix_type_sizes[ERTS_ALC_FIX_TYPE_IX(ERTS_ALC_T_DRV_EV_D_STATE)] = sizeof(ErtsDrvEventDataState); @@ -519,15 +618,15 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) hdbg_init(); #endif - erts_have_sbmbc_alloc = 0; ncpu = eaiop->ncpu; if (ncpu < 1) ncpu = 1; + erts_tsd_key_create(&erts_allctr_prelock_tsd_key); + erts_sys_alloc_init(); erts_init_utils_mem(); - set_default_sbmbc_alloc_opts(&init.sbmbc_alloc); set_default_sl_alloc_opts(&init.sl_alloc); set_default_std_alloc_opts(&init.std_alloc); set_default_ll_alloc_opts(&init.ll_alloc); @@ -543,7 +642,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) handle_args(argc, argv, &init); #ifndef ERTS_SMP - init.sbmbc_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; init.ll_alloc.thr_spec = 0; @@ -556,7 +654,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) if (init.erts_alloc_config) { /* Adjust flags that erts_alloc_config won't like */ - init.sbmbc_alloc.thr_spec = 0; init.temp_alloc.thr_spec = 0; init.sl_alloc.thr_spec = 0; init.std_alloc.thr_spec = 0; @@ -568,13 +665,22 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.fix_alloc.thr_spec = 0; } + check_disable_carrier_migration(&init.sl_alloc); + check_disable_carrier_migration(&init.std_alloc); + check_disable_carrier_migration(&init.ll_alloc); + check_disable_carrier_migration(&init.eheap_alloc); + check_disable_carrier_migration(&init.binary_alloc); + check_disable_carrier_migration(&init.ets_alloc); + check_disable_carrier_migration(&init.driver_alloc); + check_disable_carrier_migration(&init.fix_alloc); + + #ifdef ERTS_SMP /* Only temp_alloc can use thread specific interface */ if (init.temp_alloc.thr_spec) init.temp_alloc.thr_spec = erts_no_schedulers; /* Others must use thread preferred interface */ - adjust_tpref(&init.sbmbc_alloc, erts_no_schedulers); adjust_tpref(&init.sl_alloc, erts_no_schedulers); adjust_tpref(&init.std_alloc, erts_no_schedulers); adjust_tpref(&init.ll_alloc, erts_no_schedulers); @@ -593,7 +699,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) * The following allocators cannot be run with afit strategy. * Make sure they don't... */ - refuse_af_strategy(&init.sbmbc_alloc); refuse_af_strategy(&init.sl_alloc); refuse_af_strategy(&init.std_alloc); refuse_af_strategy(&init.ll_alloc); @@ -637,11 +742,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #if HALFWORD_HEAP /* Init low memory variants by cloning */ - init.sbmbc_low_alloc = init.sbmbc_alloc; - init.sbmbc_low_alloc.init.util.name_prefix = "sbmbc_low_"; - init.sbmbc_low_alloc.init.util.alloc_no = ERTS_ALC_A_SBMBC_LOW; - init.sbmbc_low_alloc.init.util.low_mem = 1; - init.std_low_alloc = init.std_alloc; init.std_low_alloc.init.util.name_prefix = "std_low_"; init.std_low_alloc.init.util.alloc_no = ERTS_ALC_A_STANDARD_LOW; @@ -654,13 +754,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) init.ll_low_alloc.init.util.force = 1; init.ll_low_alloc.init.util.low_mem = 1; - set_au_allocator(ERTS_ALC_A_SBMBC_LOW, &init.sbmbc_low_alloc, ncpu); set_au_allocator(ERTS_ALC_A_STANDARD_LOW, &init.std_low_alloc, ncpu); set_au_allocator(ERTS_ALC_A_LONG_LIVED_LOW, &init.ll_low_alloc, ncpu); #endif /* HALFWORD */ set_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, ncpu); - set_au_allocator(ERTS_ALC_A_SBMBC, &init.sbmbc_alloc, ncpu); set_au_allocator(ERTS_ALC_A_SHORT_LIVED, &init.sl_alloc, ncpu); set_au_allocator(ERTS_ALC_A_STANDARD, &init.std_alloc, ncpu); set_au_allocator(ERTS_ALC_A_LONG_LIVED, &init.ll_alloc, ncpu); @@ -687,20 +785,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) erts_mtrace_init(init.instr.mtrace, init.instr.nodename); - /* sbmbc_alloc() needs to be started first */ - start_au_allocator(ERTS_ALC_A_SBMBC, - &init.sbmbc_alloc, - &sbmbc_alloc_state); -#if HALFWORD_HEAP - start_au_allocator(ERTS_ALC_A_SBMBC_LOW, - &init.sbmbc_low_alloc, - &sbmbc_low_alloc_state); - erts_have_sbmbc_alloc = (init.sbmbc_alloc.enable - && init.sbmbc_low_alloc.enable); -#else - erts_have_sbmbc_alloc = init.sbmbc_alloc.enable; -#endif - start_au_allocator(ERTS_ALC_A_TEMPORARY, &init.temp_alloc, &temp_alloc_state); @@ -754,7 +838,7 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #ifdef DEBUG extra_block_size += install_debug_functions(); #endif - + adjust_fix_alloc_sizes(extra_block_size); } void @@ -1090,6 +1174,7 @@ get_kb_value(char *param_end, char** argv, int* ip) return ((Uint) tmp)*1024; } +#if 0 static Uint get_byte_value(char *param_end, char** argv, int* ip) { @@ -1103,6 +1188,7 @@ get_byte_value(char *param_end, char** argv, int* ip) bad_value(param, param_end, value); return (Uint) tmp; } +#endif static Uint get_amount_value(char *param_end, char** argv, int* ip) @@ -1118,17 +1204,59 @@ get_amount_value(char *param_end, char** argv, int* ip) return (Uint) tmp; } +static Uint +get_acul_value(struct au_init *auip, char *param_end, char** argv, int* ip) +{ + Sint tmp; + char *rest; + char *param = argv[*ip]+1; + char *value = get_value(param_end, argv, ip); + if (sys_strcmp(value, "de") == 0) { + switch (auip->init.util.alloc_no) { + case ERTS_ALC_A_LONG_LIVED: +#if HALFWORD_HEAP + case ERTS_ALC_A_LONG_LIVED_LOW: +#endif + return ERTS_ALC_DEFAULT_ENABLED_ACUL_LL_ALLOC; + case ERTS_ALC_A_EHEAP: + return ERTS_ALC_DEFAULT_ENABLED_ACUL_EHEAP_ALLOC; + default: + return ERTS_ALC_DEFAULT_ENABLED_ACUL; + } + } + errno = 0; + tmp = (Sint) ErtsStrToSint(value, &rest, 10); + if (errno != 0 || rest == value || tmp < 0 || 100 < tmp) + bad_value(param, param_end, value); + return (Uint) tmp; +} + static void handle_au_arg(struct au_init *auip, char* sub_param, char** argv, - int* ip) + int* ip, + int u_switch) { char *param = argv[*ip]+1; switch (sub_param[0]) { case 'a': - if(has_prefix("asbcst", sub_param)) { + if (has_prefix("acul", sub_param)) { + if (!auip->carrier_migration_allowed) { + if (!u_switch) + goto bad_switch; + else { + /* ignore */ + (void) get_acul_value(auip, sub_param + 4, argv, ip); + break; + } + } + ensure_carrier_migration_support(auip); + + auip->init.util.acul = get_acul_value(auip, sub_param + 4, argv, ip); + } + else if(has_prefix("asbcst", sub_param)) { auip->init.util.asbcst = get_kb_value(sub_param + 6, argv, ip); } else if(has_prefix("as", sub_param)) { @@ -1149,21 +1277,26 @@ handle_au_arg(struct au_init *auip, } else if (strcmp("aoff", alg) == 0) { auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_AOFF; + } + else if (strcmp("aoffcbf", alg) == 0) { + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_BF; + } + else if (strcmp("aoffcaobf", alg) == 0) { + auip->atype = AOFIRSTFIT; + auip->init.aoff.flavor = AOFF_AOBF; } else { bad_value(param, sub_param + 1, alg); } + check_disable_carrier_migration(auip); } else goto bad_switch; break; case 'e': auip->enable = get_bool_value(sub_param+1, argv, ip); -#if !HAVE_ERTS_SBMBC - if (auip->init.util.alloc_no == ERTS_ALC_A_SBMBC) { - auip->enable = 0; - } -#endif break; case 'l': if (has_prefix("lmbcs", sub_param)) { @@ -1223,18 +1356,6 @@ handle_au_arg(struct au_init *auip, if(has_prefix("sbct", sub_param)) { auip->init.util.sbct = get_kb_value(sub_param + 4, argv, ip); } - else if (has_prefix("sbmbcs", sub_param)) { -#if HAVE_ERTS_SBMBC - auip->init.util.sbmbcs = -#endif - get_byte_value(sub_param + 6, argv, ip); - } - else if (has_prefix("sbmbct", sub_param)) { -#if HAVE_ERTS_SBMBC - auip->init.util.sbmbct = -#endif - get_byte_value(sub_param + 6, argv, ip); - } else if (has_prefix("smbcs", sub_param)) { auip->default_.smbcs = 0; auip->init.util.smbcs = get_kb_value(sub_param + 5, argv, ip); @@ -1250,6 +1371,7 @@ handle_au_arg(struct au_init *auip, } else if (res == 0) { auip->thr_spec = 0; + check_disable_carrier_migration(auip); break; } goto bad_switch; @@ -1264,7 +1386,6 @@ static void handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) { struct au_init *aui[] = { - &init->sbmbc_alloc, &init->binary_alloc, &init->std_alloc, &init->ets_alloc, @@ -1291,25 +1412,22 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'M': switch (argv[i][2]) { case 'B': - handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i); - break; - case 'C': - handle_au_arg(&init->sbmbc_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0); break; case 'D': - handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0); break; case 'E': - handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->ets_alloc, &argv[i][3], argv, &i, 0); break; case 'F': - handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->fix_alloc, &argv[i][3], argv, &i, 0); break; case 'H': - handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->eheap_alloc, &argv[i][3], argv, &i, 0); break; case 'L': - handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->ll_alloc, &argv[i][3], argv, &i, 0); break; case 'M': if (has_prefix("amcbf", argv[i]+3)) { @@ -1335,13 +1453,13 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) } break; case 'R': - handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->driver_alloc, &argv[i][3], argv, &i, 0); break; case 'S': - handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->sl_alloc, &argv[i][3], argv, &i, 0); break; case 'T': - handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i); + handle_au_arg(&init->temp_alloc, &argv[i][3], argv, &i, 0); break; case 'Y': { /* sys_alloc */ if (has_prefix("tt", param+2)) { @@ -1400,9 +1518,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) else if (strcmp("max", arg) == 0) { for (a = 0; a < aui_sz; a++) aui[a]->enable = 1; -#if !HAVE_ERTS_SBMBC - init->sbmbc_alloc.enable = 0; -#endif } else if (strcmp("config", arg) == 0) { init->erts_alloc_config = 1; @@ -1430,8 +1545,8 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) for (a = 0; a < aui_sz; a++) { aui[a]->thr_spec = 0; + check_disable_carrier_migration(aui[a]); aui[a]->init.util.ramv = 0; - aui[a]->init.util.mmmbc = 10; aui[a]->init.util.lmbcs = 5*1024*1024; } } @@ -1494,7 +1609,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) argv[start + 1] = val; i = start; } - handle_au_arg(aui[a], &argv[i][3], argv, &i); + handle_au_arg(aui[a], &argv[i][3], argv, &i, 1); } } break; @@ -2040,15 +2155,11 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) return am_badarg; } - /* All alloc_util allocators except sbmbc_alloc *have* to be enabled */ + /* All alloc_util allocators *have* to be enabled */ for (ai = ERTS_ALC_A_MIN; ai <= ERTS_ALC_A_MAX; ai++) { switch (ai) { case ERTS_ALC_A_SYSTEM: - case ERTS_ALC_A_SBMBC: -#if HALFWORD_HEAP - case ERTS_ALC_A_SBMBC_LOW: -#endif break; default: if (!erts_allctrs_info[ai].enabled @@ -2088,12 +2199,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) * Often not thread safe and usually never * contain any allocated memory. */ - case ERTS_ALC_A_SBMBC: - /* Included in other allocators */ -#if HALFWORD_HEAP - case ERTS_ALC_A_SBMBC_LOW: - /* Included in other allocators */ -#endif continue; case ERTS_ALC_A_EHEAP: save = &size.processes; @@ -2128,7 +2233,6 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) if (want_tot_or_sys || want.processes || want.processes_used) { - int max_processes = erts_ptab_max(&erts_proc); UWord tmp; if (ERTS_MEM_NEED_ALL_ALCU) @@ -2138,7 +2242,7 @@ erts_memory(int *print_to_p, void *print_to_arg, void *proc, Eterm earg) fi, ERTS_ALC_NO_FIXED_SIZES); tmp = alcu_size(ERTS_ALC_A_EHEAP, NULL, 0); } - tmp += max_processes*sizeof(erts_smp_atomic_t); + tmp += erts_ptab_mem_size(&erts_proc); tmp += erts_bif_timer_memory_size(); tmp += erts_tot_link_lh_size(); @@ -2264,13 +2368,11 @@ struct aa_values { Eterm erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) { -#define MAX_AA_VALUES (23) +#define MAX_AA_VALUES (24) struct aa_values values[MAX_AA_VALUES]; Eterm res = THE_NON_VALUE; int i, length; Uint reserved_atom_space, atom_space; - int max_processes = erts_ptab_max(&erts_proc); - int max_ports = erts_ptab_max(&erts_port); if (proc) { ERTS_SMP_LC_ASSERT(ERTS_PROC_LOCK_MAIN @@ -2301,8 +2403,8 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "static"; - values[i].ui[0] = - max_ports*sizeof(erts_smp_atomic_t) /* Port table */ + values[i].ui[0] = + sizeof(ErtsPTab)*2 /* proc & port tables */ + erts_timer_wheel_memory_size(); /* Timer wheel */ i++; @@ -2381,7 +2483,12 @@ erts_allocated_areas(int *print_to_p, void *print_to_arg, void *proc) values[i].arity = 2; values[i].name = "process_table"; - values[i].ui[0] = max_processes*sizeof(Process*); + values[i].ui[0] = erts_ptab_mem_size(&erts_proc); + i++; + + values[i].arity = 2; + values[i].name = "port_table"; + values[i].ui[0] = erts_ptab_mem_size(&erts_port); i++; values[i].arity = 2; @@ -2535,7 +2642,7 @@ erts_allocator_info(int to, void *arg) as = erts_allctr_thr_spec[a].allctr[ai]; } /* Binary alloc has its own thread safety... */ - erts_alcu_info(as, 0, &to, arg, NULL, NULL); + erts_alcu_info(as, 0, 0, &to, arg, NULL, NULL); } else { switch (a) { @@ -2800,6 +2907,7 @@ reply_alloc_info(void *vair) int i; Eterm (*info_func)(Allctr_t *, int, + int, int *, void *, Uint **, @@ -2928,8 +3036,8 @@ reply_alloc_info(void *vair) allctr = erts_allctr_thr_spec[ai].allctr[0]; else allctr = erts_allctrs_info[ai].extra; - ainfo = info_func(allctr, hpp != NULL, NULL, - NULL, hpp, szp); + ainfo = info_func(allctr, air->internal, hpp != NULL, + NULL, NULL, hpp, szp); ainfo = erts_bld_tuple(hpp, szp, 3, alloc_atom, make_small(0), ainfo); } @@ -2964,7 +3072,7 @@ reply_alloc_info(void *vair) alloc_atom = erts_bld_atom(hpp, szp, (char *) ERTS_ALC_A2AD(ai)); allctr = erts_allctr_thr_spec[ai].allctr[sched_id]; - ainfo = info_func(allctr, hpp != NULL, NULL, + ainfo = info_func(allctr, air->internal, hpp != NULL, NULL, NULL, hpp, szp); ai_list = erts_bld_cons(hpp, szp, erts_bld_tuple( @@ -3020,7 +3128,8 @@ int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs, - int only_sz) + int only_sz, + int internal) { ErtsAllocInfoReq *air = aireq_alloc(); Eterm req_ai[ERTS_ALC_A_MAX+1+2] = {0}; @@ -3032,6 +3141,8 @@ erts_request_alloc_info(struct process *c_p, air->only_sz = only_sz; + air->internal = internal; + air->proc = c_p; if (is_not_internal_ref(ref)) @@ -3096,6 +3207,55 @@ erts_request_alloc_info(struct process *c_p, return 1; } +/* + * The allocator wrapper prelocking stuff below is about the locking order. + * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks + * during alloc/realloc/free. + * + * Some query functions in erl_alloc_util.c lock the allocator mutex and then + * use erts_printf that in turn may call the sys allocator through the wrappers. + * To avoid breaking locking order these query functions first "pre-locks" all + * allocator wrappers. + */ +ErtsAllocatorWrapper_t *erts_allctr_wrappers; +int erts_allctr_wrapper_prelocked = 0; +erts_tsd_key_t erts_allctr_prelock_tsd_key; + +void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper) +{ + ASSERT(wrapper->lock && wrapper->unlock); + wrapper->next = erts_allctr_wrappers; + erts_allctr_wrappers = wrapper; +} + +void erts_allctr_wrapper_pre_lock(void) +{ + if (erts_allctr_wrappers) { + ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers; + for ( ; wrapper; wrapper = wrapper->next) { + wrapper->lock(); + } + ASSERT(!erts_allctr_wrapper_prelocked); + erts_allctr_wrapper_prelocked = 1; + erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)1); + } +} + +void erts_allctr_wrapper_pre_unlock(void) +{ + if (erts_allctr_wrappers) { + ErtsAllocatorWrapper_t* wrapper = erts_allctr_wrappers; + + erts_allctr_wrapper_prelocked = 0; + erts_tsd_set(erts_allctr_prelock_tsd_key, (void*)0); + for ( ; wrapper; wrapper = wrapper->next) { + wrapper->unlock(); + } + } +} + + + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Deprecated functions * * * @@ -3125,10 +3285,7 @@ void *safe_realloc(void *ptr, Uint sz) \* */ #define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") -UWord erts_alc_test(UWord op, - UWord a1, - UWord a2, - UWord a3) +UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) { switch (op >> 8) { case 0x0: return erts_alcu_test(op, a1, a2); @@ -3180,14 +3337,13 @@ UWord erts_alc_test(UWord op, init.atype = GOODFIT; init.init.util.name_prefix = (char *) a1; init.init.util.ts = a2 ? 1 : 0; - init.init.util.sbmbct = 0; if ((char **) a3) { char **argv = (char **) a3; int i = 0; while (argv[i]) { if (argv[i][0] == '-' && argv[i][1] == 't') - handle_au_arg(&init, &argv[i][2], argv, &i); + handle_au_arg(&init, &argv[i][2], argv, &i, 0); else return (UWord) NULL; i++; @@ -3307,6 +3463,11 @@ UWord erts_alc_test(UWord op, ERTS_ALC_TEST_ABORT; break; #endif /* #ifdef USE_THREADS */ +#ifdef ERTS_SMP + case 0xf13: return (UWord) 1; +#else + case 0xf13: return (UWord) 0; +#endif default: break; } diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 7168241f76..f83f6b39cf 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -75,7 +75,7 @@ Eterm erts_allocator_options(void *proc); struct process; int erts_request_alloc_info(struct process *c_p, Eterm ref, Eterm allocs, - int only_sz); + int only_sz, int internal); #define ERTS_ALLOC_INIT_DEF_OPTS_INITER {0} typedef struct { @@ -110,14 +110,6 @@ UWord erts_alc_test(UWord, #define ERTS_ALC_MIN_LONG_LIVED_TIME (10*60*1000) -#if HALFWORD_HEAP -#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ - ((NO) == ERTS_ALC_A_SBMBC || (NO) == ERTS_ALC_A_SBMBC_LOW) -#else -#define ERTS_IS_SBMBC_ALLOCATOR_NO__(NO) \ - ((NO) == ERTS_ALC_A_SBMBC) -#endif - typedef struct { int alloc_util; int enabled; @@ -145,6 +137,18 @@ typedef struct { extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; +typedef struct ErtsAllocatorWrapper_t_ { + void (*lock)(void); + void (*unlock)(void); + struct ErtsAllocatorWrapper_t_* next; +}ErtsAllocatorWrapper_t; +ErtsAllocatorWrapper_t *erts_allctr_wrappers; +extern int erts_allctr_wrapper_prelocked; +extern erts_tsd_key_t erts_allctr_prelock_tsd_key; +void erts_allctr_wrapper_prelock_init(ErtsAllocatorWrapper_t* wrapper); +void erts_allctr_wrapper_pre_lock(void); +void erts_allctr_wrapper_pre_unlock(void); + void erts_alloc_register_scheduler(void *vesdp); #ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, @@ -198,6 +202,7 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size); void erts_free(ErtsAlcType_t type, void *ptr); void *erts_alloc_fnf(ErtsAlcType_t type, Uint size); void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size); +int erts_is_allctr_wrapper_prelocked(void); #endif /* #if !ERTS_ALC_DO_INLINE */ @@ -268,6 +273,13 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) size); } +ERTS_ALC_INLINE +int erts_is_allctr_wrapper_prelocked(void) +{ + return erts_allctr_wrapper_prelocked /* locked */ + && !!erts_tsd_get(erts_allctr_prelock_tsd_key); /* by me */ +} + #endif /* #if ERTS_ALC_DO_INLINE || defined(ERTS_ALC_INTERNAL__) */ #define ERTS_ALC_GET_THR_IX() ((int) erts_get_scheduler_id()) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 5e3615ccc2..bb5eba80be 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -76,11 +76,6 @@ allocator SYSTEM true sys_alloc -allocator SBMBC true sbmbc_alloc -+if halfword -allocator SBMBC_LOW true sbmbc_low_alloc -+endif - +if smp allocator TEMPORARY true temp_alloc @@ -146,7 +141,6 @@ class SYSTEM system_data # # <TYPE> <ALLOCATOR> <CLASS> <DESCRIPTION> -type SBMBC SBMBC SYSTEM small_block_mbc type PROC FIXED_SIZE PROCESSES proc type PORT DRIVER SYSTEM port type ATOM LONG_LIVED ATOM atom_entry @@ -156,6 +150,7 @@ type LINK_LH STANDARD PROCESSES link_lh type SUSPEND_MON STANDARD PROCESSES suspend_monitor type PEND_SUSPEND SHORT_LIVED PROCESSES pending_suspend type PROC_LIST SHORT_LIVED PROCESSES proc_list +type EXTRA_ROOT SHORT_LIVED PROCESSES extra_root type FUN_ENTRY LONG_LIVED CODE fun_entry type ATOM_TXT LONG_LIVED ATOM atom_text type BEAM_REGISTER EHEAP PROCESSES beam_register @@ -272,6 +267,7 @@ type CODE_IX_LOCK_Q SHORT_LIVED SYSTEM code_ix_lock_q type PROC_INTERVAL LONG_LIVED SYSTEM process_interval type BUSY_CALLER_TAB SHORT_LIVED SYSTEM busy_caller_table type BUSY_CALLER SHORT_LIVED SYSTEM busy_caller +type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; @@ -350,7 +346,6 @@ type SSB SHORT_LIVED PROCESSES ssb +if halfword -type SBMBC_LOW SBMBC_LOW SYSTEM small_block_mbc_low type DDLL_PROCESS STANDARD_LOW SYSTEM ddll_processes type MONITOR_LH STANDARD_LOW PROCESSES monitor_lh type NLINK_LH STANDARD_LOW PROCESSES nlink_lh @@ -365,6 +360,7 @@ type MONITOR_SH STANDARD_LOW PROCESSES monitor_sh type NLINK_SH STANDARD_LOW PROCESSES nlink_sh type AINFO_REQ STANDARD_LOW SYSTEM alloc_info_request type SCHED_WTIME_REQ STANDARD_LOW SYSTEM sched_wall_time_request +type GC_INFO_REQ STANDARD_LOW SYSTEM gc_info_request +else # "fullword" @@ -382,6 +378,7 @@ type MONITOR_SH FIXED_SIZE PROCESSES monitor_sh type NLINK_SH FIXED_SIZE PROCESSES nlink_sh type AINFO_REQ SHORT_LIVED SYSTEM alloc_info_request type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request +type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request +endif @@ -401,6 +398,7 @@ type POLLSET_UPDREQ SHORT_LIVED SYSTEM pollset_update_req type POLL_FDS LONG_LIVED SYSTEM poll_fds type POLL_RES_EVS LONG_LIVED SYSTEM poll_result_events type FD_STATUS LONG_LIVED SYSTEM fd_status +type SELECT_FDS LONG_LIVED SYSTEM select_fds +if unix diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 24f297dc58..825b68bb85 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -71,22 +71,16 @@ #define ALLOC_ZERO_EQ_NULL 0 +#ifndef ERTS_MSEG_FLG_2POW +# define ERTS_MSEG_FLG_2POW 0 +#endif +#ifndef ERTS_MSEG_FLG_NONE +# define ERTS_MSEG_FLG_NONE 0 +#endif + static int atoms_initialized = 0; static int initialized = 0; -int erts_have_sbmbc_alloc; - -#if ERTS_SA_MB_CARRIERS - -#define ERTS_SACRR_UNIT_SHIFT ERTS_SUPER_ALIGN_BITS -#define ERTS_SACRR_UNIT_SZ (1 << ERTS_SACRR_UNIT_SHIFT) -#define ERTS_SACRR_UNIT_MASK ((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT) - -#define ERTS_SACRR_UNIT_FLOOR(X) ((X) & ERTS_SACRR_UNIT_MASK) -#define ERTS_SACRR_UNIT_CEILING(X) ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK) - -#endif - #define INV_SYS_ALLOC_CARRIER_MASK ((UWord) (sys_alloc_carrier_size - 1)) #define SYS_ALLOC_CARRIER_MASK (~INV_SYS_ALLOC_CARRIER_MASK) #define SYS_ALLOC_CARRIER_FLOOR(X) ((X) & SYS_ALLOC_CARRIER_MASK) @@ -110,13 +104,12 @@ static Uint max_mseg_carriers; #define ONE_GIGA (1000000000) -#define INC_CC(CC) ((CC).no == ONE_GIGA - 1 \ - ? ((CC).giga_no++, (CC).no = 0) \ - : (CC).no++) +#define ERTS_ALC_CC_GIGA_VAL(CC) ((CC) / ONE_GIGA) +#define ERTS_ALC_CC_VAL(CC) ((CC) % ONE_GIGA) -#define DEC_CC(CC) ((CC).no == 0 \ - ? ((CC).giga_no--, (CC).no = ONE_GIGA - 1) \ - : (CC).no--) +#define INC_CC(CC) ((CC)++) + +#define DEC_CC(CC) ((CC)--) /* Multi block carrier (MBC) memory layout in R16: @@ -173,14 +166,6 @@ MBC after deallocating first block: #define SET_BLK_SZ_FTR(B, SZ) \ (((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ)) -#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) -#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) -#define LAST_BLK_HDR_FLG (((UWord) 1) << 2) - -/* Special flag combo for (allocated) SBC blocks -*/ -#define SBC_BLK_HDR_FLG (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) - #define SET_MBC_ABLK_SZ(B, SZ) \ (ASSERT(((SZ) & FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ)) @@ -222,18 +207,7 @@ MBC after deallocating first block: ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->u.carrier = (C)) - -# define ABLK_TO_MBC(B) \ - (ASSERT(IS_MBC_BLK(B) && IS_ALLOCED_BLK(B)), \ - (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \ - (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT)))) - -# define FBLK_TO_MBC(B) \ - (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ - (B)->u.carrier) -# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) - # define IS_MBC_FIRST_ABLK(AP,B) \ ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) @@ -272,10 +246,6 @@ MBC after deallocating first block: (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) -# define BLK_TO_MBC(B) ((B)->carrier) -# define ABLK_TO_MBC(B) BLK_TO_MBC(B) -# define FBLK_TO_MBC(B) BLK_TO_MBC(B) - # define IS_MBC_FIRST_BLK(AP,B) \ ((char*)(B) == (char*)((B)->carrier) + MBC_HEADER_SIZE(AP)) # define IS_MBC_FIRST_ABLK(AP,B) IS_MBC_FIRST_BLK(AP,B) @@ -300,8 +270,6 @@ MBC after deallocating first block: ((B)->bhdr & PREV_FREE_BLK_HDR_FLG) #define IS_PREV_BLK_ALLOCED(B) \ (!IS_PREV_BLK_FREE((B))) -#define IS_FREE_BLK(B) \ - (ASSERT(!IS_SBC_BLK(B)), (B)->bhdr & THIS_FREE_BLK_HDR_FLG) #define IS_ALLOCED_BLK(B) \ (!IS_FREE_BLK((B))) #define IS_LAST_BLK(B) \ @@ -318,13 +286,6 @@ MBC after deallocating first block: #define GET_BLK_HDR_FLGS(B) \ ((B)->bhdr & FLG_MASK) -#define IS_SBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) -#define IS_MBC_BLK(B) \ - (!IS_SBC_BLK((B))) - -#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B)) - #define NXT_BLK(B) \ (ASSERT(IS_MBC_BLK(B)), \ (Block_t *) (((char *) (B)) + MBC_BLK_SZ((B)))) @@ -338,9 +299,96 @@ MBC after deallocating first block: /* Carriers ... */ -#define SBC_HEADER_SIZE (UNIT_CEILING(sizeof(Carrier_t) + ABLK_HDR_SZ) \ - - ABLK_HDR_SZ) -#define MBC_HEADER_SIZE(AP) SBC_HEADER_SIZE +/* #define ERTS_ALC_CPOOL_DEBUG */ + +#if defined(DEBUG) && !defined(ERTS_ALC_CPOOL_DEBUG) +# define ERTS_ALC_CPOOL_DEBUG +#endif + +#ifndef ERTS_SMP +# undef ERTS_ALC_CPOOL_DEBUG +#endif + +#ifdef ERTS_ALC_CPOOL_DEBUG +# define ERTS_ALC_CPOOL_ASSERT(A) \ + ((void) ((A) \ + ? 1 \ + : (erts_alcu_assert_failed(#A, \ + (char *) __FILE__, \ + __LINE__, \ + (char *) __func__), \ + 0))) +#else +# define ERTS_ALC_CPOOL_ASSERT(A) ((void) 1) +#endif + +#ifdef ERTS_SMP +#define ERTS_ALC_IS_CPOOL_ENABLED(A) ((A)->cpool.util_limit) +#else +#define ERTS_ALC_IS_CPOOL_ENABLED(A) (0) +#endif + +#ifdef ERTS_SMP + +#define ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON 1000 +#define ERTS_ALC_CPOOL_ALLOC_OP_INC 8 +#define ERTS_ALC_CPOOL_FREE_OP_DEC 10 + +#define ERTS_ALC_CPOOL_ALLOC_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon < ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) { \ + (A)->cpool.disable_abandon += ERTS_ALC_CPOOL_ALLOC_OP_INC; \ + if ((A)->cpool.disable_abandon > ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON) \ + (A)->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; \ + } \ +} while (0) + + +#if ERTS_ALC_CPOOL_ALLOC_OP_INC >= ERTS_ALC_CPOOL_FREE_OP_DEC +# error "Implementation assume ERTS_ALC_CPOOL_ALLOC_OP_INC < ERTS_ALC_CPOOL_FREE_OP_DEC" +#endif + +#define ERTS_ALC_CPOOL_REALLOC_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon) { \ + (A)->cpool.disable_abandon -= (ERTS_ALC_CPOOL_FREE_OP_DEC \ + - ERTS_ALC_CPOOL_ALLOC_OP_INC); \ + if ((A)->cpool.disable_abandon < 0) \ + (A)->cpool.disable_abandon = 0; \ + } \ +} while (0) + +#define ERTS_ALC_CPOOL_FREE_OP(A) \ +do { \ + if ((A)->cpool.disable_abandon) { \ + (A)->cpool.disable_abandon -= ERTS_ALC_CPOOL_FREE_OP_DEC; \ + if ((A)->cpool.disable_abandon < 0) \ + (A)->cpool.disable_abandon = 0; \ + } \ +} while (0) + +#else +#define ERTS_ALC_CPOOL_ALLOC_OP(A) +#define ERTS_ALC_CPOOL_REALLOC_OP(A) +#define ERTS_ALC_CPOOL_FREE_OP(A) +#endif + +#define ERTS_CRR_ALCTR_FLG_IN_POOL (((erts_aint_t) 1) << 0) +#define ERTS_CRR_ALCTR_FLG_BUSY (((erts_aint_t) 1) << 1) + +#ifdef ERTS_SMP +#define SBC_HEADER_SIZE \ + (UNIT_CEILING(sizeof(Carrier_t) \ + - sizeof(ErtsAlcCPoolData_t) \ + + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) +#else +#define SBC_HEADER_SIZE \ + (UNIT_CEILING(sizeof(Carrier_t) \ + + ABLK_HDR_SZ) \ + - ABLK_HDR_SZ) +#endif +#define MBC_HEADER_SIZE(AP) ((AP)->mbc_header_size) #define MSEG_CARRIER_HDR_FLAG (((UWord) 1) << 0) @@ -352,7 +400,8 @@ MBC after deallocating first block: #define SCH_SBC SBC_CARRIER_HDR_FLAG #define SET_CARRIER_HDR(C, Sz, F, AP) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), (C)->allctr = (AP)) + (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \ + erts_smp_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP))) #define BLK_TO_SBC(B) \ ((Carrier_t *) (((char *) (B)) - SBC_HEADER_SIZE)) @@ -385,6 +434,7 @@ MBC after deallocating first block: #define CFLG_FORCE_SYS_ALLOC (1 << 3) #define CFLG_FORCE_SIZE (1 << 4) #define CFLG_MAIN_CARRIER (1 << 5) +#define CFLG_NO_CPOOL (1 << 6) #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG static void check_blk_carrier(Allctr_t *, Block_t *); @@ -412,11 +462,7 @@ static void check_blk_carrier(Allctr_t *, Block_t *); ASSERT(((AP)->mbcs.curr.norm.sys_alloc.no \ && (AP)->mbcs.curr.norm.sys_alloc.size) \ || (!(AP)->mbcs.curr.norm.sys_alloc.no \ - && !(AP)->mbcs.curr.norm.sys_alloc.size)); \ - ASSERT(((AP)->sbmbcs.curr.small_block.no \ - && (AP)->sbmbcs.curr.small_block.size) \ - || (!(AP)->sbmbcs.curr.small_block.no \ - && !(AP)->sbmbcs.curr.small_block.size)) + && !(AP)->mbcs.curr.norm.sys_alloc.size)); #else #define DEBUG_CHECK_CARRIER_NO_SZ(AP) @@ -487,17 +533,6 @@ do { \ + (AP)->mbcs.curr.norm.sys_alloc.size) -#define STAT_SBMBC_ALLOC(AP, CSZ) \ -do { \ - (AP)->sbmbcs.curr.small_block.no++; \ - (AP)->sbmbcs.curr.small_block.size += (CSZ); \ - if ((AP)->sbmbcs.max.no < (AP)->sbmbcs.curr.small_block.no) \ - (AP)->sbmbcs.max.no = (AP)->sbmbcs.curr.small_block.no; \ - if ((AP)->sbmbcs.max.size < (AP)->sbmbcs.curr.small_block.size) \ - (AP)->sbmbcs.max.size = (AP)->sbmbcs.curr.small_block.size; \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ -} while (0) - #define STAT_MSEG_MBC_ALLOC(AP, CSZ) \ do { \ (AP)->mbcs.curr.norm.mseg.no++; \ @@ -514,13 +549,19 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_SBMBC_FREE(AP, CSZ) \ +#define STAT_MBC_CPOOL_FETCH(AP, CRR) \ do { \ - ASSERT((AP)->sbmbcs.curr.small_block.no > 0); \ - (AP)->sbmbcs.curr.small_block.no--; \ - ASSERT((AP)->sbmbcs.curr.small_block.size >= (CSZ)); \ - (AP)->sbmbcs.curr.small_block.size -= (CSZ); \ - DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) \ + STAT_MSEG_MBC_ALLOC((AP), csz__); \ + else \ + STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \ + (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks; \ + if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \ + (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \ + (AP)->mbcs.blocks.curr.size += (CRR)->cpool.blocks_size; \ + if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \ + (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \ } while (0) #define STAT_MSEG_MBC_FREE(AP, CSZ) \ @@ -541,28 +582,88 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_BLK_ALLOC(AP, BSZ, FLGS) \ +#define STAT_MBC_CPOOL_INSERT(AP, CRR) \ do { \ - CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ - ? &(AP)->sbmbcs \ - : &(AP)->mbcs); \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) \ + STAT_MSEG_MBC_FREE((AP), csz__); \ + else \ + STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \ + >= (CRR)->cpool.blocks); \ + (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks; \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \ + >= (CRR)->cpool.blocks_size); \ + (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size; \ +} while (0) + +#ifdef ERTS_SMP +#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) \ +do { \ + (CRR)->cpool.blocks++; \ + (CRR)->cpool.blocks_size += (BSZ); \ +} while (0) +#else +#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) ((void) (CRR)) /* Get rid of warning */ +#endif + +#define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \ +do { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ cstats__->blocks.curr.no++; \ if (cstats__->blocks.max.no < cstats__->blocks.curr.no) \ cstats__->blocks.max.no = cstats__->blocks.curr.no; \ cstats__->blocks.curr.size += (BSZ); \ if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \ cstats__->blocks.max.size = cstats__->blocks.curr.size; \ + STAT_MBC_BLK_ALLOC_CRR((CRR), (BSZ)); \ } while (0) -#define STAT_MBC_BLK_FREE(AP, BSZ, FLGS) \ +static ERTS_INLINE int +stat_cpool_mbc_blk_free(Allctr_t *allctr, + Carrier_t *crr, + Carrier_t **busy_pcrr_pp, + UWord blksz) +{ +#ifdef ERTS_SMP + + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0); + crr->cpool.blocks--; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size >= blksz); + crr->cpool.blocks_size -= blksz; + + if (!busy_pcrr_pp || !*busy_pcrr_pp) + return 0; + + ERTS_ALC_CPOOL_ASSERT(crr == *busy_pcrr_pp); + +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_dec_read_nob(&allctr->cpool.stat.no_blocks) >= 0); + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_add_read_nob(&allctr->cpool.stat.blocks_size, + -((erts_aint_t) blksz)) >= 0); +#else + erts_atomic_dec_nob(&allctr->cpool.stat.no_blocks); + erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + -((erts_aint_t) blksz)); +#endif + + return 1; +#else + return 0; +#endif +} + +#define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS) \ do { \ - CarriersStats_t *cstats__ = (((FLGS) & ERTS_ALCU_FLG_SBMBC) \ - ? &(AP)->sbmbcs \ - : &(AP)->mbcs); \ - ASSERT(cstats__->blocks.curr.no > 0); \ - cstats__->blocks.curr.no--; \ - ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ - cstats__->blocks.curr.size -= (BSZ); \ + if (!stat_cpool_mbc_blk_free((AP), (CRR), (BPCRRPP), (BSZ))) { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + ASSERT(cstats__->blocks.curr.no > 0); \ + cstats__->blocks.curr.no--; \ + ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ + cstats__->blocks.curr.size -= (BSZ); \ + } \ } while (0) /* Debug stuff... */ @@ -588,17 +689,21 @@ do { \ #ifdef DEBUG #ifdef USE_THREADS +# ifdef ERTS_SMP +# define IS_ACTUALLY_BLOCKING (erts_thr_progress_is_blocking()) +# else +# define IS_ACTUALLY_BLOCKING 0 +# endif #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) \ do { \ - if (!(A)->thread_safe) { \ - if (!(A)->debug.saved_tid) { \ + if (!(A)->thread_safe && !IS_ACTUALLY_BLOCKING) { \ + if (!(A)->debug.saved_tid) { \ (A)->debug.tid = erts_thr_self(); \ (A)->debug.saved_tid = 1; \ } \ else { \ ERTS_SMP_LC_ASSERT( \ - ethr_equal_tids((A)->debug.tid, erts_thr_self()) \ - || erts_thr_progress_is_blocking()); \ + ethr_equal_tids((A)->debug.tid, erts_thr_self())); \ } \ } \ } while (0) @@ -609,13 +714,42 @@ do { \ #define ERTS_ALCU_DBG_CHK_THR_ACCESS(A) #endif - static void make_name_atoms(Allctr_t *allctr); static Block_t *create_carrier(Allctr_t *, Uint, UWord); -static void destroy_carrier(Allctr_t *, Block_t *); -static void mbc_free(Allctr_t *allctr, void *p); +static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **); +static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp); +static void dealloc_block(Allctr_t *, void *, int); + +/* internal data... */ + +#if 0 + +static ERTS_INLINE void * +internal_alloc(UWord size) +{ + void *res = erts_sys_alloc(0, NULL, size); + if (!res) + erts_alloc_enomem(ERTS_ALC_T_UNDEF, size); + return res; +} + +static ERTS_INLINE void * +internal_realloc(void *ptr, UWord size) +{ + void *res = erts_sys_realloc(0, NULL, ptr, size); + if (!res) + erts_alloc_enomem(ERTS_ALC_T_UNDEF, size); + return res; +} +static ERTS_INLINE void +internal_free(void *ptr) +{ + erts_sys_free(0, NULL, ptr); +} + +#endif /* mseg ... */ @@ -791,10 +925,35 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) } } -static Block_t *create_sbmbc(Allctr_t *allctr, Uint umem_sz); -static void destroy_sbmbc(Allctr_t *allctr, Block_t *blk); -static Block_t *create_carrier(Allctr_t *, Uint, UWord); -static void destroy_carrier(Allctr_t *, Block_t *); +#ifdef ERTS_SMP + +static ERTS_INLINE void +clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + if (crr) { + erts_aint_t max_size; + erts_aint_t new_val; + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + new_val = (((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + +#ifdef ERTS_ALC_CPOOL_DEBUG + { + erts_aint_t old_val = new_val|ERTS_CRR_ALCTR_FLG_BUSY; + + ERTS_ALC_CPOOL_ASSERT(old_val + == erts_smp_atomic_xchg_relb(&crr->allctr, + new_val)); + } +#else + erts_smp_atomic_set_relb(&crr->allctr, new_val); +#endif + } +} + +#endif #if 0 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ @@ -816,13 +975,154 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before) #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) #endif -erts_aint32_t -erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +static void *mbc_alloc(Allctr_t *allctr, Uint size); + +#ifdef ERTS_SMP +typedef struct { + ErtsAllctrDDBlock_t ddblock__; /* must be first */ + ErtsAlcType_t fix_type; +} ErtsAllctrFixDDBlock_t; +#endif + +static ERTS_INLINE void +dealloc_fix_block(Allctr_t *allctr, + ErtsAlcType_t type, + void *ptr, + int dec_cc_on_redirect) +{ +#ifdef ERTS_SMP + /* May be redirected... */ + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; +#endif + dealloc_block(allctr, ptr, dec_cc_on_redirect); +} + +static ERTS_INLINE void +sched_fix_shrink(Allctr_t *allctr, int on) +{ + if (on && !allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 1; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 1); + } + else if (!on && allctr->fix_shrink_scheduled) { + allctr->fix_shrink_scheduled = 0; + erts_set_aux_work_timeout(allctr->ix, + (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + 0); + } +} + +static ERTS_INLINE void +fix_cpool_check_shrink(Allctr_t *allctr, + ErtsAlcType_t type, + ErtsAlcFixList_t *fix, + Carrier_t **busy_pcrr_pp) +{ + if (fix->u.cpool.shrink_list > 0) { + if (fix->list_size == 0) + fix->u.cpool.shrink_list = 0; + else { + void *p; +#ifdef ERTS_SMP + if (busy_pcrr_pp) { + clear_busy_pool_carrier(allctr, *busy_pcrr_pp); + *busy_pcrr_pp = NULL; + } +#endif + fix->u.cpool.shrink_list--; + p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + + fix->u.cpool.allocated--; + dealloc_fix_block(allctr, type, p, 0); + } + } +} + +static ERTS_INLINE void * +fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) +{ + void *res; + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + res = fix->list; + if (res) { + fix->list = *((void **) res); + fix->list_size--; + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + fix->u.cpool.used++; + fix_cpool_check_shrink(allctr, type, fix, NULL); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (size >= allctr->sbc_threshold) { + Block_t *blk; + blk = create_carrier(allctr, size, CFLG_SBC); + res = blk ? BLK2UMEM(blk) : NULL; + } + else + res = mbc_alloc(allctr, size); + if (res) { + fix->u.cpool.used++; + fix->u.cpool.allocated++; + } + return res; +} + +static ERTS_INLINE void +fix_cpool_free(Allctr_t *allctr, + ErtsAlcType_t type, + void *p, + Carrier_t **busy_pcrr_pp) +{ + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + fix->u.cpool.used--; + + if ((!busy_pcrr_pp || !*busy_pcrr_pp) + && !fix->u.cpool.shrink_list + && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix->list; + fix->list = p; + fix->list_size++; + sched_fix_shrink(allctr, 1); + } + else { + Block_t *blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, busy_pcrr_pp); + fix->u.cpool.allocated--; + fix_cpool_check_shrink(allctr, type, fix, busy_pcrr_pp); + } +} + +static ERTS_INLINE erts_aint32_t +fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) { int all_empty = 1; erts_aint32_t res = 0; int ix, o; - ErtsAlcFixList_t *fix = allctr->fix; int flush = flgs == 0; #ifdef USE_THREADS @@ -831,56 +1131,204 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) #endif for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + ErtsAlcType_t type; ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { - fix[ix].limit = fix[ix].max_used; - if (fix[ix].limit < fix[ix].used) - fix[ix].limit = fix[ix].used; - fix[ix].max_used = fix[ix].used; - ASSERT(fix[ix].limit >= 0); - - } - if (flush) { - fix[ix].limit = 0; - fix[ix].max_used = fix[ix].used; - ASSERT(fix[ix].limit >= 0); + if (flush) + fix->u.cpool.shrink_list = fix->list_size; + else if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix->u.cpool.shrink_list = fix->u.cpool.min_list_size; + fix->u.cpool.min_list_size = fix->list_size; } + type = (ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE); for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { - Block_t *blk; void *ptr; - if (!flush && fix[ix].limit >= fix[ix].allocated) + if (fix->u.cpool.shrink_list == 0) break; - if (fix[ix].list_size == 0) + if (fix->list_size == 0) { + fix->u.cpool.shrink_list = 0; break; - ptr = fix[ix].list; - fix[ix].list = *((void **) ptr); - fix[ix].list_size--; + } + ptr = fix->list; + fix->list = *((void **) ptr); + fix->list_size--; + fix->u.cpool.shrink_list--; + fix->u.cpool.allocated--; + dealloc_fix_block(allctr, type, ptr, 0); + } + if (fix->u.cpool.min_list_size > fix->list_size) + fix->u.cpool.min_list_size = fix->list_size; + if (fix->list_size != 0) { + if (fix->u.cpool.shrink_list > 0) + res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; + all_empty = 0; + } + } + + if (all_empty) + sched_fix_shrink(allctr, 0); + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_unlock(&allctr->mutex); +#endif + + return res; +} + +static ERTS_INLINE void * +fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) +{ + ErtsAlcFixList_t *fix; + void *res; - blk = UMEM2BLK(ptr); + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix->u.nocpool.used++; + res = fix->list; + if (res) { + fix->list_size--; + fix->list = *((void **) res); + if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) { + Block_t *blk; + void *p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + blk = UMEM2BLK(p); if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, ptr); + mbc_free(allctr, p, NULL); + fix->u.nocpool.allocated--; + } + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return res; + } + if (size < 2*sizeof(UWord)) + size += sizeof(UWord); + if (fix->u.nocpool.limit < fix->u.nocpool.used) + fix->u.nocpool.limit = fix->u.nocpool.used; + if (fix->u.nocpool.max_used < fix->u.nocpool.used) + fix->u.nocpool.max_used = fix->u.nocpool.used; + fix->u.nocpool.allocated++; + + if (size >= allctr->sbc_threshold) { + Block_t *blk; + blk = create_carrier(allctr, size, CFLG_SBC); + res = blk ? BLK2UMEM(blk) : NULL; + } + else + res = mbc_alloc(allctr, size); - fix[ix].allocated--; + if (!res) { + fix->u.nocpool.allocated--; + fix->u.nocpool.used--; + } + return res; +} + +static ERTS_INLINE void +fix_nocpool_free(Allctr_t *allctr, + ErtsAlcType_t type, + void *p) +{ + Block_t *blk; + ErtsAlcFixList_t *fix; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + fix->u.nocpool.used--; + if (fix->u.nocpool.allocated < fix->u.nocpool.limit + && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { + *((void **) p) = fix->list; + fix->list = p; + fix->list_size++; + sched_fix_shrink(allctr, 1); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + return; + } + fix->u.nocpool.allocated--; + if (fix->list && fix->u.nocpool.allocated > fix->u.nocpool.limit) { + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, NULL); + p = fix->list; + fix->list = *((void **) p); + fix->list_size--; + fix->u.nocpool.allocated--; + } + + blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, NULL); + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); +} + +static ERTS_INLINE erts_aint32_t +fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + int all_empty = 1; + erts_aint32_t res = 0; + int ix, o; + int flush = flgs == 0; + +#ifdef USE_THREADS + if (allctr->thread_safe) + erts_mtx_lock(&allctr->mutex); +#endif + + for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); + if (flgs & ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM) { + fix->u.nocpool.limit = fix->u.nocpool.max_used; + if (fix->u.nocpool.limit < fix->u.nocpool.used) + fix->u.nocpool.limit = fix->u.nocpool.used; + fix->u.nocpool.max_used = fix->u.nocpool.used; + ASSERT(fix->u.nocpool.limit >= 0); + + } + if (flush) { + fix->u.nocpool.limit = 0; + fix->u.nocpool.max_used = fix->u.nocpool.used; + ASSERT(fix->u.nocpool.limit >= 0); } - if (fix[ix].list_size != 0) { - if (fix[ix].limit < fix[ix].allocated) + for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { + void *ptr; + + if (!flush && fix->u.nocpool.limit >= fix->u.nocpool.allocated) + break; + if (fix->list_size == 0) + break; + ptr = fix->list; + fix->list = *((void **) ptr); + fix->list_size--; + dealloc_block(allctr, ptr, 0); + fix->u.nocpool.allocated--; + } + if (fix->list_size != 0) { + if (fix->u.nocpool.limit < fix->u.nocpool.allocated) res |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; all_empty = 0; } ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } - if (all_empty && allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 0; - erts_set_aux_work_timeout(allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 0); - } + if (all_empty) + sched_fix_shrink(allctr, 0); #ifdef USE_THREADS if (allctr->thread_safe) @@ -890,11 +1338,18 @@ erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) return res; } -#ifdef ERTS_SMP +erts_aint32_t +erts_alcu_fix_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) +{ + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return fix_cpool_alloc_shrink(allctr, flgs); + else + return fix_nocpool_alloc_shrink(allctr, flgs); +} -#define ERTS_ALCU_DD_FIX_TYPE_OFFS \ - ((sizeof(ErtsAllctrDDBlock_t)-1)/sizeof(UWord) + 1) +static void dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned); +#ifdef ERTS_SMP static ERTS_INLINE Allctr_t* get_pref_allctr(void *extra) @@ -910,6 +1365,9 @@ get_pref_allctr(void *extra) return tspec->allctr[pref_ix]; } +#define ERTS_ALC_TS_PREF_LOCK_IF_USED (1) +#define ERTS_ALC_TS_PREF_LOCK_NO (0) + /* SMP note: * get_used_allctr() must be safe WITHOUT locking the allocator while * concurrent threads may be updating adjacent blocks. @@ -918,22 +1376,85 @@ get_pref_allctr(void *extra) * the "PREV_FREE" flag bit. */ static ERTS_INLINE Allctr_t* -get_used_allctr(void *extra, void *p, UWord *sizep) +get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, + Carrier_t **busy_pcrr_pp) { Block_t* blk = UMEM2BLK(p); - Carrier_t* crr; + Carrier_t *crr; + erts_aint_t iallctr; + Allctr_t *used_allctr; + + *busy_pcrr_pp = NULL; if (IS_SBC_BLK(blk)) { crr = BLK_TO_SBC(blk); if (sizep) *sizep = SBC_BLK_SZ(blk) - ABLK_HDR_SZ; + iallctr = erts_smp_atomic_read_dirty(&crr->allctr); } else { crr = ABLK_TO_MBC(blk); + if (sizep) *sizep = MBC_ABLK_SZ(blk) - ABLK_HDR_SZ; + if (!ERTS_ALC_IS_CPOOL_ENABLED(pref_allctr)) + iallctr = erts_smp_atomic_read_dirty(&crr->allctr); + else { + int locked_pref_allctr = 0; + iallctr = erts_smp_atomic_read_ddrb(&crr->allctr); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock + && pref_allctr->thread_safe) { + used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + if (pref_allctr == used_allctr) { + erts_mtx_lock(&pref_allctr->mutex); + locked_pref_allctr = 1; + } + } + + while ((iallctr & ((~FLG_MASK)|ERTS_CRR_ALCTR_FLG_IN_POOL)) + == (((erts_aint_t) pref_allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL)) { + erts_aint_t act; + + ERTS_ALC_CPOOL_ASSERT(!(iallctr & ERTS_CRR_ALCTR_FLG_BUSY)); + act = erts_smp_atomic_cmpxchg_ddrb(&crr->allctr, + iallctr|ERTS_CRR_ALCTR_FLG_BUSY, + iallctr); + if (act == iallctr) { + *busy_pcrr_pp = crr; + break; + } + iallctr = act; + } + + used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock) { + if (locked_pref_allctr && used_allctr != pref_allctr) { + /* Was taken out of pool; now owned by someone else */ + erts_mtx_unlock(&pref_allctr->mutex); + } + } + + ERTS_ALC_CPOOL_ASSERT( + (((iallctr & ~FLG_MASK) == (erts_aint_t) pref_allctr) + ? (((iallctr & FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL) + || ((iallctr & FLG_MASK) == 0)) + : 1)); + + return used_allctr; + } } - return crr->allctr; + + used_allctr = (Allctr_t *) (iallctr & ~FLG_MASK); + + if (ERTS_ALC_TS_PREF_LOCK_IF_USED == pref_lock + && used_allctr == pref_allctr + && pref_allctr->thread_safe) { + erts_mtx_lock(&pref_allctr->mutex); + } + + return used_allctr; } static void @@ -1028,7 +1549,7 @@ check_insert_marker(ErtsAllctrDDQueue_t *ddq, erts_aint_t ilast) } static ERTS_INLINE int -ddq_enqueue(ErtsAlcType_t type, ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) +ddq_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) { int last_elem; int um_refc_ix = 0; @@ -1129,6 +1650,59 @@ store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq) } } +static void +check_pending_dealloc_carrier(Allctr_t *allctr, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work); + +static void +handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) +{ + ErtsAlcType_t type; + + type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; + + ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type + && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + fix_nocpool_free(allctr, type, ptr); + else { + Block_t *blk = UMEM2BLK(ptr); + Carrier_t *busy_pcrr_p; + Allctr_t *used_allctr; + + if (IS_SBC_BLK(blk)) { + busy_pcrr_p = NULL; + goto doit; + } + + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, + NULL, &busy_pcrr_p); + if (used_allctr == allctr) { + doit: + fix_cpool_free(allctr, type, ptr, &busy_pcrr_p); + clear_busy_pool_carrier(allctr, busy_pcrr_p); + } + else { + /* Carrier migrated; need to redirect block to new owner... */ + int cinit = used_allctr->dd.ix - allctr->dd.ix; + + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + + DEC_CC(allctr->calls.this_free); + + ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) + erts_alloc_notify_delayed_dealloc(used_allctr->ix); + } + } +} + +static void +schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr); + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, int allctr_locked, @@ -1160,7 +1734,6 @@ handle_delayed_dealloc(Allctr_t *allctr, while (1) { Block_t *blk; void *ptr; - int ix; if (use_limit && ++ops > ops_limit) { if (ddq->head.first != ddq->head.unref_end) { @@ -1189,52 +1762,36 @@ handle_delayed_dealloc(Allctr_t *allctr, res = 1; - INC_CC(allctr->calls.this_free); + blk = UMEM2BLK(ptr); + if (IS_FREE_LAST_MBC_BLK(blk)) { + /* + * A multiblock carrier that previously has been migrated away + * from us and now is back to be deallocated. For more info + * see schedule_dealloc_carrier(). + * + * Note that we cannot use FBLK_TO_MBC(blk) since it + * data has been overwritten by the queue. + */ + Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); + ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); + ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) + != (erts_smp_atomic_read_nob(&crr->allctr) + & ~FLG_MASK)); - if (fix) { - ErtsAlcType_t type; - - type = (ErtsAlcType_t) ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS]; - ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used--; - if (fix[ix].allocated < fix[ix].limit - && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { - *((void **) ptr) = fix[ix].list; - fix[ix].list = ptr; - fix[ix].list_size++; - if (!allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 1; - erts_set_aux_work_timeout( - allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 1); - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - continue; - } - fix[ix].allocated--; - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - blk = UMEM2BLK(ptr); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, ptr); - ptr = fix[ix].list; - fix[ix].list = *((void **) ptr); - fix[ix].list_size--; - fix[ix].allocated--; - } + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + + schedule_dealloc_carrier(allctr, crr); } + else { - blk = UMEM2BLK(ptr); + INC_CC(allctr->calls.this_free); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, ptr); - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); + if (fix) + handle_delayed_fix_dealloc(allctr, ptr); + else + dealloc_block(allctr, ptr, 1); + } } if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { @@ -1244,6 +1801,12 @@ handle_delayed_dealloc(Allctr_t *allctr, store_earliest_thr_prgr(thr_prgr_p, ddq); } + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + check_pending_dealloc_carrier(allctr, + need_thr_progress, + thr_prgr_p, + need_more_work); + if (allctr->thread_safe && !allctr_locked) erts_mtx_unlock(&allctr->mutex); return res; @@ -1256,15 +1819,61 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, int cinit) { if (allctr->fix) - ((UWord *) ptr)[ERTS_ALCU_DD_FIX_TYPE_OFFS] = (UWord) type; + ((ErtsAllctrFixDDBlock_t*) ptr)->fix_type = type; - if (ddq_enqueue(type, &allctr->dd.q, ptr, cinit)) + if (ddq_enqueue(&allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(allctr->ix); } #endif #ifdef ERTS_SMP +static void +set_new_allctr_abandon_limit(Allctr_t *allctr); +static void +abandon_carrier(Allctr_t *allctr, Carrier_t *crr); + + +static ERTS_INLINE void +check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) +{ + Carrier_t *crr; + + if (busy_pcrr_pp && *busy_pcrr_pp) + return; + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return; + + allctr->cpool.check_limit_count--; + if (--allctr->cpool.check_limit_count <= 0) + set_new_allctr_abandon_limit(allctr); + + if (!erts_thr_progress_is_managed_thread()) + return; + + if (allctr->cpool.disable_abandon) + return; + + if (allctr->mbcs.blocks.curr.size > allctr->cpool.abandon_limit) + return; + + + crr = FBLK_TO_MBC(fblk); + + if (allctr->main_carrier == crr) + return; + + if (crr->cpool.blocks_size > crr->cpool.abandon_limit) + return; + + if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID + && !erts_thr_progress_has_reached(crr->cpool.thr_prgr)) + return; + + abandon_carrier(allctr, crr); +} + void erts_alcu_check_delayed_dealloc(Allctr_t *allctr, int limit, @@ -1282,78 +1891,88 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr, } #endif -#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ - handle_delayed_dealloc((Allctr), (Locked), 1, \ +#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ + handle_delayed_dealloc((Allctr), (Locked), 1, \ ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) +static void +dealloc_block(Allctr_t *allctr, void *ptr, int dec_cc_on_redirect) +{ + Block_t *blk = UMEM2BLK(ptr); + + ERTS_SMP_LC_ASSERT(!allctr->thread_safe + || erts_lc_mtx_is_locked(&allctr->mutex)); + + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); +#ifndef ERTS_SMP + else + mbc_free(allctr, ptr, NULL); +#else + else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbc_free(allctr, ptr, NULL); + else { + Carrier_t *busy_pcrr_p; + Allctr_t *used_allctr; + used_allctr = get_used_allctr(allctr, ERTS_ALC_TS_PREF_LOCK_NO, ptr, + NULL, &busy_pcrr_p); + if (used_allctr == allctr) { + mbc_free(allctr, ptr, &busy_pcrr_p); + clear_busy_pool_carrier(allctr, busy_pcrr_p); + } + else { + /* Carrier migrated; need to redirect block to new owner... */ + int cinit = used_allctr->dd.ix - allctr->dd.ix; + + ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); + + if (dec_cc_on_redirect) + DEC_CC(allctr->calls.this_free); + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) + erts_alloc_notify_delayed_dealloc(used_allctr->ix); + } + } +#endif +} + /* Multi block carrier alloc/realloc/free ... */ /* NOTE! mbc_alloc() may in case of memory shortage place the requested * block in a sbc. */ static ERTS_INLINE void * -mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp, Uint32 *alcu_flgsp) +mbc_alloc_block(Allctr_t *allctr, Uint size, Uint *blk_szp) { Block_t *blk; Uint get_blk_sz; - Uint sbmbct; ASSERT(size); ASSERT(size < allctr->sbc_threshold); *blk_szp = get_blk_sz = UMEMSZ2BLKSZ(allctr, size); - sbmbct = allctr->sbmbc_threshold; - if (sbmbct) { - if (get_blk_sz < sbmbct) { - *alcu_flgsp |= ERTS_ALCU_FLG_SBMBC; - if (get_blk_sz + allctr->min_block_size > sbmbct) { - /* Since we use block size to determine if blocks are - located in sbmbc or not... */ - get_blk_sz += allctr->min_block_size; - } - } - } - -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif - - blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, *alcu_flgsp); - -#ifdef ERTS_SMP - if (!blk && allctr->dd.use) { - if (ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1)) - blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0, - *alcu_flgsp); - } -#endif + blk = (*allctr->get_free_block)(allctr, get_blk_sz, NULL, 0); if (!blk) { - if ((*alcu_flgsp) & ERTS_ALCU_FLG_SBMBC) - blk = create_sbmbc(allctr, get_blk_sz); - else { - blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); + blk = create_carrier(allctr, get_blk_sz, CFLG_MBC); #if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY - if (!blk) { - /* Emergency! We couldn't create the carrier as we wanted. - Try to place it in a sys_alloced sbc. */ - blk = create_carrier(allctr, - size, - (CFLG_SBC - | CFLG_FORCE_SIZE - | CFLG_FORCE_SYS_ALLOC)); - } -#endif + if (!blk) { + /* Emergency! We couldn't create the carrier as we wanted. + Try to place it in a sys_alloced sbc. */ + blk = create_carrier(allctr, + size, + (CFLG_SBC + | CFLG_FORCE_SIZE + | CFLG_FORCE_SYS_ALLOC)); } +#endif } #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG if (IS_MBC_BLK(blk)) { - (*allctr->link_free_block)(allctr, blk, *alcu_flgsp); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, *alcu_flgsp); + (*allctr->unlink_free_block)(allctr, blk); } #endif @@ -1367,8 +1986,7 @@ mbc_alloc_finalize(Allctr_t *allctr, UWord flags, Carrier_t *crr, Uint want_blk_sz, - int valid_blk_info, - Uint32 alcu_flgs) + int valid_blk_info) { Uint blk_sz; Uint nxt_blk_sz; @@ -1400,7 +2018,7 @@ mbc_alloc_finalize(Allctr_t *allctr, SET_PREV_BLK_FREE(allctr, nxt_nxt_blk); } } - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_NOT_LAST_BLK(blk)); ASSERT(IS_FREE_BLK(nxt_blk)); @@ -1441,7 +2059,8 @@ mbc_alloc_finalize(Allctr_t *allctr, ASSERT(ABLK_TO_MBC(blk) == crr); } - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); + ERTS_ALC_CPOOL_ALLOC_OP(allctr); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == MBC_BLK_SZ(blk)); @@ -1461,8 +2080,7 @@ mbc_alloc(Allctr_t *allctr, Uint size) { Block_t *blk; Uint blk_sz; - Uint32 alcu_flgs = 0; - blk = mbc_alloc_block(allctr, size, &blk_sz, &alcu_flgs); + blk = mbc_alloc_block(allctr, size, &blk_sz); if (!blk) return NULL; if (IS_MBC_BLK(blk)) @@ -1472,35 +2090,34 @@ mbc_alloc(Allctr_t *allctr, Uint size) GET_BLK_HDR_FLGS(blk), FBLK_TO_MBC(blk), blk_sz, - 1, - alcu_flgs); + 1); return BLK2UMEM(blk); } static void -mbc_free(Allctr_t *allctr, void *p) +mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) { Uint is_first_blk; Uint is_last_blk; - Uint32 alcu_flgs = 0; Uint blk_sz; Block_t *blk; Block_t *nxt_blk; - + Carrier_t *crr; ASSERT(p); blk = UMEM2BLK(p); blk_sz = MBC_ABLK_SZ(blk); - if (blk_sz < allctr->sbmbc_threshold) - alcu_flgs |= ERTS_ALCU_FLG_SBMBC; ASSERT(IS_MBC_BLK(blk)); ASSERT(blk_sz >= allctr->min_block_size); HARD_CHECK_BLK_CARRIER(allctr, blk); - STAT_MBC_BLK_FREE(allctr, blk_sz, alcu_flgs); + crr = ABLK_TO_MBC(blk); + + ERTS_ALC_CPOOL_FREE_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, busy_pcrr_pp, blk_sz, alcu_flgs); is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk); is_last_blk = IS_LAST_BLK(blk); @@ -1509,7 +2126,7 @@ mbc_free(Allctr_t *allctr, void *p) ASSERT(!is_first_blk); /* Coalesce with previous block... */ blk = PREV_BLK(blk); - (*allctr->unlink_free_block)(allctr, blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, blk); blk_sz += MBC_FBLK_SZ(blk); is_first_blk = IS_MBC_FIRST_FBLK(allctr, blk); @@ -1525,7 +2142,7 @@ mbc_free(Allctr_t *allctr, void *p) nxt_blk = BLK_AFTER(blk, blk_sz); if (IS_FREE_BLK(nxt_blk)) { /* Coalesce with next block... */ - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); blk_sz += MBC_FBLK_SZ(nxt_blk); SET_MBC_FBLK_SZ(blk, blk_sz); @@ -1558,19 +2175,20 @@ mbc_free(Allctr_t *allctr, void *p) if (is_first_blk && is_last_blk && allctr->main_carrier != FIRST_BLK_TO_MBC(allctr, blk)) { - if (alcu_flgs & ERTS_ALCU_FLG_SBMBC) - destroy_sbmbc(allctr, blk); - else - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, busy_pcrr_pp); } else { - (*allctr->link_free_block)(allctr, blk, alcu_flgs); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); +#ifdef ERTS_SMP + check_abandon_carrier(allctr, blk, busy_pcrr_pp); +#endif } } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) +mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, + Carrier_t **busy_pcrr_pp) { void *new_p; Uint old_blk_sz; @@ -1584,11 +2202,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) Uint is_last_blk; #endif /* #ifndef MBC_REALLOC_ALWAYS_MOVES */ -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif - ASSERT(p); ASSERT(size); ASSERT(size < allctr->sbc_threshold); @@ -1602,13 +2215,13 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (alcu_flgs & ERTS_ALCU_FLG_FAIL_REALLOC_MOVE) return NULL; #else /* !MBC_REALLOC_ALWAYS_MOVES */ + +#ifdef ERTS_SMP + if (busy_pcrr_pp && *busy_pcrr_pp) + goto realloc_move; /* Don't want to use carrier in pool */ +#endif + get_blk_sz = blk_sz = UMEMSZ2BLKSZ(allctr, size); - if ((alcu_flgs & ERTS_ALCU_FLG_SBMBC) - && (blk_sz + allctr->min_block_size > allctr->sbmbc_threshold)) { - /* Since we use block size to determine if blocks are - located in sbmbc or not... */ - get_blk_sz = blk_sz + allctr->min_block_size; - } ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(IS_MBC_BLK(blk)); @@ -1656,8 +2269,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk = (*allctr->get_free_block)(allctr, get_blk_sz, cand_blk, - cand_blk_sz, - alcu_flgs); + cand_blk_sz); if (new_blk || cand_blk != blk) goto move_into_new_blk; } @@ -1679,8 +2291,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk = BLK_AFTER(blk, blk_sz); - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); + crr = ABLK_TO_MBC(blk); + + ERTS_ALC_CPOOL_REALLOC_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size); @@ -1688,7 +2303,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (IS_FREE_BLK(nxt_nxt_blk)) { /* Coalesce with next free block... */ nxt_blk_sz += MBC_FBLK_SZ(nxt_nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_nxt_blk); is_last_blk = GET_LAST_BLK_HDR_FLG(nxt_nxt_blk); } @@ -1698,12 +2313,11 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); } - crr = ABLK_TO_MBC(blk); SET_MBC_FBLK_HDR(nxt_blk, nxt_blk_sz, SBH_THIS_FREE | (is_last_blk ? SBH_LAST_BLK : 0), crr); - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_ALLOCED_BLK(blk)); @@ -1726,6 +2340,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); +#ifdef ERTS_SMP + check_abandon_carrier(allctr, nxt_blk, NULL); +#endif + return p; } @@ -1735,11 +2353,12 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) nxt_blk = BLK_AFTER(blk, old_blk_sz); nxt_blk_sz = MBC_BLK_SZ(nxt_blk); if (IS_FREE_BLK(nxt_blk) && get_blk_sz <= old_blk_sz + nxt_blk_sz) { + Carrier_t* crr = ABLK_TO_MBC(blk); /* Grow into next block... */ HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); nxt_blk_sz -= blk_sz - old_blk_sz; is_last_blk = IS_LAST_BLK(nxt_blk); @@ -1764,7 +2383,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) } } else { - Carrier_t* crr = ABLK_TO_MBC(blk); SET_MBC_ABLK_SZ(blk, blk_sz); nxt_blk = BLK_AFTER(blk, blk_sz); @@ -1775,15 +2393,15 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) else SET_BLK_SZ_FTR(nxt_blk, nxt_blk_sz); - (*allctr->link_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->link_free_block)(allctr, nxt_blk); ASSERT(IS_FREE_BLK(nxt_blk)); ASSERT(FBLK_TO_MBC(nxt_blk) == crr); } - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); - STAT_MBC_BLK_ALLOC(allctr, blk_sz, alcu_flgs); - + ERTS_ALC_CPOOL_REALLOC_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); ASSERT(blk_sz == MBC_BLK_SZ(blk)); @@ -1836,13 +2454,16 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (cand_blk_sz < get_blk_sz) { /* We wont fit in cand_blk get a new one */ +#ifdef ERTS_SMP + realloc_move: +#endif #endif /* !MBC_REALLOC_ALWAYS_MOVES */ new_p = mbc_alloc(allctr, size); if (!new_p) return NULL; sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p); + mbc_free(allctr, p, busy_pcrr_pp); return new_p; @@ -1855,8 +2476,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk = (*allctr->get_free_block)(allctr, get_blk_sz, cand_blk, - cand_blk_sz, - alcu_flgs); + cand_blk_sz); move_into_new_blk: /* * new_blk, and cand_blk have to be correctly set @@ -1870,11 +2490,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) GET_BLK_HDR_FLGS(new_blk), FBLK_TO_MBC(new_blk), blk_sz, - 1, - alcu_flgs); + 1); new_p = BLK2UMEM(new_blk); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p); + mbc_free(allctr, p, NULL); return new_p; } else { @@ -1894,7 +2513,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, new_blk, alcu_flgs); /* prev */ + (*allctr->unlink_free_block)(allctr, new_blk); /* prev */ if (is_last_blk) new_blk_flgs |= LAST_BLK_HDR_FLG; @@ -1903,7 +2522,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) if (IS_FREE_BLK(nxt_blk)) { new_blk_flgs |= GET_LAST_BLK_HDR_FLG(nxt_blk); new_blk_sz += MBC_FBLK_SZ(nxt_blk); - (*allctr->unlink_free_block)(allctr, nxt_blk, alcu_flgs); + (*allctr->unlink_free_block)(allctr, nxt_blk); } } @@ -1928,10 +2547,10 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) new_blk_flgs, crr, blk_sz, - 0, - alcu_flgs); + 0); - STAT_MBC_BLK_FREE(allctr, old_blk_sz, alcu_flgs); + ERTS_ALC_CPOOL_FREE_OP(allctr); + STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); return new_p; } @@ -1939,6 +2558,647 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs) #endif /* !MBC_REALLOC_ALWAYS_MOVES */ } +#ifdef ERTS_SMP + +#define ERTS_ALC_MAX_DEALLOC_CARRIER 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 10 +#define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 +#define ERTS_ALC_CPOOL_MAX_NO_CARRIERS 5 +#define ERTS_ALC_CPOOL_INSERT_ALLOWED_OFFSET 100 +#define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 + +#define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0) +#define ERTS_ALC_CPOOL_PTR_DEL_MRK (((erts_aint_t) 1) << 1) + +#define ERTS_ALC_CPOOL_PTR_MRKS \ + (ERTS_ALC_CPOOL_PTR_MOD_MRK | ERTS_ALC_CPOOL_PTR_DEL_MRK) + +/* + * When setting multiple mod markers we always + * set mod markers in pointer order and always + * on next pointers before prev pointers. + */ + +typedef union { + ErtsAlcCPoolData_t sentinel; + char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAlcCPoolData_t))]; +} ErtsAlcCrrPool_t; + +#if ERTS_ALC_A_INVALID != 0 +# error "Carrier pool implementation assumes ERTS_ALC_A_INVALID == 0" +#endif +#if ERTS_ALC_A_MIN <= ERTS_ALC_A_INVALID +# error "Carrier pool implementation assumes ERTS_ALC_A_MIN > ERTS_ALC_A_INVALID" +#endif + +/* + * The pool is only allowed to be manipulated by managed + * threads except in the alloc_SUITE:cpool case. In this + * test case carrier_pool[ERTS_ALC_A_INVALID] will be + * used. + */ + +static ErtsAlcCrrPool_t carrier_pool[ERTS_ALC_A_MAX+1] erts_align_attribute(ERTS_CACHE_LINE_SIZE); + +#define ERTS_ALC_CPOOL_MAX_BACKOFF (1 << 8) + +static int +backoff(int n) +{ + int i; + + for (i = 0; i < n; i++) + ERTS_SPIN_BODY; + + if (n >= ERTS_ALC_CPOOL_MAX_BACKOFF) + return ERTS_ALC_CPOOL_MAX_BACKOFF; + else + return n << 1; +} + +static int +cpool_dbg_is_in_pool(Allctr_t *allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *cpdp = sentinel; + Carrier_t *tmp_crr; + + while (1) { + cpdp = (ErtsAlcCPoolData_t *) (erts_atomic_read_ddrb(&cpdp->next) & ~FLG_MASK); + if (cpdp == sentinel) + return 0; + tmp_crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + if (tmp_crr == crr) + return 1; + } +} + +static int +cpool_is_empty(Allctr_t *allctr) +{ + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + return ((erts_atomic_read_rb(&sentinel->next) == (erts_aint_t) sentinel) + && (erts_atomic_read_rb(&sentinel->prev) == (erts_aint_t) sentinel)); +} + +static ERTS_INLINE ErtsAlcCPoolData_t * +cpool_aint2cpd(erts_aint_t aint) +{ + return (ErtsAlcCPoolData_t *) (aint & ~ERTS_ALC_CPOOL_PTR_MRKS); +} + +static ERTS_INLINE erts_aint_t +cpool_read(erts_atomic_t *aptr) +{ + return erts_atomic_read_acqb(aptr); +} + +static ERTS_INLINE void +cpool_init(erts_atomic_t *aptr, erts_aint_t val) +{ + erts_atomic_set_nob(aptr, val); +} + +static ERTS_INLINE void +cpool_set_mod_marked(erts_atomic_t *aptr, erts_aint_t new, erts_aint_t old) +{ +#ifdef ERTS_ALC_CPOOL_DEBUG + erts_aint_t act = erts_atomic_xchg_relb(aptr, new); + ERTS_ALC_CPOOL_ASSERT(act == (old | ERTS_ALC_CPOOL_PTR_MOD_MRK)); +#else + erts_atomic_set_relb(aptr, new); +#endif +} + + +static ERTS_INLINE erts_aint_t +cpool_try_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp) +{ + ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0); + return erts_atomic_cmpxchg_nob(aptr, exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, exp); +} + +static ERTS_INLINE erts_aint_t +cpool_mod_mark_exp(erts_atomic_t *aptr, erts_aint_t exp) +{ + int b; + erts_aint_t act; + ERTS_ALC_CPOOL_ASSERT((exp & ERTS_ALC_CPOOL_PTR_MOD_MRK) == 0); + while (1) { + act = erts_atomic_cmpxchg_nob(aptr, + exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, + exp); + if (act == exp) + return exp; + b = 1; + do { + if ((act & ~ERTS_ALC_CPOOL_PTR_MOD_MRK) != exp) + return act; + b = backoff(b); + act = erts_atomic_read_nob(aptr); + } while (act != exp); + } +} + +static ERTS_INLINE erts_aint_t +cpool_mod_mark(erts_atomic_t *aptr) +{ + int b; + erts_aint_t act, exp; + act = cpool_read(aptr); + while (1) { + b = 1; + while (act & ERTS_ALC_CPOOL_PTR_MOD_MRK) { + b = backoff(b); + act = erts_atomic_read_nob(aptr); + } + exp = act; + act = erts_atomic_cmpxchg_acqb(aptr, + exp | ERTS_ALC_CPOOL_PTR_MOD_MRK, + exp); + if (act == exp) + return exp; + } +} + +static void +cpool_insert(Allctr_t *allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *cpd1p, *cpd2p; + erts_aint_t val; + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + == (erts_aint_t) allctr); + + erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + (erts_aint_t) crr->cpool.blocks_size); + erts_atomic_add_nob(&allctr->cpool.stat.no_blocks, + (erts_aint_t) crr->cpool.blocks); + erts_atomic_add_nob(&allctr->cpool.stat.carriers_size, + (erts_aint_t) CARRIER_SZ(crr)); + erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); + + erts_smp_atomic_set_nob(&crr->allctr, + ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); + + /* + * We search in 'next' direction and begin by passing + * one element before trying to insert. This in order to + * avoid contention with threads fetching elements. + */ + + val = cpool_read(&sentinel->next); + + /* Find a predecessor to be, and set mod marker on its next ptr */ + + while (1) { + cpd1p = cpool_aint2cpd(val); + if (cpd1p == sentinel) { + val = cpool_mod_mark(&cpd1p->next); + break; + } + val = cpool_read(&cpd1p->next); + if (!(val & ERTS_ALC_CPOOL_PTR_MRKS)) { + erts_aint_t tmp = cpool_try_mod_mark_exp(&cpd1p->next, val); + if (tmp == val) { + val = tmp; + break; + } + val = tmp; + } + } + + /* Set mod marker on prev ptr of the to be successor */ + + cpd2p = cpool_aint2cpd(val); + + cpool_init(&crr->cpool.next, (erts_aint_t) cpd2p); + cpool_init(&crr->cpool.prev, (erts_aint_t) cpd1p); + + val = (erts_aint_t) cpd1p; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&cpd2p->prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Write pointers to this element in successor and predecessor */ + + cpool_set_mod_marked(&cpd1p->next, + (erts_aint_t) &crr->cpool, + (erts_aint_t) cpd2p); + cpool_set_mod_marked(&cpd2p->prev, + (erts_aint_t) &crr->cpool, + (erts_aint_t) cpd1p); +} + +static void +cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) +{ + ErtsAlcCPoolData_t *cpd1p, *cpd2p; + erts_aint_t val; +#ifdef ERTS_ALC_CPOOL_DEBUG + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; +#endif + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + ERTS_ALC_CPOOL_ASSERT(sentinel != &crr->cpool); + + /* Set mod marker on next ptr of our predecessor */ + + val = (erts_aint_t) &crr->cpool; + while (1) { + erts_aint_t tmp; + cpd1p = cpool_aint2cpd(cpool_read(&crr->cpool.prev)); + tmp = cpool_mod_mark_exp(&cpd1p->next, val); + if (tmp == val) + break; + } + + /* Set mod marker on our next ptr */ + + val = cpool_mod_mark(&crr->cpool.next); + + /* Set mod marker on the prev ptr of our successor */ + + cpd2p = cpool_aint2cpd(val); + + val = (erts_aint_t) &crr->cpool; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&cpd2p->prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Set mod marker on our prev ptr */ + + val = (erts_aint_t) cpd1p; + + while (1) { + int b; + erts_aint_t tmp; + + tmp = cpool_mod_mark_exp(&crr->cpool.prev, val); + if (tmp == val) + break; + b = 1; + do { + b = backoff(b); + tmp = cpool_read(&cpd2p->prev); + } while (tmp != val); + } + + /* Write pointers past this element in predecessor and successor */ + + cpool_set_mod_marked(&cpd1p->next, + (erts_aint_t) cpd2p, + (erts_aint_t) &crr->cpool); + cpool_set_mod_marked(&cpd2p->prev, + (erts_aint_t) cpd1p, + (erts_aint_t) &crr->cpool); + + /* Repleace mod markers with delete markers on this element */ + cpool_set_mod_marked(&crr->cpool.next, + ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_DEL_MRK, + ((erts_aint_t) cpd2p) | ERTS_ALC_CPOOL_PTR_MOD_MRK); + cpool_set_mod_marked(&crr->cpool.prev, + ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_DEL_MRK, + ((erts_aint_t) cpd1p) | ERTS_ALC_CPOOL_PTR_MOD_MRK); + + crr->cpool.thr_prgr = erts_thr_progress_later(NULL); + + erts_atomic_add_nob(&prev_allctr->cpool.stat.blocks_size, + -((erts_aint_t) crr->cpool.blocks_size)); + erts_atomic_add_nob(&prev_allctr->cpool.stat.no_blocks, + -((erts_aint_t) crr->cpool.blocks)); + erts_atomic_add_nob(&prev_allctr->cpool.stat.carriers_size, + -((erts_aint_t) CARRIER_SZ(crr))); + erts_atomic_dec_wb(&prev_allctr->cpool.stat.no_carriers); + +} + +static Carrier_t * +cpool_fetch(Allctr_t *allctr, UWord size) +{ + int i; + Carrier_t *crr; + ErtsAlcCPoolData_t *cpdp; + ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + || erts_thr_progress_is_managed_thread()); + + i = 0; + + /* First; check our own pending dealloc carrier list... */ + crr = allctr->cpool.dc_list.last; + while (crr && i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { + if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + unlink_carrier(&allctr->cpool.dc_list, crr); +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, + ((erts_aint_t) allctr)) + == (((erts_aint_t) allctr) & ~FLG_MASK)); +#else + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); +#endif + return crr; + } + crr = crr->prev; + i++; + } + + /* ... then the pool ... */ + + /* + * We search in 'prev' direction and begin by passing + * one element before trying to fetch. This in order to + * avoid contention with threads inserting elements. + */ + + cpdp = cpool_aint2cpd(cpool_read(&sentinel->prev)); + if (cpdp == sentinel) + return NULL; + + while (i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { + erts_aint_t exp; + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) { + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) + return NULL; + i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; /* Last one to inspect */ + } + crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + exp = erts_smp_atomic_read_rb(&crr->allctr); + if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL|ERTS_CRR_ALCTR_FLG_BUSY)) + == ERTS_CRR_ALCTR_FLG_IN_POOL) + && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { + erts_aint_t act; + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~FLG_MASK)), crr); + return crr; + } + } + i++; + } + return NULL; +} + +static void +check_pending_dealloc_carrier(Allctr_t *allctr, + int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, + int *need_more_work) +{ + Carrier_t *crr = allctr->cpool.dc_list.first; + + if (crr) { + ErtsThrPrgrVal current = erts_thr_progress_current(); + int i = 0; + + do { + Carrier_t *dcrr; + + if (!erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr)) + break; + + dcrr = crr; + crr = crr->next; + dealloc_carrier(allctr, dcrr, 1); + i++; + } while (crr && i < ERTS_ALC_MAX_DEALLOC_CARRIER); + + allctr->cpool.dc_list.first = crr; + if (!crr) + allctr->cpool.dc_list.last = NULL; + else { + crr->prev = NULL; + + if (need_more_work) { + ERTS_ALC_CPOOL_ASSERT(need_thr_progress && thr_prgr_p); + if (erts_thr_progress_has_reached_this(current, crr->cpool.thr_prgr)) + *need_more_work = 1; + else { + *need_thr_progress = 1; + if (*thr_prgr_p == ERTS_THR_PRGR_INVALID + || erts_thr_progress_cmp(crr->cpool.thr_prgr, + *thr_prgr_p) < 0) { + *thr_prgr_p = crr->cpool.thr_prgr; + } + } + } + } + } +} + +static void +schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + Allctr_t *orig_allctr; + int check_pending_dealloc; + erts_aint_t max_size; + + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + dealloc_carrier(allctr, crr, 1); + return; + } + + orig_allctr = crr->cpool.orig_allctr; + + if (allctr != orig_allctr) { + Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr); + int cinit = orig_allctr->dd.ix - allctr->dd.ix; + + /* + * We send the carrier to its origin for deallocation. + * This in order: + * - not to complicate things for the thread specific + * instances of mseg_alloc, and + * - to ensure that we always only reuse empty carriers + * originating from our own thread specific mseg_alloc + * instance which is beneficial on NUMA systems. + * + * The receiver will recognize that this is a carrier to + * deallocate (and not a block which is the common case) + * since the block is an mbc block that is free and last + * in the carrier. + */ + ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); + + ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); + ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) + == (erts_smp_atomic_read_nob(&crr->allctr) + & ~FLG_MASK)); + + if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) + erts_alloc_notify_delayed_dealloc(orig_allctr->ix); + return; + } + + if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID + || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { + dealloc_carrier(allctr, crr, 1); + return; + } + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + crr->next = NULL; + crr->prev = allctr->cpool.dc_list.last; + if (allctr->cpool.dc_list.last) { + check_pending_dealloc = 1; + allctr->cpool.dc_list.last->next = crr; + } + else { + check_pending_dealloc = 0; + allctr->cpool.dc_list.first = crr; + } + allctr->cpool.dc_list.last = crr; + if (check_pending_dealloc) + check_pending_dealloc_carrier(allctr, NULL, NULL, NULL); + erts_alloc_ensure_handle_delayed_dealloc_call(allctr->ix); +} + +static ERTS_INLINE void +cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) +{ + erts_atomic_init_nob(&crr->cpool.next, ERTS_AINT_NULL); + erts_atomic_init_nob(&crr->cpool.prev, ERTS_AINT_NULL); + crr->cpool.orig_allctr = allctr; + crr->cpool.thr_prgr = ERTS_THR_PRGR_INVALID; + erts_atomic_init_nob(&crr->cpool.max_size, 0); + crr->cpool.blocks = 0; + crr->cpool.blocks_size = 0; + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + crr->cpool.abandon_limit = 0; + else { + UWord csz = CARRIER_SZ(crr); + UWord limit = csz*allctr->cpool.util_limit; + if (limit > csz) + limit /= 100; + else + limit = (csz/100)*allctr->cpool.util_limit; + crr->cpool.abandon_limit = limit; + } +} + +static void +set_new_allctr_abandon_limit(Allctr_t *allctr) +{ + UWord limit; + UWord csz; + + allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; + + csz = allctr->mbcs.curr.norm.mseg.size; + csz += allctr->mbcs.curr.norm.sys_alloc.size; + + limit = csz*allctr->cpool.util_limit; + if (limit > csz) + limit /= 100; + else + limit = (csz/100)*allctr->cpool.util_limit; + + allctr->cpool.abandon_limit = limit; +} + +static void +abandon_carrier(Allctr_t *allctr, Carrier_t *crr) +{ + erts_aint_t max_size; + + STAT_MBC_CPOOL_INSERT(allctr, crr); + + unlink_carrier(&allctr->mbc_list, crr); + + allctr->remove_mbc(allctr, crr); + + max_size = (erts_aint_t) allctr->largest_fblk_in_mbc(allctr, crr); + erts_atomic_set_nob(&crr->cpool.max_size, max_size); + + cpool_insert(allctr, crr); + + set_new_allctr_abandon_limit(allctr); +} + +static void +cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp) +{ + int i; + UWord noc = 0, csz = 0, nob = 0, bsz = 0; + + /* + * We try to get consistent values, but after + * ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS failed + * tries we give up and present what we got... + */ + for (i = 0; i <= ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS; i++) { + UWord tnoc, tcsz, tnob, tbsz; + + tnoc = (UWord) (nocp + ? erts_atomic_read_nob(&allctr->cpool.stat.no_carriers) + : 0); + tcsz = (UWord) (cszp + ? erts_atomic_read_nob(&allctr->cpool.stat.carriers_size) + : 0); + tnob = (UWord) (nobp + ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks) + : 0); + tbsz = (UWord) (bszp + ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size) + : 0); + if (tnoc == noc && tcsz == csz && tnob == nob && tbsz == bsz) + break; + noc = tnoc; + csz = tcsz; + nob = tnob; + bsz = tbsz; + ERTS_THR_READ_MEMORY_BARRIER; + } + + if (nocp) + *nocp = noc; + if (cszp) + *cszp = csz; + if (nobp) + *nobp = nob; + if (bszp) + *bszp = bsz; +} + + +#endif /* ERTS_SMP */ + #ifdef DEBUG #if ERTS_SA_MB_CARRIERS @@ -1982,81 +3242,6 @@ static void CHECK_1BLK_CARRIER(Allctr_t* A, int SBC, int MSEGED, Carrier_t* C, #endif static Block_t * -create_sbmbc(Allctr_t *allctr, Uint umem_sz) -{ - Block_t *blk; - Uint blk_sz; - Uint crr_sz = allctr->sbmbc_size; - Carrier_t *crr; - -#if HALFWORD_HEAP - if (allctr->mseg_opt.low_mem) - crr = erts_alloc(ERTS_ALC_T_SBMBC_LOW, crr_sz); - else -#endif - crr = erts_alloc(ERTS_ALC_T_SBMBC, crr_sz); - - INC_CC(allctr->calls.sbmbc_alloc); - SET_CARRIER_HDR(crr, crr_sz, SCH_SYS_ALLOC|SCH_MBC, allctr); - - blk = MBC_TO_FIRST_BLK(allctr, crr); - - blk_sz = UNIT_FLOOR(crr_sz - MBC_HEADER_SIZE(allctr)); - - SET_MBC_FBLK_HDR(blk, blk_sz, SBH_THIS_FREE|SBH_LAST_BLK, crr); - - link_carrier(&allctr->sbmbc_list, crr); - - STAT_SBMBC_ALLOC(allctr, crr_sz); - CHECK_1BLK_CARRIER(allctr, 0, 0, crr, crr_sz, blk, blk_sz); - if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); - - DEBUG_SAVE_ALIGNMENT(crr); - return blk; -} - -static void -destroy_sbmbc(Allctr_t *allctr, Block_t *blk) -{ - Uint crr_sz; - Carrier_t *crr; - - ASSERT(IS_MBC_BLK(blk)); - ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); - - crr = FIRST_BLK_TO_MBC(allctr, blk); - crr_sz = CARRIER_SZ(crr); - -#ifdef DEBUG - if (!allctr->stopped) { - ASSERT(IS_LAST_BLK(blk)); - -#ifdef ERTS_ALLOC_UTIL_HARD_DEBUG - (*allctr->link_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); - HARD_CHECK_BLK_CARRIER(allctr, blk); - (*allctr->unlink_free_block)(allctr, blk, ERTS_ALCU_FLG_SBMBC); -#endif - } -#endif - - STAT_SBMBC_FREE(allctr, crr_sz); - - unlink_carrier(&allctr->sbmbc_list, crr); - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr, ERTS_ALCU_FLG_SBMBC); - - INC_CC(allctr->calls.sbmbc_free); - -#if HALFWORD_HEAP - if (allctr->mseg_opt.low_mem) - erts_free(ERTS_ALC_T_SBMBC_LOW, crr); - else -#endif - erts_free(ERTS_ALC_T_SBMBC, crr); -} - -static Block_t * create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) { Block_t *blk; @@ -2085,6 +3270,24 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) blk_sz = UMEMSZ2BLKSZ(allctr, umem_sz); +#ifdef ERTS_SMP + allctr->cpool.disable_abandon = ERTS_ALC_CPOOL_MAX_DISABLE_ABANDON; + + if ((flags & (CFLG_MBC|CFLG_NO_CPOOL)) == CFLG_MBC + && ERTS_ALC_IS_CPOOL_ENABLED(allctr) + && erts_thr_progress_is_managed_thread()) { + crr = cpool_fetch(allctr, blk_sz); + if (crr) { + STAT_MBC_CPOOL_FETCH(allctr, crr); + link_carrier(&allctr->mbc_list, crr); + (*allctr->add_mbc)(allctr, crr); + blk = (*allctr->get_free_block)(allctr, blk_sz, NULL, 0); + ASSERT(blk); + return blk; + } + } +#endif + #if HAVE_ERTS_MSEG if (flags & CFLG_FORCE_SYS_ALLOC) @@ -2212,11 +3415,15 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) allctr->main_carrier = crr; } +#ifdef ERTS_SMP + cpool_init_carrier_data(allctr, crr); +#endif + link_carrier(&allctr->mbc_list, crr); CHECK_1BLK_CARRIER(allctr, 0, is_mseg, crr, crr_sz, blk, blk_sz); if (allctr->creating_mbc) - (*allctr->creating_mbc)(allctr, crr, 0); + (*allctr->creating_mbc)(allctr, crr); } @@ -2364,19 +3571,27 @@ resize_carrier(Allctr_t *allctr, Block_t *old_blk, Uint umem_sz, UWord flags) } static void -destroy_carrier(Allctr_t *allctr, Block_t *blk) +dealloc_carrier(Allctr_t *allctr, Carrier_t *crr, int superaligned) { - Uint crr_sz; - Carrier_t *crr; - int is_mbc; #if HAVE_ERTS_MSEG - Uint is_mseg = 0; - Uint mseg_flags = ERTS_MSEG_FLG_NONE; + if (IS_MSEG_CARRIER(crr)) + alcu_mseg_dealloc(allctr, crr, CARRIER_SZ(crr), + (superaligned + ? ERTS_MSEG_FLG_2POW + : ERTS_MSEG_FLG_NONE)); + else #endif + alcu_sys_free(allctr, crr, superaligned); +} + +static void +destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) +{ + Uint crr_sz; + Carrier_t *crr; if (IS_SBC_BLK(blk)) { Uint blk_sz = SBC_BLK_SZ(blk); - is_mbc = 0; crr = BLK_TO_SBC(blk); crr_sz = CARRIER_SZ(crr); @@ -2386,7 +3601,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) #if HAVE_ERTS_MSEG if (IS_MSEG_CARRIER(crr)) { - is_mseg++; ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); STAT_MSEG_SBC_FREE(allctr, crr_sz, blk_sz); } @@ -2396,9 +3610,9 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) unlink_carrier(&allctr->sbc_list, crr); + dealloc_carrier(allctr, crr, 0); } else { - is_mbc = 1; ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); crr = FIRST_BLK_TO_MBC(allctr, blk); crr_sz = CARRIER_SZ(crr); @@ -2415,30 +3629,35 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk) } #endif -#if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(crr)) { - is_mseg++; - ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); - STAT_MSEG_MBC_FREE(allctr, crr_sz); - mseg_flags = ERTS_MSEG_FLG_2POW; + if (allctr->destroying_mbc) + (*allctr->destroying_mbc)(allctr, crr); + +#ifdef ERTS_SMP + if (busy_pcrr_pp && *busy_pcrr_pp) { + ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); + *busy_pcrr_pp = NULL; + cpool_delete(allctr, allctr, crr); } else #endif - STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); - - unlink_carrier(&allctr->mbc_list, crr); - if (allctr->destroying_mbc) - (*allctr->destroying_mbc)(allctr, crr, 0); - } - - + { + unlink_carrier(&allctr->mbc_list, crr); #if HAVE_ERTS_MSEG - if (is_mseg) { - alcu_mseg_dealloc(allctr, crr, crr_sz, mseg_flags); - } - else + if (IS_MSEG_CARRIER(crr)) { + ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); + STAT_MSEG_MBC_FREE(allctr, crr_sz); + } + else +#endif + STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); + } + +#ifdef ERTS_SMP + schedule_dealloc_carrier(allctr, crr); +#else + dealloc_carrier(allctr, crr, 1); #endif - alcu_sys_free(allctr, crr, is_mbc); + } } @@ -2472,19 +3691,19 @@ static struct { Eterm lmbcs; Eterm smbcs; Eterm mbcgs; - Eterm sbmbcs; - Eterm sbmbct; + Eterm acul; #if HAVE_ERTS_MSEG Eterm mmc; #endif Eterm ycs; - /* Eterm sbmbcs; */ - Eterm fix_types; Eterm mbcs; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif Eterm sbcs; Eterm sys_alloc_carriers_size; @@ -2509,8 +3728,6 @@ static struct { Eterm mseg_dealloc; Eterm mseg_realloc; #endif - Eterm sbmbc_alloc; - Eterm sbmbc_free; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -2529,12 +3746,6 @@ static erts_mtx_t init_atoms_mtx; static void init_atoms(Allctr_t *allctr) { - -#ifdef USE_THREADS - if (allctr && allctr->thread_safe) - erts_mtx_unlock(&allctr->mutex); -#endif - erts_mtx_lock(&init_atoms_mtx); if (!atoms_initialized) { @@ -2572,19 +3783,19 @@ init_atoms(Allctr_t *allctr) AM_INIT(lmbcs); AM_INIT(smbcs); AM_INIT(mbcgs); - AM_INIT(sbmbcs); - AM_INIT(sbmbct); + AM_INIT(acul); #if HAVE_ERTS_MSEG AM_INIT(mmc); #endif AM_INIT(ycs); - /*AM_INIT(sbmbcs);*/ - AM_INIT(fix_types); AM_INIT(mbcs); +#ifdef ERTS_SMP + AM_INIT(mbcs_pool); +#endif AM_INIT(sbcs); AM_INIT(sys_alloc_carriers_size); @@ -2609,8 +3820,6 @@ init_atoms(Allctr_t *allctr) AM_INIT(mseg_dealloc); AM_INIT(mseg_realloc); #endif - AM_INIT(sbmbc_free); - AM_INIT(sbmbc_alloc); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -2625,18 +3834,13 @@ init_atoms(Allctr_t *allctr) fix_type_atoms[ix] = am_atom_put(name, len); } } - - if (allctr) { + if (allctr && !allctr->atoms_initialized) { make_name_atoms(allctr); (*allctr->init_atoms)(); -#ifdef USE_THREADS - if (allctr->thread_safe) - erts_mtx_lock(&allctr->mutex); -#endif allctr->atoms_initialized = 1; } @@ -2663,19 +3867,22 @@ ensure_atoms_initialized(Allctr_t *allctr) * that would fit a small when size check is done may need to be built * as a big when the actual build is performed. Caller is required to * HRelease after build. + * + * Note, bld_unstable_uint() should have been called bld_unstable_uword() + * but we do not want to rename it... */ static ERTS_INLINE Eterm -bld_unstable_uint(Uint **hpp, Uint *szp, Uint ui) +bld_unstable_uint(Uint **hpp, Uint *szp, UWord ui) { Eterm res = THE_NON_VALUE; if (szp) - *szp += BIG_UINT_HEAP_SIZE; + *szp += BIG_UWORD_HEAP_SIZE(~((UWord) 0)); if (hpp) { if (IS_USMALL(0, ui)) res = make_small(ui); else { - res = uint_to_big(ui, *hpp); - *hpp += BIG_UINT_HEAP_SIZE; + res = uword_to_big(ui, *hpp); + *hpp += BIG_UWORD_HEAP_SIZE(ui); } } return res; @@ -2701,8 +3908,24 @@ add_4tup(Uint **hpp, Uint *szp, Eterm *lp, bld_cons(hpp, szp, bld_tuple(hpp, szp, 4, el1, el2, el3, el4), *lp); } +static ERTS_INLINE void +add_fix_types(Allctr_t *allctr, int internal, Uint **hpp, Uint *szp, + Eterm *lp, Eterm fix) +{ + if (allctr->fix) { + if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, lp, am.fix_types, fix); + else if (internal) + add_3tup(hpp, szp, lp, + am.fix_types, + erts_bld_uword(hpp, szp, ~((UWord) 0)), + fix); + } +} + static Eterm sz_info_fix(Allctr_t *allctr, + int internal, int *print_to_p, void *print_to_arg, Uint **hpp, @@ -2710,36 +3933,67 @@ sz_info_fix(Allctr_t *allctr, { Eterm res; int ix; - ErtsAlcFixList_t *fix = allctr->fix; - ASSERT(fix); + ASSERT(allctr->fix); res = NIL; - for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { - ErtsAlcType_t n = ix + ERTS_ALC_N_MIN_A_FIXED_SIZE; - Uint alloced = (fix[ix].type_size * fix[ix].allocated); - Uint used = fix[ix].type_size*fix[ix].used; - - if (print_to_p) { - int to = *print_to_p; - void *arg = print_to_arg; - erts_print(to, - arg, - "fix type: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(n), - alloced, - used); - } + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + + if (internal) { + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + UWord alloced = fix->type_size * fix->u.cpool.allocated; + UWord used = fix->type_size * fix->u.cpool.used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type internal: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE + + ix), + alloced, + used); + } - if (hpp || szp) { - add_3tup(hpp, szp, &res, - fix_type_atoms[ix], - bld_unstable_uint(hpp, szp, alloced), - bld_unstable_uint(hpp, szp, used)); + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } } } + else { + + for (ix = ERTS_ALC_NO_FIXED_SIZES-1; ix >= 0; ix--) { + ErtsAlcFixList_t *fix = &allctr->fix[ix]; + UWord alloced = fix->type_size * fix->u.nocpool.allocated; + UWord used = fix->type_size*fix->u.nocpool.used; + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + erts_print(to, + arg, + "fix type: %s %bpu %bpu\n", + (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE + + ix), + alloced, + used); + } + if (hpp || szp) { + add_3tup(hpp, szp, &res, + fix_type_atoms[ix], + bld_unstable_uint(hpp, szp, alloced), + bld_unstable_uint(hpp, szp, used)); + } + } + } return res; } @@ -2753,9 +4007,7 @@ sz_info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_size = (cs == &allctr->sbmbcs - ? cs->curr.small_block.size - : cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size); + UWord curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { int to = *print_to_p; @@ -2769,7 +4021,7 @@ sz_info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers size: %beu %bpu %bpu\n", + "%scarriers size: %bpu %bpu %bpu\n", prefix, curr_size, cs->max.size, @@ -2793,6 +4045,62 @@ sz_info_carriers(Allctr_t *allctr, return res; } +#ifdef ERTS_SMP + +static Eterm +info_cpool(Allctr_t *allctr, + int sz_only, + char *prefix, + int *print_to_p, + void *print_to_arg, + Uint **hpp, + Uint *szp) +{ + Eterm res = THE_NON_VALUE; + UWord noc, csz, nob, bsz; + + noc = csz = nob = bsz = ~0; + if (print_to_p || hpp) { + if (sz_only) + cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + else + cpool_read_stat(allctr, &noc, &csz, &nob, &bsz); + } + + if (print_to_p) { + int to = *print_to_p; + void *arg = print_to_arg; + if (!sz_only) + erts_print(to, arg, "%sblocks: %bpu\n", prefix, nob); + erts_print(to, arg, "%sblocks size: %bpu\n", prefix, bsz); + if (!sz_only) + erts_print(to, arg, "%scarriers: %bpu\n", prefix, noc); + erts_print(to, arg, "%scarriers size: %bpu\n", prefix, csz); + } + + if (hpp || szp) { + res = NIL; + add_2tup(hpp, szp, &res, + am.carriers_size, + bld_unstable_uint(hpp, szp, csz)); + if (!sz_only) + add_2tup(hpp, szp, &res, + am.carriers, + bld_unstable_uint(hpp, szp, noc)); + add_2tup(hpp, szp, &res, + am.blocks_size, + bld_unstable_uint(hpp, szp, bsz)); + if (!sz_only) + add_2tup(hpp, szp, &res, + am.blocks, + bld_unstable_uint(hpp, szp, nob)); + } + + return res; +} + +#endif /* ERTS_SMP */ + static Eterm info_carriers(Allctr_t *allctr, CarriersStats_t *cs, @@ -2803,17 +4111,10 @@ info_carriers(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; - Uint curr_no, curr_size; - int small_block = cs == &allctr->sbmbcs; - - if (small_block) { - curr_no = cs->curr.small_block.no; - curr_size = cs->curr.small_block.size; - } - else { - curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; - curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; - } + UWord curr_no, curr_size; + + curr_no = cs->curr.norm.mseg.no + cs->curr.norm.sys_alloc.no; + curr_size = cs->curr.norm.mseg.size + cs->curr.norm.sys_alloc.size; if (print_to_p) { int to = *print_to_p; @@ -2834,75 +4135,67 @@ info_carriers(Allctr_t *allctr, cs->blocks.max_ever.size); erts_print(to, arg, - "%scarriers: %beu %bpu %bpu\n", + "%scarriers: %bpu %bpu %bpu\n", prefix, curr_no, cs->max.no, cs->max_ever.no); - if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers: %bpu\n", - prefix, - cs->curr.norm.mseg.no); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers: %bpu\n", - prefix, - cs->curr.norm.sys_alloc.no); - } erts_print(to, arg, - "%scarriers size: %beu %bpu %bpu\n", + "%smseg carriers: %bpu\n", + prefix, + cs->curr.norm.mseg.no); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.no); + erts_print(to, + arg, + "%scarriers size: %bpu %bpu %bpu\n", prefix, curr_size, cs->max.size, cs->max_ever.size); - if (!small_block) { #if HAVE_ERTS_MSEG - erts_print(to, - arg, - "%smseg carriers size: %bpu\n", - prefix, - cs->curr.norm.mseg.size); -#endif - erts_print(to, - arg, - "%ssys_alloc carriers size: %bpu\n", - prefix, - cs->curr.norm.sys_alloc.size); - } + erts_print(to, + arg, + "%smseg carriers size: %bpu\n", + prefix, + cs->curr.norm.mseg.size); +#endif + erts_print(to, + arg, + "%ssys_alloc carriers size: %bpu\n", + prefix, + cs->curr.norm.sys_alloc.size); } if (hpp || szp) { res = NIL; - if (!small_block) { - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.size)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers_size, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers_size, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.size)); #endif - } add_4tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, curr_size), bld_unstable_uint(hpp, szp, cs->max.size), bld_unstable_uint(hpp, szp, cs->max_ever.size)); - if (!small_block) { - add_2tup(hpp, szp, &res, - am.sys_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); + add_2tup(hpp, szp, &res, + am.sys_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.sys_alloc.no)); #if HAVE_ERTS_MSEG - add_2tup(hpp, szp, &res, - am.mseg_alloc_carriers, - bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); + add_2tup(hpp, szp, &res, + am.mseg_alloc_carriers, + bld_unstable_uint(hpp, szp, cs->curr.norm.mseg.no)); #endif - } add_4tup(hpp, szp, &res, am.carriers, bld_unstable_uint(hpp, szp, curr_no), @@ -2961,16 +4254,10 @@ info_calls(Allctr_t *allctr, if (print_to_p) { #define PRINT_CC_4(TO, TOA, NAME, CC) \ - if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s calls: %b32u\n", NAME, CC.no); \ - else \ - erts_print(TO, TOA, "%s calls: %b32u%09lu\n", NAME, CC.giga_no, CC.no) + erts_print(TO, TOA, "%s calls: %b64u\n", NAME, CC) #define PRINT_CC_5(TO, TOA, PRFX, NAME, CC) \ - if ((CC).giga_no == 0) \ - erts_print(TO, TOA, "%s%s calls: %b32u\n",PRFX,NAME,CC.no); \ - else \ - erts_print(TO, TOA, "%s%s calls: %b32u%09lu\n",PRFX,NAME,CC.giga_no,CC.no) + erts_print(TO, TOA, "%s%s calls: %b64u\n",PRFX,NAME,CC) char *prefix = allctr->name_prefix; int to = *print_to_p; @@ -2980,9 +4267,6 @@ info_calls(Allctr_t *allctr, PRINT_CC_5(to, arg, prefix, "free", allctr->calls.this_free); PRINT_CC_5(to, arg, prefix, "realloc", allctr->calls.this_realloc); - PRINT_CC_4(to, arg, "sbmbc_alloc", allctr->calls.sbmbc_alloc); - PRINT_CC_4(to, arg, "sbmbc_free", allctr->calls.sbmbc_free); - #if HAVE_ERTS_MSEG PRINT_CC_4(to, arg, "mseg_alloc", allctr->calls.mseg_alloc); PRINT_CC_4(to, arg, "mseg_dealloc", allctr->calls.mseg_dealloc); @@ -3009,50 +4293,42 @@ info_calls(Allctr_t *allctr, add_3tup(hpp, szp, &res, am.sys_realloc, - bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_realloc))); add_3tup(hpp, szp, &res, am.sys_free, - bld_unstable_uint(hpp, szp, allctr->calls.sys_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_free.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_free)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_free))); add_3tup(hpp, szp, &res, am.sys_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sys_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.sys_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.sys_alloc))); #if HAVE_ERTS_MSEG add_3tup(hpp, szp, &res, am.mseg_realloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_realloc))); add_3tup(hpp, szp, &res, am.mseg_dealloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_dealloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_dealloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_dealloc))); add_3tup(hpp, szp, &res, am.mseg_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.mseg_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.mseg_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.mseg_alloc))); #endif add_3tup(hpp, szp, &res, - am.sbmbc_free, - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_free.no)); - add_3tup(hpp, szp, &res, - am.sbmbc_alloc, - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.sbmbc_alloc.no)); - add_3tup(hpp, szp, &res, allctr->name.realloc, - bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_realloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_realloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_realloc))); add_3tup(hpp, szp, &res, allctr->name.free, - bld_unstable_uint(hpp, szp, allctr->calls.this_free.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_free.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_free)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_free))); add_3tup(hpp, szp, &res, allctr->name.alloc, - bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.giga_no), - bld_unstable_uint(hpp, szp, allctr->calls.this_alloc.no)); + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->calls.this_alloc)), + bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->calls.this_alloc))); } return res; @@ -3066,6 +4342,7 @@ info_options(Allctr_t *allctr, Uint *szp) { Eterm res = THE_NON_VALUE; + int acul; if (!allctr) { if (print_to_p) @@ -3077,6 +4354,12 @@ info_options(Allctr_t *allctr, return res; } +#ifdef ERTS_SMP + acul = allctr->cpool.util_limit; +#else + acul = 0; +#endif + if (print_to_p) { char topt[21]; /* Enough for any 64-bit integer */ if (allctr->t) @@ -3106,8 +4389,7 @@ info_options(Allctr_t *allctr, "option lmbcs: %beu\n" "option smbcs: %beu\n" "option mbcgs: %beu\n" - "option sbmbcs: %beu\n" - "option sbmbct: %beu\n", + "option acul: %d\n", topt, allctr->ramv ? "true" : "false", #if HALFWORD_HEAP @@ -3128,8 +4410,7 @@ info_options(Allctr_t *allctr, allctr->largest_mbc_size, allctr->smallest_mbc_size, allctr->mbc_growth_stages, - allctr->sbmbc_size, - allctr->sbmbc_threshold); + acul); } res = (*allctr->info_options)(allctr, "option ", print_to_p, print_to_arg, @@ -3137,11 +4418,8 @@ info_options(Allctr_t *allctr, if (hpp || szp) { add_2tup(hpp, szp, &res, - am.sbmbct, - bld_uint(hpp, szp, allctr->sbmbc_threshold)); - add_2tup(hpp, szp, &res, - am.sbmbcs, - bld_uint(hpp, szp, allctr->sbmbc_size)); + am.acul, + bld_uint(hpp, szp, (UWord) acul)); add_2tup(hpp, szp, &res, am.mbcgs, bld_uint(hpp, szp, allctr->mbc_growth_stages)); @@ -3264,17 +4542,21 @@ erts_alcu_info_options(Allctr_t *allctr, { Eterm res; + if (hpp || szp) + ensure_atoms_initialized(allctr); #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { + erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); + } #endif - if (hpp || szp) - ensure_atoms_initialized(allctr); res = info_options(allctr, print_to_p, print_to_arg, hpp, szp); #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); + erts_allctr_wrapper_pre_unlock(); + } #endif return res; } @@ -3283,13 +4565,17 @@ erts_alcu_info_options(Allctr_t *allctr, Eterm erts_alcu_sz_info(Allctr_t *allctr, + int internal, int begin_max_period, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { - Eterm res, sbmbcs, mbcs, sbcs, fix = THE_NON_VALUE; + Eterm res, mbcs, sbcs, fix = THE_NON_VALUE; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif res = THE_NON_VALUE; @@ -3301,67 +4587,81 @@ erts_alcu_sz_info(Allctr_t *allctr, return am_false; } + if (hpp || szp) + ensure_atoms_initialized(allctr); + #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { + erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); + } #endif ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - if (hpp || szp) - ensure_atoms_initialized(allctr); - /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); if (allctr->fix) - fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); - sbmbcs = sz_info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, - print_to_arg, hpp, szp); + fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = sz_info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbcs_pool = info_cpool(allctr, 1, "mbcs_pool ", print_to_p, + print_to_arg, hpp, szp); + else + mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ +#endif sbcs = sz_info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); if (hpp || szp) { res = NIL; add_2tup(hpp, szp, &res, am.sbcs, sbcs); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); +#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); - add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); - if (allctr->fix) - add_2tup(hpp, szp, &res, am.fix_types, fix); + add_fix_types(allctr, internal, hpp, szp, &res, fix); } if (begin_max_period) { - reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); + erts_allctr_wrapper_pre_unlock(); + } #endif return res; } + Eterm erts_alcu_info(Allctr_t *allctr, + int internal, int begin_max_period, int *print_to_p, void *print_to_arg, Uint **hpp, Uint *szp) { - Eterm res, sett, sbmbcs, mbcs, sbcs, calls, fix = THE_NON_VALUE; + Eterm res, sett, mbcs, sbcs, calls, fix = THE_NON_VALUE; +#ifdef ERTS_SMP + Eterm mbcs_pool; +#endif res = THE_NON_VALUE; @@ -3373,22 +4673,23 @@ erts_alcu_info(Allctr_t *allctr, return am_false; } + if (hpp || szp) + ensure_atoms_initialized(allctr); + #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { + erts_allctr_wrapper_pre_lock(); erts_mtx_lock(&allctr->mutex); + } #endif ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - if (hpp || szp) - ensure_atoms_initialized(allctr); - /* Update sbc values not continously updated */ allctr->sbcs.blocks.curr.no = allctr->sbcs.curr.norm.mseg.no + allctr->sbcs.curr.norm.sys_alloc.no; allctr->sbcs.blocks.max.no = allctr->sbcs.max.no; - update_max_ever_values(&allctr->sbmbcs); update_max_ever_values(&allctr->mbcs); update_max_ever_values(&allctr->sbcs); @@ -3402,11 +4703,16 @@ erts_alcu_info(Allctr_t *allctr, sett = info_options(allctr, print_to_p, print_to_arg, hpp, szp); if (allctr->fix) - fix = sz_info_fix(allctr, print_to_p, print_to_arg, hpp, szp); - sbmbcs = info_carriers(allctr, &allctr->sbmbcs, "sbmbcs ", print_to_p, - print_to_arg, hpp, szp); + fix = sz_info_fix(allctr, internal, print_to_p, print_to_arg, hpp, szp); mbcs = info_carriers(allctr, &allctr->mbcs, "mbcs ", print_to_p, print_to_arg, hpp, szp); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + mbcs_pool = info_cpool(allctr, 0, "mbcs_pool ", print_to_p, + print_to_arg, hpp, szp); + else + mbcs_pool = THE_NON_VALUE; /* shut up annoying warning... */ +#endif sbcs = info_carriers(allctr, &allctr->sbcs, "sbcs ", print_to_p, print_to_arg, hpp, szp); calls = info_calls(allctr, print_to_p, print_to_arg, hpp, szp); @@ -3416,10 +4722,12 @@ erts_alcu_info(Allctr_t *allctr, add_2tup(hpp, szp, &res, am.calls, calls); add_2tup(hpp, szp, &res, am.sbcs, sbcs); +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + add_2tup(hpp, szp, &res, am.mbcs_pool, mbcs_pool); +#endif add_2tup(hpp, szp, &res, am.mbcs, mbcs); - add_2tup(hpp, szp, &res, am.sbmbcs, sbmbcs); - if (allctr->fix) - add_2tup(hpp, szp, &res, am.fix_types, fix); + add_fix_types(allctr, internal, hpp, szp, &res, fix); add_2tup(hpp, szp, &res, am.options, sett); add_3tup(hpp, szp, &res, am.versions, @@ -3428,15 +4736,16 @@ erts_alcu_info(Allctr_t *allctr, } if (begin_max_period) { - reset_max_values(&allctr->sbmbcs); reset_max_values(&allctr->mbcs); reset_max_values(&allctr->sbcs); } #ifdef USE_THREADS - if (allctr->thread_safe) + if (allctr->thread_safe) { erts_mtx_unlock(&allctr->mutex); + erts_allctr_wrapper_pre_unlock(); + } #endif return res; @@ -3454,22 +4763,37 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t * size->carriers = allctr->mbcs.curr.norm.mseg.size; size->carriers += allctr->mbcs.curr.norm.sys_alloc.size; - size->carriers += allctr->sbmbcs.curr.small_block.size; size->carriers += allctr->sbcs.curr.norm.mseg.size; size->carriers += allctr->sbcs.curr.norm.sys_alloc.size; size->blocks = allctr->mbcs.blocks.curr.size; - size->blocks += allctr->sbmbcs.blocks.curr.size; size->blocks += allctr->sbcs.blocks.curr.size; +#ifdef ERTS_SMP + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + UWord csz, bsz; + cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + size->blocks += bsz; + size->carriers += csz; + } +#endif + if (fi) { int ix; for (ix = 0; ix < fisz; ix++) { if (allctr->fix) { - fi[ix].allocated += (allctr->fix[ix].type_size - * allctr->fix[ix].allocated); - fi[ix].used += (allctr->fix[ix].type_size - * allctr->fix[ix].used); + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.cpool.used); + } + else { + fi[ix].allocated += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.allocated); + fi[ix].used += (allctr->fix[ix].type_size + * allctr->fix[ix].u.nocpool.used); + } } } } @@ -3487,7 +4811,6 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) { Allctr_t *allctr = (Allctr_t *) extra; void *res; - ErtsAlcFixList_t *fix; ASSERT(initialized); @@ -3505,56 +4828,21 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size) INC_CC(allctr->calls.this_alloc); - fix = allctr->fix; - if (fix) { - int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used++; - res = fix[ix].list; - if (res) { - fix[ix].list_size--; - fix[ix].list = *((void **) res); - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - void *p = fix[ix].list; - Block_t *blk; - fix[ix].list = *((void **) p); - fix[ix].list_size--; - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - fix[ix].allocated--; - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - return res; - } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); - if (fix[ix].limit < fix[ix].used) - fix[ix].limit = fix[ix].used; - if (fix[ix].max_used < fix[ix].used) - fix[ix].max_used = fix[ix].used; - fix[ix].allocated++; + if (allctr->fix) { + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + return fix_cpool_alloc(allctr, type, size); + else + return fix_nocpool_alloc(allctr, type, size); } if (size >= allctr->sbc_threshold) { Block_t *blk; -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif blk = create_carrier(allctr, size, CFLG_SBC); res = blk ? BLK2UMEM(blk) : NULL; } else res = mbc_alloc(allctr, size); - if (!res && fix) { - int ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - fix[ix].allocated--; - fix[ix].used--; - } return res; } @@ -3623,9 +4911,22 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) if (pref_allctr->thread_safe) erts_mtx_lock(&pref_allctr->mutex); +#ifdef ERTS_SMP + ASSERT(pref_allctr->dd.use); + ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); +#endif + ERTS_ALCU_DBG_CHK_THR_ACCESS(pref_allctr); res = do_erts_alcu_alloc(type, pref_allctr, size); + +#ifdef ERTS_SMP + if (!res && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { + /* Cleaned up a bit more; try one more time... */ + res = do_erts_alcu_alloc(type, pref_allctr, size); + } +#endif + if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); @@ -3642,9 +4943,9 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size) /* ------------------------------------------------------------------------- */ static ERTS_INLINE void -do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) +do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p, + Carrier_t **busy_pcrr_pp) { - int ix; Allctr_t *allctr = (Allctr_t *) extra; ASSERT(initialized); @@ -3656,57 +4957,28 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (p) { - ErtsAlcFixList_t *fix = allctr->fix; - Block_t *blk; INC_CC(allctr->calls.this_free); - if (fix) { - ix = type - ERTS_ALC_N_MIN_A_FIXED_SIZE; - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); - fix[ix].used--; - if (fix[ix].allocated < fix[ix].limit - && fix[ix].list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { - *((void **) p) = fix[ix].list; - fix[ix].list = p; - fix[ix].list_size++; - if (!allctr->fix_shrink_scheduled) { - allctr->fix_shrink_scheduled = 1; - erts_set_aux_work_timeout( - allctr->ix, - (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), - 1); - } - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); - return; - } - fix[ix].allocated--; - if (fix[ix].list && fix[ix].allocated > fix[ix].limit) { - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - p = fix[ix].list; - fix[ix].list = *((void **) p); - fix[ix].list_size--; - fix[ix].allocated--; - } + if (allctr->fix) { + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) + fix_cpool_free(allctr, type, p, busy_pcrr_pp); + else + fix_nocpool_free(allctr, type, p); + } + else { + Block_t *blk = UMEM2BLK(p); + if (IS_SBC_BLK(blk)) + destroy_carrier(allctr, blk, NULL); + else + mbc_free(allctr, p, busy_pcrr_pp); } - - blk = UMEM2BLK(p); - if (IS_SBC_BLK(blk)) - destroy_carrier(allctr, blk); - else - mbc_free(allctr, p); - ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } } void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p) { - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } #ifdef USE_THREADS @@ -3716,7 +4988,7 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p) { Allctr_t *allctr = (Allctr_t *) extra; erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); erts_mtx_unlock(&allctr->mutex); } @@ -3738,7 +5010,7 @@ erts_alcu_free_thr_spec(ErtsAlcType_t type, void *extra, void *p) if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - do_erts_alcu_free(type, allctr, p); + do_erts_alcu_free(type, allctr, p, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -3748,10 +5020,12 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) { if (p) { + Carrier_t *busy_pcrr_p; Allctr_t *pref_allctr, *used_allctr; pref_allctr = get_pref_allctr(extra); - used_allctr = get_used_allctr(extra, p, NULL); + used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_IF_USED, + p, NULL, &busy_pcrr_p); if (pref_allctr != used_allctr) enqueue_dealloc_other_instance(type, used_allctr, @@ -3759,12 +5033,11 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p) (used_allctr->dd.ix - pref_allctr->dd.ix)); else { - if (used_allctr->thread_safe) - erts_mtx_lock(&used_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); - do_erts_alcu_free(type, used_allctr, p); - if (used_allctr->thread_safe) - erts_mtx_unlock(&used_allctr->mutex); + do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); } } } @@ -3780,7 +5053,8 @@ do_erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size, - Uint32 alcu_flgs) + Uint32 alcu_flgs, + Carrier_t **busy_pcrr_pp) { Allctr_t *allctr = (Allctr_t *) extra; Block_t *blk; @@ -3805,7 +5079,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, #if ALLOC_ZERO_EQ_NULL if (!size) { ASSERT(p); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, busy_pcrr_pp); INC_CC(allctr->calls.this_realloc); DEC_CC(allctr->calls.this_free); return NULL; @@ -3816,32 +5090,9 @@ do_erts_alcu_realloc(ErtsAlcType_t type, blk = UMEM2BLK(p); - if (allctr->sbmbc_threshold > 0) { - Uint old_sz, new_sz, lim; - lim = allctr->sbmbc_threshold; - old_sz = BLK_SZ(blk); - new_sz = UMEMSZ2BLKSZ(allctr, size); - if ((old_sz < lim && lim <= new_sz) - || (new_sz < lim && lim <= old_sz)) { - /* *Need* to move it... */ - - INC_CC(allctr->calls.this_realloc); - res = do_erts_alcu_alloc(type, extra, size); - DEC_CC(allctr->calls.this_alloc); - - sys_memcpy(res, p, MIN(size, old_sz - ABLK_HDR_SZ)); - - do_erts_alcu_free(type, extra, p); - DEC_CC(allctr->calls.this_free); - return res; - } - if (old_sz < lim) - alcu_flgs |= ERTS_ALCU_FLG_SBMBC; - } - if (size < allctr->sbc_threshold) { if (IS_MBC_BLK(blk)) - res = mbc_realloc(allctr, p, size, alcu_flgs); + res = mbc_realloc(allctr, p, size, alcu_flgs, busy_pcrr_pp); else { Uint used_sz = SBC_HEADER_SIZE + ABLK_HDR_SZ + size; Uint crr_sz; @@ -3880,16 +5131,12 @@ do_erts_alcu_realloc(ErtsAlcType_t type, sys_memcpy((void*) res, (void*) p, MIN(SBC_BLK_SZ(blk) - ABLK_HDR_SZ, size)); - destroy_carrier(allctr, blk); + destroy_carrier(allctr, blk, NULL); } } } else { Block_t *new_blk; -#ifdef ERTS_SMP - if (allctr->dd.use) - ERTS_ALCU_HANDLE_DD_IN_OP(allctr, 1); -#endif if(IS_SBC_BLK(blk)) { do_carrier_resize: #if HALFWORD_HEAP @@ -3908,7 +5155,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, sys_memcpy((void *) res, (void *) p, MIN(MBC_ABLK_SZ(blk) - ABLK_HDR_SZ, size)); - mbc_free(allctr, p); + mbc_free(allctr, p, busy_pcrr_pp); } else res = NULL; @@ -3922,7 +5169,7 @@ void * erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size) { void *res; - res = do_erts_alcu_realloc(type, extra, p, size, 0); + res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL); DEBUG_CHECK_ALIGNMENT(res); return res; } @@ -3943,7 +5190,7 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } DEBUG_CHECK_ALIGNMENT(res); return res; @@ -3958,7 +5205,7 @@ erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size) Allctr_t *allctr = (Allctr_t *) extra; void *res; erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_realloc(type, extra, ptr, size, 0); + res = do_erts_alcu_realloc(type, extra, ptr, size, 0, NULL); erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); return res; @@ -3982,7 +5229,7 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size) if (cpy_size > size) cpy_size = size; sys_memcpy(res, p, cpy_size); - do_erts_alcu_free(type, extra, p); + do_erts_alcu_free(type, extra, p, NULL); } erts_mtx_unlock(&allctr->mutex); DEBUG_CHECK_ALIGNMENT(res); @@ -4009,7 +5256,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra, if (allctr->thread_safe) erts_mtx_lock(&allctr->mutex); - res = do_erts_alcu_realloc(type, allctr, ptr, size, 0); + res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); @@ -4052,7 +5299,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra, if (cpy_size > size) cpy_size = size; sys_memcpy(res, ptr, cpy_size); - do_erts_alcu_free(type, allctr, ptr); + do_erts_alcu_free(type, allctr, ptr, NULL); if (allctr->thread_safe) erts_mtx_unlock(&allctr->mutex); } @@ -4069,40 +5316,64 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, void *res; Allctr_t *pref_allctr, *used_allctr; UWord old_user_size; + Carrier_t *busy_pcrr_p; +#ifdef ERTS_SMP + int retried; +#endif if (!p) return erts_alcu_alloc_thr_pref(type, extra, size); pref_allctr = get_pref_allctr(extra); - used_allctr = get_used_allctr(extra, p, &old_user_size); + + if (pref_allctr->thread_safe) + erts_mtx_lock(&pref_allctr->mutex); + +#ifdef ERTS_SMP + ASSERT(pref_allctr->dd.use); + ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1); + retried = 0; +restart: +#endif + + used_allctr = get_used_allctr(pref_allctr, ERTS_ALC_TS_PREF_LOCK_NO, + p, &old_user_size, &busy_pcrr_p); ASSERT(used_allctr && pref_allctr); if (!force_move && used_allctr == pref_allctr) { - if (used_allctr->thread_safe) - erts_mtx_lock(&used_allctr->mutex); ERTS_ALCU_DBG_CHK_THR_ACCESS(used_allctr); res = do_erts_alcu_realloc(type, used_allctr, p, size, - 0); - if (used_allctr->thread_safe) - erts_mtx_unlock(&used_allctr->mutex); + 0, + &busy_pcrr_p); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); +#ifdef ERTS_SMP + if (!res && !retried && ERTS_ALCU_HANDLE_DD_IN_OP(pref_allctr, 1)) { + /* Cleaned up a bit more; try one more time... */ + retried = 1; + goto restart; + } +#endif + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); } else { - if (pref_allctr->thread_safe) - erts_mtx_lock(&pref_allctr->mutex); res = do_erts_alcu_alloc(type, pref_allctr, size); - if (pref_allctr->thread_safe && used_allctr != pref_allctr) { - erts_mtx_unlock(&pref_allctr->mutex); - } - if (res) { - DEBUG_CHECK_ALIGNMENT(res); + if (!res) + goto unlock_ts_return; + else { - sys_memcpy(res, p, MIN(size,old_user_size)); + DEBUG_CHECK_ALIGNMENT(res); if (used_allctr != pref_allctr) { + if (pref_allctr->thread_safe) + erts_mtx_unlock(&pref_allctr->mutex); + + sys_memcpy(res, p, MIN(size, old_user_size)); + enqueue_dealloc_other_instance(type, used_allctr, p, @@ -4110,8 +5381,14 @@ realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size, - pref_allctr->dd.ix)); } else { - do_erts_alcu_free(type, used_allctr, p); + + sys_memcpy(res, p, MIN(size, old_user_size)); + + do_erts_alcu_free(type, used_allctr, p, &busy_pcrr_p); ASSERT(pref_allctr == used_allctr); + clear_busy_pool_carrier(used_allctr, busy_pcrr_p); + + unlock_ts_return: if (pref_allctr->thread_safe) erts_mtx_unlock(&pref_allctr->mutex); } @@ -4219,13 +5496,23 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #if ERTS_SMP if (init->tpref) { Uint sz = ABLK_HDR_SZ; - sz += ERTS_ALCU_DD_FIX_TYPE_OFFS*sizeof(UWord); - if (init->fix) - sz += sizeof(UWord); + sz += (init->fix ? + sizeof(ErtsAllctrFixDDBlock_t) : sizeof(ErtsAllctrDDBlock_t)); sz = UNIT_CEILING(sz); if (sz > allctr->min_block_size) allctr->min_block_size = sz; } + + allctr->cpool.dc_list.first = NULL; + allctr->cpool.dc_list.last = NULL; + allctr->cpool.abandon_limit = 0; + allctr->cpool.disable_abandon = 0; + erts_atomic_init_nob(&allctr->cpool.stat.blocks_size, 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_blocks, 0); + erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); + allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; + allctr->cpool.util_limit = init->acul; #endif allctr->sbc_threshold = init->sbct; @@ -4245,27 +5532,6 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) } #endif - - - allctr->sbmbc_threshold = init->sbmbct; - - if (!erts_have_sbmbc_alloc - || ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no)) - allctr->sbmbc_threshold = 0; - - if (!allctr->sbmbc_threshold) - allctr->sbmbc_size = 0; - else { - Uint min_size; - allctr->sbmbc_size = init->sbmbcs; - min_size = allctr->sbmbc_threshold; - min_size += allctr->min_block_size; - min_size += MBC_HEADER_SIZE(allctr); - if (allctr->sbmbc_size < min_size) - allctr->sbmbc_size = min_size; - } - - #if HAVE_ERTS_MSEG if (allctr->mseg_opt.abs_shrink_th > ~((UWord) 0) / 100) allctr->mseg_opt.abs_shrink_th = ~((UWord) 0) / 100; @@ -4277,16 +5543,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) #ifdef ERTS_ENABLE_LOCK_COUNT erts_mtx_init_x_opt(&allctr->mutex, - ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) - ? "sbmbc_alloc" - : "alcu_allocator", + "alcu_allocator", make_small(allctr->alloc_no), ERTS_LCNT_LT_ALLOC); #else erts_mtx_init_x(&allctr->mutex, - ERTS_IS_SBMBC_ALLOCATOR_NO__(allctr->alloc_no) - ? "sbmbc_alloc" - : "alcu_allocator", + "alcu_allocator", make_small(allctr->alloc_no)); #endif /*ERTS_ENABLE_LOCK_COUNT*/ @@ -4305,6 +5567,8 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!allctr->get_next_mbc_size) allctr->get_next_mbc_size = get_next_mbc_size; + if (allctr->mbc_header_size < sizeof(Carrier_t)) + goto error; #ifdef ERTS_SMP allctr->dd.use = 0; if (init->tpref) { @@ -4313,6 +5577,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->dd.ix = init->ix; } #endif + allctr->mbc_header_size = (UNIT_CEILING(allctr->mbc_header_size + + ABLK_HDR_SZ) + - ABLK_HDR_SZ); if (allctr->main_carrier_size) { Block_t *blk; @@ -4321,6 +5588,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->main_carrier_size, CFLG_MBC | CFLG_FORCE_SIZE + | CFLG_NO_CPOOL #if !HALFWORD_HEAP && !ERTS_SUPER_ALIGNED_MSEG_ONLY | CFLG_FORCE_SYS_ALLOC #endif @@ -4328,7 +5596,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (!blk) goto error; - (*allctr->link_free_block)(allctr, blk, 0); + (*allctr->link_free_block)(allctr, blk); HARD_CHECK_BLK_CARRIER(allctr, blk); @@ -4339,13 +5607,24 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->fix = init->fix; allctr->fix_shrink_scheduled = 0; for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) { - allctr->fix[i].max_used = 0; - allctr->fix[i].limit = 0; allctr->fix[i].type_size = init->fix_type_size[i]; allctr->fix[i].list_size = 0; allctr->fix[i].list = NULL; - allctr->fix[i].allocated = 0; - allctr->fix[i].used = 0; +#ifdef ERTS_SMP + ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t)); +#endif + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + allctr->fix[i].u.cpool.min_list_size = 0; + allctr->fix[i].u.cpool.shrink_list = 0; + allctr->fix[i].u.cpool.allocated = 0; + allctr->fix[i].u.cpool.used = 0; + } + else { + allctr->fix[i].u.nocpool.max_used = 0; + allctr->fix[i].u.nocpool.limit = 0; + allctr->fix[i].u.nocpool.allocated = 0; + allctr->fix[i].u.nocpool.used = 0; + } } } @@ -4370,11 +5649,9 @@ erts_alcu_stop(Allctr_t *allctr) allctr->stopped = 1; while (allctr->sbc_list.first) - destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first)); + destroy_carrier(allctr, SBC2BLK(allctr, allctr->sbc_list.first), NULL); while (allctr->mbc_list.first) - destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first)); - while (allctr->sbmbc_list.first) - destroy_sbmbc(allctr, MBC_TO_FIRST_BLK(allctr, allctr->sbmbc_list.first)); + destroy_carrier(allctr, MBC_TO_FIRST_BLK(allctr, allctr->mbc_list.first), NULL); #ifdef USE_THREADS if (allctr->thread_safe) @@ -4388,6 +5665,14 @@ erts_alcu_stop(Allctr_t *allctr) void erts_alcu_init(AlcUInit_t *init) { +#ifdef ERTS_SMP + int i; + for (i = 0; i <= ERTS_ALC_A_MAX; i++) { + ErtsAlcCPoolData_t *sentinel = &carrier_pool[i].sentinel; + erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); + erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); + } +#endif ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); @@ -4415,43 +5700,70 @@ erts_alcu_init(AlcUInit_t *init) * to erts_alcu_test() * \* */ -unsigned long -erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_alcu_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x000: return (unsigned long) BLK_SZ((Block_t *) a1); - case 0x001: return (unsigned long) BLK_UMEM_SZ((Block_t *) a1); - case 0x002: return (unsigned long) IS_PREV_BLK_FREE((Block_t *) a1); - case 0x003: return (unsigned long) IS_FREE_BLK((Block_t *) a1); - case 0x004: return (unsigned long) IS_LAST_BLK((Block_t *) a1); - case 0x005: return (unsigned long) UMEM2BLK((void *) a1); - case 0x006: return (unsigned long) BLK2UMEM((Block_t *) a1); - case 0x007: return (unsigned long) IS_SB_CARRIER((Carrier_t *) a1); - case 0x008: return (unsigned long) IS_SBC_BLK((Block_t *) a1); - case 0x009: return (unsigned long) IS_MB_CARRIER((Carrier_t *) a1); - case 0x00a: return (unsigned long) IS_MSEG_CARRIER((Carrier_t *) a1); - case 0x00b: return (unsigned long) CARRIER_SZ((Carrier_t *) a1); - case 0x00c: return (unsigned long) SBC2BLK((Allctr_t *) a1, + case 0x000: return (UWord) BLK_SZ((Block_t *) a1); + case 0x001: return (UWord) BLK_UMEM_SZ((Block_t *) a1); + case 0x002: return (UWord) IS_PREV_BLK_FREE((Block_t *) a1); + case 0x003: return (UWord) IS_FREE_BLK((Block_t *) a1); + case 0x004: return (UWord) IS_LAST_BLK((Block_t *) a1); + case 0x005: return (UWord) UMEM2BLK((void *) a1); + case 0x006: return (UWord) BLK2UMEM((Block_t *) a1); + case 0x007: return (UWord) IS_SB_CARRIER((Carrier_t *) a1); + case 0x008: return (UWord) IS_SBC_BLK((Block_t *) a1); + case 0x009: return (UWord) IS_MB_CARRIER((Carrier_t *) a1); + case 0x00a: return (UWord) IS_MSEG_CARRIER((Carrier_t *) a1); + case 0x00b: return (UWord) CARRIER_SZ((Carrier_t *) a1); + case 0x00c: return (UWord) SBC2BLK((Allctr_t *) a1, (Carrier_t *) a2); - case 0x00d: return (unsigned long) BLK_TO_SBC((Block_t *) a2); - case 0x00e: return (unsigned long) MBC_TO_FIRST_BLK((Allctr_t *) a1, + case 0x00d: return (UWord) BLK_TO_SBC((Block_t *) a2); + case 0x00e: return (UWord) MBC_TO_FIRST_BLK((Allctr_t *) a1, (Carrier_t *) a2); - case 0x00f: return (unsigned long) FIRST_BLK_TO_MBC((Allctr_t *) a1, + case 0x00f: return (UWord) FIRST_BLK_TO_MBC((Allctr_t *) a1, (Block_t *) a2); - case 0x010: return (unsigned long) ((Allctr_t *) a1)->mbc_list.first; - case 0x011: return (unsigned long) ((Allctr_t *) a1)->mbc_list.last; - case 0x012: return (unsigned long) ((Allctr_t *) a1)->sbc_list.first; - case 0x013: return (unsigned long) ((Allctr_t *) a1)->sbc_list.last; - case 0x014: return (unsigned long) ((Carrier_t *) a1)->next; - case 0x015: return (unsigned long) ((Carrier_t *) a1)->prev; - case 0x016: return (unsigned long) ABLK_HDR_SZ; - case 0x017: return (unsigned long) ((Allctr_t *) a1)->min_block_size; - case 0x018: return (unsigned long) NXT_BLK((Block_t *) a1); - case 0x019: return (unsigned long) PREV_BLK((Block_t *) a1); - case 0x01a: return (unsigned long) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); - case 0x01b: return (unsigned long) sizeof(Unit_t); - default: ASSERT(0); return ~((unsigned long) 0); + case 0x010: return (UWord) ((Allctr_t *) a1)->mbc_list.first; + case 0x011: return (UWord) ((Allctr_t *) a1)->mbc_list.last; + case 0x012: return (UWord) ((Allctr_t *) a1)->sbc_list.first; + case 0x013: return (UWord) ((Allctr_t *) a1)->sbc_list.last; + case 0x014: return (UWord) ((Carrier_t *) a1)->next; + case 0x015: return (UWord) ((Carrier_t *) a1)->prev; + case 0x016: return (UWord) ABLK_HDR_SZ; + case 0x017: return (UWord) ((Allctr_t *) a1)->min_block_size; + case 0x018: return (UWord) NXT_BLK((Block_t *) a1); + case 0x019: return (UWord) PREV_BLK((Block_t *) a1); + case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); + case 0x01b: return (UWord) sizeof(Unit_t); + case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1); + case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; + case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; +#ifdef ERTS_SMP + case 0x01f: return (UWord) sizeof(ErtsAlcCrrPool_t); + case 0x020: + SET_CARRIER_HDR((Carrier_t *) a2, 0, SCH_SYS_ALLOC|SCH_MBC, (Allctr_t *) a1); + cpool_init_carrier_data((Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x021: + cpool_insert((Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x022: + cpool_delete((Allctr_t *) a1, (Allctr_t *) a1, (Carrier_t *) a2); + return (UWord) a2; + case 0x023: return (UWord) cpool_is_empty((Allctr_t *) a1); + case 0x024: return (UWord) cpool_dbg_is_in_pool((Allctr_t *) a1, (Carrier_t *) a2); +#else + case 0x01f: return (UWord) 0; + case 0x020: return (UWord) 0; + case 0x021: return (UWord) 0; + case 0x022: return (UWord) 0; + case 0x023: return (UWord) 0; + case 0x024: return (UWord) 0; +#endif + + default: ASSERT(0); return ~((UWord) 0); } + return 0; } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ @@ -4459,6 +5771,20 @@ erts_alcu_test(unsigned long op, unsigned long a1, unsigned long a2) \* */ void +erts_alcu_assert_failed(char* expr, char* file, int line, char *func) +{ + fflush(stdout); + fprintf(stderr, "%s:%d:%s(): Assertion failed: %s\n", + file, line, func, expr); + fflush(stderr); +#if defined(__WIN__) || defined(__WIN32__) + DebugBreak(); +#else + abort(); +#endif +} + +void erts_alcu_verify_unused(Allctr_t *allctr) { UWord no; @@ -4466,12 +5792,10 @@ erts_alcu_verify_unused(Allctr_t *allctr) no = allctr->sbcs.curr.norm.mseg.no; no += allctr->sbcs.curr.norm.sys_alloc.no; no += allctr->mbcs.blocks.curr.no; - no += allctr->sbmbcs.blocks.curr.no; if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; - sz += allctr->sbmbcs.blocks.curr.size; erl_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" @@ -4605,7 +5929,7 @@ check_blk_carrier(Allctr_t *allctr, Block_t *iblk) cl = &allctr->mbc_list; } -#if 0 /* FIXIT sbmbc */ +#ifdef DEBUG if (cl->first == crr) { ASSERT(!crr->prev); } diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index e975dc34a7..70ecf28172 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2012. All Rights Reserved. + * Copyright Ericsson AB 2002-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -55,8 +55,7 @@ typedef struct { UWord lmbcs; UWord smbcs; UWord mbcgs; - UWord sbmbct; - UWord sbmbcs; + int acul; void *fix; size_t *fix_type_size; @@ -100,8 +99,7 @@ typedef struct { 10*1024*1024, /* (bytes) lmbcs: largest mbc size */\ 1024*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ - 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + 0, /* (%) acul: abandon carrier utilization limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -134,8 +132,7 @@ typedef struct { 1024*1024, /* (bytes) lmbcs: largest mbc size */\ 128*1024, /* (bytes) smbcs: smallest mbc size */\ 10, /* (amount) mbcgs: mbc growth stages */\ - 256, /* (bytes) sbmbct: small block mbc threshold */\ - 8*1024, /* (bytes) sbmbcs: small block mbc size */ \ + 0, /* (%) acul: abandon carrier utilization limit */\ /* --- Data not options -------------------------------------------- */\ NULL, /* (ptr) fix */\ NULL /* (ptr) fix_type_size */\ @@ -165,8 +162,8 @@ void erts_alcu_free_thr_pref(ErtsAlcType_t, void *, void *); #endif Eterm erts_alcu_au_info_options(int *, void *, Uint **, Uint *); Eterm erts_alcu_info_options(Allctr_t *, int *, void *, Uint **, Uint *); -Eterm erts_alcu_sz_info(Allctr_t *, int, int *, void *, Uint **, Uint *); -Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *); +Eterm erts_alcu_sz_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); +Eterm erts_alcu_info(Allctr_t *, int, int, int *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); @@ -181,7 +178,6 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define ERL_ALLOC_UTIL_IMPL__ #define ERTS_ALCU_FLG_FAIL_REALLOC_MOVE (((Uint32) 1) << 0) -#define ERTS_ALCU_FLG_SBMBC (((Uint32) 1) << 1) #ifdef USE_THREADS #define ERL_THREADS_EMU_INTERNAL__ @@ -234,6 +230,11 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); # define MBC_ABLK_OFFSET_BITS 9 /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ # endif +# define ERTS_SACRR_UNIT_SHIFT ERTS_SUPER_ALIGN_BITS +# define ERTS_SACRR_UNIT_SZ (1 << ERTS_SACRR_UNIT_SHIFT) +# define ERTS_SACRR_UNIT_MASK ((~(UWord)0) << ERTS_SACRR_UNIT_SHIFT) +# define ERTS_SACRR_UNIT_FLOOR(X) ((X) & ERTS_SACRR_UNIT_MASK) +# define ERTS_SACRR_UNIT_CEILING(X) ERTS_SACRR_UNIT_FLOOR((X) + ~ERTS_SACRR_UNIT_MASK) # define ERTS_SA_MB_CARRIERS 1 #else # define ERTS_SA_MB_CARRIERS 0 @@ -249,10 +250,8 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); # define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS) # define MBC_ABLK_OFFSET_MASK (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) # define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~FLG_MASK) -# define HAVE_ERTS_SBMBC 0 #else # define MBC_ABLK_SZ_MASK (~FLG_MASK) -# define HAVE_ERTS_SBMBC 1 #endif #define MBC_ABLK_SZ(B) (ASSERT_EXPR(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) @@ -262,18 +261,37 @@ erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #define CARRIER_SZ(C) \ ((C)->chdr & CARRIER_SZ_MASK) -extern int erts_have_sbmbc_alloc; - typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; +#ifdef ERTS_SMP + +typedef struct { + erts_atomic_t next; + erts_atomic_t prev; + Allctr_t *orig_allctr; + ErtsThrPrgrVal thr_prgr; + erts_atomic_t max_size; + UWord abandon_limit; + UWord blocks; + UWord blocks_size; +} ErtsAlcCPoolData_t; + +#endif + typedef struct Carrier_t_ Carrier_t; struct Carrier_t_ { UWord chdr; Carrier_t *next; Carrier_t *prev; - Allctr_t *allctr; + erts_smp_atomic_t allctr; +#ifdef ERTS_SMP + ErtsAlcCPoolData_t cpool; /* Overwritten by block if sbc */ +#endif }; +#define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ + ((Allctr_t *) (erts_smp_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + typedef struct { Carrier_t *first; Carrier_t *last; @@ -291,12 +309,46 @@ typedef struct { #endif } Block_t; +#define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) +#define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) +#define LAST_BLK_HDR_FLG (((UWord) 1) << 2) + +#define SBC_BLK_HDR_FLG /* Special flag combo for (allocated) SBC blocks */\ + (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) + +/* + * FREE_LAST_MBC_BLK_HDR_FLGS is a special flag combo used for + * distinguishing empty mbc's from allocated blocks in + * handle_delayed_dealloc(). + */ +#define FREE_LAST_MBC_BLK_HDR_FLGS (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) + +#define IS_FREE_LAST_MBC_BLK(B) \ + (((B)->bhdr & FLG_MASK) == FREE_LAST_MBC_BLK_HDR_FLGS) + +#define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) +#define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) +#define IS_FREE_BLK(B) (ASSERT(IS_MBC_BLK(B)), \ + (B)->bhdr & THIS_FREE_BLK_HDR_FLG) + +#if MBC_ABLK_OFFSET_BITS +# define FBLK_TO_MBC(B) (ASSERT(IS_MBC_BLK(B) && IS_FREE_BLK(B)), \ + (B)->u.carrier) +# define ABLK_TO_MBC(B) \ + (ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \ + (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \ + (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT)))) +# define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) +#else +# define FBLK_TO_MBC(B) ((B)->carrier) +# define ABLK_TO_MBC(B) ((B)->carrier) +# define BLK_TO_MBC(B) ((B)->carrier) +#endif +#define MBC_BLK_SZ(B) (IS_FREE_BLK(B) ? MBC_FBLK_SZ(B) : MBC_ABLK_SZ(B)) + typedef UWord FreeBlkFtr_t; /* Footer of a free block */ -typedef struct { - UWord giga_no; - UWord no; -} CallCounter_t; +typedef Uint64 CallCounter_t; typedef struct { UWord no; @@ -309,7 +361,6 @@ typedef struct { StatValues_t mseg; StatValues_t sys_alloc; } norm; - StatValues_t small_block; } curr; StatValues_t max; StatValues_t max_ever; @@ -369,10 +420,20 @@ typedef struct { size_t type_size; SWord list_size; void *list; - SWord max_used; - SWord limit; - SWord allocated; - SWord used; + union { + struct { + SWord max_used; + SWord limit; + SWord allocated; + SWord used; + } nocpool; + struct { + int min_list_size; + int shrink_list; + UWord allocated; + UWord used; + } cpool; + } u; } ErtsAlcFixList_t; struct Allctr_t_ { @@ -420,37 +481,56 @@ struct Allctr_t_ { Uint largest_mbc_size; Uint smallest_mbc_size; Uint mbc_growth_stages; - Uint sbmbc_threshold; - Uint sbmbc_size; #if HAVE_ERTS_MSEG ErtsMsegOpt_t mseg_opt; #endif /* */ + Uint mbc_header_size; Uint min_mbc_size; Uint min_mbc_first_free_size; Uint min_block_size; /* Carriers */ - CarrierList_t sbmbc_list; CarrierList_t mbc_list; CarrierList_t sbc_list; +#ifdef ERTS_SMP + struct { + CarrierList_t dc_list; + UWord abandon_limit; + int disable_abandon; + int check_limit_count; + int util_limit; + struct { + erts_atomic_t blocks_size; + erts_atomic_t no_blocks; + erts_atomic_t carriers_size; + erts_atomic_t no_carriers; + } stat; + } cpool; +#endif /* Main carrier (if there is one) */ Carrier_t * main_carrier; /* Callback functions (first 4 are mandatory) */ Block_t * (*get_free_block) (Allctr_t *, Uint, - Block_t *, Uint, Uint32); - void (*link_free_block) (Allctr_t *, Block_t *, Uint32); - void (*unlink_free_block) (Allctr_t *, Block_t *, Uint32); + Block_t *, Uint); + void (*link_free_block) (Allctr_t *, Block_t *); + void (*unlink_free_block) (Allctr_t *, Block_t *); Eterm (*info_options) (Allctr_t *, char *, int *, void *, Uint **, Uint *); Uint (*get_next_mbc_size) (Allctr_t *); - void (*creating_mbc) (Allctr_t *, Carrier_t *, Uint32); - void (*destroying_mbc) (Allctr_t *, Carrier_t *, Uint32); + void (*creating_mbc) (Allctr_t *, Carrier_t *); + void (*destroying_mbc) (Allctr_t *, Carrier_t *); + + /* The three callbacks below are needed to support carrier migration */ + void (*add_mbc) (Allctr_t *, Carrier_t *); + void (*remove_mbc) (Allctr_t *, Carrier_t *); + UWord (*largest_fblk_in_mbc) (Allctr_t *, Carrier_t *); + void (*init_atoms) (void); #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -482,8 +562,6 @@ struct Allctr_t_ { CallCounter_t this_alloc; CallCounter_t this_free; CallCounter_t this_realloc; - CallCounter_t sbmbc_alloc; - CallCounter_t sbmbc_free; CallCounter_t mseg_alloc; CallCounter_t mseg_dealloc; CallCounter_t mseg_realloc; @@ -494,8 +572,7 @@ struct Allctr_t_ { CarriersStats_t sbcs; CarriersStats_t mbcs; - CarriersStats_t sbmbcs; - + #ifdef DEBUG #ifdef USE_THREADS struct { @@ -512,7 +589,9 @@ void erts_alcu_stop(Allctr_t *); void erts_alcu_verify_unused(Allctr_t *); void erts_alcu_verify_unused_ts(Allctr_t *allctr); -unsigned long erts_alcu_test(unsigned long, unsigned long, unsigned long); +UWord erts_alcu_test(UWord, UWord, UWord); + +void erts_alcu_assert_failed(char* expr, char* file, int line, char *func); #ifdef DEBUG int is_sbc_blk(Block_t*); diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 6ce209085c..4e6c8b317e 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -28,11 +28,17 @@ * * This module is a callback-module for erl_alloc_util.c * - * Algorithm: The tree nodes are free-blocks ordered in address order. + * AOFF Algorithm: + * The tree nodes are ordered in address order. * Every node also keeps the size of the largest block in its - * sub-tree ('max_size'). By that we can start from root and keep + * sub-tree ('max_sz'). By that we can start from root and keep * left (for low addresses) while dismissing entire sub-trees with * too small blocks. + * Bestfit within carrier: + * The only difference for "bestfit within carrier" is the tree + * sorting order. Blocks within the same carrier are sorted + * wrt size instead of address. The 'max_sz' field is maintained + * in order to dismiss entire carriers with too small blocks. * * Authors: Rickard Green/Sverker Eriksson */ @@ -62,6 +68,15 @@ # define LEFT_VISITED_FLG (((Uint) 1) << 2) # define RIGHT_VISITED_FLG (((Uint) 1) << 3) #endif +#ifdef DEBUG +# define IS_BF_FLG (((Uint) 1) << 4) +#endif + +#define IS_TREE_NODE(N) (((AOFF_RBTree_t *) (N))->flags & TREE_NODE_FLG) +#define IS_LIST_ELEM(N) (!IS_TREE_NODE(((AOFF_RBTree_t *) (N)))) + +#define SET_TREE_NODE(N) (((AOFF_RBTree_t *) (N))->flags |= TREE_NODE_FLG) +#define SET_LIST_ELEM(N) (((AOFF_RBTree_t *) (N))->flags &= ~TREE_NODE_FLG) #define IS_RED(N) (((AOFF_RBTree_t *) (N)) \ && ((AOFF_RBTree_t *) (N))->flags & RED_FLG) @@ -85,21 +100,55 @@ typedef struct AOFF_RBTree_t_ AOFF_RBTree_t; struct AOFF_RBTree_t_ { Block_t hdr; - Uint flags; AOFF_RBTree_t *parent; AOFF_RBTree_t *left; AOFF_RBTree_t *right; - Uint max_sz; /* of all blocks in this sub-tree */ + Uint32 flags; + Uint32 max_sz; /* of all blocks in this sub-tree */ }; #define AOFF_BLK_SZ(B) MBC_FBLK_SZ(&(B)->hdr) +/* BF block nodes keeps list of all with equal size + */ +typedef struct { + AOFF_RBTree_t t; + AOFF_RBTree_t *next; +}AOFF_RBTreeList_t; + +#define LIST_NEXT(N) (((AOFF_RBTreeList_t*) (N))->next) +#define LIST_PREV(N) (((AOFF_RBTreeList_t*) (N))->t.parent) + +typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; + +struct AOFF_Carrier_t_ { + Carrier_t crr; + AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ + AOFF_RBTree_t* root; /* Root of my block tree */ +}; +#define RBT_NODE_TO_MBC(PTR) ((AOFF_Carrier_t*)((char*)(PTR) - offsetof(AOFF_Carrier_t, rbt_node))) + +/* + To support carrier migration we keep two kinds of rb-trees: + 1. One tree of carriers for each allocator instance. + 2. One tree of free blocks for each carrier. + Both trees use the same node structure AOFF_RBTree_t and implementation. + Carrier nodes thus contain a phony Block_t header 'rbt_node.hdr'. + The size value of such a phony block is the size of the largest free block in + that carrier, i.e same as 'max_sz' of the root node of its block tree. +*/ + #ifdef HARD_DEBUG -static AOFF_RBTree_t * check_tree(AOFF_RBTree_t* root, Uint); +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) rbt_assert_is_member(ROOT,NODE) +# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) check_tree(CRR, FLV, ROOT, SZ) +static AOFF_RBTree_t * check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint); +#else +# define HARD_CHECK_IS_MEMBER(ROOT,NODE) +# define HARD_CHECK_TREE(CRR,FLV,ROOT,SZ) #endif -/* Calculate 'max_size' of tree node x by only looking at the direct children - * of x and x itself. +/* Calculate 'max_sz' of tree node x by only looking at 'max_sz' of the + * direct children of x and the size x itself. */ static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x) { @@ -113,7 +162,7 @@ static ERTS_INLINE Uint node_max_size(AOFF_RBTree_t *x) return sz; } -/* Set new possibly lower 'max_size' of node and propagate change toward root +/* Set new possibly lower 'max_sz' of node and propagate change toward root */ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, AOFF_RBTree_t* stop_at) @@ -132,32 +181,53 @@ static ERTS_INLINE void lower_max_size(AOFF_RBTree_t *node, else ASSERT(new_max == old_max); } +static ERTS_INLINE SWord cmp_blocks(enum AOFF_Flavor flavor, + AOFF_RBTree_t* lhs, AOFF_RBTree_t* rhs) +{ + ASSERT(lhs != rhs); + ASSERT(flavor == AOFF_AOFF || FBLK_TO_MBC(&lhs->hdr) == FBLK_TO_MBC(&rhs->hdr)); + if (flavor != AOFF_AOFF) { + SWord diff = (SWord)AOFF_BLK_SZ(lhs) - (SWord)AOFF_BLK_SZ(rhs); + if (diff || flavor == AOFF_BF) return diff; + } + return (char*)lhs - (char*)rhs; +} + +static ERTS_INLINE SWord cmp_cand_blk(enum AOFF_Flavor flavor, + Block_t* cand_blk, AOFF_RBTree_t* rhs) +{ + if (flavor != AOFF_AOFF) { + if (BLK_TO_MBC(cand_blk) == FBLK_TO_MBC(&rhs->hdr)) { + SWord diff = (SWord)MBC_BLK_SZ(cand_blk) - (SWord)MBC_FBLK_SZ(&rhs->hdr); + if (diff || flavor == AOFF_BF) return diff; + } + } + return (char*)cand_blk - (char*)rhs; +} + /* Prototypes of callback functions */ -static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint, Uint32 flags); -static void aoff_link_free_block(Allctr_t *, Block_t*, Uint32 flags); -static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags); +static Block_t* aoff_get_free_block(Allctr_t *, Uint, Block_t *, Uint); +static void aoff_link_free_block(Allctr_t *, Block_t*); +static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *del); +static void aoff_creating_mbc(Allctr_t*, Carrier_t*); +static void aoff_destroying_mbc(Allctr_t*, Carrier_t*); +static void aoff_add_mbc(Allctr_t*, Carrier_t*); +static void aoff_remove_mbc(Allctr_t*, Carrier_t*); +static UWord aoff_largest_fblk_in_mbc(Allctr_t*, Carrier_t*); + +/* Generic tree functions used by both carrier and block trees. */ +static void rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del); +static void rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk); +static AOFF_RBTree_t* rbt_search(AOFF_RBTree_t* root, Uint size); +#ifdef HARD_DEBUG +static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node); +#endif static Eterm info_options(Allctr_t *, char *, int *, void *, Uint **, Uint *); static void init_atoms(void); - -#ifdef DEBUG - -/* Destroy all tree fields */ -#define DESTROY_TREE_NODE(N) \ - sys_memset((void *) (((Block_t *) (N)) + 1), \ - 0xff, \ - (sizeof(AOFF_RBTree_t) - sizeof(Block_t))) - -#else - -#define DESTROY_TREE_NODE(N) - -#endif - - static int atoms_initialized = 0; void @@ -184,9 +254,12 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); + alc->flavor = aoffinit->flavor; + allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = sizeof(AOFF_RBTree_t); + allctr->min_block_size = (aoffinit->flavor == AOFF_BF ? + sizeof(AOFF_RBTreeList_t):sizeof(AOFF_RBTree_t)); allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; @@ -195,12 +268,15 @@ erts_aoffalc_start(AOFFAllctr_t *alc, allctr->get_free_block = aoff_get_free_block; allctr->link_free_block = aoff_link_free_block; - allctr->unlink_free_block = aoff_unlink_free_block; + allctr->unlink_free_block = aoff_unlink_free_block; allctr->info_options = info_options; allctr->get_next_mbc_size = NULL; - allctr->creating_mbc = NULL; - allctr->destroying_mbc = NULL; + allctr->creating_mbc = aoff_creating_mbc; + allctr->destroying_mbc = aoff_destroying_mbc; + allctr->add_mbc = aoff_add_mbc; + allctr->remove_mbc = aoff_remove_mbc; + allctr->largest_fblk_in_mbc = aoff_largest_fblk_in_mbc; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -303,10 +379,7 @@ replace(AOFF_RBTree_t **root, AOFF_RBTree_t *x, AOFF_RBTree_t *y) y->parent = x->parent; y->right = x->right; y->left = x->left; - - y->max_sz = x->max_sz; - lower_max_size(y, NULL); - DESTROY_TREE_NODE(x); + y->max_sz = x->max_sz; } static void @@ -403,21 +476,72 @@ tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk) } static void -aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) +aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) +{ + AOFFAllctr_t* alc = (AOFFAllctr_t*)allctr; + AOFF_RBTree_t* del = (AOFF_RBTree_t*)blk; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); + + ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + + if (alc->flavor == AOFF_BF) { + ASSERT(del->flags & IS_BF_FLG); + if (IS_LIST_ELEM(del)) { + /* Remove from list */ + ASSERT(LIST_PREV(del)); + ASSERT(LIST_PREV(del)->flags & IS_BF_FLG); + LIST_NEXT(LIST_PREV(del)) = LIST_NEXT(del); + if (LIST_NEXT(del)) { + ASSERT(LIST_NEXT(del)->flags & IS_BF_FLG); + LIST_PREV(LIST_NEXT(del)) = LIST_PREV(del); + } + return; + } + else if (LIST_NEXT(del)) { + /* Replace tree node by next element in list... */ + + ASSERT(AOFF_BLK_SZ(LIST_NEXT(del)) == AOFF_BLK_SZ(del)); + ASSERT(IS_LIST_ELEM(LIST_NEXT(del))); + + replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); + + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + return; + } + } + + rbt_delete(&crr->root, (AOFF_RBTree_t*)del); + + HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, 0); + + /* Update the carrier tree with a potentially new (lower) max_sz + */ + if (crr->root) { + if (crr->rbt_node.hdr.bhdr == crr->root->max_sz) { + return; + } + ASSERT(crr->rbt_node.hdr.bhdr > crr->root->max_sz); + crr->rbt_node.hdr.bhdr = crr->root->max_sz; + } + else { + crr->rbt_node.hdr.bhdr = 0; + } + lower_max_size(&crr->rbt_node, NULL); +} + + +static void +rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &alc->sbmbc_root : &alc->mbc_root); Uint spliced_is_black; - AOFF_RBTree_t *x, *y, *z = (AOFF_RBTree_t *) del; + AOFF_RBTree_t *x, *y, *z = del; AOFF_RBTree_t null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ - null_x.parent = NULL; + HARD_CHECK_IS_MEMBER(*root, del); -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif + null_x.parent = NULL; /* Remove node from tree... */ @@ -461,7 +585,9 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) } if (y != z) { /* We spliced out the successor of z; replace z by the successor */ + ASSERT(z != &null_x); replace(root, z, y); + lower_max_size(y, NULL); } if (spliced_is_black) { @@ -572,28 +698,48 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *del, Uint32 flags) RBT_ASSERT(!null_x.right); } } +} - DESTROY_TREE_NODE(del); +static void +aoff_link_free_block(Allctr_t *allctr, Block_t *block) +{ + AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; + AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; + AOFF_RBTree_t *crr_node; + AOFF_Carrier_t *blk_crr = (AOFF_Carrier_t*) FBLK_TO_MBC(block); + Uint blk_sz = AOFF_BLK_SZ(blk); -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); + ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); + HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); + + rbt_insert(alc->flavor, &blk_crr->root, blk); + + /* Update the carrier tree with a potentially new (larger) max_sz + */ + crr_node = &blk_crr->rbt_node; + if (blk_sz > crr_node->hdr.bhdr) { + ASSERT(blk_sz == blk_crr->root->max_sz); + crr_node->hdr.bhdr = blk_sz; + while (blk_sz > crr_node->max_sz) { + crr_node->max_sz = blk_sz; + crr_node = crr_node->parent; + if (!crr_node) break; + } + } + HARD_CHECK_TREE(&blk_crr->crr, alc->flavor, blk_crr->root, 0); } static void -aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +rbt_insert(enum AOFF_Flavor flavor, AOFF_RBTree_t** root, AOFF_RBTree_t* blk) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; - AOFF_RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &alc->sbmbc_root : &alc->mbc_root); Uint blk_sz = AOFF_BLK_SZ(blk); -#ifdef HARD_DEBUG - check_tree(*root, 0); +#ifdef DEBUG + blk->flags = (flavor == AOFF_BF) ? IS_BF_FLG : 0; +#else + blk->flags = 0; #endif - - blk->flags = 0; blk->left = NULL; blk->right = NULL; blk->max_sz = blk_sz; @@ -606,10 +752,12 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) else { AOFF_RBTree_t *x = *root; while (1) { + SWord diff; if (x->max_sz < blk_sz) { x->max_sz = blk_sz; } - if (blk < x) { + diff = cmp_blocks(flavor, blk, x); + if (diff < 0) { if (!x->left) { blk->parent = x; x->left = blk; @@ -617,7 +765,7 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } x = x->left; } - else { + else if (diff > 0) { if (!x->right) { blk->parent = x; x->right = blk; @@ -625,7 +773,18 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } x = x->right; } - + else { + ASSERT(flavor == AOFF_BF); + ASSERT(blk->flags & IS_BF_FLG); + ASSERT(x->flags & IS_BF_FLG); + SET_LIST_ELEM(blk); + LIST_NEXT(blk) = LIST_NEXT(x); + LIST_PREV(blk) = x; + if (LIST_NEXT(x)) + LIST_PREV(LIST_NEXT(x)) = blk; + LIST_NEXT(x) = blk; + return; + } } /* Insert block into size tree */ @@ -635,38 +794,63 @@ aoff_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) if (IS_RED(blk->parent)) tree_insert_fixup(root, blk); } - -#ifdef HARD_DEBUG - check_tree(*root, 0); -#endif + if (flavor == AOFF_BF) { + SET_TREE_NODE(blk); + LIST_NEXT(blk) = NULL; + } } -static Block_t * -aoff_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, Uint32 flags) +static AOFF_RBTree_t* +rbt_search(AOFF_RBTree_t* root, Uint size) { - AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; - AOFF_RBTree_t *x = ((flags & ERTS_ALCU_FLG_SBMBC) - ? alc->sbmbc_root : alc->mbc_root); - AOFF_RBTree_t *blk = NULL; -#ifdef HARD_DEBUG - AOFF_RBTree_t* dbg_blk = check_tree(x, size); -#endif + AOFF_RBTree_t* x = root; - ASSERT(!cand_blk || cand_size >= size); - - while (x) { + ASSERT(x); + for (;;) { if (x->left && x->left->max_sz >= size) { x = x->left; } else if (AOFF_BLK_SZ(x) >= size) { - blk = x; - break; + return x; } else { x = x->right; + if (!x) { + return NULL; + } } } +} + +static Block_t * +aoff_get_free_block(Allctr_t *allctr, Uint size, + Block_t *cand_blk, Uint cand_size) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_RBTree_t *crr_node = alc->mbc_root; + AOFF_Carrier_t* crr; + AOFF_RBTree_t *blk = NULL; +#ifdef HARD_DEBUG + AOFF_RBTree_t* dbg_blk; +#endif + + ASSERT(!cand_blk || cand_size >= size); + + /* Get first-fit carrier + */ + if (!crr_node || !(blk=rbt_search(crr_node, size))) { + return NULL; + } + crr = RBT_NODE_TO_MBC(blk); + + /* Get block within carrier tree + */ +#ifdef HARD_DEBUG + dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->flavor, crr->root, size); +#endif + + blk = rbt_search(crr->root, size); + ASSERT(blk); #ifdef HARD_DEBUG ASSERT(blk == dbg_blk); @@ -675,15 +859,87 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cand_blk < &blk->hdr) { + if (cand_blk && cmp_cand_blk(alc->flavor, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } - aoff_unlink_free_block(allctr, (Block_t *) blk, flags); + aoff_unlink_free_block(allctr, (Block_t *) blk); return (Block_t *) blk; } +static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t **root = &alc->mbc_root; + + HARD_CHECK_TREE(NULL, 0, *root, 0); + + /* Link carrier in address order tree + */ + crr->rbt_node.hdr.bhdr = 0; + rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + + /* aoff_link_free_block will add free block later */ + crr->root = NULL; + + HARD_CHECK_TREE(NULL, 0, *root, 0); +} + +static void aoff_destroying_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t *root = alc->mbc_root; + + if (crr->rbt_node.parent || &crr->rbt_node == root) { + aoff_remove_mbc(allctr, carrier); + } + /*else already removed */ +} + +static void aoff_add_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t **root = &alc->mbc_root; + + HARD_CHECK_TREE(NULL, 0, *root, 0); + + /* Link carrier in address order tree + */ + rbt_insert(AOFF_AOFF, root, &crr->rbt_node); + + HARD_CHECK_TREE(NULL, 0, *root, 0); +} + +static void aoff_remove_mbc(Allctr_t *allctr, Carrier_t *carrier) +{ + AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + AOFF_RBTree_t **root = &alc->mbc_root; + + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); + HARD_CHECK_TREE(NULL, 0, *root, 0); + + rbt_delete(root, &crr->rbt_node); + crr->rbt_node.parent = NULL; + crr->rbt_node.left = NULL; + crr->rbt_node.right = NULL; + crr->rbt_node.max_sz = crr->rbt_node.hdr.bhdr; + + HARD_CHECK_TREE(NULL, 0, *root, 0); +} + +static UWord aoff_largest_fblk_in_mbc(Allctr_t* allctr, Carrier_t* carrier) +{ + AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; + + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(carrier)); + ASSERT(crr->rbt_node.hdr.bhdr == (crr->root ? crr->root->max_sz : 0)); + return crr->rbt_node.hdr.bhdr; +} /* * info_options() @@ -692,6 +948,8 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, static struct { Eterm as; Eterm aoff; + Eterm aoffcaobf; + Eterm aoffcbf; #ifdef DEBUG Eterm end_of_atoms; #endif @@ -720,6 +978,8 @@ init_atoms(void) #endif AM_INIT(as); AM_INIT(aoff); + AM_INIT(aoffcaobf); + AM_INIT(aoffcbf); #ifdef DEBUG for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) { @@ -749,14 +1009,17 @@ info_options(Allctr_t *allctr, Uint **hpp, Uint *szp) { + AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; Eterm res = THE_NON_VALUE; + const char* flavor_str[3] = {"aoff", "aoffcaobf", "aoffcbf"}; + Eterm flavor_atom[3] = {am.aoff, am.aoffcaobf, am.aoffcbf}; if (print_to_p) { erts_print(*print_to_p, print_to_arg, "%sas: %s\n", prefix, - "aoff"); + flavor_str[alc->flavor]); } if (hpp || szp) { @@ -766,7 +1029,7 @@ info_options(Allctr_t *allctr, __FILE__, __LINE__);; res = NIL; - add_2tup(hpp, szp, &res, am.as, am.aoff); + add_2tup(hpp, szp, &res, am.as, flavor_atom[alc->flavor]); } return res; @@ -780,19 +1043,28 @@ info_options(Allctr_t *allctr, * to erts_aoffalc_test() * \* */ -unsigned long -erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_aoffalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x500: return (unsigned long) 0; /* IS_AOBF */ - case 0x501: return (unsigned long) ((AOFFAllctr_t *) a1)->mbc_root; - case 0x502: return (unsigned long) ((AOFF_RBTree_t *) a1)->parent; - case 0x503: return (unsigned long) ((AOFF_RBTree_t *) a1)->left; - case 0x504: return (unsigned long) ((AOFF_RBTree_t *) a1)->right; - case 0x506: return (unsigned long) IS_BLACK((AOFF_RBTree_t *) a1); - case 0x508: return (unsigned long) 1; /* IS_AOFF */ - case 0x509: return (unsigned long) ((AOFF_RBTree_t *) a1)->max_sz; - default: ASSERT(0); return ~((unsigned long) 0); + case 0x500: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_AOBF; + case 0x501: { + AOFF_RBTree_t *node = ((AOFFAllctr_t *) a1)->mbc_root; + Uint size = (Uint) a2; + node = node ? rbt_search(node, size) : NULL; + return (UWord) (node ? RBT_NODE_TO_MBC(node)->root : NULL); + } + case 0x502: return (UWord) ((AOFF_RBTree_t *) a1)->parent; + case 0x503: return (UWord) ((AOFF_RBTree_t *) a1)->left; + case 0x504: return (UWord) ((AOFF_RBTree_t *) a1)->right; + case 0x505: return (UWord) LIST_NEXT(a1); + case 0x506: return (UWord) IS_BLACK((AOFF_RBTree_t *) a1); + case 0x507: return (UWord) IS_TREE_NODE((AOFF_RBTree_t *) a1); + case 0x508: return (UWord) 0; /* IS_BF_ALGO */ + case 0x509: return (UWord) ((AOFF_RBTree_t *) a1)->max_sz; + case 0x50a: return (UWord) ((AOFFAllctr_t *) a1)->flavor == AOFF_BF; + case 0x50b: return (UWord) LIST_PREV(a1); + default: ASSERT(0); return ~((UWord) 0); } } @@ -804,6 +1076,16 @@ erts_aoffalc_test(unsigned long op, unsigned long a1, unsigned long a2) #ifdef HARD_DEBUG +static int rbt_assert_is_member(AOFF_RBTree_t* root, AOFF_RBTree_t* node) +{ + while (node != root) { + ASSERT(node->parent); + ASSERT(node->parent->left == node || node->parent->right == node); + node = node->parent; + } + return 1; +} + #define IS_LEFT_VISITED(FB) ((FB)->flags & LEFT_VISITED_FLG) #define IS_RIGHT_VISITED(FB) ((FB)->flags & RIGHT_VISITED_FLG) @@ -840,16 +1122,19 @@ static void print_tree(AOFF_RBTree_t*); */ static AOFF_RBTree_t * -check_tree(AOFF_RBTree_t* root, Uint size) +check_tree(Carrier_t* within_crr, enum AOFF_Flavor flavor, AOFF_RBTree_t* root, Uint size) { AOFF_RBTree_t *res = NULL; Sint blacks; Sint curr_blacks; AOFF_RBTree_t *x; + Carrier_t* crr; + Uint depth, max_depth, node_cnt; #ifdef PRINT_TREE print_tree(root); #endif + ASSERT(within_crr || flavor == AOFF_AOFF); if (!root) return res; @@ -859,12 +1144,16 @@ check_tree(AOFF_RBTree_t* root, Uint size) ASSERT(!x->parent); curr_blacks = 1; blacks = -1; + depth = 1; + max_depth = 0; + node_cnt = 0; while (x) { if (!IS_LEFT_VISITED(x)) { SET_LEFT_VISITED(x); if (x->left) { x = x->left; + ++depth; if (IS_BLACK(x)) curr_blacks++; continue; @@ -880,6 +1169,7 @@ check_tree(AOFF_RBTree_t* root, Uint size) SET_RIGHT_VISITED(x); if (x->right) { x = x->right; + ++depth; if (IS_BLACK(x)) curr_blacks++; continue; @@ -891,6 +1181,30 @@ check_tree(AOFF_RBTree_t* root, Uint size) } } + ++node_cnt; + if (depth > max_depth) + max_depth = depth; + + if (within_crr) { + crr = FBLK_TO_MBC(&x->hdr); + ASSERT(crr == within_crr); + ASSERT((char*)x > (char*)crr); + ASSERT(((char*)x + AOFF_BLK_SZ(x)) <= ((char*)crr + CARRIER_SZ(crr))); + + } + if (flavor == AOFF_BF) { + AOFF_RBTree_t* y = x; + AOFF_RBTree_t* nxt = LIST_NEXT(y); + ASSERT(IS_TREE_NODE(x)); + while (nxt) { + ASSERT(IS_LIST_ELEM(nxt)); + ASSERT(AOFF_BLK_SZ(nxt) == AOFF_BLK_SZ(x)); + ASSERT(FBLK_TO_MBC(&nxt->hdr) == within_crr); + ASSERT(LIST_PREV(nxt) == y); + y = nxt; + nxt = LIST_NEXT(nxt); + } + } if (IS_RED(x)) { ASSERT(IS_BLACK(x->right)); @@ -901,13 +1215,13 @@ check_tree(AOFF_RBTree_t* root, Uint size) if (x->left) { ASSERT(x->left->parent == x); - ASSERT(x->left < x); + ASSERT(cmp_blocks(flavor, x->left, x) < 0); ASSERT(x->left->max_sz <= x->max_sz); } if (x->right) { ASSERT(x->right->parent == x); - ASSERT(x->right > x); + ASSERT(cmp_blocks(flavor, x->right, x) > 0); ASSERT(x->right->max_sz <= x->max_sz); } ASSERT(x->max_sz >= AOFF_BLK_SZ(x)); @@ -916,7 +1230,7 @@ check_tree(AOFF_RBTree_t* root, Uint size) || x->max_sz == (x->right ? x->right->max_sz : 0)); if (size && AOFF_BLK_SZ(x) >= size) { - if (!res || x < res) { + if (!res || cmp_blocks(flavor, x, res) < 0) { res = x; } } @@ -926,10 +1240,11 @@ check_tree(AOFF_RBTree_t* root, Uint size) if (IS_BLACK(x)) curr_blacks--; x = x->parent; - + --depth; } - + ASSERT(depth == 0 || (!root && depth==1)); ASSERT(curr_blacks == 0); + ASSERT((1 << (max_depth/2)) <= node_cnt); UNSET_LEFT_VISITED(root); UNSET_RIGHT_VISITED(root); @@ -954,9 +1269,9 @@ print_tree_aux(AOFF_RBTree_t *x, int indent) for (i = 0; i < indent; i++) { putc(' ', stderr); } - fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%lu\r\n", + fprintf(stderr, "%s: sz=%lu addr=0x%lx max_size=%u\r\n", IS_BLACK(x) ? "BLACK" : "RED", - AOFF_BLK_SZ(x), (Uint)x, x->max_sz); + AOFF_BLK_SZ(x), (Uint)x, (unsigned)x->max_sz); print_tree_aux(x->left, indent + INDENT_STEP); } } diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 6fa626f723..25b344c6a8 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -27,8 +27,14 @@ typedef struct AOFFAllctr_t_ AOFFAllctr_t; +enum AOFF_Flavor { + AOFF_AOFF = 0, + AOFF_AOBF = 1, + AOFF_BF = 2 +}; + typedef struct { - int dummy; + enum AOFF_Flavor flavor; } AOFFAllctrInit_t; #define ERTS_DEFAULT_AOFF_ALLCTR_INIT {0/*dummy*/} @@ -51,10 +57,10 @@ struct AOFFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ struct AOFF_RBTree_t_* mbc_root; - struct AOFF_RBTree_t_* sbmbc_root; + enum AOFF_Flavor flavor; }; -unsigned long erts_aoffalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_aoffalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_AOFF_ALLOC_IMPL) && !defined(ERL_AOFF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index 054d1a48f6..e6d72f569b 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -583,6 +583,20 @@ int erts_async_ready_clean(void *varq, void *val) #endif /* +** Generate a fair async key prom an ErlDrvPort +** The port data gives a fair distribution grom port pointer +** to unsigned integer - to be used in key for driver_async below. +*/ +unsigned int driver_async_port_key(ErlDrvPort port) +{ + ErlDrvTermData td = driver_mk_port(port); + if (td == (ErlDrvTermData) NIL) { + return 0; + } + return (unsigned int) (UWord) internal_port_data(td); +} + +/* ** Schedule async_invoke on a worker thread ** NOTE will be syncrounous when threads are unsupported ** return values: diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index 5625622e3f..41f449bb28 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -89,21 +89,21 @@ static RBTree_t * check_tree(RBTree_t, int, Uint); #endif -static void tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags); +static void tree_delete(Allctr_t *allctr, Block_t *del); /* Prototypes of callback functions */ /* "address order best fit" specific callback functions */ static Block_t * aobf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void aobf_link_free_block (Allctr_t *, Block_t *, Uint32); + Block_t *, Uint); +static void aobf_link_free_block (Allctr_t *, Block_t *); #define aobf_unlink_free_block tree_delete /* "best fit" specific callback functions */ static Block_t * bf_get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void bf_link_free_block (Allctr_t *, Block_t *, Uint32); -static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *, Uint32); + Block_t *, Uint); +static void bf_link_free_block (Allctr_t *, Block_t *); +static ERTS_INLINE void bf_unlink_free_block (Allctr_t *, Block_t *); static Eterm info_options (Allctr_t *, char *, int *, @@ -179,6 +179,7 @@ erts_bfalc_start(BFAllctr_t *bfallctr, bfallctr->address_order = bfinit->ao; + allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = (bfinit->ao @@ -207,6 +208,9 @@ erts_bfalc_start(BFAllctr_t *bfallctr, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = NULL; allctr->destroying_mbc = NULL; + allctr->add_mbc = NULL; + allctr->remove_mbc = NULL; + allctr->largest_fblk_in_mbc = NULL; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -407,13 +411,11 @@ tree_insert_fixup(RBTree_t **root, RBTree_t *blk) * callback function in the address order case. */ static void -tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags) +tree_delete(Allctr_t *allctr, Block_t *del) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; Uint spliced_is_black; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x, *y, *z = (RBTree_t *) del; RBTree_t null_x; /* null_x is used to get the fixup started when we splice out a node without children. */ @@ -586,12 +588,10 @@ tree_delete(Allctr_t *allctr, Block_t *del, Uint32 flags) \* */ static void -aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +aobf_link_free_block(Allctr_t *allctr, Block_t *block) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *blk = (RBTree_t *) block; Uint blk_sz = BF_BLK_SZ(blk); @@ -647,21 +647,18 @@ aobf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) #if 0 /* tree_delete() is directly used instead */ static void -aobf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +aobf_unlink_free_block(Allctr_t *allctr, Block_t *block) { - tree_delete(allctr, block, flags); + tree_delete(allctr, block); } #endif static Block_t * aobf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, - Uint32 flags) + Block_t *cand_blk, Uint cand_size) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -694,7 +691,7 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, return NULL; /* cand_blk was better */ } - aobf_unlink_free_block(allctr, (Block_t *) blk, flags); + aobf_unlink_free_block(allctr, (Block_t *) blk); return (Block_t *) blk; } @@ -705,12 +702,10 @@ aobf_get_free_block(Allctr_t *allctr, Uint size, \* */ static void -bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +bf_link_free_block(Allctr_t *allctr, Block_t *block) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *blk = (RBTree_t *) block; Uint blk_sz = BF_BLK_SZ(blk); @@ -779,12 +774,10 @@ bf_link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static ERTS_INLINE void -bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +bf_unlink_free_block(Allctr_t *allctr, Block_t *block) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x = (RBTree_t *) block; if (IS_LIST_ELEM(x)) { @@ -812,7 +805,7 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } else { /* Remove from tree */ - tree_delete(allctr, block, flags); + tree_delete(allctr, block); } DESTROY_LIST_ELEM(x); @@ -821,13 +814,10 @@ bf_unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) static Block_t * bf_get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, - Uint32 flags) + Block_t *cand_blk, Uint cand_size) { BFAllctr_t *bfallctr = (BFAllctr_t *) allctr; - RBTree_t **root = ((flags & ERTS_ALCU_FLG_SBMBC) - ? &bfallctr->sbmbc_root - : &bfallctr->mbc_root); + RBTree_t **root = &bfallctr->mbc_root; RBTree_t *x = *root; RBTree_t *blk = NULL; Uint blk_sz; @@ -867,7 +857,7 @@ bf_get_free_block(Allctr_t *allctr, Uint size, the tree node */ blk = LIST_NEXT(blk) ? LIST_NEXT(blk) : blk; - bf_unlink_free_block(allctr, (Block_t *) blk, flags); + bf_unlink_free_block(allctr, (Block_t *) blk); return (Block_t *) blk; } @@ -972,20 +962,22 @@ info_options(Allctr_t *allctr, * to erts_bfalc_test() * \* */ -unsigned long -erts_bfalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_bfalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x200: return (unsigned long) ((BFAllctr_t *) a1)->address_order; - case 0x201: return (unsigned long) ((BFAllctr_t *) a1)->mbc_root; - case 0x202: return (unsigned long) ((RBTree_t *) a1)->parent; - case 0x203: return (unsigned long) ((RBTree_t *) a1)->left; - case 0x204: return (unsigned long) ((RBTree_t *) a1)->right; - case 0x205: return (unsigned long) ((RBTreeList_t *) a1)->next; - case 0x206: return (unsigned long) IS_BLACK((RBTree_t *) a1); - case 0x207: return (unsigned long) IS_TREE_NODE((RBTree_t *) a1); - case 0x208: return (unsigned long) 0; /* IS_AOFF */ - default: ASSERT(0); return ~((unsigned long) 0); + case 0x200: return (UWord) ((BFAllctr_t *) a1)->address_order; /* IS_AOBF */ + case 0x201: return (UWord) ((BFAllctr_t *) a1)->mbc_root; + case 0x202: return (UWord) ((RBTree_t *) a1)->parent; + case 0x203: return (UWord) ((RBTree_t *) a1)->left; + case 0x204: return (UWord) ((RBTree_t *) a1)->right; + case 0x205: return (UWord) LIST_NEXT(a1); + case 0x206: return (UWord) IS_BLACK((RBTree_t *) a1); + case 0x207: return (UWord) IS_TREE_NODE((RBTree_t *) a1); + case 0x208: return (UWord) 1; /* IS_BF_ALGO */ + case 0x20a: return (UWord) !((BFAllctr_t *) a1)->address_order; /* IS_BF */ + case 0x20b: return (UWord) LIST_PREV(a1); + default: ASSERT(0); return ~((UWord) 0); } } diff --git a/erts/emulator/beam/erl_bestfit_alloc.h b/erts/emulator/beam/erl_bestfit_alloc.h index 0c29662852..870439e886 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.h +++ b/erts/emulator/beam/erl_bestfit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2011. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -55,11 +55,10 @@ struct BFAllctr_t_ { Allctr_t allctr; /* Has to be first! */ RBTree_t * mbc_root; - RBTree_t * sbmbc_root; int address_order; }; -unsigned long erts_bfalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_bfalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_BF_ALLOC_IMPL) && !defined(ERL_BF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 379d3eecc6..673dfc658c 100755 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -30,6 +30,7 @@ #include "bif.h" #include "big.h" #include "erl_version.h" +#include "erl_compile_flags.h" #include "erl_db_util.h" #include "erl_message.h" #include "erl_binary.h" @@ -59,6 +60,7 @@ static Export* alloc_info_trap = NULL; static Export* alloc_sizes_trap = NULL; static Export *gather_sched_wall_time_res_trap; +static Export *gather_gc_info_res_trap; #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -114,6 +116,9 @@ static char erts_system_version[] = ("Erlang " ERLANG_OTP_RELEASE #ifdef VALGRIND " [valgrind-compiled]" #endif +#ifdef ERTS_FRMPTR + " [frame-pointer]" +#endif #ifdef USE_DTRACE " [dtrace]" #endif @@ -486,27 +491,6 @@ collect_one_suspend_monitor(ErtsSuspendMonitor *smon, void *vsmicp) } } - -static void one_link_size(ErtsLink *lnk, void *vpu) -{ - Uint *pu = vpu; - *pu += ERTS_LINK_SIZE*sizeof(Uint); - if(!IS_CONST(lnk->pid)) - *pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint); - if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { - erts_doforall_links(ERTS_LINK_ROOT(lnk),&one_link_size,vpu); - } -} -static void one_mon_size(ErtsMonitor *mon, void *vpu) -{ - Uint *pu = vpu; - *pu += ERTS_MONITOR_SIZE*sizeof(Uint); - if(!IS_CONST(mon->pid)) - *pu += NC_HEAP_SIZE(mon->pid)*sizeof(Uint); - if(!IS_CONST(mon->ref)) - *pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint); -} - /* * process_info/[1,2] */ @@ -1420,41 +1404,8 @@ process_info_aux(Process *BIF_P, } case am_memory: { /* Memory consumed in bytes */ - ErlMessage *mp; - Uint size = 0; Uint hsz = 3; - struct saved_calls *scb; - size += sizeof(Process); - - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - - erts_doforall_links(ERTS_P_LINKS(rp), &one_link_size, &size); - erts_doforall_monitors(ERTS_P_MONITORS(rp), &one_mon_size, &size); - size += (rp->heap_sz + rp->mbuf_sz) * sizeof(Eterm); - if (rp->old_hend && rp->old_heap) - size += (rp->old_hend - rp->old_heap) * sizeof(Eterm); - - size += rp->msg.len * sizeof(ErlMessage); - - for (mp = rp->msg.first; mp; mp = mp->next) - if (mp->data.attached) - size += erts_msg_attached_data_size(mp)*sizeof(Eterm); - - if (rp->arg_reg != rp->def_arg_reg) { - size += rp->arity * sizeof(rp->arg_reg[0]); - } - - if (rp->psd) - size += sizeof(ErtsPSD); - - scb = ERTS_PROC_GET_SAVED_CALLS_BUF(rp); - if (scb) { - size += (sizeof(struct saved_calls) - + (scb->len-1) * sizeof(scb->ct[0])); - } - - size += erts_dicts_mem_size(rp); - + Uint size = erts_process_memory(rp); (void) erts_bld_uint(NULL, &hsz, size); hp = HAlloc(BIF_P, hsz); res = erts_bld_uint(&hp, NULL, size); @@ -1721,13 +1672,22 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ sel = *tp++; - if (sel == am_allocator_sizes) { + if (sel == am_memory_internal) { + switch (arity) { + case 3: + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1, 1)) + return am_true; + default: + goto badarg; + } + } + else if (sel == am_allocator_sizes) { switch (arity) { case 2: ERTS_BIF_PREP_TRAP1(ret, alloc_sizes_trap, BIF_P, *tp); return ret; case 3: - if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1)) + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 1, 0)) return am_true; default: goto badarg; @@ -1786,7 +1746,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ ERTS_BIF_PREP_TRAP1(ret, alloc_info_trap, BIF_P, *tp); return ret; case 3: - if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0)) + if (erts_request_alloc_info(BIF_P, tp[0], tp[1], 0, 0)) return am_true; default: goto badarg; @@ -2096,6 +2056,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #elif defined(ERTS_ENABLE_LOCK_COUNT) ERTS_DECL_AM(lcnt); BIF_RET(AM_lcnt); +#elif defined(ERTS_FRMPTR) + ERTS_DECL_AM(frmptr); + BIF_RET(AM_frmptr); #else BIF_RET(am_opt); #endif @@ -2615,74 +2578,14 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) hp = hsz ? HAlloc(BIF_P, hsz) : NULL; res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit); BIF_RET(res); - } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) { -#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ - || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ - || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) - int i; - char **str; -#endif -#ifdef ETHR_NATIVE_ATOMIC32_IMPL - erts_printf("32-bit native atomics: %s\n", - ETHR_NATIVE_ATOMIC32_IMPL); - str = ethr_native_atomic32_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_atomic32_%s()\n", str[i]); -#endif -#ifdef ETHR_NATIVE_ATOMIC64_IMPL - erts_printf("64-bit native atomics: %s\n", - ETHR_NATIVE_ATOMIC64_IMPL); - str = ethr_native_atomic64_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_atomic64_%s()\n", str[i]); -#endif -#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL - if (ethr_have_native_dw_atomic()) { - erts_printf("Double word native atomics: %s\n", - ETHR_NATIVE_DW_ATOMIC_IMPL); - str = ethr_native_dw_atomic_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_dw_atomic_%s()\n", str[i]); - str = ethr_native_su_dw_atomic_ops(); - for (i = 0; str[i]; i++) - erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]); - } -#endif -#ifdef ETHR_NATIVE_SPINLOCK_IMPL - erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL); -#endif -#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL - erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL); -#endif -#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ - ? "yes" : "no")); -#endif -#ifdef ETHR_X86_OUT_OF_ORDER - erts_printf("x86" -#ifdef ARCH_64 - "_64" -#endif - " out of order\n"); -#endif -#ifdef ETHR_SPARC_TSO - erts_printf("Sparc TSO\n"); -#endif -#ifdef ETHR_SPARC_PSO - erts_printf("Sparc PSO\n"); -#endif -#ifdef ETHR_SPARC_RMO - erts_printf("Sparc RMO\n"); -#endif -#if defined(ETHR_PPC_HAVE_LWSYNC) - erts_printf("Have lwsync instruction: yes\n"); -#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) - erts_printf("Have lwsync instruction: no\n"); -#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) - erts_printf("Have lwsync instruction: %s (runtime test)\n", - ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no"); -#endif - BIF_RET(am_true); + } else if (ERTS_IS_ATOM_STR("ethread_info", BIF_ARG_1)) { + BIF_RET(erts_get_ethread_info(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("emu_args", BIF_ARG_1)) { + BIF_RET(erts_get_emu_args(BIF_P)); + } + else if (ERTS_IS_ATOM_STR("beam_jump_table", BIF_ARG_1)) { + BIF_RET(erts_beam_jump_table() ? am_true : am_false); } else if (ERTS_IS_ATOM_STR("dynamic_trace", BIF_ARG_1)) { #if defined(USE_DTRACE) @@ -2708,6 +2611,31 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(am_true); } #endif + else if (ERTS_IS_ATOM_STR("compile_info",BIF_ARG_1)) { + Uint sz; + Eterm res = NIL, tup, text; + Eterm *hp = HAlloc(BIF_P, 3*(2 + 3) + /* three 2-tuples and three cons */ + 2*(strlen(erts_build_flags_CONFIG_H) + + strlen(erts_build_flags_CFLAGS) + + strlen(erts_build_flags_LDFLAGS))); + + sz = strlen(erts_build_flags_CONFIG_H); + text = buf_to_intlist(&hp, erts_build_flags_CONFIG_H, sz, NIL); + tup = TUPLE2(hp, am_config_h, text); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + sz = strlen(erts_build_flags_CFLAGS); + text = buf_to_intlist(&hp, erts_build_flags_CFLAGS, sz, NIL); + tup = TUPLE2(hp, am_cflags, text); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + sz = strlen(erts_build_flags_LDFLAGS); + text = buf_to_intlist(&hp, erts_build_flags_LDFLAGS, sz, NIL); + tup = TUPLE2(hp, am_ldflags, text); hp += 3; + res = CONS(hp, tup, res); hp += 2; + + BIF_RET(res); + } BIF_ERROR(BIF_P, BADARG); } @@ -2861,12 +2789,10 @@ erts_bld_port_info(Eterm **hpp, ErlOffHeap *ohp, Uint *szp, Port *prt, Eterm ite included though). */ Uint size = 0; - ErlHeapFragment* bp; - erts_doforall_links(ERTS_P_LINKS(prt), &one_link_size, &size); + erts_doforall_links(ERTS_P_LINKS(prt), &erts_one_link_size, &size); - for (bp = prt->bp; bp; bp = bp->next) - size += sizeof(ErlHeapFragment) + (bp->alloc_size - 1)*sizeof(Eterm); + size += erts_port_data_size(prt); if (prt->linebuf) size += sizeof(LineBuf) + prt->linebuf->ovsiz; @@ -3153,18 +3079,10 @@ BIF_RETTYPE statistics_1(BIF_ALIST_1) res = TUPLE2(hp, cs, SMALL_ZERO); BIF_RET(res); } else if (BIF_ARG_1 == am_garbage_collection) { - Uint hsz = 4; - ErtsGCInfo gc_info; - Eterm gcs; - Eterm recl; - erts_gc_info(&gc_info); - (void) erts_bld_uint(NULL, &hsz, gc_info.garbage_collections); - (void) erts_bld_uint(NULL, &hsz, gc_info.reclaimed); - hp = HAlloc(BIF_P, hsz); - gcs = erts_bld_uint(&hp, NULL, gc_info.garbage_collections); - recl = erts_bld_uint(&hp, NULL, gc_info.reclaimed); - res = TUPLE3(hp, gcs, recl, SMALL_ZERO); - BIF_RET(res); + res = erts_gc_info_request(BIF_P); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_gc_info_res_trap, BIF_P, res); } else if (BIF_ARG_1 == am_reductions) { Uint reds; Uint diff; @@ -4132,6 +4050,8 @@ erts_bif_info_init(void) alloc_sizes_trap = erts_export_put(am_erlang, am_alloc_sizes, 1); gather_sched_wall_time_res_trap = erts_export_put(am_erlang, am_gather_sched_wall_time_result, 1); + gather_gc_info_res_trap + = erts_export_put(am_erlang, am_gather_gc_info_result, 1); process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 44fa41c7b6..109c54fd7f 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -423,57 +423,164 @@ BIF_RETTYPE erts_internal_port_info_2(BIF_ALIST_2) } } +/* + * The erlang:port_set_data()/erlang:port_get_data() operations should + * be viewed as operations on a table (inet_db) with data values + * associated with port identifier keys. That is, these operations are + * *not* signals to/from ports. + */ + +#if (TAG_PRIMARY_IMMED1 & 0x3) == 0 +# error "erlang:port_set_data()/erlang:port_get_data() needs to be rewritten!" +#endif + +typedef struct { + ErtsThrPrgrLaterOp later_op; + Uint hsize; + Eterm data; + ErlOffHeap off_heap; + Eterm heap[1]; +} ErtsPortDataHeap; -BIF_RETTYPE erts_internal_port_set_data_2(BIF_ALIST_2) +static void +free_port_data_heap(void *vpdhp) { - Eterm ref; + erts_cleanup_offheap(&((ErtsPortDataHeap *) vpdhp)->off_heap); + erts_free(ERTS_ALC_T_PORT_DATA_HEAP, vpdhp); +} + +static ERTS_INLINE void +cleanup_old_port_data(erts_aint_t data) +{ + if ((data & 0x3) != 0) { + ASSERT(is_immed((Eterm) data)); + } + else { +#ifdef ERTS_SMP + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + size_t size; + ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; + size = sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm) - 1); + erts_schedule_thr_prgr_later_cleanup_op(free_port_data_heap, + (void *) pdhp, + &pdhp->later_op, + size); +#else + free_port_data_heap((void *) data); +#endif + } +} + +void +erts_init_port_data(Port *prt) +{ + erts_smp_atomic_init_nob(&prt->data, (erts_aint_t) am_undefined); +} + +void +erts_cleanup_port_data(Port *prt) +{ + ASSERT(erts_atomic32_read_nob(&prt->state) & ERTS_PORT_SFLGS_INVALID_LOOKUP); + cleanup_old_port_data(erts_smp_atomic_read_nob(&prt->data)); + erts_smp_atomic_set_nob(&prt->data, (erts_aint_t) THE_NON_VALUE); +} + +Uint +erts_port_data_size(Port *prt) +{ + erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data); + + if ((data & 0x3) != 0) { + ASSERT(is_immed((Eterm) (UWord) data)); + return (Uint) 0; + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + return (Uint) sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm)-1); + } +} + +ErlOffHeap * +erts_port_data_offheap(Port *prt) +{ + erts_aint_t data = erts_smp_atomic_read_ddrb(&prt->data); + + if ((data & 0x3) != 0) { + ASSERT(is_immed((Eterm) (UWord) data)); + return NULL; + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + return &pdhp->off_heap; + } +} + +BIF_RETTYPE port_set_data_2(BIF_ALIST_2) +{ + /* + * This is not a signal. See comment above. + */ + erts_aint_t data; Port* prt; prt = lookup_port(BIF_P, BIF_ARG_1); if (!prt) - BIF_RET(am_badarg); + BIF_ERROR(BIF_P, BADARG); - switch (erts_port_set_data(BIF_P, prt, BIF_ARG_2, &ref)) { - case ERTS_PORT_OP_CALLER_EXIT: - case ERTS_PORT_OP_BADARG: - case ERTS_PORT_OP_DROPPED: - BIF_RET(am_badarg); - case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(ref)); - BIF_RET(ref); - case ERTS_PORT_OP_DONE: - BIF_RET(am_true); - default: - ERTS_INTERNAL_ERROR("Unexpected erts_port_set_data() result"); - BIF_RET(am_internal_error); + if (is_immed(BIF_ARG_2)) { + data = (erts_aint_t) BIF_ARG_2; + ASSERT((data & 0x3) != 0); } + else { + ErtsPortDataHeap *pdhp; + Uint hsize; + Eterm *hp; + + hsize = size_object(BIF_ARG_2); + pdhp = erts_alloc(ERTS_ALC_T_PORT_DATA_HEAP, + sizeof(ErtsPortDataHeap) + hsize*(sizeof(Eterm)-1)); + hp = &pdhp->heap[0]; + pdhp->off_heap.first = NULL; + pdhp->off_heap.overhead = 0; + pdhp->data = copy_struct(BIF_ARG_2, hsize, &hp, &pdhp->off_heap); + data = (erts_aint_t) pdhp; + ASSERT((data & 0x3) == 0); + } + + data = erts_smp_atomic_xchg_wb(&prt->data, data); + + cleanup_old_port_data(data); + + BIF_RET(am_true); } -BIF_RETTYPE erts_internal_port_get_data_1(BIF_ALIST_1) +BIF_RETTYPE port_get_data_1(BIF_ALIST_1) { - Eterm retval; + /* + * This is not a signal. See comment above. + */ + Eterm res; + erts_aint_t data; Port* prt; prt = lookup_port(BIF_P, BIF_ARG_1); if (!prt) - BIF_RET(am_badarg); + BIF_ERROR(BIF_P, BADARG); - switch (erts_port_get_data(BIF_P, prt, &retval)) { - case ERTS_PORT_OP_CALLER_EXIT: - case ERTS_PORT_OP_BADARG: - case ERTS_PORT_OP_DROPPED: - BIF_RET(am_badarg); - case ERTS_PORT_OP_SCHEDULED: - ASSERT(is_internal_ref(retval)); - BIF_RET(retval); - case ERTS_PORT_OP_DONE: - ASSERT(is_not_internal_ref(retval)); - BIF_RET(retval); - default: - ERTS_INTERNAL_ERROR("Unexpected erts_port_get_data() result"); - BIF_RET(am_internal_error); + data = erts_smp_atomic_read_ddrb(&prt->data); + + if ((data & 0x3) != 0) { + res = (Eterm) (UWord) data; + ASSERT(is_immed(res)); + } + else { + ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; + Eterm *hp = HAlloc(BIF_P, pdhp->hsize); + res = copy_struct(pdhp->data, pdhp->hsize, &hp, &MSO(BIF_P)); } + + BIF_RET(res); } /* diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 41c98bbffb..06fbbea123 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -1224,9 +1224,7 @@ trace_info_func(Process* p, Eterm func_spec, Eterm key) match_spec_meta = NIL; } if (r & FUNC_TRACE_COUNT_TRACE) { - c = count < 0 ? - erts_make_integer(-count-1, p) : - erts_make_integer(count, p); + c = erts_make_integer(count, p); } if (r & FUNC_TRACE_TIME_TRACE) { ct = call_time; @@ -2014,6 +2012,7 @@ void erts_system_monitor_clear(Process *c_p) { #endif erts_set_system_monitor(NIL); erts_system_monitor_long_gc = 0; + erts_system_monitor_long_schedule = 0; erts_system_monitor_large_heap = 0; erts_system_monitor_flags.busy_port = 0; erts_system_monitor_flags.busy_dist_port = 0; @@ -2038,12 +2037,17 @@ static Eterm system_monitor_get(Process *p) Uint hsz = 3 + (erts_system_monitor_flags.busy_dist_port ? 2 : 0) + (erts_system_monitor_flags.busy_port ? 2 : 0); Eterm long_gc = NIL; + Eterm long_schedule = NIL; Eterm large_heap = NIL; if (erts_system_monitor_long_gc != 0) { hsz += 2+3; (void) erts_bld_uint(NULL, &hsz, erts_system_monitor_long_gc); } + if (erts_system_monitor_long_schedule != 0) { + hsz += 2+3; + (void) erts_bld_uint(NULL, &hsz, erts_system_monitor_long_schedule); + } if (erts_system_monitor_large_heap != 0) { hsz += 2+3; (void) erts_bld_uint(NULL, &hsz, erts_system_monitor_large_heap); @@ -2053,6 +2057,10 @@ static Eterm system_monitor_get(Process *p) if (erts_system_monitor_long_gc != 0) { long_gc = erts_bld_uint(&hp, NULL, erts_system_monitor_long_gc); } + if (erts_system_monitor_long_schedule != 0) { + long_schedule = erts_bld_uint(&hp, NULL, + erts_system_monitor_long_schedule); + } if (erts_system_monitor_large_heap != 0) { large_heap = erts_bld_uint(&hp, NULL, erts_system_monitor_large_heap); } @@ -2061,6 +2069,10 @@ static Eterm system_monitor_get(Process *p) Eterm t = TUPLE2(hp, am_long_gc, long_gc); hp += 3; res = CONS(hp, t, res); hp += 2; } + if (long_schedule != NIL) { + Eterm t = TUPLE2(hp, am_long_schedule, long_schedule); hp += 3; + res = CONS(hp, t, res); hp += 2; + } if (large_heap != NIL) { Eterm t = TUPLE2(hp, am_large_heap, large_heap); hp += 3; res = CONS(hp, t, res); hp += 2; @@ -2115,7 +2127,7 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) } if (is_not_list(list)) goto error; else { - Uint long_gc, large_heap; + Uint long_gc, long_schedule, large_heap; int busy_port, busy_dist_port; system_blocked = 1; @@ -2125,7 +2137,8 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) if (!erts_pid2proc(p, ERTS_PROC_LOCK_MAIN, monitor_pid, 0)) goto error; - for (long_gc = 0, large_heap = 0, busy_port = 0, busy_dist_port = 0; + for (long_gc = 0, long_schedule = 0, large_heap = 0, + busy_port = 0, busy_dist_port = 0; is_list(list); list = CDR(list_val(list))) { Eterm t = CAR(list_val(list)); @@ -2135,6 +2148,9 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) if (tp[1] == am_long_gc) { if (! term_to_Uint(tp[2], &long_gc)) goto error; if (long_gc < 1) long_gc = 1; + } else if (tp[1] == am_long_schedule) { + if (! term_to_Uint(tp[2], &long_schedule)) goto error; + if (long_schedule < 1) long_schedule = 1; } else if (tp[1] == am_large_heap) { if (! term_to_Uint(tp[2], &large_heap)) goto error; if (large_heap < 16384) large_heap = 16384; @@ -2150,6 +2166,7 @@ system_monitor(Process *p, Eterm monitor_pid, Eterm list) prev = system_monitor_get(p); erts_set_system_monitor(monitor_pid); erts_system_monitor_long_gc = long_gc; + erts_system_monitor_long_schedule = long_schedule; erts_system_monitor_large_heap = large_heap; erts_system_monitor_flags.busy_port = !!busy_port; erts_system_monitor_flags.busy_dist_port = !!busy_dist_port; diff --git a/erts/emulator/beam/erl_bits.c b/erts/emulator/beam/erl_bits.c index 3753b618e1..43eb691338 100644 --- a/erts/emulator/beam/erl_bits.c +++ b/erts/emulator/beam/erl_bits.c @@ -484,8 +484,16 @@ erts_bs_get_float_2(Process *p, Uint num_bits, unsigned flags, ErlBinMatchBuffer ERTS_FP_ERROR_THOROUGH(p, f32, return THE_NON_VALUE); f.fd = f32; } else { +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef ftmp; + ftmp.fd = f64; + f.fw[0] = ftmp.fw[1]; + f.fw[1] = ftmp.fw[0]; + ERTS_FP_ERROR_THOROUGH(p, f.fd, return THE_NON_VALUE); +#else ERTS_FP_ERROR_THOROUGH(p, f64, return THE_NON_VALUE); f.fd = f64; +#endif } mb->offset += num_bits; hp = HeapOnlyAlloc(p, FLOAT_SIZE_OBJECT); @@ -1014,8 +1022,13 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) #endif } else if (is_small(arg)) { u.f64 = (double) signed_val(arg); +#ifdef DOUBLE_MIDDLE_ENDIAN + a = u.i32[1]; + b = u.i32[0]; +#else a = u.i32[0]; b = u.i32[1]; +#endif } else if (is_big(arg)) { if (big_to_double(arg, &u.f64) < 0) { return 0; @@ -1118,21 +1131,42 @@ erts_new_bs_put_float(Process *c_p, Eterm arg, Uint num_bits, int flags) byte *bptr; double f64; float f32; +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef fbuf, ftmp; +#endif if (num_bits == 64) { if (is_float(arg)) { +#ifdef DOUBLE_MIDDLE_ENDIAN + FloatDef *fdp = (FloatDef*)(float_val(arg) + 1); + ftmp = *fdp; +#else bptr = (byte *) (float_val(arg) + 1); +#endif } else if (is_small(arg)) { f64 = (double) signed_val(arg); +#ifdef DOUBLE_MIDDLE_ENDIAN + ftmp.fd = f64; +#else bptr = (byte *) &f64; +#endif } else if (is_big(arg)) { if (big_to_double(arg, &f64) < 0) { return 0; } +#ifdef DOUBLE_MIDDLE_ENDIAN + ftmp.fd = f64; +#else bptr = (byte *) &f64; +#endif } else { return 0; } +#ifdef DOUBLE_MIDDLE_ENDIAN + fbuf.fw[0] = ftmp.fw[1]; + fbuf.fw[1] = ftmp.fw[0]; + bptr = fbuf.fb; +#endif } else if (num_bits == 32) { if (is_float(arg)) { FloatDef f; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 713ac0ba18..ef3749a2c4 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -2319,6 +2319,8 @@ restart: break; case matchSilent: --esp; + if (in_flags & ERTS_PAM_IGNORE_TRACE_SILENT) + break; if (*esp == am_true) { erts_smp_proc_lock(c_p, ERTS_PROC_LOCKS_ALL_MINOR); ERTS_TRACE_FLAGS(c_p) |= F_TRACE_SILENT; @@ -4971,7 +4973,8 @@ static Eterm match_spec_test(Process *p, Eterm against, Eterm spec, int trace) save_cp = p->cp; p->cp = NULL; res = erts_match_set_run(p, mps, arr, n, - ERTS_PAM_COPY_RESULT, &ret_flags); + ERTS_PAM_COPY_RESULT|ERTS_PAM_IGNORE_TRACE_SILENT, + &ret_flags); p->cp = save_cp; } else { n = 0; diff --git a/erts/emulator/beam/erl_driver.h b/erts/emulator/beam/erl_driver.h index e280563de1..b68fd46fcc 100644 --- a/erts/emulator/beam/erl_driver.h +++ b/erts/emulator/beam/erl_driver.h @@ -133,7 +133,7 @@ typedef struct { #define ERL_DRV_EXTENDED_MARKER (0xfeeeeeed) #define ERL_DRV_EXTENDED_MAJOR_VERSION 2 -#define ERL_DRV_EXTENDED_MINOR_VERSION 1 +#define ERL_DRV_EXTENDED_MINOR_VERSION 2 /* * The emulator will refuse to load a driver with different major @@ -546,6 +546,11 @@ EXTERN int erl_drv_equal_tids(ErlDrvTid tid1, ErlDrvTid tid2); EXTERN void erl_drv_thread_exit(void *resp); EXTERN int erl_drv_thread_join(ErlDrvTid, void **respp); +EXTERN char* erl_drv_mutex_name(ErlDrvMutex *mtx); +EXTERN char* erl_drv_cond_name(ErlDrvCond *cnd); +EXTERN char* erl_drv_rwlock_name(ErlDrvRWLock *rwlck); +EXTERN char* erl_drv_thread_name(ErlDrvTid tid); + /* * Misc. */ @@ -638,6 +643,8 @@ EXTERN int erl_drv_send_term(ErlDrvTermData port, int len); /* Async IO functions */ +EXTERN unsigned int driver_async_port_key(ErlDrvPort port); + EXTERN long driver_async(ErlDrvPort ix, unsigned int* key, void (*async_invoke)(void*), @@ -681,6 +688,3 @@ EXTERN int erl_drv_getenv(char *key, char *value, size_t *value_size); /* also in global.h, but driver's can't include global.h */ void dtrace_drvport_str(ErlDrvPort port, char *port_buf); - - - diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index a49a155701..4f1bba8657 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -188,6 +188,17 @@ erl_drv_mutex_destroy(ErlDrvMutex *dmtx) #endif } + +char * +erl_drv_mutex_name(ErlDrvMutex *dmtx) +{ +#ifdef USE_THREADS + return dmtx ? dmtx->name : NULL; +#else + return NULL; +#endif +} + int erl_drv_mutex_trylock(ErlDrvMutex *dmtx) { @@ -258,6 +269,15 @@ erl_drv_cond_destroy(ErlDrvCond *dcnd) #endif } +char * +erl_drv_cond_name(ErlDrvCond *dcnd) +{ +#ifdef USE_THREADS + return dcnd ? dcnd->name : NULL; +#else + return NULL; +#endif +} void erl_drv_cond_signal(ErlDrvCond *dcnd) @@ -331,6 +351,16 @@ erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck) #endif } +char * +erl_drv_rwlock_name(ErlDrvRWLock *drwlck) +{ +#ifdef USE_THREADS + return drwlck ? drwlck->name : NULL; +#else + return NULL; +#endif +} + int erl_drv_rwlock_tryrlock(ErlDrvRWLock *drwlck) { @@ -617,6 +647,18 @@ erl_drv_thread_create(char *name, #endif } +char * +erl_drv_thread_name(ErlDrvTid tid) +{ +#ifdef USE_THREADS + struct ErlDrvTid_ *dtid = (struct ErlDrvTid_ *) tid; + return dtid ? dtid->name : NULL; +#else + return NULL; +#endif +} + + ErlDrvTid erl_drv_thread_self(void) { diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 298909c921..8ba94d89e9 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -47,10 +47,6 @@ */ #define ALENGTH(a) (sizeof(a)/sizeof(a[0])) -static erts_smp_spinlock_t info_lck; -static Uint garbage_cols; /* no of garbage collections */ -static Uint reclaimed; /* no of words reclaimed in GCs */ - # define STACK_SZ_ON_HEAP(p) ((p)->hend - (p)->stop) # define OverRunCheck(P) \ if ((P)->stop < (P)->htop) { \ @@ -120,6 +116,8 @@ static void offset_rootset(Process *p, Sint offs, char* area, Uint area_size, static void offset_off_heap(Process* p, Sint offs, char* area, Uint area_size); static void offset_mqueue(Process *p, Sint offs, char* area, Uint area_size); +static void init_gc_info(ErtsGCInfo *gcip); + #ifdef HARDDEBUG static void disallow_heap_frag_ref_in_heap(Process* p); static void disallow_heap_frag_ref_in_old_heap(Process* p); @@ -137,13 +135,41 @@ static int num_heap_sizes; /* Number of heap sizes. */ Uint erts_test_long_gc_sleep; /* Only used for testing... */ +typedef struct { + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsGCInfoReq; + +#if !HALFWORD_HEAP +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(gcireq, + ErtsGCInfoReq, + 5, + ERTS_ALC_T_GC_INFO_REQ) +#else +static ERTS_INLINE ErtsGCInfoReq * +gcireq_alloc(void) +{ + return erts_alloc(ERTS_ALC_T_GC_INFO_REQ, + sizeof(ErtsGCInfoReq)); +} + +static ERTS_INLINE void +gcireq_free(ErtsGCInfoReq *ptr) +{ + erts_free(ERTS_ALC_T_GC_INFO_REQ, ptr); +} +#endif + /* * Initialize GC global data. */ void erts_init_gc(void) { - int i = 0; + int i = 0, ix; Sint max_heap_size = 0; ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); @@ -156,9 +182,6 @@ erts_init_gc(void) ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); - erts_smp_spinlock_init(&info_lck, "gc_info"); - garbage_cols = 0; - reclaimed = 0; erts_test_long_gc_sleep = 0; /* @@ -199,6 +222,16 @@ erts_init_gc(void) } } num_heap_sizes = i; + + for (ix = 0; ix < erts_no_schedulers; ix++) { + ErtsSchedulerData *esdp = ERTS_SCHEDULER_IX(ix); + init_gc_info(&esdp->gc_info); + } + +#if !HALFWORD_HEAP + init_gcireq_alloc(); +#endif + } /* @@ -287,17 +320,6 @@ erts_heap_sizes(Process* p) return res; } -void -erts_gc_info(ErtsGCInfo *gcip) -{ - if (gcip) { - erts_smp_spin_lock(&info_lck); - gcip->garbage_collections = garbage_cols; - gcip->reclaimed = reclaimed; - erts_smp_spin_unlock(&info_lck); - } -} - void erts_offset_heap(Eterm* hp, Uint sz, Sint offs, Eterm* low, Eterm* high) { @@ -378,6 +400,7 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) Uint reclaimed_now = 0; int done = 0; Uint ms1, s1, us1; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); #ifdef USE_VM_PROBES DTRACE_CHARBUF(pidbuf, DTRACE_TERM_BUF_SIZE); #endif @@ -455,11 +478,9 @@ erts_garbage_collect(Process* p, int need, Eterm* objv, int nobj) monitor_large_heap(p); } - erts_smp_spin_lock(&info_lck); - garbage_cols++; - reclaimed += reclaimed_now; - erts_smp_spin_unlock(&info_lck); - + esdp->gc_info.garbage_cols++; + esdp->gc_info.reclaimed += reclaimed_now; + FLAGS(p) &= ~F_FORCE_GC; #ifdef CHECK_FOR_HOLES @@ -1943,6 +1964,17 @@ setup_rootset(Process *p, Eterm *objv, int nobj, Rootset *rootset) ++n; } + /* + * A trapping BIF can add to rootset by setting the extra_root + * in the process_structure. + */ + if (p->extra_root != NULL) { + roots[n].v = p->extra_root->objv; + roots[n].sz = p->extra_root->sz; + ++n; + } + + ASSERT((is_nil(p->seq_trace_token) || is_tuple(follow_moved(p->seq_trace_token)) || is_atom(p->seq_trace_token))); @@ -2520,6 +2552,12 @@ offset_one_rootset(Process *p, Sint offs, char* area, Uint area_size, p->dictionary->used, offs, area, area_size); } + if (p->extra_root != NULL) { + offset_heap_ptr(p->extra_root->objv, + p->extra_root->sz, + offs, area, area_size); + } + offset_heap_ptr(&p->fvalue, 1, offs, area, area_size); offset_heap_ptr(&p->ftrace, 1, offs, area, area_size); offset_heap_ptr(&p->seq_trace_token, 1, offs, area, area_size); @@ -2543,6 +2581,110 @@ offset_rootset(Process *p, Sint offs, char* area, Uint area_size, offset_one_rootset(p, offs, area, area_size, objv, nobj); } +static void +init_gc_info(ErtsGCInfo *gcip) +{ + gcip->reclaimed = 0; + gcip->garbage_cols = 0; +} + +static void +reply_gc_info(void *vgcirp) +{ + Uint64 reclaimed = 0, garbage_cols = 0; + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsGCInfoReq *gcirp = (ErtsGCInfoReq *) vgcirp; + ErtsProcLocks rp_locks = (gcirp->req_sched == esdp->no + ? ERTS_PROC_LOCK_MAIN + : 0); + Process *rp = gcirp->proc; + Eterm ref_copy = NIL, msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz, *szp; + ErlOffHeap *ohp = NULL; + ErlHeapFragment *bp = NULL; + + ASSERT(esdp); + + reclaimed = esdp->gc_info.reclaimed; + garbage_cols = esdp->gc_info.garbage_cols; + + sz = 0; + hpp = NULL; + szp = &sz; + + while (1) { + if (hpp) + ref_copy = STORE_NC(hpp, ohp, gcirp->ref); + else + *szp += REF_THING_SIZE; + + msg = erts_bld_tuple(hpp, szp, 3, + make_small(esdp->no), + erts_bld_uint64(hpp, szp, garbage_cols), + erts_bld_uint64(hpp, szp, reclaimed)); + + msg = erts_bld_tuple(hpp, szp, 2, ref_copy, msg); + if (hpp) + break; + + hp = erts_alloc_message_heap(sz, &bp, &ohp, rp, &rp_locks); + szp = NULL; + hpp = &hp; + } + + erts_queue_message(rp, &rp_locks, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); + + if (gcirp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_smp_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&gcirp->refc) == 0) + gcireq_free(vgcirp); +} + +Eterm +erts_gc_info_request(Process *c_p) +{ + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsGCInfoReq *gcirp; + Eterm *hp; + + gcirp = gcireq_alloc(); + ref = erts_make_ref(c_p); + hp = &gcirp->ref_heap[0]; + + gcirp->proc = c_p; + gcirp->ref = STORE_NC(&hp, NULL, ref); + gcirp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&gcirp->refc, + (erts_aint32_t) erts_no_schedulers); + + erts_smp_proc_add_refc(c_p, (Sint32) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_gc_info, + (void *) gcirp); +#endif + + reply_gc_info((void *) gcirp); + + return ref; +} + #if defined(DEBUG) || defined(ERTS_OFFHEAP_DEBUG) static int diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index c2088929e9..e9d8249ee1 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -163,10 +163,10 @@ BKT_MIN_SZ(GFAllctr_t *gfallctr, int ix) /* Prototypes of callback functions */ static Block_t * get_free_block (Allctr_t *, Uint, - Block_t *, Uint, Uint32); -static void link_free_block (Allctr_t *, Block_t *, Uint32); -static void unlink_free_block (Allctr_t *, Block_t *, Uint32); -static void update_last_aux_mbc (Allctr_t *, Carrier_t *, Uint32); + Block_t *, Uint); +static void link_free_block (Allctr_t *, Block_t *); +static void unlink_free_block (Allctr_t *, Block_t *); +static void update_last_aux_mbc (Allctr_t *, Carrier_t *); static Eterm info_options (Allctr_t *, char *, int *, void *, Uint **, Uint *); static void init_atoms (void); @@ -203,8 +203,7 @@ erts_gfalc_start(GFAllctr_t *gfallctr, sys_memcpy((void *) gfallctr, (void *) &zero.allctr, sizeof(GFAllctr_t)); - init->sbmbct = 0; /* Small mbc not yet supported by goodfit */ - + allctr->mbc_header_size = sizeof(Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; allctr->min_block_size = sizeof(GFFreeBlock_t); @@ -222,7 +221,9 @@ erts_gfalc_start(GFAllctr_t *gfallctr, allctr->get_next_mbc_size = NULL; allctr->creating_mbc = update_last_aux_mbc; allctr->destroying_mbc = update_last_aux_mbc; - + allctr->add_mbc = NULL; + allctr->remove_mbc = NULL; + allctr->largest_fblk_in_mbc = NULL; allctr->init_atoms = init_atoms; #ifdef ERTS_ALLOC_UTIL_HARD_DEBUG @@ -384,7 +385,7 @@ search_bucket(Allctr_t *allctr, int ix, Uint size) static Block_t * get_free_block(Allctr_t *allctr, Uint size, - Block_t *cand_blk, Uint cand_size, Uint32 flags) + Block_t *cand_blk, Uint cand_size) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; int unsafe_bi, min_bi; @@ -403,7 +404,7 @@ get_free_block(Allctr_t *allctr, Uint size, if (blk) { if (cand_blk && cand_size <= MBC_FBLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk, flags); + unlink_free_block(allctr, blk); return blk; } if (min_bi < NO_OF_BKTS - 1) { @@ -423,14 +424,14 @@ get_free_block(Allctr_t *allctr, Uint size, ASSERT(blk); if (cand_blk && cand_size <= MBC_FBLK_SZ(blk)) return NULL; /* cand_blk was better */ - unlink_free_block(allctr, blk, flags); + unlink_free_block(allctr, blk); return blk; } static void -link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +link_free_block(Allctr_t *allctr, Block_t *block) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; @@ -451,7 +452,7 @@ link_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static void -unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) +unlink_free_block(Allctr_t *allctr, Block_t *block) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; GFFreeBlock_t *blk = (GFFreeBlock_t *) block; @@ -472,7 +473,7 @@ unlink_free_block(Allctr_t *allctr, Block_t *block, Uint32 flags) } static void -update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc, Uint32 flags) +update_last_aux_mbc(Allctr_t *allctr, Carrier_t *mbc) { GFAllctr_t *gfallctr = (GFAllctr_t *) allctr; @@ -589,16 +590,16 @@ info_options(Allctr_t *allctr, * to erts_gfalc_test() * \* */ -unsigned long -erts_gfalc_test(unsigned long op, unsigned long a1, unsigned long a2) +UWord +erts_gfalc_test(UWord op, UWord a1, UWord a2) { switch (op) { - case 0x100: return (unsigned long) BKT_IX((GFAllctr_t *) a1, (Uint) a2); - case 0x101: return (unsigned long) BKT_MIN_SZ((GFAllctr_t *) a1, (int) a2); - case 0x102: return (unsigned long) NO_OF_BKTS; - case 0x103: return (unsigned long) + case 0x100: return (UWord) BKT_IX((GFAllctr_t *) a1, (Uint) a2); + case 0x101: return (UWord) BKT_MIN_SZ((GFAllctr_t *) a1, (int) a2); + case 0x102: return (UWord) NO_OF_BKTS; + case 0x103: return (UWord) find_bucket(&((GFAllctr_t *) a1)->bucket_mask, (int) a2); - default: ASSERT(0); return ~((unsigned long) 0); + default: ASSERT(0); return ~((UWord) 0); } } diff --git a/erts/emulator/beam/erl_goodfit_alloc.h b/erts/emulator/beam/erl_goodfit_alloc.h index a554a6f466..385de0da23 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.h +++ b/erts/emulator/beam/erl_goodfit_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2010. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -82,7 +82,7 @@ struct GFAllctr_t_ { }; -unsigned long erts_gfalc_test(unsigned long, unsigned long, unsigned long); +UWord erts_gfalc_test(UWord, UWord, UWord); #endif /* #if defined(GET_ERL_GF_ALLOC_IMPL) && !defined(ERL_GF_ALLOC_IMPL__) */ diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index e6a96d427f..8c4fffa75b 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -131,8 +131,10 @@ extern void ConWaitForExit(void); static void erl_init(int ncpu, int proc_tab_sz, + int legacy_proc_tab, int port_tab_sz, - int port_tab_sz_ignore_files); + int port_tab_sz_ignore_files, + int legacy_port_tab); static erts_atomic_t exiting; @@ -280,7 +282,9 @@ erts_short_init(void) int ncpu = early_init(NULL, NULL); erl_init(ncpu, ERTS_DEFAULT_MAX_PROCESSES, + 0, ERTS_DEFAULT_MAX_PORTS, + 0, 0); erts_initialized = 1; } @@ -288,19 +292,21 @@ erts_short_init(void) static void erl_init(int ncpu, int proc_tab_sz, + int legacy_proc_tab, int port_tab_sz, - int port_tab_sz_ignore_files) + int port_tab_sz_ignore_files, + int legacy_port_tab) { init_benchmarking(); erts_init_monitors(); - erts_init_gc(); erts_init_time(); erts_init_sys_common_misc(); - erts_init_process(ncpu, proc_tab_sz); + erts_init_process(ncpu, proc_tab_sz, legacy_proc_tab); erts_init_scheduling(no_schedulers, no_schedulers_online); erts_init_cpu_topology(); /* Must be after init_scheduling */ + erts_init_gc(); /* Must be after init_scheduling */ erts_alloc_late_init(); H_MIN_SIZE = erts_next_heap_size(H_MIN_SIZE, 0); @@ -327,13 +333,14 @@ erl_init(int ncpu, init_dist(); erl_drv_thr_init(); erts_init_async(); - erts_init_io(port_tab_sz, port_tab_sz_ignore_files); + erts_init_io(port_tab_sz, port_tab_sz_ignore_files, legacy_port_tab); init_load(); erts_init_bif(); erts_init_bif_chksum(); erts_init_bif_binary(); erts_init_bif_re(); erts_init_unicode(); /* after RE to get access to PCRE unicode */ + erts_init_external(); erts_delay_trap = erts_export_put(am_erlang, am_delay_trap, 2); erts_late_init_process(); #if HAVE_ERTS_MSEG @@ -542,9 +549,12 @@ void erts_usage(void) ERTS_SCHED_THREAD_MAX_STACK_SIZE); erts_fprintf(stderr, "-spp Bool set port parallelism scheduling hint\n"); erts_fprintf(stderr, "-S n1:n2 set number of schedulers (n1), and number of\n"); - erts_fprintf(stderr, " schedulers online (n2), valid range for both\n"); - erts_fprintf(stderr, " numbers are [1-%d]\n", + erts_fprintf(stderr, " schedulers online (n2), maximum for both\n"); + erts_fprintf(stderr, " numbers is %d\n", ERTS_MAX_NO_OF_SCHEDULERS); + erts_fprintf(stderr, "-SP p1:p2 specify schedulers (p1) and schedulers online (p2)\n"); + erts_fprintf(stderr, " as percentages of logical processors configured and logical\n"); + erts_fprintf(stderr, " processors available, respectively\n"); erts_fprintf(stderr, "-t size set the maximum number of atoms the " "emulator can handle\n"); erts_fprintf(stderr, " valid range is [%d-%d]\n", @@ -624,12 +634,16 @@ early_init(int *argc, char **argv) /* int ncpuavail; int schdlrs; int schdlrs_onln; + int schdlrs_percentage = 100; + int schdlrs_onln_percentage = 100; int max_main_threads; int max_reader_groups; int reader_groups; char envbuf[21]; /* enough for any 64-bit integer */ size_t envbufsz; + erts_save_emu_args(*argc, argv); + erts_sched_compact_load = 1; erts_printf_eterm_func = erts_printf_term; erts_disable_tolerant_timeofday = 0; @@ -749,63 +763,132 @@ early_init(int *argc, char **argv) /* } break; } - case 'S' : { - int tot, onln; - char *arg = get_arg(argv[i]+2, argv[i+1], &i); - switch (sscanf(arg, "%d:%d", &tot, &onln)) { - case 0: - switch (sscanf(arg, ":%d", &onln)) { + case 'S' : + if (argv[i][2] == 'P') { + int ptot, ponln; + char *arg = get_arg(argv[i]+3, argv[i+1], &i); + switch (sscanf(arg, "%d:%d", &ptot, &ponln)) { + case 0: + switch (sscanf(arg, ":%d", &ponln)) { + case 1: + if (ponln < 0) + goto bad_SP; + ptot = 100; + goto chk_SP; + default: + goto bad_SP; + } case 1: - tot = no_schedulers; - goto chk_S; + if (ptot < 0) + goto bad_SP; + ponln = ptot < 100 ? ptot : 100; + goto chk_SP; + case 2: + if (ptot < 0 || ponln < 0) + goto bad_SP; + chk_SP: + schdlrs_percentage = ptot; + schdlrs_onln_percentage = ponln; + break; default: - goto bad_S; - } - case 1: - onln = tot < schdlrs_onln ? tot : schdlrs_onln; - case 2: - chk_S: - if (tot > 0) - schdlrs = tot; - else - schdlrs = no_schedulers + tot; - if (onln > 0) - schdlrs_onln = onln; - else - schdlrs_onln = no_schedulers_online + onln; - if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { - erts_fprintf(stderr, - "bad amount of schedulers %d\n", - tot); - erts_usage(); - } - if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) { + bad_SP: + erts_fprintf(stderr, + "bad schedulers percentage specifier %s\n", + arg); + erts_usage(); + break; + } + + VERBOSE(DEBUG_SYSTEM, + ("using %d:%d scheduler percentages\n", + schdlrs_percentage, schdlrs_onln_percentage)); + } else { + int tot, onln; + char *arg = get_arg(argv[i]+2, argv[i+1], &i); + switch (sscanf(arg, "%d:%d", &tot, &onln)) { + case 0: + switch (sscanf(arg, ":%d", &onln)) { + case 1: + tot = no_schedulers; + goto chk_S; + default: + goto bad_S; + } + case 1: + onln = tot < schdlrs_onln ? tot : schdlrs_onln; + case 2: + chk_S: + if (tot > 0) + schdlrs = tot; + else + schdlrs = no_schedulers + tot; + if (onln > 0) + schdlrs_onln = onln; + else + schdlrs_onln = no_schedulers_online + onln; + if (schdlrs < 1 || ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { + erts_fprintf(stderr, + "bad amount of schedulers %d\n", + tot); + erts_usage(); + } + if (schdlrs_onln < 1 || schdlrs < schdlrs_onln) { + erts_fprintf(stderr, + "bad amount of schedulers online %d " + "(total amount of schedulers %d)\n", + schdlrs_onln, schdlrs); + erts_usage(); + } + break; + default: + bad_S: erts_fprintf(stderr, - "bad amount of schedulers online %d " - "(total amount of schedulers %d)\n", - schdlrs_onln, schdlrs); + "bad amount of schedulers %s\n", + arg); erts_usage(); + break; } - break; - default: - bad_S: - erts_fprintf(stderr, - "bad amount of schedulers %s\n", - arg); - erts_usage(); - break; - } - VERBOSE(DEBUG_SYSTEM, - ("using %d:%d scheduler(s)\n", tot, onln)); - break; - } + VERBOSE(DEBUG_SYSTEM, + ("using %d:%d scheduler(s)\n", tot, onln)); + } + break; default: break; } } i++; } + +#ifdef ERTS_SMP + /* apply any scheduler percentages */ + if (schdlrs_percentage != 100 || schdlrs_onln_percentage != 100) { + schdlrs = schdlrs * schdlrs_percentage / 100; + schdlrs_onln = schdlrs_onln * schdlrs_onln_percentage / 100; + if (schdlrs < 1) + schdlrs = 1; + if (ERTS_MAX_NO_OF_SCHEDULERS < schdlrs) { + erts_fprintf(stderr, + "bad schedulers percentage %d " + "(total amount of schedulers %d)\n", + schdlrs_percentage, schdlrs); + erts_usage(); + } + if (schdlrs_onln < 1) + schdlrs_onln = 1; + if (schdlrs < schdlrs_onln) { + erts_fprintf(stderr, + "bad schedulers online percentage %d " + "(total amount of schedulers %d, online %d)\n", + schdlrs_onln_percentage, schdlrs, schdlrs_onln); + erts_usage(); + } + } +#else + /* Silence gcc warnings */ + (void)schdlrs_percentage; + (void)schdlrs_onln_percentage; +#endif } #ifndef USE_THREADS @@ -923,6 +1006,9 @@ erl_start(int argc, char **argv) int proc_tab_sz = ERTS_DEFAULT_MAX_PROCESSES; int port_tab_sz = ERTS_DEFAULT_MAX_PORTS; int port_tab_sz_ignore_files = 0; + int legacy_proc_tab = 0; + int legacy_port_tab = 0; + envbufsz = sizeof(envbuf); if (erts_sys_getenv_raw(ERL_MAX_ETS_TABLES_ENV, envbuf, &envbufsz) == 0) @@ -1268,31 +1354,42 @@ erl_start(int argc, char **argv) case 'P': /* set maximum number of processes */ arg = get_arg(argv[i]+2, argv[i+1], &i); - errno = 0; - proc_tab_sz = strtol(arg, NULL, 10); - if (errno != 0 - || proc_tab_sz < ERTS_MIN_PROCESSES - || ERTS_MAX_PROCESSES < proc_tab_sz) { - erts_fprintf(stderr, "bad number of processes %s\n", arg); - erts_usage(); + if (strcmp(arg, "legacy") == 0) + legacy_proc_tab = 1; + else { + errno = 0; + proc_tab_sz = strtol(arg, NULL, 10); + if (errno != 0 + || proc_tab_sz < ERTS_MIN_PROCESSES + || ERTS_MAX_PROCESSES < proc_tab_sz) { + erts_fprintf(stderr, "bad number of processes %s\n", arg); + erts_usage(); + } } break; case 'Q': /* set maximum number of ports */ arg = get_arg(argv[i]+2, argv[i+1], &i); - errno = 0; - port_tab_sz = strtol(arg, NULL, 10); - if (errno != 0 - || port_tab_sz < ERTS_MIN_PROCESSES - || ERTS_MAX_PROCESSES < port_tab_sz) { - erts_fprintf(stderr, "bad number of ports %s\n", arg); - erts_usage(); + if (strcmp(arg, "legacy") == 0) + legacy_port_tab = 1; + else { + errno = 0; + port_tab_sz = strtol(arg, NULL, 10); + if (errno != 0 + || port_tab_sz < ERTS_MIN_PROCESSES + || ERTS_MAX_PROCESSES < port_tab_sz) { + erts_fprintf(stderr, "bad number of ports %s\n", arg); + erts_usage(); + } + port_tab_sz_ignore_files = 1; } - port_tab_sz_ignore_files = 1; break; case 'S' : /* Was handled in early_init() just read past it */ - (void) get_arg(argv[i]+2, argv[i+1], &i); + if (argv[i][2] == 'P') + (void) get_arg(argv[i]+3, argv[i+1], &i); + else + (void) get_arg(argv[i]+2, argv[i+1], &i); break; case 's' : { @@ -1462,6 +1559,22 @@ erl_start(int argc, char **argv) ("suggested scheduler thread stack size %d kilo words\n", erts_sched_thread_suggested_stack_size)); } + else if (has_prefix("fwi", sub_param)) { + long val; + arg = get_arg(sub_param+3, argv[i+1], &i); + errno = 0; + val = strtol(arg, NULL, 10); + if (errno != 0 || val < 0) { + erts_fprintf(stderr, + "bad scheduler forced wakeup " + "interval %s\n", + arg); + erts_usage(); + } +#ifdef ERTS_SMP + erts_runq_supervision_interval = val; +#endif + } else { erts_fprintf(stderr, "bad scheduling option %s\n", argv[i]); erts_usage(); @@ -1642,8 +1755,10 @@ erl_start(int argc, char **argv) erl_init(ncpu, proc_tab_sz, + legacy_proc_tab, port_tab_sz, - port_tab_sz_ignore_files); + port_tab_sz_ignore_files, + legacy_port_tab); load_preloaded(); erts_end_staging_code_ix(); diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index b5b245288b..df7c443387 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -271,6 +271,18 @@ stat_upd_realloc(ErtsAlcType_t n, Uint size, Uint old_size) * stat instrumentation callback functions */ +static void stat_pre_lock(void) +{ + erts_mtx_lock(&instr_mutex); +} + +static void stat_pre_unlock(void) +{ + erts_mtx_unlock(&instr_mutex); +} + +static ErtsAllocatorWrapper_t instr_wrapper; + static void * stat_alloc(ErtsAlcType_t n, void *extra, Uint size) { @@ -278,7 +290,9 @@ stat_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint ssize; void *res; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } ssize = size + STAT_BLOCK_HEADER_SIZE; res = (*real_af->alloc)(n, real_af->extra, ssize); @@ -293,7 +307,9 @@ stat_alloc(ErtsAlcType_t n, void *extra, Uint size) res = (void *) ((StatBlock_t *) res)->mem; } - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } return res; } @@ -307,7 +323,9 @@ stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) void *sptr; void *res; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } if (ptr) { sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE); @@ -329,7 +347,9 @@ stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) res = (void *) ((StatBlock_t *) res)->mem; } - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } return res; } @@ -340,7 +360,9 @@ stat_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *sptr; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } if (ptr) { sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE); @@ -352,7 +374,9 @@ stat_free(ErtsAlcType_t n, void *extra, void *ptr) (*real_af->free)(n, real_af->extra, sptr); - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } } @@ -360,6 +384,18 @@ stat_free(ErtsAlcType_t n, void *extra, void *ptr) * map stat instrumentation callback functions */ +static void map_stat_pre_lock(void) +{ + erts_mtx_lock(&instr_x_mutex); + erts_mtx_lock(&instr_mutex); +} + +static void map_stat_pre_unlock(void) +{ + erts_mtx_unlock(&instr_mutex); + erts_mtx_unlock(&instr_x_mutex); +} + static void * map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) { @@ -367,7 +403,9 @@ map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) Uint msize; void *res; - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_mutex); + } msize = size + MAP_STAT_BLOCK_HEADER_SIZE; res = (*real_af->alloc)(n, real_af->extra, msize); @@ -388,7 +426,9 @@ map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size) res = (void *) mb->mem; } - erts_mtx_unlock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + } return res; } @@ -402,8 +442,10 @@ map_stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) void *mptr; void *res; - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_x_mutex); + erts_mtx_lock(&instr_mutex); + } if (ptr) { mptr = (void *) (((char *) ptr) - MAP_STAT_BLOCK_HEADER_SIZE); @@ -449,9 +491,10 @@ map_stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) res = (void *) mb->mem; } - - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + erts_mtx_unlock(&instr_x_mutex); + } return res; } @@ -462,8 +505,10 @@ map_stat_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *mptr; - erts_mtx_lock(&instr_x_mutex); - erts_mtx_lock(&instr_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&instr_x_mutex); + erts_mtx_lock(&instr_mutex); + } if (ptr) { MapStatBlock_t *mb; @@ -486,8 +531,10 @@ map_stat_free(ErtsAlcType_t n, void *extra, void *ptr) (*real_af->free)(n, real_af->extra, mptr); - erts_mtx_unlock(&instr_mutex); - erts_mtx_unlock(&instr_x_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&instr_mutex); + erts_mtx_unlock(&instr_x_mutex); + } } @@ -496,8 +543,10 @@ static void dump_memory_map_to_stream(FILE *fp) ErtsAlcType_t n; MapStatBlock_t *bp; int lock = !ERTS_IS_CRASH_DUMPING; - if (lock) + if (lock) { + ASSERT(!erts_is_allctr_wrapper_prelocked()); erts_mtx_lock(&instr_mutex); + } /* Write header */ @@ -1155,6 +1204,7 @@ erts_instr_get_type_info(Process *proc) Uint erts_instr_init(int stat, int map_stat) { + Uint extra_sz; int i; am_tot = NULL; @@ -1186,8 +1236,6 @@ erts_instr_init(int stat, int map_stat) sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1)); for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) - continue; if (erts_allctrs_info[i].enabled) stats->ap[i] = &stats->a[i]; else @@ -1201,27 +1249,28 @@ erts_instr_init(int stat, int map_stat) erts_instr_memory_map = 1; erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) - continue; erts_allctrs[i].alloc = map_stat_alloc; erts_allctrs[i].realloc = map_stat_realloc; erts_allctrs[i].free = map_stat_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } - return MAP_STAT_BLOCK_HEADER_SIZE; + instr_wrapper.lock = map_stat_pre_lock; + instr_wrapper.unlock = map_stat_pre_unlock; + extra_sz = MAP_STAT_BLOCK_HEADER_SIZE; } else { erts_instr_stat = 1; for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { - if (ERTS_IS_SBMBC_ALLOCATOR_NO__(i)) - continue; erts_allctrs[i].alloc = stat_alloc; erts_allctrs[i].realloc = stat_realloc; erts_allctrs[i].free = stat_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } - return STAT_BLOCK_HEADER_SIZE; + instr_wrapper.lock = stat_pre_lock; + instr_wrapper.unlock = stat_pre_unlock; + extra_sz = STAT_BLOCK_HEADER_SIZE; } - + erts_allctr_wrapper_prelock_init(&instr_wrapper); + return extra_sz; } diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 69bb4be717..2114d0c001 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2012. All Rights Reserved. + * Copyright Ericsson AB 2005-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -143,21 +143,20 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "ptimer_pre_alloc_lock", "address", }, { "btm_pre_alloc_lock", NULL, }, { "dist_entry_out_queue", "address" }, + { "port_sched_lock", "port_id" }, + { "port_table", NULL }, #endif { "mtrace_op", NULL }, { "instr_x", NULL }, { "instr", NULL }, { "alcu_allocator", "index" }, - { "sbmbc_alloc", "index" }, { "mseg", NULL }, #if HALFWORD_HEAP { "pmmap", NULL }, #endif #ifdef ERTS_SMP - { "port_sched_lock", "port_id" }, { "port_task_pre_alloc_lock", "address" }, { "proclist_pre_alloc_lock", "address" }, - { "port_table", NULL }, { "xports_list_pre_alloc_lock", "address" }, { "inet_buffer_stack_lock", NULL }, { "gc_info", NULL }, @@ -181,6 +180,11 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "efile_drv dtrace mutex", NULL }, #endif { "mtrace_buf", NULL }, +#ifdef __WIN32__ +#ifdef ERTS_SMP + { "sys_gethrtime", NULL }, +#endif +#endif { "erts_alloc_hard_debug", NULL } }; diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 70e592cc5f..244a2b26db 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -1024,3 +1024,23 @@ Eterm erts_debug_dump_links_1(BIF_ALIST_1) } } } + +void erts_one_link_size(ErtsLink *lnk, void *vpu) +{ + Uint *pu = vpu; + *pu += ERTS_LINK_SIZE*sizeof(Uint); + if(!IS_CONST(lnk->pid)) + *pu += NC_HEAP_SIZE(lnk->pid)*sizeof(Uint); + if (lnk->type != LINK_NODE && ERTS_LINK_ROOT(lnk) != NULL) { + erts_doforall_links(ERTS_LINK_ROOT(lnk),&erts_one_link_size,vpu); + } +} +void erts_one_mon_size(ErtsMonitor *mon, void *vpu) +{ + Uint *pu = vpu; + *pu += ERTS_MONITOR_SIZE*sizeof(Uint); + if(!IS_CONST(mon->pid)) + *pu += NC_HEAP_SIZE(mon->pid)*sizeof(Uint); + if(!IS_CONST(mon->ref)) + *pu += NC_HEAP_SIZE(mon->ref)*sizeof(Uint); +} diff --git a/erts/emulator/beam/erl_monitors.h b/erts/emulator/beam/erl_monitors.h index 6a360a2336..fb11dbbd22 100644 --- a/erts/emulator/beam/erl_monitors.h +++ b/erts/emulator/beam/erl_monitors.h @@ -170,6 +170,8 @@ ErtsSuspendMonitor *erts_lookup_suspend_monitor(ErtsSuspendMonitor *root, Eterm pid); void erts_delete_suspend_monitor(ErtsSuspendMonitor **root, Eterm pid); void erts_init_monitors(void); +void erts_one_link_size(ErtsLink *lnk, void *vpu); +void erts_one_mon_size(ErtsMonitor *mon, void *vpu); #define erts_doforall_monitors erts_sweep_monitors #define erts_doforall_links erts_sweep_links diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index e538ba30c2..c8bb126687 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -222,6 +222,8 @@ static byte *tracep; static byte *endp; static SysTimeval last_tv; +static ErtsAllocatorWrapper_t mtrace_wrapper; + #if ERTS_MTRACE_SEGMENT_ID >= ERTS_ALC_A_MIN || ERTS_MTRACE_SEGMENT_ID < 0 #error ERTS_MTRACE_SEGMENT_ID >= ERTS_ALC_A_MIN || ERTS_MTRACE_SEGMENT_ID < 0 #endif @@ -555,6 +557,8 @@ write_trace_header(char *nodename, char *pid, char *hostname) return 1; } +static void mtrace_pre_lock(void); +static void mtrace_pre_unlock(void); static void *mtrace_alloc(ErtsAlcType_t, void *, Uint); static void *mtrace_realloc(ErtsAlcType_t, void *, void *, Uint); static void mtrace_free(ErtsAlcType_t, void *, void *); @@ -635,12 +639,16 @@ erts_mtrace_install_wrapper_functions(void) erts_allctrs[i].free = mtrace_free; erts_allctrs[i].extra = (void *) &real_allctrs[i]; } + mtrace_wrapper.lock = mtrace_pre_lock; + mtrace_wrapper.unlock = mtrace_pre_unlock; + erts_allctr_wrapper_prelock_init(&mtrace_wrapper); } } void erts_mtrace_stop(void) { + ASSERT(!erts_is_allctr_wrapper_prelocked()); erts_mtx_lock(&mtrace_op_mutex); erts_mtx_lock(&mtrace_buf_mutex); if (erts_mtrace_enabled) { @@ -677,6 +685,7 @@ erts_mtrace_stop(void) void erts_mtrace_exit(Uint32 exit_value) { + ASSERT(!erts_is_allctr_wrapper_prelocked()); erts_mtx_lock(&mtrace_op_mutex); erts_mtx_lock(&mtrace_buf_mutex); if (erts_mtrace_enabled) { @@ -935,18 +944,33 @@ write_free_entry(byte tag, erts_mtx_unlock(&mtrace_buf_mutex); } +static void mtrace_pre_lock(void) +{ + erts_mtx_lock(&mtrace_op_mutex); +} + +static void mtrace_pre_unlock(void) +{ + erts_mtx_unlock(&mtrace_op_mutex); +} + + static void * mtrace_alloc(ErtsAlcType_t n, void *extra, Uint size) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *res; - erts_mtx_lock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&mtrace_op_mutex); + } res = (*real_af->alloc)(n, real_af->extra, size); write_alloc_entry(ERTS_MT_ALLOC_BDY_TAG, res, n, 0, size); - erts_mtx_unlock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&mtrace_op_mutex); + } return res; } @@ -957,12 +981,16 @@ mtrace_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *res; - erts_mtx_lock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&mtrace_op_mutex); + } res = (*real_af->realloc)(n, real_af->extra, ptr, size); write_realloc_entry(ERTS_MT_REALLOC_BDY_TAG, res, n, 0, ptr, size); - erts_mtx_unlock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_unlock(&mtrace_op_mutex); + } return res; @@ -973,10 +1001,14 @@ mtrace_free(ErtsAlcType_t n, void *extra, void *ptr) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; - erts_mtx_lock(&mtrace_op_mutex); + if (!erts_is_allctr_wrapper_prelocked()) { + erts_mtx_lock(&mtrace_op_mutex); + } (*real_af->free)(n, real_af->extra, ptr); - write_free_entry(ERTS_MT_FREE_BDY_TAG, n, 0, ptr); + if (!erts_is_allctr_wrapper_prelocked()) { + write_free_entry(ERTS_MT_FREE_BDY_TAG, n, 0, ptr); + } erts_mtx_unlock(&mtrace_op_mutex); } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index d4c2b5bdcc..48f8be8dd3 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -310,9 +310,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, Process* rp; Process* c_p; ErlHeapFragment* frags; -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) - ErtsProcLocks rp_had_locks; -#endif Eterm receiver = to_pid->pid; int flush_me = 0; int scheduler = erts_get_scheduler_id() != 0; @@ -332,10 +329,6 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #endif } -#if defined(ERTS_ENABLE_LOCK_CHECK) && defined(ERTS_SMP) - rp_had_locks = rp_locks; -#endif - rp = (scheduler ? erts_proc_lookup(receiver) : erts_pid2proc_opt(c_p, ERTS_PROC_LOCK_MAIN, diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index ebfba065d1..c6d136f951 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2012. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1371,6 +1371,7 @@ setup_reference_table(void) /* Insert all ports */ max = erts_ptab_max(&erts_port); for (i = 0; i < max; i++) { + ErlOffHeap *ohp; erts_aint32_t state; Port *prt; @@ -1389,8 +1390,9 @@ setup_reference_table(void) if (ERTS_P_MONITORS(prt)) insert_monitors(ERTS_P_MONITORS(prt), prt->common.id); /* Insert port data */ - for(hfp = prt->bp; hfp; hfp = hfp->next) - insert_offheap(&(hfp->off_heap), HEAP_REF, prt->common.id); + ohp = erts_port_data_offheap(prt); + if (ohp) + insert_offheap(ohp, HEAP_REF, prt->common.id); /* Insert controller */ if (prt->dist_entry) insert_dist_entry(prt->dist_entry, diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index 377aa72ed5..ad3f104a68 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -167,8 +167,7 @@ struct _erl_drv_port { #endif erts_atomic_t connected; /* A connected process */ Eterm caller; /* Current caller. */ - Eterm data; /* Data associated with port. */ - ErlHeapFragment* bp; /* Heap fragment holding data (NULL if imm data). */ + erts_smp_atomic_t data; /* Data associated with port. */ Uint bytes_in; /* Number of bytes read */ Uint bytes_out; /* Number of bytes written */ @@ -189,6 +188,12 @@ struct _erl_drv_port { int reds; /* Only used while executing driver callbacks */ }; + +void erts_init_port_data(Port *); +void erts_cleanup_port_data(Port *); +Uint erts_port_data_size(Port *); +ErlOffHeap *erts_port_data_offheap(Port *); + #define ERTS_PORT_GET_CONNECTED(PRT) \ ((Eterm) erts_atomic_read_nob(&(PRT)->connected)) #define ERTS_PORT_SET_CONNECTED(PRT, PID) \ @@ -326,8 +331,6 @@ extern erts_smp_atomic_t erts_bytes_in; /* no bytes sent into the system */ #define ERTS_PORT_REDS_CONTROL (CONTEXT_REDS/100) #define ERTS_PORT_REDS_CALL (CONTEXT_REDS/50) #define ERTS_PORT_REDS_INFO (CONTEXT_REDS/100) -#define ERTS_PORT_REDS_SET_DATA (CONTEXT_REDS/100) -#define ERTS_PORT_REDS_GET_DATA (CONTEXT_REDS/100) #define ERTS_PORT_REDS_TERMINATE (CONTEXT_REDS/50) void print_port_info(Port *, int, void *); @@ -794,8 +797,6 @@ struct binary; #define ERTS_P2P_SIG_TYPE_INFO 7 #define ERTS_P2P_SIG_TYPE_LINK 8 #define ERTS_P2P_SIG_TYPE_UNLINK 9 -#define ERTS_P2P_SIG_TYPE_SET_DATA 10 -#define ERTS_P2P_SIG_TYPE_GET_DATA 11 #define ERTS_P2P_SIG_TYPE_BITS 4 #define ERTS_P2P_SIG_TYPE_MASK \ @@ -810,6 +811,7 @@ struct binary; #define ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT ERTS_P2P_SIG_DATA_FLG(4) #define ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK ERTS_P2P_SIG_DATA_FLG(5) #define ERTS_P2P_SIG_DATA_FLG_SCHED ERTS_P2P_SIG_DATA_FLG(6) +#define ERTS_P2P_SIG_DATA_FLG_ASYNC ERTS_P2P_SIG_DATA_FLG(7) struct ErtsProc2PortSigData_ { int flags; @@ -856,10 +858,6 @@ struct ErtsProc2PortSigData_ { struct { Eterm from; } unlink; - struct { - ErlHeapFragment *bp; - Eterm data; - } set_data; } u; } ; @@ -919,6 +917,7 @@ erts_schedule_proc2port_signal(Process *, Eterm *, ErtsProc2PortSigData *, int, + ErtsPortTaskHandle *, ErtsProc2PortSigCallback); int erts_deliver_port_exit(Port *, Eterm, Eterm, int); @@ -932,6 +931,7 @@ int erts_deliver_port_exit(Port *, Eterm, Eterm, int); #define ERTS_PORT_SIG_FLG_BROKEN_LINK ERTS_P2P_SIG_DATA_FLG_BROKEN_LINK #define ERTS_PORT_SIG_FLG_BAD_OUTPUT ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT #define ERTS_PORT_SIG_FLG_FORCE_SCHED ERTS_P2P_SIG_DATA_FLG_SCHED +#define ERTS_PORT_SIG_FLG_ASYNC ERTS_P2P_SIG_DATA_FLG_ASYNC /* ERTS_PORT_SIG_FLG_FORCE_IMM_CALL only when crash dumping... */ #define ERTS_PORT_SIG_FLG_FORCE_IMM_CALL ERTS_P2P_SIG_DATA_FLG_BAD_OUTPUT @@ -953,7 +953,5 @@ ErtsPortOpResult erts_port_unlink(Process *, Port *, Eterm, Eterm *); ErtsPortOpResult erts_port_control(Process *, Port *, unsigned int, Eterm, Eterm *); ErtsPortOpResult erts_port_call(Process *, Port *, unsigned int, Eterm, Eterm *); ErtsPortOpResult erts_port_info(Process *, Port *, Eterm, Eterm *); -ErtsPortOpResult erts_port_set_data(Process *, Port *, Eterm, Eterm *); -ErtsPortOpResult erts_port_get_data(Process *, Port *, Eterm *); #endif diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 0ed08bee01..547a42beb2 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -552,6 +552,19 @@ set_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) } } +static ERTS_INLINE void +set_tmp_handle(ErtsPortTask *ptp, ErtsPortTaskHandle *pthp) +{ + ptp->u.alive.handle = NULL; + if (pthp) { + /* + * IMPORTANT! Task either need to be aborted, or task handle + * need to be detached before thread progress has been made. + */ + erts_smp_atomic_set_relb(pthp, (erts_aint_t) ptp); + } +} + /* * Busy port queue management @@ -1182,6 +1195,13 @@ erl_drv_consume_timeslice(ErlDrvPort dprt, int percent) return 1; } +void +erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *pthp) +{ + ERTS_SMP_LC_ASSERT(erts_thr_progress_lc_is_delaying()); + reset_port_task_handle(pthp); +} + /* * Abort a scheduled task. */ @@ -1206,7 +1226,7 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp) ERTS_SMP_READ_MEMORY_BARRIER; old_state = erts_smp_atomic32_read_nob(&ptp->state); if (old_state == ERTS_PT_STATE_SCHEDULED) { - ASSERT(saved_pthp == pthp); + ASSERT(!saved_pthp || saved_pthp == pthp); } #endif @@ -1227,9 +1247,6 @@ erts_port_task_abort(ErtsPortTaskHandle *pthp) &erts_port_task_outstanding_io_tasks) > 0); erts_smp_atomic_dec_relb(&erts_port_task_outstanding_io_tasks); break; - case ERTS_PORT_TASK_PROC_SIG: - ERTS_INTERNAL_ERROR("Aborted process to port signal"); - break; default: break; } @@ -1404,7 +1421,6 @@ erts_port_task_schedule(Eterm id, } case ERTS_PORT_TASK_PROC_SIG: { va_list argp; - ASSERT(!pthp); va_start(argp, type); sigdp = va_arg(argp, ErtsProc2PortSigData *); ptp = p2p_sig_data_to_task(sigdp); @@ -1412,7 +1428,7 @@ erts_port_task_schedule(Eterm id, ptp->u.alive.flags |= va_arg(argp, int); va_end(argp); if (!(ptp->u.alive.flags & ERTS_PT_FLG_NOSUSPEND)) - set_handle(ptp, pthp); + set_tmp_handle(ptp, pthp); else { ns_pthlp = erts_alloc(ERTS_ALC_T_PT_HNDL_LIST, sizeof(ErtsPortTaskHandleList)); @@ -1578,6 +1594,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) int fpe_was_unmasked; erts_aint32_t state; int active; + Uint64 start_time = 0; ERTS_SMP_LC_ASSERT(erts_smp_lc_runq_is_locked(runq)); @@ -1639,6 +1656,10 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reset_handle(ptp); + if (erts_system_monitor_long_schedule != 0) { + start_time = erts_timestamp_millis(); + } + ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(pp)); ERTS_SMP_CHK_NO_PROC_LOCKS; ASSERT(pp->drv_ptr); @@ -1707,6 +1728,14 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds += erts_port_driver_callback_epilogue(pp, &state); + if (start_time != 0) { + Sint64 diff = erts_timestamp_millis() - start_time; + if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { + monitor_long_schedule_port(pp,ptp->type,(Uint) diff); + } + } + start_time = 0; + aborted_port_task: schedule_port_task_free(ptp); @@ -1809,6 +1838,16 @@ release_port(void *vport) { erts_port_dec_refc((Port *) vport); } + +static void +schedule_release_port(void *vport) { + Port *pp = (Port*)vport; + /* This is only used when a port release was ordered from a non-scheduler */ + erts_schedule_thr_prgr_later_op(release_port, + (void *) pp, + &pp->common.u.release); +} + #endif static void @@ -1912,18 +1951,21 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) break; case ERTS_PORT_TASK_INPUT: erts_stale_drv_select(pp->common.id, + ERTS_Port2ErlDrvPort(pp), ptp->u.alive.td.io.event, DO_READ, 1); break; case ERTS_PORT_TASK_OUTPUT: erts_stale_drv_select(pp->common.id, + ERTS_Port2ErlDrvPort(pp), ptp->u.alive.td.io.event, DO_WRITE, 1); break; case ERTS_PORT_TASK_EVENT: erts_stale_drv_select(pp->common.id, + ERTS_Port2ErlDrvPort(pp), ptp->u.alive.td.io.event, 0, 1); @@ -2001,10 +2043,15 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) * Schedule cleanup of port structure... */ #ifdef ERTS_SMP - /* Has to be more or less immediate to release any driver */ - erts_schedule_thr_prgr_later_op(release_port, - (void *) pp, - &pp->common.u.release); + /* We might not be a scheduler, eg. traceing to port we are sys_msg_dispatcher */ + if (!erts_get_scheduler_data()) { + erts_schedule_misc_aux_work(1, schedule_release_port, (void*)pp); + } else { + /* Has to be more or less immediate to release any driver */ + erts_schedule_thr_prgr_later_op(release_port, + (void *) pp, + &pp->common.u.release); + } #else pp->cleanup = 1; #endif diff --git a/erts/emulator/beam/erl_port_task.h b/erts/emulator/beam/erl_port_task.h index ae6cd69ae2..e4d964146e 100644 --- a/erts/emulator/beam/erl_port_task.h +++ b/erts/emulator/beam/erl_port_task.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2012. All Rights Reserved. + * Copyright Ericsson AB 2006-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -243,7 +243,9 @@ int erts_port_task_execute(ErtsRunQueue *, Port **); void erts_port_task_init(void); #endif +void erts_port_task_tmp_handle_detach(ErtsPortTaskHandle *); int erts_port_task_abort(ErtsPortTaskHandle *); + void erts_port_task_abort_nosuspend_tasks(Port *); int erts_port_task_schedule(Eterm, diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index f9ef4eac65..434d5ca147 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -246,6 +246,10 @@ static erts_smp_atomic32_t function_calls; #ifdef ERTS_SMP static erts_smp_atomic32_t doing_sys_schedule; static erts_smp_atomic32_t no_empty_run_queues; +long erts_runq_supervision_interval = 0; +static ethr_event runq_supervision_event; +static erts_tid_t runq_supervisor_tid; +static erts_atomic_t runq_supervisor_sleeping; #else /* !ERTS_SMP */ ErtsSchedulerData *erts_scheduler_data; #endif @@ -267,6 +271,7 @@ static Uint last_exact_reductions; Uint erts_default_process_flags; Eterm erts_system_monitor; Eterm erts_system_monitor_long_gc; +Uint erts_system_monitor_long_schedule; Eterm erts_system_monitor_large_heap; struct erts_system_monitor_flags_t erts_system_monitor_flags; @@ -485,7 +490,7 @@ release_process(void *vproc) /* initialize the scheduler */ void -erts_init_process(int ncpu, int proc_tab_size) +erts_init_process(int ncpu, int proc_tab_size, int legacy_proc_tab) { #ifdef ERTS_SMP @@ -505,7 +510,8 @@ erts_init_process(int ncpu, int proc_tab_size) (ErtsPTabElementCommon *) &erts_invalid_process.common, proc_tab_size, sizeof(Process), - "process_table"); + "process_table", + legacy_proc_tab); last_reductions = 0; last_exact_reductions = 0; @@ -1326,6 +1332,17 @@ erts_alloc_notify_delayed_dealloc(int ix) ERTS_SSI_AUX_WORK_DD); } +void +erts_alloc_ensure_handle_delayed_dealloc_call(int ix) +{ +#ifdef DEBUG + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(!esdp || ix == (int) esdp->no); +#endif + set_aux_work_flags_wakeup_nob(ERTS_SCHED_SLEEP_INFO_IX(ix-1), + ERTS_SSI_AUX_WORK_DD); +} + static ERTS_INLINE erts_aint32_t handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) { @@ -1882,7 +1899,7 @@ erts_set_aux_work_timeout(int ix, erts_aint32_t type, int enable) ERTS_DBG_CHK_AUX_WORK_VAL(type); ERTS_DBG_CHK_AUX_WORK_VAL(erts_atomic32_read_nob(&aux_work_tmo->type[ix])); -// erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable); + /* erts_fprintf(stderr, "t(%d, 0x%x, %d)\n", ix, type, enable); */ if (!enable) { old = erts_atomic32_read_band_mb(&aux_work_tmo->type[ix], ~type); @@ -2032,7 +2049,13 @@ empty_runq(ErtsRunQueue *rq) */ ASSERT(0 <= empty && empty < 2*erts_no_run_queues); #endif - erts_smp_atomic32_inc_relb(&no_empty_run_queues); + if (!erts_runq_supervision_interval) + erts_smp_atomic32_inc_relb(&no_empty_run_queues); + else { + erts_smp_atomic32_inc_mb(&no_empty_run_queues); + if (erts_atomic_read_nob(&runq_supervisor_sleeping)) + ethr_event_set(&runq_supervision_event); + } } } @@ -2049,7 +2072,14 @@ non_empty_runq(ErtsRunQueue *rq) */ ASSERT(0 < empty && empty <= 2*erts_no_run_queues); #endif - erts_smp_atomic32_dec_relb(&no_empty_run_queues); + if (!erts_runq_supervision_interval) + erts_smp_atomic32_dec_relb(&no_empty_run_queues); + else { + erts_aint32_t no; + no = erts_smp_atomic32_dec_read_mb(&no_empty_run_queues); + if (no > 0 && erts_atomic_read_nob(&runq_supervisor_sleeping)) + ethr_event_set(&runq_supervision_event); + } } } @@ -2653,7 +2683,6 @@ try_inc_no_active_runqs(int active) return 0; } - static ERTS_INLINE int chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) { @@ -4319,6 +4348,53 @@ set_wakeup_other_data(void) } } +static int +no_runqs_to_supervise(void) +{ + int used; + erts_aint32_t nerq = erts_smp_atomic32_read_acqb(&no_empty_run_queues); + if (nerq <= 0) + return 0; + get_no_runqs(NULL, &used); + if (nerq >= used) + return 0; + return used; +} + +static void * +runq_supervisor(void *unused) +{ + while (1) { + int ix, no_rqs; + + erts_milli_sleep(erts_runq_supervision_interval); + no_rqs = no_runqs_to_supervise(); + if (!no_rqs) { + erts_atomic_set_nob(&runq_supervisor_sleeping, 1); + while (1) { + ethr_event_reset(&runq_supervision_event); + no_rqs = no_runqs_to_supervise(); + if (no_rqs) { + erts_atomic_set_nob(&runq_supervisor_sleeping, 0); + break; + } + ethr_event_wait(&runq_supervision_event); + } + } + + for (ix = 0; ix < no_rqs; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + if (ERTS_RUNQ_FLGS_GET(rq) & ERTS_RUNQ_FLG_NONEMPTY) { + erts_smp_runq_lock(rq); + if (rq->len != 0) + wake_scheduler_on_empty_runq(rq); /* forced wakeup... */ + erts_smp_runq_unlock(rq); + } + } + } + return NULL; +} + #endif void @@ -4670,6 +4746,8 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online) esdp->reductions = 0; init_sched_wall_time(&esdp->sched_wall_time); + erts_port_task_handle_init(&esdp->nosuspend_port_task_handle); + } init_misc_aux_work(); @@ -5740,6 +5818,22 @@ erts_start_schedulers(void) ethr_thr_opts opts = ETHR_THR_OPTS_DEFAULT_INITER; opts.detached = 1; + +#ifdef ERTS_SMP + if (erts_runq_supervision_interval) { + opts.suggested_stack_size = 16; + erts_atomic_init_nob(&runq_supervisor_sleeping, 0); + if (0 != ethr_event_init(&runq_supervision_event)) + erl_exit(1, "Failed to create run-queue supervision event\n"); + if (0 != ethr_thr_create(&runq_supervisor_tid, + runq_supervisor, + NULL, + &opts)) + erl_exit(1, "Failed to create run-queue supervision thread\n"); + + } +#endif + opts.suggested_stack_size = erts_sched_thread_suggested_stack_size; if (wanted < 1) @@ -7143,7 +7237,7 @@ erts_sched_stat_modify(int what) break; case ERTS_SCHED_STAT_MODIFY_DISABLE: erts_smp_thr_progress_block(); - erts_sched_stat.enabled = 1; + erts_sched_stat.enabled = 0; erts_smp_thr_progress_unblock(); break; case ERTS_SCHED_STAT_MODIFY_CLEAR: @@ -7498,6 +7592,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->htop = p->heap; p->heap_sz = sz; p->catches = 0; + p->extra_root = NULL; p->bin_vheap_sz = p->min_vheap_size; p->bin_old_vheap_sz = p->min_vheap_size; @@ -8930,6 +9025,12 @@ erts_continue_exit_process(Process *p) if (pbt) erts_free(ERTS_ALC_T_BPD, (void *) pbt); + if (p->extra_root != NULL) { + (p->extra_root->cleanup)(p->extra_root); /* Should deallocate + whole structure */ + p->extra_root = NULL; + } + delete_process(p); #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 3d3579fa7e..8e5467f196 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -387,6 +387,10 @@ struct ErtsRunQueue_ { } ports; }; +#ifdef ERTS_SMP +extern long erts_runq_supervision_interval; +#endif + typedef union { ErtsRunQueue runq; char align[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsRunQueue))]; @@ -420,6 +424,11 @@ typedef struct { } ErtsSchedWallTime; typedef struct { + Uint64 reclaimed; + Uint64 garbage_cols; +} ErtsGCInfo; + +typedef struct { int sched; erts_aint32_t aux_work; } ErtsDelayedAuxWorkWakeupJob; @@ -507,6 +516,8 @@ struct ErtsSchedulerData_ { Uint64 reductions; ErtsSchedWallTime sched_wall_time; + ErtsGCInfo gc_info; + ErtsPortTaskHandle nosuspend_port_task_handle; #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC erts_alloc_verify_func_t verify_unused_temp_alloc; @@ -692,6 +703,14 @@ struct ErtsPendingSuspend_ { #endif + +typedef struct ErlExtraRootSet_ ErlExtraRootSet; +struct ErlExtraRootSet_ { + Eterm *objv; + Uint sz; + void (*cleanup)(ErlExtraRootSet *); +}; + /* Defines to ease the change of memory architecture */ # define HEAP_START(p) (p)->heap # define HEAP_TOP(p) (p)->htop @@ -785,6 +804,8 @@ struct process { ErlMessageQueue msg; /* Message queue */ + ErlExtraRootSet *extra_root; /* Used by trapping BIF's */ + union { ErtsBifTimer *bif_timers; /* Bif timers aiming at this process */ void *terminate; @@ -1002,6 +1023,7 @@ extern erts_smp_rwmtx_t erts_cpu_bind_rwmtx; */ extern Eterm erts_system_monitor; extern Uint erts_system_monitor_long_gc; +extern Uint erts_system_monitor_long_schedule; extern Uint erts_system_monitor_large_heap; struct erts_system_monitor_flags_t { unsigned int busy_port : 1; @@ -1125,6 +1147,7 @@ void erts_early_init_scheduling(int); void erts_init_scheduling(int, int); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Eterm erts_gc_info_request(Process *c_p); Uint64 erts_get_proc_interval(void); Uint64 erts_ensure_later_proc_interval(Uint64); Uint64 erts_step_proc_interval(void); @@ -1332,6 +1355,7 @@ int erts_is_multi_scheduling_blocked(void); Eterm erts_multi_scheduling_blockers(Process *); void erts_start_schedulers(void); void erts_alloc_notify_delayed_dealloc(int); +void erts_alloc_ensure_handle_delayed_dealloc_call(int); void erts_smp_notify_check_children_needed(void); #endif #if ERTS_USE_ASYNC_READY_Q @@ -1351,7 +1375,7 @@ void erts_schedule_multi_misc_aux_work(int ignore_self, erts_aint32_t erts_set_aux_work_timeout(int, erts_aint32_t, int); void erts_sched_notify_check_cpu_bind(void); Uint erts_active_schedulers(void); -void erts_init_process(int, int); +void erts_init_process(int, int, int); Eterm erts_process_status(Process *, ErtsProcLocks, Process *, Eterm); Uint erts_run_queues_len(Uint *); void erts_add_to_runq(Process *); @@ -1421,6 +1445,8 @@ Eterm erts_debug_reader_groups_map(Process *c_p, int groups); Uint erts_debug_nbalance(void); int erts_debug_wait_deallocations(Process *c_p); +Uint erts_process_memory(Process *c_p); + #ifdef ERTS_SMP # define ERTS_GET_SCHEDULER_DATA_FROM_PROC(PROC) ((PROC)->scheduler_data) # define ERTS_PROC_GET_SCHDATA(PROC) ((PROC)->scheduler_data) @@ -1964,6 +1990,7 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) } } + #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* #ifdef ERTS_SMP */ diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index a93229c473..6cd0d23b97 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -76,6 +76,43 @@ erts_deep_process_dump(int to, void *to_arg) dump_binaries(to, to_arg, all_binaries); } +Uint erts_process_memory(Process *p) { + ErlMessage *mp; + Uint size = 0; + struct saved_calls *scb; + size += sizeof(Process); + + ERTS_SMP_MSGQ_MV_INQ2PRIVQ(p); + + erts_doforall_links(ERTS_P_LINKS(p), &erts_one_link_size, &size); + erts_doforall_monitors(ERTS_P_MONITORS(p), &erts_one_mon_size, &size); + size += (p->heap_sz + p->mbuf_sz) * sizeof(Eterm); + if (p->old_hend && p->old_heap) + size += (p->old_hend - p->old_heap) * sizeof(Eterm); + + size += p->msg.len * sizeof(ErlMessage); + + for (mp = p->msg.first; mp; mp = mp->next) + if (mp->data.attached) + size += erts_msg_attached_data_size(mp)*sizeof(Eterm); + + if (p->arg_reg != p->def_arg_reg) { + size += p->arity * sizeof(p->arg_reg[0]); + } + + if (p->psd) + size += sizeof(ErtsPSD); + + scb = ERTS_PROC_GET_SAVED_CALLS_BUF(p); + if (scb) { + size += (sizeof(struct saved_calls) + + (scb->len-1) * sizeof(scb->ct[0])); + } + + size += erts_dicts_mem_size(p); + return size; +} + static void dump_process_info(int to, void *to_arg, Process *p) { diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index 5bbc71c659..fa5482b841 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -34,6 +34,8 @@ typedef struct ErtsPTabListBifData_ ErtsPTabListBifData; +#define ERTS_PTAB_NEW_MAX_RESERVE_FAIL 1000 + #define ERTS_PTAB_LIST_BIF_TAB_INSPECT_INDICES_PER_RED 25 #define ERTS_PTAB_LIST_BIF_TAB_CHUNK_SIZE 1000 #define ERTS_PTAB_LIST_BIF_MIN_START_REDS \ @@ -415,6 +417,27 @@ last_data_cmp(Uint64 ld1, Uint64 ld2) #define ERTS_PTAB_LastData2EtermData(LD) \ ((Eterm) ((LD) & ~(~((Uint64) 0) << ERTS_PTAB_ID_DATA_SIZE))) +static ERTS_INLINE Uint32 +ix_to_free_id_data_ix(ErtsPTab *ptab, Uint32 ix) +{ + Uint32 dix; + + dix = ((ix & ptab->r.o.dix_cl_mask) << ptab->r.o.dix_cl_shift); + dix += ((ix >> ptab->r.o.dix_cli_shift) & ptab->r.o.dix_cli_mask); + ASSERT(0 <= dix && dix < ptab->r.o.max); + return dix; +} + +UWord +erts_ptab_mem_size(ErtsPTab *ptab) +{ + UWord size = ptab->r.o.max*sizeof(erts_smp_atomic_t); + if (ptab->r.o.free_id_data) + size += ptab->r.o.max*sizeof(erts_smp_atomic32_t); + return size; +} + + void erts_ptab_init_table(ErtsPTab *ptab, ErtsAlcType_t atype, @@ -422,10 +445,11 @@ erts_ptab_init_table(ErtsPTab *ptab, ErtsPTabElementCommon *invalid_element, int size, UWord element_size, - char *name) + char *name, + int legacy) { - size_t tab_sz; - int bits; + size_t tab_sz, alloc_sz; + Uint32 bits, cl, cli, ix, ix_per_cache_line, tab_cache_lines; char *tab_end; erts_smp_atomic_t *tab_entry; erts_smp_rwmtx_opt_t rwmtx_opts = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; @@ -448,7 +472,10 @@ erts_ptab_init_table(ErtsPTab *ptab, ptab->r.o.max = size; tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic_t)); - ptab->r.o.tab = erts_alloc_permanent_cache_aligned(atype, tab_sz); + alloc_sz = tab_sz; + if (!legacy) + alloc_sz += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t)); + ptab->r.o.tab = erts_alloc_permanent_cache_aligned(atype, alloc_sz); tab_end = ((char *) ptab->r.o.tab) + tab_sz; tab_entry = ptab->r.o.tab; while (tab_end > ((char *) tab_entry)) { @@ -456,25 +483,17 @@ erts_ptab_init_table(ErtsPTab *ptab, tab_entry++; } - ptab->r.o.tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; - ptab->r.o.pix_per_cache_line = (ERTS_CACHE_LINE_SIZE - / sizeof(erts_smp_atomic_t)); + tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; + ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic_t)); ASSERT((ptab->r.o.max & (ptab->r.o.max - 1)) == 0); /* power of 2 */ - ASSERT((ptab->r.o.pix_per_cache_line - & (ptab->r.o.pix_per_cache_line - 1)) == 0); /* power of 2 */ - ASSERT((ptab->r.o.tab_cache_lines - & (ptab->r.o.tab_cache_lines - 1)) == 0); /* power of 2 */ - - ptab->r.o.pix_mask - = (1 << bits) - 1; - ptab->r.o.pix_cl_mask - = ptab->r.o.tab_cache_lines-1; - ptab->r.o.pix_cl_shift - = erts_fit_in_bits_int32(ptab->r.o.pix_per_cache_line-1); - ptab->r.o.pix_cli_shift - = erts_fit_in_bits_int32(ptab->r.o.pix_cl_mask); - ptab->r.o.pix_cli_mask - = (1 << (bits - ptab->r.o.pix_cli_shift)) - 1; + ASSERT((ix_per_cache_line & (ix_per_cache_line - 1)) == 0); /* power of 2 */ + ASSERT((tab_cache_lines & (tab_cache_lines - 1)) == 0); /* power of 2 */ + + ptab->r.o.pix_mask = (1 << bits) - 1; + ptab->r.o.pix_cl_mask = tab_cache_lines-1; + ptab->r.o.pix_cl_shift = erts_fit_in_bits_int32(ix_per_cache_line-1); + ptab->r.o.pix_cli_shift = erts_fit_in_bits_int32(ptab->r.o.pix_cl_mask); + ptab->r.o.pix_cli_mask = (1 << (bits - ptab->r.o.pix_cli_shift)) - 1; ASSERT(ptab->r.o.pix_cl_shift + ptab->r.o.pix_cli_shift == bits); @@ -482,6 +501,46 @@ erts_ptab_init_table(ErtsPTab *ptab, ptab->r.o.invalid_data = erts_ptab_id2data(ptab, invalid_element->id); ptab->r.o.release_element = release_element; + if (legacy) { + ptab->r.o.free_id_data = NULL; + ptab->r.o.dix_cl_mask = 0; + ptab->r.o.dix_cl_shift = 0; + ptab->r.o.dix_cli_shift = 0; + ptab->r.o.dix_cli_mask = 0; + } + else { + + tab_sz = ERTS_ALC_CACHE_LINE_ALIGN_SIZE(size*sizeof(erts_smp_atomic32_t)); + ptab->r.o.free_id_data = (erts_smp_atomic32_t *) tab_end; + + tab_cache_lines = tab_sz/ERTS_CACHE_LINE_SIZE; + ix_per_cache_line = (ERTS_CACHE_LINE_SIZE/sizeof(erts_smp_atomic32_t)); + + ptab->r.o.dix_cl_mask = tab_cache_lines-1; + ptab->r.o.dix_cl_shift = erts_fit_in_bits_int32(ix_per_cache_line-1); + ptab->r.o.dix_cli_shift = erts_fit_in_bits_int32(ptab->r.o.dix_cl_mask); + ptab->r.o.dix_cli_mask = (1 << (bits - ptab->r.o.dix_cli_shift)) - 1; + + ASSERT((ix_per_cache_line & (ix_per_cache_line - 1)) == 0); /* power of 2 */ + ASSERT((tab_cache_lines & (tab_cache_lines - 1)) == 0); /* power of 2 */ + + ASSERT(ptab->r.o.dix_cl_shift + ptab->r.o.dix_cli_shift == bits); + + ix = 0; + for (cl = 0; cl < tab_cache_lines; cl++) { + for (cli = 0; cli < ix_per_cache_line; cli++) { + erts_smp_atomic32_init_nob(&ptab->r.o.free_id_data[ix], + cli*tab_cache_lines+cl); + ASSERT(erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data); + ix++; + } + } + + erts_smp_atomic32_init_nob(&ptab->vola.tile.aid_ix, -1); + erts_smp_atomic32_init_nob(&ptab->vola.tile.fid_ix, -1); + + } + erts_smp_interval_init(&ptab->list.data.interval); ptab->list.data.deleted.start = NULL; ptab->list.data.deleted.end = NULL; @@ -522,9 +581,7 @@ erts_ptab_new_element(ErtsPTab *ptab, void *init_arg, void (*init_ptab_el)(void *, Eterm)) { - int pix; - Uint64 ld, exp_ld; - Eterm data; + Uint32 pix, ix, data; erts_aint32_t count; erts_aint_t invalid = (erts_aint_t) ptab->r.o.invalid_element; @@ -551,62 +608,110 @@ erts_ptab_new_element(ErtsPTab *ptab, ptab_el->u.alive.started_interval = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); - ld = last_data_read_acqb(ptab); + if (ptab->r.o.free_id_data) { + do { + ix = (Uint32) erts_smp_atomic32_inc_read_acqb(&ptab->vola.tile.aid_ix); + ix = ix_to_free_id_data_ix(ptab, ix); - /* Reserve slot */ - while (1) { - ld++; - pix = erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld)); - if (erts_smp_atomic_read_nob(&ptab->r.o.tab[pix]) == ERTS_AINT_NULL) { - erts_aint_t val; - val = erts_smp_atomic_cmpxchg_relb(&ptab->r.o.tab[pix], - invalid, - ERTS_AINT_NULL); + data = erts_smp_atomic32_xchg_nob(&ptab->r.o.free_id_data[ix], + (erts_aint32_t)ptab->r.o.invalid_data); + }while ((Eterm)data == ptab->r.o.invalid_data); + + init_ptab_el(init_arg, (Eterm) data); + +#ifdef ERTS_SMP + erts_smp_atomic32_init_nob(&ptab_el->refc, 1); +#endif + + pix = erts_ptab_data2pix(ptab, (Eterm) data); + +#ifdef DEBUG + ASSERT(ERTS_AINT_NULL == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], + (erts_aint_t) ptab_el)); +#else + erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); +#endif + + erts_ptab_runlock(ptab); - if (ERTS_AINT_NULL == val) - break; - } } + else { + int rlocked = ERTS_PTAB_NEW_MAX_RESERVE_FAIL; + Uint64 ld, exp_ld; + /* Deprecated legacy algorithm... */ - data = ERTS_PTAB_LastData2EtermData(ld); + restart: + + ptab_el->u.alive.started_interval + = erts_smp_current_interval_nob(erts_ptab_interval(ptab)); + + ld = last_data_read_acqb(ptab); + + /* Reserve slot */ + while (1) { + ld++; + pix = erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld)); + if (erts_smp_atomic_read_nob(&ptab->r.o.tab[pix]) + == ERTS_AINT_NULL) { + erts_aint_t val; + val = erts_smp_atomic_cmpxchg_relb(&ptab->r.o.tab[pix], + invalid, + ERTS_AINT_NULL); + + if (ERTS_AINT_NULL == val) + break; + } + if (rlocked && --rlocked == 0) { + erts_ptab_runlock(ptab); + erts_ptab_rwlock(ptab); + goto restart; + } + } - if (data == ptab->r.o.invalid_data) { - /* Do not use invalid data; fix it... */ - ld += ptab->r.o.max; - ASSERT(pix == erts_ptab_data2pix(ptab, - ERTS_PTAB_LastData2EtermData(ld))); data = ERTS_PTAB_LastData2EtermData(ld); - ASSERT(data != ptab->r.o.invalid_data); - } - exp_ld = last_data_read_nob(ptab); + if (data == ptab->r.o.invalid_data) { + /* Do not use invalid data; fix it... */ + ld += ptab->r.o.max; + ASSERT(pix == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); + data = ERTS_PTAB_LastData2EtermData(ld); + ASSERT(data != ptab->r.o.invalid_data); + } + + exp_ld = last_data_read_nob(ptab); - /* Move last data forward */ - while (1) { - Uint64 act_ld; - if (last_data_cmp(ld, exp_ld) < 0) - break; - act_ld = last_data_cmpxchg_relb(ptab, ld, exp_ld); - if (act_ld == exp_ld) - break; - exp_ld = act_ld; - } + /* Move last data forward */ + while (1) { + Uint64 act_ld; + if (last_data_cmp(ld, exp_ld) < 0) + break; + act_ld = last_data_cmpxchg_relb(ptab, ld, exp_ld); + if (act_ld == exp_ld) + break; + exp_ld = act_ld; + } - init_ptab_el(init_arg, data); + init_ptab_el(init_arg, data); #ifdef ERTS_SMP - erts_smp_atomic32_init_nob(&ptab_el->refc, 1); + erts_smp_atomic32_init_nob(&ptab_el->refc, 1); #endif - /* Move into slot reserved */ + /* Move into slot reserved */ #ifdef DEBUG - ASSERT(invalid == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], + ASSERT(invalid == erts_smp_atomic_xchg_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el)); #else - erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); + erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], (erts_aint_t) ptab_el); #endif - erts_ptab_runlock(ptab); + if (rlocked) + erts_ptab_runlock(ptab); + else + erts_ptab_rwunlock(ptab); + + } return 1; } @@ -647,7 +752,9 @@ erts_ptab_delete_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el) { int maybe_save; - int pix = erts_ptab_id2pix(ptab, ptab_el->id); + Uint32 pix, ix, data; + + pix = erts_ptab_id2pix(ptab, ptab_el->id); ASSERT(erts_get_scheduler_id()); /* *Need* to be a scheduler */ @@ -660,6 +767,29 @@ erts_ptab_delete_element(ErtsPTab *ptab, erts_smp_atomic_set_relb(&ptab->r.o.tab[pix], ERTS_AINT_NULL); + if (ptab->r.o.free_id_data) { + Uint32 prev_data; + /* Next data for this slot... */ + data = (Uint32) erts_ptab_id2data(ptab, ptab_el->id); + data += ptab->r.o.max; + data &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + if (data == ptab->r.o.invalid_data) { /* make sure not invalid */ + data += ptab->r.o.max; + data &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + } + ASSERT(data != ptab->r.o.invalid_data); + ASSERT(pix == erts_ptab_data2pix(ptab, data)); + + do { + ix = (Uint32) erts_smp_atomic32_inc_read_relb(&ptab->vola.tile.fid_ix); + ix = ix_to_free_id_data_ix(ptab, ix); + + prev_data = erts_smp_atomic32_cmpxchg_nob(&ptab->r.o.free_id_data[ix], + data, + ptab->r.o.invalid_data); + }while ((Eterm)prev_data != ptab->r.o.invalid_data); + } + ASSERT(erts_smp_atomic32_read_nob(&ptab->vola.tile.count) > 0); erts_smp_atomic32_dec_relb(&ptab->vola.tile.count); @@ -1270,6 +1400,31 @@ erts_ptab_init(void) * Debug stuff */ +static void assert_ptab_consistency(ErtsPTab *ptab) +{ +#ifdef DEBUG + if (ptab->r.o.free_id_data) { + Uint32 ix, pix, data; + int free_pids = 0; + int null_slots = 0; + + for (ix=0; ix < ptab->r.o.max; ix++) { + if (erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]) != ptab->r.o.invalid_data) { + ++free_pids; + data = erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[ix]); + pix = erts_ptab_data2pix(ptab, (Eterm) data); + ASSERT(erts_ptab_pix2intptr_nob(ptab, pix) == ERTS_AINT_NULL); + } + if (erts_smp_atomic_read_nob(&ptab->r.o.tab[ix]) == ERTS_AINT_NULL) { + ++null_slots; + } + } + ASSERT(free_pids == null_slots); + ASSERT(free_pids == ptab->r.o.max - erts_smp_atomic32_read_nob(&ptab->vola.tile.count)); + } +#endif +} + Sint erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) { @@ -1280,45 +1435,93 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) erts_ptab_rwlock(ptab); - if (!set) - ld = last_data_read_nob(ptab); - else { + assert_ptab_consistency(ptab); - ld = (Uint64) next; - data = ERTS_PTAB_LastData2EtermData(ld); - if (ptab->r.o.invalid_data == data) { - ld += ptab->r.o.max; - ASSERT(erts_ptab_data2pix(ptab, data) - == erts_ptab_data2pix(ptab, - ERTS_PTAB_LastData2EtermData(ld))); + if (ptab->r.o.free_id_data) { + Uint32 id_ix, dix; + + if (set) { + Uint32 i, max_ix, num, stop_id_ix; + max_ix = ptab->r.o.max - 1; + num = next; + id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix); + + for (i=0; i <= max_ix; ++i) { + Uint32 pix; + ++num; + num &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + if (num == ptab->r.o.invalid_data) { + num += ptab->r.o.max; + num &= ~(~((Uint32) 0) << ERTS_PTAB_ID_DATA_SIZE); + } + pix = erts_ptab_data2pix(ptab, num); + if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) { + ++id_ix; + dix = ix_to_free_id_data_ix(ptab, id_ix); + erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], num); + ASSERT(pix == erts_ptab_data2pix(ptab, num)); + } + } + erts_smp_atomic32_set_nob(&ptab->vola.tile.fid_ix, id_ix); + + /* Write invalid_data in rest of free_id_data[]: */ + stop_id_ix = (1 + erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix)) & max_ix; + while (1) { + id_ix = (id_ix+1) & max_ix; + if (id_ix == stop_id_ix) + break; + dix = ix_to_free_id_data_ix(ptab, id_ix); + erts_smp_atomic32_set_nob(&ptab->r.o.free_id_data[dix], + ptab->r.o.invalid_data); + } } - last_data_set_relb(ptab, ld); + id_ix = (Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.aid_ix) + 1; + dix = ix_to_free_id_data_ix(ptab, id_ix); + res = (Sint) erts_smp_atomic32_read_nob(&ptab->r.o.free_id_data[dix]); } + else { + /* Deprecated legacy algorithm... */ + if (!set) + ld = last_data_read_nob(ptab); + else { - while (1) { - int pix; - ld++; - pix = (int) (ld % ptab->r.o.max); - if (first_pix < 0) - first_pix = pix; - else if (pix == first_pix) { - res = -1; - break; - } - if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) { + ld = (Uint64) next; data = ERTS_PTAB_LastData2EtermData(ld); if (ptab->r.o.invalid_data == data) { ld += ptab->r.o.max; ASSERT(erts_ptab_data2pix(ptab, data) == erts_ptab_data2pix(ptab, ERTS_PTAB_LastData2EtermData(ld))); + } + last_data_set_relb(ptab, ld); + } + + while (1) { + int pix; + ld++; + pix = (int) (ld % ptab->r.o.max); + if (first_pix < 0) + first_pix = pix; + else if (pix == first_pix) { + res = -1; + break; + } + if (ERTS_AINT_NULL == erts_ptab_pix2intptr_nob(ptab, pix)) { data = ERTS_PTAB_LastData2EtermData(ld); + if (ptab->r.o.invalid_data == data) { + ld += ptab->r.o.max; + ASSERT(erts_ptab_data2pix(ptab, data) + == erts_ptab_data2pix(ptab, + ERTS_PTAB_LastData2EtermData(ld))); + data = ERTS_PTAB_LastData2EtermData(ld); + } + res = data; + break; } - res = data; - break; } } + assert_ptab_consistency(ptab); erts_ptab_rwunlock(ptab); return res; diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h index 7fa1251900..e3e05f14af 100644 --- a/erts/emulator/beam/erl_ptab.h +++ b/erts/emulator/beam/erl_ptab.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -94,18 +94,23 @@ typedef struct { erts_smp_atomic_t last_data; #endif erts_smp_atomic32_t count; + erts_smp_atomic32_t aid_ix; + erts_smp_atomic32_t fid_ix; } ErtsPTabVolatileData; typedef struct { erts_smp_atomic_t *tab; + erts_smp_atomic32_t *free_id_data; Uint32 max; - Uint32 tab_cache_lines; - Uint32 pix_per_cache_line; Uint32 pix_mask; Uint32 pix_cl_mask; Uint32 pix_cl_shift; Uint32 pix_cli_mask; Uint32 pix_cli_shift; + Uint32 dix_cl_mask; + Uint32 dix_cl_shift; + Uint32 dix_cli_mask; + Uint32 dix_cli_shift; ErtsPTabElementCommon *invalid_element; Eterm invalid_data; void (*release_element)(void *); @@ -179,7 +184,8 @@ void erts_ptab_init_table(ErtsPTab *ptab, ErtsPTabElementCommon *invalid_element, int size, UWord element_size, - char *name); + char *name, + int legacy); int erts_ptab_new_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el, void *init_arg, @@ -187,6 +193,7 @@ int erts_ptab_new_element(ErtsPTab *ptab, void erts_ptab_delete_element(ErtsPTab *ptab, ErtsPTabElementCommon *ptab_el); int erts_ptab_initialized(ErtsPTab *ptab); +UWord erts_ptab_mem_size(ErtsPTab *ptab); ERTS_GLB_INLINE erts_interval_t *erts_ptab_interval(ErtsPTab *ptab); ERTS_GLB_INLINE int erts_ptab_max(ErtsPTab *ptab); diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index 1aeecf2b06..5f392944c2 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011-2012. All Rights Reserved. + * Copyright Ericsson AB 2011-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 848877d43e..fa015ee4b9 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1999-2012. All Rights Reserved. + * Copyright Ericsson AB 1999-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -2268,7 +2268,134 @@ trace_gc(Process *p, Eterm what) #undef LOCAL_HEAP_SIZE } +void +monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint time) +{ + ErlHeapFragment *bp; + ErlOffHeap *off_heap; +#ifndef ERTS_SMP + Process *monitor_p; +#endif + Uint hsz; + Eterm *hp, list, in_mfa = am_undefined, out_mfa = am_undefined; + Eterm in_tpl, out_tpl, tmo_tpl, tmo, msg; + + +#ifndef ERTS_SMP + ASSERT(is_internal_pid(system_monitor)); + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p || p == monitor_p) { + return; + } +#endif + /* + * Size: {monitor, pid, long_schedule, [{timeout, T}, {in, {M,F,A}},{out,{M,F,A}}]} -> + * 5 (top tuple of 4), (3 (elements) * 2 (cons)) + 3 (timeout tuple of 2) + size of Timeout + + * (2 * 3 (in/out tuple of 2)) + + * 0 (unknown) or 4 (MFA tuple of 3) + 0 (unknown) or 4 (MFA tuple of 3) + * = 20 + (in_fp != NULL) ? 4 : 0 + (out_fp != NULL) ? 4 : 0 + size of Timeout + */ + hsz = 20 + ((in_fp != NULL) ? 4 : 0) + ((out_fp != NULL) ? 4 : 0); + (void) erts_bld_uint(NULL, &hsz, time); + hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); + tmo = erts_bld_uint(&hp, NULL, time); + if (in_fp != NULL) { + in_mfa = TUPLE3(hp,(Eterm) in_fp[0], (Eterm) in_fp[1], make_small(in_fp[2])); + hp +=4; + } + if (out_fp != NULL) { + out_mfa = TUPLE3(hp,(Eterm) out_fp[0], (Eterm) out_fp[1], make_small(out_fp[2])); + hp +=4; + } + tmo_tpl = TUPLE2(hp,am_timeout, tmo); + hp += 3; + in_tpl = TUPLE2(hp,am_in,in_mfa); + hp += 3; + out_tpl = TUPLE2(hp,am_out,out_mfa); + hp += 3; + list = CONS(hp,out_tpl,NIL); + hp += 2; + list = CONS(hp,in_tpl,list); + hp += 2; + list = CONS(hp,tmo_tpl,list); + hp += 2; + msg = TUPLE4(hp, am_monitor, p->common.id, am_long_schedule, list); + hp += 5; +#ifdef ERTS_SMP + enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, p->common.id, NIL, msg, bp); +#else + erts_queue_message(monitor_p, NULL, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); +#endif +} +void +monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time) +{ + ErlHeapFragment *bp; + ErlOffHeap *off_heap; +#ifndef ERTS_SMP + Process *monitor_p; +#endif + Uint hsz; + Eterm *hp, list, op; + Eterm op_tpl, tmo_tpl, tmo, msg; + + +#ifndef ERTS_SMP + ASSERT(is_internal_pid(system_monitor)); + monitor_p = erts_proc_lookup(system_monitor); + if (!monitor_p) { + return; + } +#endif + /* + * Size: {monitor, port, long_schedule, [{timeout, T}, {op, Operation}]} -> + * 5 (top tuple of 4), (2 (elements) * 2 (cons)) + 3 (timeout tuple of 2) + * + size of Timeout + 3 (op tuple of 2 atoms) + * = 15 + size of Timeout + */ + hsz = 15; + (void) erts_bld_uint(NULL, &hsz, time); + + hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); + switch (type) { + case ERTS_PORT_TASK_PROC_SIG: op = am_proc_sig; break; + case ERTS_PORT_TASK_TIMEOUT: op = am_timeout; break; + case ERTS_PORT_TASK_INPUT: op = am_input; break; + case ERTS_PORT_TASK_OUTPUT: op = am_output; break; + case ERTS_PORT_TASK_EVENT: op = am_event; break; + case ERTS_PORT_TASK_DIST_CMD: op = am_dist_cmd; break; + default: op = am_undefined; break; + } + + tmo = erts_bld_uint(&hp, NULL, time); + + op_tpl = TUPLE2(hp,am_port_op,op); + hp += 3; + + tmo_tpl = TUPLE2(hp,am_timeout, tmo); + hp += 3; + + list = CONS(hp,op_tpl,NIL); + hp += 2; + list = CONS(hp,tmo_tpl,list); + hp += 2; + msg = TUPLE4(hp, am_monitor, pp->common.id, am_long_schedule, list); + hp += 5; +#ifdef ERTS_SMP + enqueue_sys_msg(SYS_MSG_TYPE_SYSMON, pp->common.id, NIL, msg, bp); +#else + erts_queue_message(monitor_p, NULL, bp, msg, NIL +#ifdef USE_VM_PROBES + , NIL +#endif + ); +#endif +} void monitor_long_gc(Process *p, Uint time) { @@ -3011,6 +3138,7 @@ sys_msg_disp_failure(ErtsSysMsgQ *smqp, Eterm receiver) case SYS_MSG_TYPE_SYSMON: if (receiver == NIL && !erts_system_monitor_long_gc + && !erts_system_monitor_long_schedule && !erts_system_monitor_large_heap && !erts_system_monitor_flags.busy_port && !erts_system_monitor_flags.busy_dist_port) diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 50fb27aab0..853c6cb0d8 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2012. All Rights Reserved. + * Copyright Ericsson AB 2012-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -83,6 +83,8 @@ void erts_system_profile_setup_active_schedulers(void); /* system_monitor */ void monitor_long_gc(Process *p, Uint time); +void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time); +void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time); void monitor_large_heap(Process *p); void monitor_generic(Process *p, Eterm type, Eterm spec); Uint erts_trace_flag2bit(Eterm flag); diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index fa53fd0937..e00440b905 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -724,7 +724,7 @@ L_Again: /* Restart with sublist, old listend was pushed on stack */ hp = HAlloc(p, 2); obj = CDR(objp); ioterm = CONS(hp, rest_term, obj); - //(*left) = 0; + /* (*left) = 0; */ goto done; } if (rest_term != NIL) { @@ -1723,14 +1723,14 @@ static BIF_RETTYPE do_bif_utf8_to_list(Process *p, if (b_sz) { ErlSubBin *sb; Eterm orig; - ERTS_DECLARE_DUMMY(Uint offset); + Uint offset; ASSERT(state != ERTS_UTF8_OK); hp = HAlloc(p, ERL_SUB_BIN_SIZE); sb = (ErlSubBin *) hp; ERTS_GET_REAL_BIN(orig_bin, orig, offset, bitoffs, bitsize); sb->thing_word = HEADER_SUB_BIN; sb->size = b_sz; - sb->offs = num_bytes_to_process + num_processed_bytes; + sb->offs = offset + num_bytes_to_process + num_processed_bytes; sb->orig = orig; sb->bitoffs = bitoffs; sb->bitsize = bitsize; diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index a2064bd8a3..80d29d554a 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -24,6 +24,8 @@ #include "erl_smp.h" #include "erl_printf.h" +struct process; + typedef struct { #ifdef DEBUG int smp_api; @@ -155,6 +157,9 @@ Uint32 block_hash(byte *, unsigned, Uint32); Uint32 make_hash2(Eterm); Uint32 make_hash(Eterm); +void erts_save_emu_args(int argc, char **argv); +Eterm erts_get_emu_args(struct process *c_p); +Eterm erts_get_ethread_info(struct process * c_p); Eterm erts_bld_atom(Uint **hpp, Uint *szp, char *str); Eterm erts_bld_uint(Uint **hpp, Uint *szp, Uint ui); diff --git a/erts/emulator/beam/erl_zlib.c b/erts/emulator/beam/erl_zlib.c index f73d48b6c2..47fd92988e 100644 --- a/erts/emulator/beam/erl_zlib.c +++ b/erts/emulator/beam/erl_zlib.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009. All Rights Reserved. + * Copyright Ericsson AB 2009-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -44,6 +44,48 @@ void erl_zlib_zfree_callback (voidpf opaque, voidpf ptr) erts_free(ERTS_ALC_T_ZLIB, ptr); } +/* + * Initialize a z_stream with a source, to later *chunk* data into a destination + * Returns Z_OK or Error. + */ +int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, + uLong sourceLen, int level) +{ + streamp->next_in = (Bytef*)source; + streamp->avail_in = (uInt)sourceLen; + streamp->total_out = streamp->avail_out = 0; + streamp->next_out = NULL; + erl_zlib_alloc_init(streamp); + return deflateInit(streamp, level); +} +/* + * Deflate a chunk, The destination length is the limit. + * Returns Z_OK if more to process, Z_STREAM_END if we are done. + */ +int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen) +{ + int err; + uLongf last_tot = streamp->total_out; + + streamp->next_out = dest; + streamp->avail_out = (uInt)*destLen; + + if ((uLong)streamp->avail_out != *destLen) return Z_BUF_ERROR; + + err = deflate(streamp, Z_FINISH); + *destLen = streamp->total_out - last_tot; + return err; +} + + +/* + * When we are done, free up the deflate structure + * Retyurns Z_OK or Error + */ +int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp) +{ + return deflateEnd(streamp); +} int ZEXPORT erl_zlib_compress2 (Bytef* dest, uLongf* destLen, const Bytef* source, uLong sourceLen, diff --git a/erts/emulator/beam/erl_zlib.h b/erts/emulator/beam/erl_zlib.h index 9054a5e428..5ac849d21c 100644 --- a/erts/emulator/beam/erl_zlib.h +++ b/erts/emulator/beam/erl_zlib.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2009. All Rights Reserved. + * Copyright Ericsson AB 2009-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -31,6 +31,14 @@ (s)->zfree = erl_zlib_zfree_callback; \ } while (0) +/* + * Chunked interface, used by term_to_binary among others. + */ +int ZEXPORT erl_zlib_deflate_start(z_stream *streamp, const Bytef* source, + uLong sourceLen, int level); +int ZEXPORT erl_zlib_deflate_chunk(z_stream *streamp, Bytef* dest, uLongf* destLen); +int ZEXPORT erl_zlib_deflate_finish(z_stream *streamp); + /* Use instead of compress */ #define erl_zlib_compress(dest,destLen,source,sourceLen) \ diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 5ce0d97c74..1c88765381 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -81,7 +81,11 @@ * */ +static Export term_to_binary_trap_export; + static byte* enc_term(ErtsAtomCacheMap *, Eterm, byte*, Uint32, struct erl_off_heap_header** off_heap); +static int enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, + struct erl_off_heap_header** off_heap, Sint *reds, byte **res); static Uint is_external_string(Eterm obj, int* p_is_string); static byte* enc_atom(ErtsAtomCacheMap *, Eterm, byte*, Uint32); static byte* enc_pid(ErtsAtomCacheMap *, Eterm, byte*, Uint32); @@ -89,9 +93,31 @@ static byte* dec_term(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); static byte* dec_atom(ErtsDistExternal *, byte*, Eterm*); static byte* dec_pid(ErtsDistExternal *, Eterm**, byte*, ErlOffHeap*, Eterm*); static Sint decoded_size(byte *ep, byte* endp, int internal_tags); +static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1); +static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, + Binary *context_b); static Uint encode_size_struct2(ErtsAtomCacheMap *, Eterm, unsigned); +static int encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, + unsigned dflags, Sint *reds, Uint *res); + +void erts_init_external(void) { +#if 1 /* In R16 */ + erts_init_trap_export(&term_to_binary_trap_export, + am_erlang, am_term_to_binary_trap, 1, + &term_to_binary_trap_1); +#else + sys_memset((void *) &term_to_binary_trap_export, 0, sizeof(Export)); + term_to_binary_trap_export.address = &term_to_binary_trap_export.code[3]; + term_to_binary_trap_export.code[0] = am_erlang; + term_to_binary_trap_export.code[1] = am_term_to_binary_trap; + term_to_binary_trap_export.code[2] = 1; + term_to_binary_trap_export.code[3] = (BeamInstr) em_apply_bif; + term_to_binary_trap_export.code[4] = (BeamInstr) &term_to_binary_trap_1; +#endif + return; +} #define ERTS_MAX_INTERNAL_ATOM_CACHE_ENTRIES 255 @@ -1009,10 +1035,28 @@ BIF_RETTYPE erts_debug_dist_ext_to_term_2(BIF_ALIST_2) BIF_ERROR(BIF_P, BADARG); } - +static BIF_RETTYPE term_to_binary_trap_1(BIF_ALIST_1) +{ + Eterm *tp = tuple_val(BIF_ARG_1); + Eterm Term = tp[1]; + Eterm bt = tp[2]; + Binary *bin = ((ProcBin *) binary_val(bt))->val; + Eterm res = erts_term_to_binary_int(BIF_P, Term, 0, 0,bin); + if (is_tuple(res)) { + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + BIF_RET(res); + } +} + BIF_RETTYPE term_to_binary_1(BIF_ALIST_1) { - return erts_term_to_binary(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS); + Eterm res = erts_term_to_binary_int(BIF_P, BIF_ARG_1, 0, TERM_TO_BINARY_DFLAGS, NULL); + if (is_tuple(res)) { + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + BIF_RET(res); + } } BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) @@ -1022,6 +1066,8 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) Eterm Flags = BIF_ARG_2; int level = 0; Uint flags = TERM_TO_BINARY_DFLAGS; + Eterm res; + Binary *bin = NULL; while (is_list(Flags)) { Eterm arg = CAR(list_val(Flags)); @@ -1058,7 +1104,12 @@ BIF_RETTYPE term_to_binary_2(BIF_ALIST_2) goto error; } - return erts_term_to_binary(p, Term, level, flags); + res = erts_term_to_binary_int(p, Term, level, flags, bin); + if (is_tuple(res)) { + BIF_TRAP1(&term_to_binary_trap_export,BIF_P,res); + } else { + BIF_RET(res); + } } static uLongf binary2term_uncomp_size(byte* data, Sint size) @@ -1335,16 +1386,13 @@ external_size_2(BIF_ALIST_2) } } -Eterm -erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) +static Eterm +erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint flags) { - Uint size; Eterm bin; size_t real_size; byte* endp; - size = encode_size_struct2(NULL, Term, flags) + 1 /* VERSION_MAGIC */; - if (level != 0) { byte buf[256]; byte* bytes = buf; @@ -1414,6 +1462,327 @@ erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) } } +Eterm +erts_term_to_binary(Process* p, Eterm Term, int level, Uint flags) { + Uint size; + size = encode_size_struct2(NULL, Term, flags) + 1 /* VERSION_MAGIC */; + return erts_term_to_binary_simple(p, Term, size, level, flags); +} + +/* Define for testing */ +/* #define EXTREME_TTB_TRAPPING 1 */ + +#ifndef EXTREME_TTB_TRAPPING +#define TERM_TO_BINARY_LOOP_FACTOR 500 +#define TERM_TO_BINARY_SIZE_FACTOR 500000 +#define TERM_TO_BINARY_COMPRESS_CHUNK 500000 +#else +#define TERM_TO_BINARY_LOOP_FACTOR 1 +#define TERM_TO_BINARY_SIZE_FACTOR 10 +#define TERM_TO_BINARY_COMPRESS_CHUNK 10 +#endif + + +typedef enum { TTBSize, TTBEncode, TTBCompress } TTBState; +typedef struct { + Uint flags; + int level; +} TTBSizeContext; + +typedef struct { + Uint flags; + int level; + Binary *result_bin; +} TTBEncodeContext; + +typedef struct { + Uint real_size; + Uint dest_len; + byte *dbytes; + Binary *result_bin; + Binary *destination_bin; + z_stream stream; +} TTBCompressContext; + +typedef struct { + int alive; + TTBState state; + union { + TTBSizeContext sc; + TTBEncodeContext ec; + TTBCompressContext cc; + } s; +} TTBContext; + +static void context_destructor(Binary *context_bin) +{ + TTBContext *context = ERTS_MAGIC_BIN_DATA(context_bin); + if (context->alive) { + context->alive = 0; + switch (context->state) { + case TTBSize: + break; + case TTBEncode: + if (context->s.ec.result_bin != NULL) { /* Set to NULL if ever made alive! */ + ASSERT(erts_refc_read(&(context->s.ec.result_bin->refc),0) == 0); + erts_bin_free(context->s.ec.result_bin); + context->s.ec.result_bin = NULL; + } + break; + case TTBCompress: + erl_zlib_deflate_finish(&(context->s.cc.stream)); + + if (context->s.cc.destination_bin != NULL) { /* Set to NULL if ever made alive! */ + ASSERT(erts_refc_read(&(context->s.cc.destination_bin->refc),0) == 0); + erts_bin_free(context->s.cc.destination_bin); + context->s.cc.destination_bin = NULL; + } + + if (context->s.cc.result_bin != NULL) { /* Set to NULL if ever made alive! */ + ASSERT(erts_refc_read(&(context->s.cc.result_bin->refc),0) == 0); + erts_bin_free(context->s.cc.result_bin); + context->s.cc.result_bin = NULL; + } + break; + } + } +} + +static Eterm erts_term_to_binary_int(Process* p, Eterm Term, int level, Uint flags, + Binary *context_b) +{ + Eterm *hp; + Eterm res; + Eterm c_term; +#ifndef EXTREME_TTB_TRAPPING + Sint reds = (Sint) (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_LOOP_FACTOR); +#else + Sint reds = 20; /* For testing */ +#endif + Sint initial_reds = reds; + TTBContext c_buff; + TTBContext *context = &c_buff; + +#define EXPORT_CONTEXT() \ + do { \ + if (context_b == NULL) { \ + context_b = erts_create_magic_binary(sizeof(TTBContext), \ + context_destructor); \ + context = ERTS_MAGIC_BIN_DATA(context_b); \ + memcpy(context,&c_buff,sizeof(TTBContext)); \ + } \ + } while (0) + +#define RETURN_STATE() \ + do { \ + hp = HAlloc(p, PROC_BIN_SIZE+3); \ + c_term = erts_mk_magic_binary_term(&hp, &MSO(p), context_b); \ + res = TUPLE2(hp, Term, c_term); \ + BUMP_ALL_REDS(p); \ + return res; \ + } while (0); + + + if (context_b == NULL) { + /* Setup enough to get started */ + context->state = TTBSize; + context->alive = 1; + context->s.sc.flags = flags; + context->s.sc.level = level; + } else { + context = ERTS_MAGIC_BIN_DATA(context_b); + } + /* Initialization done, now we will go through the states */ + for (;;) { + switch (context->state) { + case TTBSize: + { + Uint size; + Binary *result_bin; + int level; + Uint flags; + /* Try for fast path */ + if (encode_size_struct_int(p, NULL, Term, context->s.sc.flags, &reds, &size) < 0) { + EXPORT_CONTEXT(); + /* Same state */ + RETURN_STATE(); + } + ++size; /* VERSION_MAGIC */ + /* Move these to next state */ + flags = context->s.sc.flags; + level = context->s.sc.level; + if (size <= ERL_ONHEAP_BIN_LIMIT) { + /* Finish in one go */ + res = erts_term_to_binary_simple(p, Term, size, + level, flags); + BUMP_REDS(p, size / TERM_TO_BINARY_SIZE_FACTOR); + return res; + } + + result_bin = erts_bin_nrml_alloc(size); + result_bin->flags = 0; + result_bin->orig_size = size; + erts_refc_init(&result_bin->refc, 0); + result_bin->orig_bytes[0] = VERSION_MAGIC; + /* Next state immediately, no need to export context */ + context->state = TTBEncode; + context->s.ec.flags = flags; + context->s.ec.level = level; + context->s.ec.result_bin = result_bin; + break; + } + case TTBEncode: + { + byte *endp; + byte *bytes = (byte *) context->s.ec.result_bin->orig_bytes; + size_t real_size; + Binary *result_bin; + + flags = context->s.ec.flags; + if (enc_term_int(p,NULL,Term, bytes+1, flags, NULL, &reds, &endp) < 0) { + EXPORT_CONTEXT(); + RETURN_STATE(); + } + real_size = endp - bytes; + result_bin = erts_bin_realloc(context->s.ec.result_bin,real_size); + level = context->s.ec.level; + BUMP_REDS(p, (initial_reds - reds) / TERM_TO_BINARY_LOOP_FACTOR); + if (level == 0 || real_size < 6) { /* We are done */ + ProcBin* pb; + return_normal: + context->s.ec.result_bin = NULL; + context->alive = 0; + pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = real_size; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; + pb->val = result_bin; + pb->bytes = (byte*) result_bin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); + erts_refc_inc(&result_bin->refc, 1); + if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + erts_bin_free(context_b); + } + return make_binary(pb); + } + /* Continue with compression... */ + /* To make absolutely sure that zlib does not barf on a reallocated context, + we make sure it's "exported" before doing anything compession-like */ + EXPORT_CONTEXT(); + bytes = (byte *) result_bin->orig_bytes; /* result_bin is reallocated */ + if (erl_zlib_deflate_start(&(context->s.cc.stream),bytes+1,real_size-1,level) + != Z_OK) { + goto return_normal; + } + context->state = TTBCompress; + context->s.cc.real_size = real_size; + context->s.cc.result_bin = result_bin; + + result_bin = erts_bin_nrml_alloc(real_size); + result_bin->flags = 0; + result_bin->orig_size = real_size; + erts_refc_init(&result_bin->refc, 0); + result_bin->orig_bytes[0] = VERSION_MAGIC; + + context->s.cc.destination_bin = result_bin; + context->s.cc.dest_len = 0; + context->s.cc.dbytes = (byte *) result_bin->orig_bytes+6; + break; + } + case TTBCompress: + { + uLongf tot_dest_len = context->s.cc.real_size - 6; + uLongf left = (tot_dest_len - context->s.cc.dest_len); + uLongf this_time = (left > TERM_TO_BINARY_COMPRESS_CHUNK) ? + TERM_TO_BINARY_COMPRESS_CHUNK : + left; + Binary *result_bin; + ProcBin *pb; + Uint max = (ERTS_BIF_REDS_LEFT(p) * TERM_TO_BINARY_COMPRESS_CHUNK) / CONTEXT_REDS; + + if (max < this_time) { + this_time = max + 1; /* do not set this_time to 0 */ + } + + res = erl_zlib_deflate_chunk(&(context->s.cc.stream), context->s.cc.dbytes, &this_time); + context->s.cc.dbytes += this_time; + context->s.cc.dest_len += this_time; + switch (res) { + case Z_OK: + if (context->s.cc.dest_len >= tot_dest_len) { + goto no_use_compressing; + } + RETURN_STATE(); + case Z_STREAM_END: + { + byte *dbytes = (byte *) context->s.cc.destination_bin->orig_bytes + 1; + + dbytes[0] = COMPRESSED; + put_int32(context->s.cc.real_size-1,dbytes+1); + erl_zlib_deflate_finish(&(context->s.cc.stream)); + result_bin = erts_bin_realloc(context->s.cc.destination_bin, + context->s.cc.dest_len+6); + context->s.cc.destination_bin = NULL; + pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = context->s.cc.dest_len+6; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; + pb->val = result_bin; + pb->bytes = (byte*) result_bin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); + erts_refc_inc(&result_bin->refc, 1); + erts_bin_free(context->s.cc.result_bin); + context->s.cc.result_bin = NULL; + context->alive = 0; + BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); + if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + erts_bin_free(context_b); + } + return make_binary(pb); + } + default: /* Compression error, revert to uncompressed binary (still in + context) */ + no_use_compressing: + result_bin = context->s.cc.result_bin; + context->s.cc.result_bin = NULL; + pb = (ProcBin *) HAlloc(p, PROC_BIN_SIZE); + pb->thing_word = HEADER_PROC_BIN; + pb->size = context->s.cc.real_size; + pb->next = MSO(p).first; + MSO(p).first = (struct erl_off_heap_header*)pb; + pb->val = result_bin; + pb->bytes = (byte*) result_bin->orig_bytes; + pb->flags = 0; + OH_OVERHEAD(&(MSO(p)), pb->size / sizeof(Eterm)); + erts_refc_inc(&result_bin->refc, 1); + erl_zlib_deflate_finish(&(context->s.cc.stream)); + erts_bin_free(context->s.cc.destination_bin); + context->s.cc.destination_bin = NULL; + context->alive = 0; + BUMP_REDS(p, (this_time * CONTEXT_REDS) / TERM_TO_BINARY_COMPRESS_CHUNK); + if (context_b && erts_refc_read(&context_b->refc,0) == 0) { + erts_bin_free(context_b); + } + return make_binary(pb); + } + } + } + } +#undef EXPORT_CONTEXT +#undef RETURN_STATE +} + + + + + + + + /* * This function fills ext with the external format of atom. * If it's an old atom we just supply an index, otherwise @@ -1678,32 +2047,71 @@ dec_pid(ErtsDistExternal *edep, Eterm** hpp, byte* ep, ErlOffHeap* off_heap, Ete #define ENC_PATCH_FUN_SIZE ((Eterm) 2) #define ENC_LAST_ARRAY_ELEMENT ((Eterm) 3) +/* Free extra rootset (used when trapping) */ +static void cleanup_ttb_extra_root(ErlExtraRootSet *rs) +{ + if (rs->objv != NULL) { + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs->objv); + } + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs); +} + +/* Same as above, but we have an extra "stack" beyond GC reach, i.e. an array of two extra roots */ +static void cleanup_ttb_extra_root_2(ErlExtraRootSet *rs) +{ + if (rs->objv != NULL) { + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs->objv); + } + if (rs[1].objv != NULL) { + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs[1].objv); + } + + erts_free(ERTS_ALC_T_EXTRA_ROOT, rs); +} + static byte* enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, struct erl_off_heap_header** off_heap) { - DECLARE_WSTACK(s); + byte *res; + (void) enc_term_int(NULL, acmp, obj, ep, dflags, off_heap, NULL, &res); + return res; +} + +static int +enc_term_int(Process *p,ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, + struct erl_off_heap_header** off_heap, Sint *reds, byte **res) +{ + DECLARE_ESTACK(s); + DECLARE_WSTACK(com); Uint n; Uint i; Uint j; Uint* ptr; Eterm val; FloatDef f; -#if HALFWORD_HEAP - UWord wobj; -#endif + int count_reds = (p != NULL && reds != NULL); + Sint r = 0; + if (count_reds) { + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_EXTRA_ROOT); + WSTACK_CHANGE_ALLOCATOR(com, ERTS_ALC_T_EXTRA_ROOT); + r = *reds; + } + + if (p && p->extra_root) { /* restore saved stacks and byte pointer */ + ESTACK_RESTORE(s,p->extra_root[0].objv, p->extra_root[0].sz); + obj = ESTACK_POP(s); + WSTACK_RESTORE(com, p->extra_root[1].objv, p->extra_root[1].sz); + ep = (byte *) WSTACK_POP(com); + } goto L_jump_start; outer_loop: - while (!WSTACK_ISEMPTY(s)) { -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else - obj = WSTACK_POP(s); -#endif - switch (val = WSTACK_POP(s)) { + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); + switch (val = WSTACK_POP(com)) { case ENC_TERM: break; case ENC_ONE_CONS: @@ -1714,45 +2122,57 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, obj = CAR(cons); tl = CDR(cons); - WSTACK_PUSH(s, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); - WSTACK_PUSH(s, tl); + WSTACK_PUSH(com, is_list(tl) ? ENC_ONE_CONS : ENC_TERM); + ESTACK_PUSH(s, tl); } break; case ENC_PATCH_FUN_SIZE: + /* obj will be discarded, it was NIL */ { -#if HALFWORD_HEAP - byte* size_p = (byte *) wobj; -#else - byte* size_p = (byte *) obj; -#endif + byte* size_p = (byte *) WSTACK_POP(com); put_int32(ep - size_p, size_p); } goto outer_loop; case ENC_LAST_ARRAY_ELEMENT: + /* obj is the tuple */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - obj = *ptr; + Eterm* ptr = tuple_val(obj); + i = arityval(*ptr); + obj = ptr[i]; } break; default: /* ENC_LAST_ARRAY_ELEMENT+1 and upwards */ { -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - obj = *ptr++; - WSTACK_PUSH(s, val-1); - WSTACK_PUSH(s, (UWord) ptr); + Eterm* ptr = tuple_val(obj); + i = arityval(*ptr); + ESTACK_PUSH(s, obj); /* put back tuple and next element index */ + WSTACK_PUSH(com, val-1); + obj = ptr[i - (val - ENC_LAST_ARRAY_ELEMENT)]; /* the index is counting down */ } break; } L_jump_start: + + if (count_reds && --r == 0) { + *reds = r; + ESTACK_PUSH(s,obj); /* push back current object, to be popped on restore */ + WSTACK_PUSH(com,((UWord) ep)); + if (p->extra_root == NULL) { + /* NB. Allocate an array of two "extra-roots", of which only the first element + is seen and handled by the GC. Index 1 holds the Wstack. */ + p->extra_root = erts_alloc(ERTS_ALC_T_EXTRA_ROOT, sizeof(ErlExtraRootSet)*2); + p->extra_root->objv = NULL; + p->extra_root->sz = 0; + p->extra_root->cleanup = cleanup_ttb_extra_root_2; + p->extra_root[1].objv = NULL; + p->extra_root[1].sz = 0; + p->extra_root[1].cleanup = NULL; /* Never used */ + } + ESTACK_SAVE(s, p->extra_root[0].objv, p->extra_root[0].sz); + WSTACK_SAVE(com, p->extra_root[1].objv, (p->extra_root[1].sz)); + return -1; + } switch(tag_val_def(obj)) { case NIL_DEF: *ep++ = NIL_EXT; @@ -1896,8 +2316,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, ep += 4; } if (i > 0) { - WSTACK_PUSH(s, ENC_LAST_ARRAY_ELEMENT+i-1); - WSTACK_PUSH(s, (UWord) ptr); + WSTACK_PUSH(com, ENC_LAST_ARRAY_ELEMENT+i-1); + ESTACK_PUSH(s, obj); } break; @@ -1905,7 +2325,7 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, GET_DOUBLE(obj, f); if (dflags & DFLAG_NEW_FLOATS) { *ep++ = NEW_FLOAT_EXT; -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) put_int32(f.fw[0], ep); ep += 4; put_int32(f.fw[1], ep); @@ -2041,8 +2461,9 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, int ei; *ep++ = NEW_FUN_EXT; - WSTACK_PUSH(s, ENC_PATCH_FUN_SIZE); - WSTACK_PUSH(s, (UWord) ep); /* Position for patching in size */ + WSTACK_PUSH(com, (UWord) ep); /* Position for patching in size */ + WSTACK_PUSH(com, ENC_PATCH_FUN_SIZE); + ESTACK_PUSH(s,NIL); /* Will be thrown away */ ep += 4; *ep = funp->arity; ep += 1; @@ -2059,8 +2480,8 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, fun_env: for (ei = funp->num_free-1; ei > 0; ei--) { - WSTACK_PUSH(s, ENC_TERM); - WSTACK_PUSH(s, (UWord) funp->env[ei]); + WSTACK_PUSH(com, ENC_TERM); + ESTACK_PUSH(s, (UWord) funp->env[ei]); } if (funp->num_free != 0) { obj = funp->env[0]; @@ -2103,8 +2524,17 @@ enc_term(ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Uint32 dflags, break; } } - DESTROY_WSTACK(s); - return ep; + DESTROY_ESTACK(s); + DESTROY_WSTACK(com); + if (p && p->extra_root) { + cleanup_ttb_extra_root_2(p->extra_root); + p->extra_root = NULL; + } + if (count_reds) { + *reds = r; + } + *res = ep; + return 0; } static @@ -2374,7 +2804,7 @@ dec_term_atom_common: volatile unsigned long *fpexnp = erts_get_current_fp_exception(); #endif -#ifdef WORDS_BIGENDIAN +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) ff.fw[0] = get_int32(ep); ep += 4; ff.fw[1] = get_int32(ep); @@ -2892,51 +3322,47 @@ dec_term_atom_common: to a sequence of bytes N.B. That this must agree with to_external2() above!!! (except for cached atoms) */ +static Uint encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) { + Uint res; + (void) encode_size_struct_int(NULL, acmp, obj, dflags, NULL, &res); + return res; +} -static Uint -encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) +static int +encode_size_struct_int(Process *p, ErtsAtomCacheMap *acmp, Eterm obj, + unsigned dflags, Sint *reds, Uint *res) { - DECLARE_WSTACK(s); + DECLARE_ESTACK(s); Uint m, i, arity; Uint result = 0; -#if HALFWORD_HEAP - UWord wobj = 0; -#endif + int count_reds = (p != NULL && reds != 0); + Sint r = 0; + + if (count_reds) { + ESTACK_CHANGE_ALLOCATOR(s, ERTS_ALC_T_EXTRA_ROOT); + r = *reds; + } + + if (p && p->extra_root) { /* restore saved stack */ + ESTACK_RESTORE(s,p->extra_root->objv, p->extra_root->sz + 1); + result = ESTACK_POP(s); /*Untagged, beyond p->extra_root->sz */ + obj = ESTACK_POP(s); + + } goto L_jump_start; outer_loop: - while (!WSTACK_ISEMPTY(s)) { -#if HALFWORD_HEAP - obj = (Eterm) (wobj = WSTACK_POP(s)); -#else - obj = WSTACK_POP(s); -#endif + while (!ESTACK_ISEMPTY(s)) { + obj = ESTACK_POP(s); handle_popped_obj: - if (is_CP(obj)) { /* Does not look for CP, looks for "no tag" */ -#if HALFWORD_HEAP - Eterm* ptr = (Eterm *) wobj; -#else - Eterm* ptr = (Eterm *) obj; -#endif - /* - * Pointer into a tuple. - */ - obj = *ptr--; - if (!is_header(obj)) { - WSTACK_PUSH(s, (UWord)ptr); - } else { - /* Reached tuple header */ - ASSERT(header_is_arityval(obj)); - goto outer_loop; - } - } else if (is_list(obj)) { + if (is_list(obj)) { Eterm* cons = list_val(obj); Eterm tl; tl = CDR(cons); obj = CAR(cons); - WSTACK_PUSH(s, tl); + ESTACK_PUSH(s, tl); } else if (is_nil(obj)) { result++; goto outer_loop; @@ -2948,6 +3374,20 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } L_jump_start: + if (count_reds && --r == 0) { + *reds = r; + ESTACK_PUSH(s,obj); /* push back current object */ + ESTACK_PUSH(s,result); /* Untagged, will be out of GC reach */ + if (p->extra_root == NULL) { + p->extra_root = erts_alloc(ERTS_ALC_T_EXTRA_ROOT, sizeof(ErlExtraRootSet)); + p->extra_root->objv = NULL; + p->extra_root->sz = 0; + p->extra_root->cleanup = cleanup_ttb_extra_root; + } + ESTACK_SAVE(s, p->extra_root->objv, p->extra_root->sz); + --p->extra_root->sz; /* Hide result from GC */ + return -1; + } switch (tag_val_def(obj)) { case NIL_DEF: result++; @@ -3034,20 +3474,24 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) case TUPLE_DEF: { Eterm* ptr = tuple_val(obj); - + Uint i; arity = arityval(*ptr); if (arity <= 0xff) { result += 1 + 1; } else { result += 1 + 4; } - ptr += arity; -#if HALFWORD_HEAP - obj = (Eterm) (wobj = (UWord) ptr); -#else - obj = (Eterm) ptr; -#endif - goto handle_popped_obj; + for (i = 1; i <= arity; ++i) { + if (is_list(ptr[i])) { + if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { + result += m + 2 + 1; + } else { + result += 5; + } + } + ESTACK_PUSH(s,ptr[i]); + } + goto outer_loop; } break; case FLOAT_DEF: @@ -3105,14 +3549,14 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) if (is_not_list(obj)) { /* Push any non-list terms on the stack */ - WSTACK_PUSH(s, obj); + ESTACK_PUSH(s, obj); } else { /* Lists must be handled specially. */ if ((m = is_string(obj)) && (m < MAX_STRING_LEN)) { result += m + 2 + 1; } else { result += 5; - WSTACK_PUSH(s, obj); + ESTACK_PUSH(s, obj); } } } @@ -3143,19 +3587,16 @@ encode_size_struct2(ErtsAtomCacheMap *acmp, Eterm obj, unsigned dflags) } } - DESTROY_WSTACK(s); - return result; -} - -static int is_valid_utf8_atom(byte* bytes, Uint nbytes) -{ - byte* err_pos; - Uint num_chars; - - /*SVERK Do we really need to validate correct utf8? */ - return nbytes <= MAX_ATOM_SZ_LIMIT - && erts_analyze_utf8(bytes, nbytes, &err_pos, &num_chars, NULL) == ERTS_UTF8_OK - && num_chars <= MAX_ATOM_CHARACTERS; + DESTROY_ESTACK(s); + if (p && p->extra_root) { + cleanup_ttb_extra_root(p->extra_root); + p->extra_root = NULL; + } + if (count_reds) { + *reds = r; + } + *res = result; + return 0; } static Sint @@ -3235,7 +3676,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(2); n = get_int16(ep); ep += 2; - if (!is_valid_utf8_atom(ep, n)) { + if (n > MAX_ATOM_SZ_LIMIT) { return -1; } SKIP(n+atom_extra_skip); @@ -3254,7 +3695,7 @@ decoded_size(byte *ep, byte* endp, int internal_tags) CHKSIZE(1); n = get_int8(ep); ep++; - if (!is_valid_utf8_atom(ep, n)) { + if (n > MAX_ATOM_SZ_LIMIT) { return -1; } SKIP(n+atom_extra_skip); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 012c1c7e6a..bacd5a5752 100755 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -376,7 +376,7 @@ extern int stackdump_on_exit; */ -void erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end); +void erl_grow_stack(ErtsAlcType_t a_type, Eterm** start, Eterm** sp, Eterm** end); #define ESTK_CONCAT(a,b) a##b #define ESTK_SUBSCRIPT(s,i) *((Eterm *)((byte *)ESTK_CONCAT(s,_start) + (i))) #define DEF_ESTACK_SIZE (16) @@ -385,20 +385,79 @@ void erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end); Eterm ESTK_CONCAT(s,_default_stack)[DEF_ESTACK_SIZE]; \ Eterm* ESTK_CONCAT(s,_start) = ESTK_CONCAT(s,_default_stack); \ Eterm* ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start); \ - Eterm* ESTK_CONCAT(s,_end) = ESTK_CONCAT(s,_start) + DEF_ESTACK_SIZE + Eterm* ESTK_CONCAT(s,_end) = ESTK_CONCAT(s,_start) + DEF_ESTACK_SIZE;\ + ErtsAlcType_t ESTK_CONCAT(s,_alloc_type) = ERTS_ALC_T_ESTACK + +#define ESTACK_CHANGE_ALLOCATOR(s,t) \ +do { \ + if (ESTK_CONCAT(s,_start) != ESTK_CONCAT(s,_default_stack)) { \ + erl_exit(1, "Internal error - trying to change allocator " \ + "type of active estack\n"); \ + } \ + ESTK_CONCAT(s,_alloc_type) = (t); \ + } while (0) + +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'v'. 'v' and 'vsize' are changed by this macro. If + * 'v' points to anything, it should have been allocated by a previous + * call to this macro. Be careful to set a correct allocator prior to + * saving. + * 'v' can be any lvalue pointer, it will point to an array of UWord + * after calling this macro. + */ +#define ESTACK_SAVE(s,v,vsize) /* v and vsize are "name parameters" */ \ +do { \ + Uint _esz = ESTACK_COUNT(s); \ + if (ESTK_CONCAT(s,_start) == ESTK_CONCAT(s,_default_stack)) { \ + if ((v) == NULL) { \ + (v) = erts_alloc(ESTK_CONCAT(s,_alloc_type), \ + DEF_ESTACK_SIZE * sizeof(Eterm)); \ + } \ + memcpy((v),ESTK_CONCAT(s,_start),_esz*sizeof(Eterm)); \ + } else { \ + (v) = (void *) ESTK_CONCAT(s,_start); \ + } \ + (vsize) = _esz; \ + } while (0) + +/* + * Use on empty stack, only the allocator can be changed before this + * The vector parameter is reset to NULL if the vector is moved to stack, + * otherwise it's kept for reuse, so a saved and restored vector might + * need freeing using the correct allocator parameter. + * 'v' can be any lvalue pointer, it's cast to an (Eterm *). + */ +#define ESTACK_RESTORE(s, v, vsize) /*v is a "name parameter"*/ \ +do { \ + if ((vsize) > DEF_ESTACK_SIZE) { \ + Uint _ca = DEF_ESTACK_SIZE; \ + while (_ca < (vsize)) \ + _ca = _ca * 2; \ + ESTK_CONCAT(s,_start) = (Eterm *) (v); \ + ESTK_CONCAT(s,_end) = ((Eterm *)(v)) + _ca; \ + ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start) + (vsize); \ + (v) = NULL; \ + } else { \ + memcpy(ESTK_CONCAT(s,_start),(v),(vsize)*sizeof(Eterm));\ + ESTK_CONCAT(s,_sp) = ESTK_CONCAT(s,_start) + (vsize); \ + } \ + } while (0) + +#define ESTACK_IS_STATIC(s) (ESTK_CONCAT(s,_start) == ESTK_CONCAT(s,_default_stack)) #define DESTROY_ESTACK(s) \ do { \ if (ESTK_CONCAT(s,_start) != ESTK_CONCAT(s,_default_stack)) { \ - erts_free(ERTS_ALC_T_ESTACK, ESTK_CONCAT(s,_start)); \ + erts_free(ESTK_CONCAT(s,_alloc_type), ESTK_CONCAT(s,_start)); \ } \ } while(0) #define ESTACK_PUSH(s, x) \ do { \ if (ESTK_CONCAT(s,_sp) == ESTK_CONCAT(s,_end)) { \ - erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \ - &ESTK_CONCAT(s,_end)); \ + erl_grow_stack(ESTK_CONCAT(s,_alloc_type),&ESTK_CONCAT(s,_start), \ + &ESTK_CONCAT(s,_sp), &ESTK_CONCAT(s,_end)); \ } \ *ESTK_CONCAT(s,_sp)++ = (x); \ } while(0) @@ -406,8 +465,8 @@ do { \ #define ESTACK_PUSH2(s, x, y) \ do { \ if (ESTK_CONCAT(s,_sp) > ESTK_CONCAT(s,_end) - 2) { \ - erl_grow_stack(&ESTK_CONCAT(s,_start), &ESTK_CONCAT(s,_sp), \ - &ESTK_CONCAT(s,_end)); \ + erl_grow_stack(ESTK_CONCAT(s,_alloc_type),&ESTK_CONCAT(s,_start), \ + &ESTK_CONCAT(s,_sp), &ESTK_CONCAT(s,_end)); \ } \ *ESTK_CONCAT(s,_sp)++ = (x); \ *ESTK_CONCAT(s,_sp)++ = (y); \ @@ -430,7 +489,7 @@ do { \ #define ESTACK_POP(s) (*(--ESTK_CONCAT(s,_sp))) -void erl_grow_wstack(UWord** start, UWord** sp, UWord** end); +void erl_grow_wstack(ErtsAlcType_t a_type, UWord** start, UWord** sp, UWord** end); #define WSTK_CONCAT(a,b) a##b #define WSTK_SUBSCRIPT(s,i) *((UWord *)((byte *)WSTK_CONCAT(s,_start) + (i))) #define DEF_WSTACK_SIZE (16) @@ -439,20 +498,79 @@ void erl_grow_wstack(UWord** start, UWord** sp, UWord** end); UWord WSTK_CONCAT(s,_default_stack)[DEF_WSTACK_SIZE]; \ UWord* WSTK_CONCAT(s,_start) = WSTK_CONCAT(s,_default_stack); \ UWord* WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start); \ - UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE + UWord* WSTK_CONCAT(s,_end) = WSTK_CONCAT(s,_start) + DEF_WSTACK_SIZE; \ + ErtsAlcType_t WSTK_CONCAT(s,_alloc_type) = ERTS_ALC_T_ESTACK + +#define WSTACK_CHANGE_ALLOCATOR(s,t) \ +do { \ + if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) { \ + erl_exit(1, "Internal error - trying to change allocator " \ + "type of active wstack\n"); \ + } \ + WSTK_CONCAT(s,_alloc_type) = (t); \ + } while (0) #define DESTROY_WSTACK(s) \ do { \ if (WSTK_CONCAT(s,_start) != WSTK_CONCAT(s,_default_stack)) { \ - erts_free(ERTS_ALC_T_ESTACK, WSTK_CONCAT(s,_start)); \ + erts_free(WSTK_CONCAT(s,_alloc_type), WSTK_CONCAT(s,_start)); \ } \ } while(0) +/* + * Do not free the stack after this, it may have pointers into what + * was saved in 'v'. 'v' and 'vsize' are changed by this macro. If + * 'v' points to anything, it should have been allocated by a previous + * call to this macro. Be careful to set a correct allocator prior to + * saving. + * 'v' can be any lvalue pointer, it will point to an array of UWord + * after calling this macro. + */ +#define WSTACK_SAVE(s,v,vsize) /* v and vsize are "name parameters" */ \ +do { \ + Uint _wsz = WSTACK_COUNT(s); \ + if (WSTK_CONCAT(s,_start) == WSTK_CONCAT(s,_default_stack)) { \ + if ((v) == NULL) { \ + (v) = erts_alloc(WSTK_CONCAT(s,_alloc_type), \ + DEF_WSTACK_SIZE * sizeof(UWord)); \ + } \ + memcpy((v),WSTK_CONCAT(s,_start),_wsz*sizeof(UWord)); \ + } else { \ + (v) = (void *) WSTK_CONCAT(s,_start); \ + } \ + (vsize) = _wsz; \ + } while (0) + +/* + * Use on empty stack, only the allocator can be changed before this + * The vector parameter is reset to NULL if the vector is moved to stack, + * otherwise it's kept for reuse, so a saved and restored vector might + * need freeing using the correct allocator parameter. + * 'v' can be any lvalue pointer, it's cast to an (UWord *). + */ +#define WSTACK_RESTORE(s, v, vsize) /*v is a "name parameter"*/ \ +do { \ + if ((vsize) > DEF_WSTACK_SIZE) { \ + Uint _ca = DEF_WSTACK_SIZE; \ + while (_ca < (vsize)) \ + _ca = _ca * 2; \ + WSTK_CONCAT(s,_start) = (UWord *) (v); \ + WSTK_CONCAT(s,_end) = ((UWord *)(v)) + _ca; \ + WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start) + (vsize); \ + (v) = NULL; \ + } else { \ + memcpy(WSTK_CONCAT(s,_start),(v),(vsize)*sizeof(UWord));\ + WSTK_CONCAT(s,_sp) = WSTK_CONCAT(s,_start) + (vsize); \ + } \ + } while (0) + +#define WSTACK_IS_STATIC(s) (WSTK_CONCAT(s,_start) == WSTK_CONCAT(s,_default_stack)) + #define WSTACK_PUSH(s, x) \ do { \ if (WSTK_CONCAT(s,_sp) == WSTK_CONCAT(s,_end)) { \ - erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ - &WSTK_CONCAT(s,_end)); \ + erl_grow_wstack(WSTK_CONCAT(s,_alloc_type), &WSTK_CONCAT(s,_start), \ + &WSTK_CONCAT(s,_sp), &WSTK_CONCAT(s,_end)); \ } \ *WSTK_CONCAT(s,_sp)++ = (x); \ } while(0) @@ -460,8 +578,8 @@ do { \ #define WSTACK_PUSH2(s, x, y) \ do { \ if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 2) { \ - erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ - &WSTK_CONCAT(s,_end)); \ + erl_grow_wstack(WSTK_CONCAT(s,_alloc_type), &WSTK_CONCAT(s,_start), \ + &WSTK_CONCAT(s,_sp), &WSTK_CONCAT(s,_end)); \ } \ *WSTK_CONCAT(s,_sp)++ = (x); \ *WSTK_CONCAT(s,_sp)++ = (y); \ @@ -470,8 +588,8 @@ do { \ #define WSTACK_PUSH3(s, x, y, z) \ do { \ if (WSTK_CONCAT(s,_sp) > WSTK_CONCAT(s,_end) - 3) { \ - erl_grow_wstack(&WSTK_CONCAT(s,_start), &WSTK_CONCAT(s,_sp), \ - &WSTK_CONCAT(s,_end)); \ + erl_grow_wstack(WSTK_CONCAT(s,_alloc_type), &WSTK_CONCAT(s,_start), \ + &WSTK_CONCAT(s,_sp), &WSTK_CONCAT(s,_end)); \ } \ *WSTK_CONCAT(s,_sp)++ = (x); \ *WSTK_CONCAT(s,_sp)++ = (y); \ @@ -688,12 +806,6 @@ void MD5Final(unsigned char [16], MD5_CTX *); /* ggc.c */ - -typedef struct { - Uint garbage_collections; - Uint reclaimed; -} ErtsGCInfo; - void erts_gc_info(ErtsGCInfo *gcip); void erts_init_gc(void); int erts_garbage_collect(Process*, int, Eterm*, int); @@ -724,15 +836,15 @@ int erts_add_driver_entry(ErlDrvEntry *drv, DE_Handle *handle, int driver_list_l void erts_destroy_driver(erts_driver_t *drv); int erts_save_suspend_process_on_port(Port*, Process*); Port *erts_open_driver(erts_driver_t*, Eterm, char*, SysDriverOpts*, int *, int *); -void erts_init_io(int, int); +void erts_init_io(int, int, int); void erts_raw_port_command(Port*, byte*, Uint); void driver_report_exit(ErlDrvPort, int); LineBuf* allocate_linebuf(int); int async_ready(Port *, void*); -ErtsPortNames *erts_get_port_names(Eterm); +ErtsPortNames *erts_get_port_names(Eterm, ErlDrvPort); void erts_free_port_names(ErtsPortNames *); Uint erts_port_ioq_size(Port *pp); -void erts_stale_drv_select(Eterm, ErlDrvEvent, int, int); +void erts_stale_drv_select(Eterm, ErlDrvPort, ErlDrvEvent, int, int); Port *erts_get_heart_port(void); @@ -746,6 +858,8 @@ void erl_drv_thr_init(void); /* utils.c */ void erts_cleanup_offheap(ErlOffHeap *offheap); +Uint64 erts_timestamp_millis(void); + Export* erts_find_function(Eterm, Eterm, unsigned int, ErtsCodeIndex); Eterm store_external_or_ref_in_proc_(Process *, Eterm); @@ -777,6 +891,9 @@ Sint erts_re_set_loop_limit(Sint limit); void erts_init_bif_binary(void); Sint erts_binary_set_loop_limit(Sint limit); +/* external.c */ +void erts_init_external(void); + /* erl_unicode.c */ void erts_init_unicode(void); Sint erts_unicode_set_loop_limit(Sint limit); @@ -820,7 +937,7 @@ char* Sint_to_buf(Sint, struct Sint_buf*); #define ERTS_IOLIST_OVERFLOW 1 #define ERTS_IOLIST_TYPE 2 -Eterm buf_to_intlist(Eterm**, char*, size_t, Eterm); /* most callers pass plain char*'s */ +Eterm buf_to_intlist(Eterm**, const char*, size_t, Eterm); /* most callers pass plain char*'s */ #define ERTS_IOLIST_TO_BUF_OVERFLOW (~((ErlDrvSizeT) 0)) #define ERTS_IOLIST_TO_BUF_TYPE_ERROR (~((ErlDrvSizeT) 1)) @@ -901,9 +1018,10 @@ Eterm erts_match_set_lint(Process *p, Eterm matchexpr); extern void erts_match_set_release_result(Process* p); enum erts_pam_run_flags { - ERTS_PAM_TMP_RESULT=0, - ERTS_PAM_COPY_RESULT=1, - ERTS_PAM_CONTIGUOUS_TUPLE=2 + ERTS_PAM_TMP_RESULT=1, + ERTS_PAM_COPY_RESULT=2, + ERTS_PAM_CONTIGUOUS_TUPLE=4, + ERTS_PAM_IGNORE_TRACE_SILENT=8 }; extern Eterm erts_match_set_run(Process *p, Binary *mpsp, Eterm *args, int num_args, @@ -925,6 +1043,8 @@ extern erts_driver_t vanilla_driver; extern erts_driver_t spawn_driver; extern erts_driver_t fd_driver; +int erts_beam_jump_table(void); + /* Should maybe be placed in erl_message.h, but then we get an include mess. */ ERTS_GLB_INLINE Eterm * erts_alloc_message_heap_state(Uint size, @@ -1100,6 +1220,13 @@ erts_alloc_message_heap(Uint size, # define UnUseTmpHeapNoproc(Size) /* Nothing */ #endif /* HEAP_ON_C_STACK */ +ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); +ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); +ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf); +ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, + Eterm module, Eterm function, int arity, + char *process_buf, char *mfa_buf); + #if ERTS_GLB_INLINE_INCL_FUNC_DEF #include "dtrace-wrapper.h" diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 43918a7141..c1e66b59af 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -364,9 +364,8 @@ static Port *create_port(char *name, ERTS_P_LINKS(prt) = NULL; ERTS_P_MONITORS(prt) = NULL; prt->linebuf = NULL; - prt->bp = NULL; prt->suspended = NULL; - prt->data = am_undefined; + erts_init_port_data(prt); prt->port_data_lock = NULL; prt->control_flags = 0; prt->bytes_in = 0; @@ -1442,6 +1441,7 @@ erts_schedule_proc2port_signal(Process *c_p, Eterm *refp, ErtsProc2PortSigData *sigdp, int task_flags, + ErtsPortTaskHandle *pthp, ErtsProc2PortSigCallback callback) { int sched_res; @@ -1491,7 +1491,7 @@ erts_schedule_proc2port_signal(Process *c_p, /* Schedule port close call for later execution... */ sched_res = erts_port_task_schedule(prt->common.id, - NULL, + pthp, ERTS_PORT_TASK_PROC_SIG, sigdp, callback, @@ -1629,6 +1629,7 @@ bad_port_signal(Process *c_p, refp, sigdp, 0, + NULL, port_badsig); } @@ -1838,8 +1839,11 @@ erts_port_output(Process *c_p, ErlIOVec *evp = NULL; char *buf = NULL; int force_immediate_call = (flags & ERTS_PORT_SIG_FLG_FORCE_IMM_CALL); + int async_nosuspend; + ErtsPortTaskHandle *ns_pthp; ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP + | ERTS_PORT_SIG_FLG_ASYNC | ERTS_PORT_SIG_FLG_NOSUSPEND | ERTS_PORT_SIG_FLG_FORCE | ERTS_PORT_SIG_FLG_FORCE_IMM_CALL)) == 0); @@ -1861,6 +1865,12 @@ erts_port_output(Process *c_p, ? ERTS_PORT_OP_DROPPED : ERTS_PORT_OP_BUSY); + async_nosuspend = ((flags & (ERTS_PORT_SIG_FLG_ASYNC + | ERTS_PORT_SIG_FLG_NOSUSPEND + | ERTS_PORT_SIG_FLG_FORCE)) + == (ERTS_PORT_SIG_FLG_ASYNC + | ERTS_PORT_SIG_FLG_NOSUSPEND)); + try_call = (force_immediate_call /* crash dumping */ || !(sched_flags & (invalid_flags | ERTS_PTS_FLGS_FORCE_SCHEDULE_OP))); @@ -1995,6 +2005,15 @@ erts_port_output(Process *c_p, return ERTS_PORT_OP_DONE; case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: sched_flags = try_call_state.sched_flags; + if (async_nosuspend + && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { + driver_free_binary(cbin); + if (evp != &ev) + erts_free(ERTS_ALC_T_TMP, evp); + return ((sched_flags & ERTS_PTS_FLG_EXIT) + ? ERTS_PORT_OP_DROPPED + : ERTS_PORT_OP_BUSY); + } case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: /* Schedule outputv() call instead... */ break; @@ -2142,6 +2161,13 @@ erts_port_output(Process *c_p, return ERTS_PORT_OP_DONE; case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: sched_flags = try_call_state.sched_flags; + if (async_nosuspend + && (sched_flags & (busy_flgs|ERTS_PTS_FLG_EXIT))) { + erts_free(ERTS_ALC_T_TMP, buf); + return ((sched_flags & ERTS_PTS_FLG_EXIT) + ? ERTS_PORT_OP_DROPPED + : ERTS_PORT_OP_BUSY); + } case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: /* Schedule outputv() call instead... */ break; @@ -2163,20 +2189,33 @@ erts_port_output(Process *c_p, task_flags = ERTS_PT_FLG_WAIT_BUSY; sigdp->flags |= flags; + ns_pthp = NULL; if (flags & (ERTS_P2P_SIG_DATA_FLG_FORCE|ERTS_P2P_SIG_DATA_FLG_NOSUSPEND)) { task_flags = 0; if (flags & ERTS_P2P_SIG_DATA_FLG_FORCE) sigdp->flags &= ~ERTS_P2P_SIG_DATA_FLG_NOSUSPEND; + else if (async_nosuspend) { + ErtsSchedulerData *esdp = (c_p + ? ERTS_PROC_GET_SCHDATA(c_p) + : erts_get_scheduler_data()); + ASSERT(esdp); + ns_pthp = &esdp->nosuspend_port_task_handle; + sigdp->flags &= ~ERTS_P2P_SIG_DATA_FLG_NOSUSPEND; + } else if (flags & ERTS_P2P_SIG_DATA_FLG_NOSUSPEND) task_flags = ERTS_PT_FLG_NOSUSPEND; } + ASSERT(ns_pthp || !async_nosuspend); + ASSERT(async_nosuspend || !ns_pthp); + res = erts_schedule_proc2port_signal(c_p, prt, c_p ? c_p->common.id : ERTS_INVALID_PID, refp, sigdp, task_flags, + ns_pthp, port_sig_callback); if (res != ERTS_PORT_OP_SCHEDULED) { @@ -2187,9 +2226,23 @@ erts_port_output(Process *c_p, return res; } - if (!(sched_flags & ERTS_PTS_FLG_EXIT) && (sched_flags & busy_flgs)) - return ERTS_PORT_OP_BUSY_SCHEDULED; - + if (!(flags & ERTS_PORT_SIG_FLG_FORCE)) { + sched_flags = erts_smp_atomic32_read_acqb(&prt->sched.flags); + if (!(sched_flags & ERTS_PTS_FLG_BUSY_PORT)) { + if (async_nosuspend) + erts_port_task_tmp_handle_detach(ns_pthp); + } + else { + if (!async_nosuspend) + return ERTS_PORT_OP_BUSY_SCHEDULED; + else { + if (erts_port_task_abort(ns_pthp) == 0) + return ERTS_PORT_OP_BUSY; + else + erts_port_task_tmp_handle_detach(ns_pthp); + } + } + } return res; bad_value: @@ -2285,6 +2338,7 @@ erts_port_exit(Process *c_p, ErlHeapFragment *bp = NULL; ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP + | ERTS_PORT_SIG_FLG_ASYNC | ERTS_PORT_SIG_FLG_BROKEN_LINK | ERTS_PORT_SIG_FLG_FORCE_SCHED)) == 0); @@ -2345,6 +2399,7 @@ erts_port_exit(Process *c_p, refp, sigdp, 0, + NULL, port_sig_exit); if (res == ERTS_PORT_OP_DROPPED) { @@ -2460,7 +2515,8 @@ erts_port_connect(Process *c_p, !refp, am_connect); - ASSERT((flags & ~ERTS_PORT_SIG_FLG_BANG_OP) == 0); + ASSERT((flags & ~(ERTS_PORT_SIG_FLG_BANG_OP + | ERTS_PORT_SIG_FLG_ASYNC)) == 0); if (is_not_internal_pid(connect)) connect_id = NIL; /* Fail in op (for signal order) */ @@ -2499,6 +2555,7 @@ erts_port_connect(Process *c_p, refp, sigdp, 0, + NULL, port_sig_connect); } @@ -2555,6 +2612,7 @@ erts_port_unlink(Process *c_p, Port *prt, Eterm from, Eterm *refp) refp, sigdp, 0, + NULL, port_sig_unlink); } @@ -2644,11 +2702,13 @@ erts_port_link(Process *c_p, Port *prt, Eterm to, Eterm *refp) refp, sigdp, 0, + NULL, port_sig_link); } void erts_init_io(int port_tab_size, - int port_tab_size_ignore_files) + int port_tab_size_ignore_files, + int legacy_port_tab) { ErlDrvEntry** dp; UWord common_element_size; @@ -2691,7 +2751,8 @@ void erts_init_io(int port_tab_size, (ErtsPTabElementCommon *) &erts_invalid_port.common, port_tab_size, common_element_size, /* Doesn't need to be excact */ - "port_table"); + "port_table", + legacy_port_tab); erts_smp_atomic_init_nob(&erts_bytes_out, 0); erts_smp_atomic_init_nob(&erts_bytes_in, 0); @@ -3371,11 +3432,8 @@ terminate_port(Port *prt) erts_free(ERTS_ALC_T_LINEBUF, (void *) prt->linebuf); prt->linebuf = NULL; } - if (prt->bp != NULL) { - free_message_buffer(prt->bp); - prt->bp = NULL; - prt->data = am_undefined; - } + + erts_cleanup_port_data(prt); if (prt->psd) erts_free(ERTS_ALC_T_PRTSD, prt->psd); @@ -3625,6 +3683,10 @@ erts_port_command(Process *c_p, ASSERT(port); flags |= ERTS_PORT_SIG_FLG_BANG_OP; + if (!erts_port_synchronous_ops) { + flags |= ERTS_PORT_SIG_FLG_ASYNC; + refp = NULL; + } if (is_tuple_arity(command, 2)) { Eterm cntd; @@ -3632,21 +3694,14 @@ erts_port_command(Process *c_p, cntd = tp[1]; if (is_internal_pid(cntd)) { if (tp[2] == am_close) { - if (!erts_port_synchronous_ops) - refp = NULL; flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND; return erts_port_exit(c_p, flags, port, cntd, am_normal, refp); } else if (is_tuple_arity(tp[2], 2)) { tp = tuple_val(tp[2]); if (tp[1] == am_command) { - if (!(flags & ERTS_PORT_SIG_FLG_NOSUSPEND) - && !erts_port_synchronous_ops) - refp = NULL; return erts_port_output(c_p, flags, port, cntd, tp[2], refp); } else if (tp[1] == am_connect) { - if (!erts_port_synchronous_ops) - refp = NULL; flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND; return erts_port_connect(c_p, flags, port, cntd, tp[2], refp); } @@ -3655,8 +3710,6 @@ erts_port_command(Process *c_p, } /* badsig */ - if (!erts_port_synchronous_ops) - refp = NULL; flags &= ~ERTS_PORT_SIG_FLG_NOSUSPEND; return bad_port_signal(c_p, flags, port, c_p->common.id, refp, am_command); } @@ -4053,6 +4106,7 @@ erts_port_control(Process* c_p, retvalp, sigdp, 0, + NULL, port_sig_control); if (res != ERTS_PORT_OP_SCHEDULED) { cleanup_scheduled_control(binp, bufp); @@ -4147,7 +4201,7 @@ port_sig_call(Port *prt, ErlOffHeap *ohp; Process *rp; ErtsProcLocks rp_locks = 0; - Uint hsz; + Sint hsz; rp = erts_proc_lookup_raw(sigdp->caller); if (!rp) @@ -4264,7 +4318,7 @@ erts_port_call(Process* c_p, switch (try_call_res) { case ERTS_TRY_IMM_DRV_CALL_OK: { Eterm *hp, *hp_end; - Uint hsz; + Sint hsz; unsigned ret_flags = 0U; Eterm term; @@ -4333,6 +4387,7 @@ erts_port_call(Process* c_p, retvalp, sigdp, 0, + NULL, port_sig_call); if (res != ERTS_PORT_OP_SCHEDULED) { cleanup_scheduled_call(bufp); @@ -4499,228 +4554,10 @@ erts_port_info(Process* c_p, retvalp, sigdp, 0, + NULL, port_sig_info); } -static int -port_sig_set_data(Port *prt, - erts_aint32_t state, - int op, - ErtsProc2PortSigData *sigdp) -{ - ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY); - - if (op == ERTS_PROC2PORT_SIG_EXEC) { - if (prt->bp) - free_message_buffer(prt->bp); - prt->bp = sigdp->u.set_data.bp; - prt->data = sigdp->u.set_data.data; - port_sched_op_reply(sigdp->caller, sigdp->ref, am_true); - } - else { - if (sigdp->u.set_data.bp) - free_message_buffer(sigdp->u.set_data.bp); - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); - } - return ERTS_PORT_REDS_SET_DATA; -} - -ErtsPortOpResult -erts_port_set_data(Process* c_p, - Port *prt, - Eterm data, - Eterm *refp) -{ - ErtsPortOpResult res; - Eterm set_data; - ErlHeapFragment *bp; - ErtsProc2PortSigData *sigdp; - ErtsTryImmDrvCallResult try_call_res; - ErtsTryImmDrvCallState try_call_state - = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( - c_p, - prt, - ERTS_PORT_SFLGS_INVALID_LOOKUP, - 0, - !refp, - am_set_data); - - if (is_immed(data)) { - set_data = data; - bp = NULL; - } - else { - Eterm *hp; - Uint sz = size_object(data); - bp = new_message_buffer(sz); - hp = bp->mem; - set_data = copy_struct(data, sz, &hp, &bp->off_heap); - } - - try_call_res = try_imm_drv_call(&try_call_state); - switch (try_call_res) { - case ERTS_TRY_IMM_DRV_CALL_OK: - if (prt->bp) - free_message_buffer(prt->bp); - prt->bp = bp; - prt->data = set_data; - finalize_imm_drv_call(&try_call_state); - BUMP_REDS(c_p, ERTS_PORT_REDS_SET_DATA); - return ERTS_PORT_OP_DONE; - case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: - return ERTS_PORT_OP_DROPPED; - case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: - case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: - /* Schedule call instead... */ - break; - } - - sigdp = erts_port_task_alloc_p2p_sig_data(); - sigdp->flags = ERTS_P2P_SIG_TYPE_SET_DATA; - sigdp->u.set_data.data = set_data; - sigdp->u.set_data.bp = bp; - - res = erts_schedule_proc2port_signal(c_p, - prt, - c_p->common.id, - refp, - sigdp, - 0, - port_sig_set_data); - if (res != ERTS_PORT_OP_SCHEDULED && bp) - free_message_buffer(bp); - return res; -} - -static int -port_sig_get_data(Port *prt, - erts_aint32_t state, - int op, - ErtsProc2PortSigData *sigdp) -{ - ASSERT(sigdp->flags & ERTS_P2P_SIG_DATA_FLG_REPLY); - if (op != ERTS_PROC2PORT_SIG_EXEC) - port_sched_op_reply(sigdp->caller, sigdp->ref, am_badarg); - else { - Process *rp; - ErtsProcLocks rp_locks = 0; - - rp = erts_proc_lookup_raw(sigdp->caller); - if (rp) { - Uint hsz; - Eterm *hp, *hp_start; - Eterm data, msg; - ErlHeapFragment *bp; - ErlOffHeap *ohp; - - hsz = ERTS_QUEUE_PORT_SCHED_OP_REPLY_SIZE; - hsz += 3; - if (prt->bp) - hsz += prt->bp->used_size; - - hp_start = hp = erts_alloc_message_heap(hsz, - &bp, - &ohp, - rp, - &rp_locks); - - if (is_immed(prt->data)) - data = prt->data; - else - data = copy_struct(prt->data, - prt->bp->used_size, - &hp, - &bp->off_heap); - - - - msg = TUPLE2(hp, am_ok, data); - hp += 3; - - queue_port_sched_op_reply(rp, - &rp_locks, - hp_start, - hp, - hsz, - bp, - sigdp->ref, - msg); - if (rp_locks) - erts_smp_proc_unlock(rp, rp_locks); - } - } - return ERTS_PORT_REDS_GET_DATA; -} - -ErtsPortOpResult -erts_port_get_data(Process* c_p, - Port *prt, - Eterm *retvalp) -{ - ErtsProc2PortSigData *sigdp; - ErtsTryImmDrvCallResult try_call_res; - ErtsTryImmDrvCallState try_call_state - = ERTS_INIT_TRY_IMM_DRV_CALL_STATE( - c_p, - prt, - ERTS_PORT_SFLGS_INVALID_LOOKUP, - 0, - 0, - am_get_data); - - try_call_res = try_imm_drv_call(&try_call_state); - switch (try_call_res) { - case ERTS_TRY_IMM_DRV_CALL_OK: { - Eterm *hp; - Eterm data; - ErlHeapFragment *bp; - Uint sz; - if (is_immed(prt->data)) { - bp = NULL; - data = prt->data; - } - else { - bp = new_message_buffer(prt->bp->used_size); - data = copy_struct(prt->data, - prt->bp->used_size, - &hp, - &bp->off_heap); - } - finalize_imm_drv_call(&try_call_state); - if (is_immed(data)) - sz = 0; - else - sz = bp->used_size; - - hp = HAlloc(c_p, sz + 3); - if (is_not_immed(data)) { - data = copy_struct(data, bp->used_size, &hp, &MSO(c_p)); - free_message_buffer(bp); - } - *retvalp = TUPLE2(hp, am_ok, data); - BUMP_REDS(c_p, ERTS_PORT_REDS_GET_DATA); - return ERTS_PORT_OP_DONE; - } - case ERTS_TRY_IMM_DRV_CALL_INVALID_PORT: - return ERTS_PORT_OP_DROPPED; - case ERTS_TRY_IMM_DRV_CALL_INVALID_SCHED_FLAGS: - case ERTS_TRY_IMM_DRV_CALL_BUSY_LOCK: - /* Schedule call instead... */ - break; - } - - sigdp = erts_port_task_alloc_p2p_sig_data(); - sigdp->flags = ERTS_P2P_SIG_TYPE_GET_DATA; - - return erts_schedule_proc2port_signal(c_p, - prt, - c_p->common.id, - retvalp, - sigdp, - 0, - port_sig_get_data); -} - typedef struct { int to; void *arg; @@ -4977,7 +4814,8 @@ int async_ready(Port *p, void* data) static void report_missing_drv_callback(Port *p, char *drv_type, char *callback) { - ErtsPortNames *pnp = erts_get_port_names(p->common.id); + ErtsPortNames *pnp = erts_get_port_names(p->common.id, + ERTS_Port2ErlDrvPort(p)); char *unknown = "<unknown>"; char *drv_name = pnp->driver_name ? pnp->driver_name : unknown; char *prt_name = pnp->name ? pnp->name : unknown; @@ -4992,15 +4830,25 @@ report_missing_drv_callback(Port *p, char *drv_type, char *callback) void erts_stale_drv_select(Eterm port, + ErlDrvPort drv_port, ErlDrvEvent hndl, int mode, int deselect) { char *type; - ErlDrvPort drv_port = ERTS_Port2ErlDrvPort(erts_port_lookup_raw(port)); - ErtsPortNames *pnp = erts_get_port_names(port); + ErtsPortNames *pnp; erts_dsprintf_buf_t *dsbufp; + if (drv_port == ERTS_INVALID_ERL_DRV_PORT) { + Port *prt = erts_port_lookup_raw(port); + if (prt) + drv_port = ERTS_Port2ErlDrvPort(prt); + else + drv_port = ERTS_INVALID_ERL_DRV_PORT; + } + + pnp = erts_get_port_names(port, drv_port); + switch (mode) { case ERL_DRV_READ | ERL_DRV_WRITE: type = "Input/Output"; @@ -5035,12 +4883,16 @@ erts_stale_drv_select(Eterm port, } ErtsPortNames * -erts_get_port_names(Eterm id) +erts_get_port_names(Eterm id, ErlDrvPort drv_port) { - Port *prt = erts_port_lookup_raw(id); + Port *prt; ErtsPortNames *pnp; ASSERT(is_nil(id) || is_internal_port(id)); + prt = ERTS_ErlDrvPort2Port(drv_port); + if (prt == ERTS_INVALID_ERL_DRV_PORT) + prt = erts_port_lookup_raw(id); + if (!prt) { pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, sizeof(ErtsPortNames)); pnp->name = NULL; @@ -5052,6 +4904,7 @@ erts_get_port_names(Eterm id) size_t pnp_len = sizeof(ErtsPortNames); #ifndef DEBUG pnp_len += 100; /* In most cases 100 characters will be enough... */ + ASSERT(prt->common.id == id); #endif pnp = erts_alloc(ERTS_ALC_T_PORT_NAMES, pnp_len); do { @@ -5941,10 +5794,6 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, return driver_output2(ix, hbuf, hlen, NULL, 0); size = vec->size - skip; /* Size of remaining bytes in vector */ - ASSERT(hlen >= 0); /* debug only */ - if (hlen < 0) - hlen = 0; - prt = erts_drvport2port_state(ix, &state); if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; @@ -5967,7 +5816,7 @@ int driver_outputv(ErlDrvPort ix, char* hbuf, ErlDrvSizeT hlen, binv++; n--; } else { - iov->iov_base += skip; + iov->iov_base = ((char *)(iov->iov_base)) + skip; iov->iov_len -= skip; skip = 0; } @@ -6412,7 +6261,7 @@ int driver_enqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) n--; } else { - iov->iov_base += skip; + iov->iov_base = ((char *)(iov->iov_base)) + skip; iov->iov_len -= skip; skip = 0; } @@ -6477,7 +6326,7 @@ int driver_pushqv(ErlDrvPort ix, ErlIOVec* vec, ErlDrvSizeT skip) n--; } else { - iov->iov_base += skip; + iov->iov_base = ((char *)(iov->iov_base)) + skip; iov->iov_len -= skip; skip = 0; } @@ -6536,7 +6385,7 @@ ErlDrvSizeT driver_deq(ErlDrvPort ix, ErlDrvSizeT size) q->v_head++; } else { - q->v_head->iov_base += size; + q->v_head->iov_base = ((char *)(q->v_head->iov_base)) + size; q->v_head->iov_len -= size; size = 0; } diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index a8f15fdc38..bd2be7afca 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -185,15 +185,15 @@ erts_set_hole_marker(Eterm* ptr, Uint sz) * Helper function for the ESTACK macros defined in global.h. */ void -erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end) +erl_grow_stack(ErtsAlcType_t a_type, Eterm** start, Eterm** sp, Eterm** end) { Uint old_size = (*end - *start); Uint new_size = old_size * 2; Uint sp_offs = *sp - *start; if (new_size > 2 * DEF_ESTACK_SIZE) { - *start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(Eterm)); + *start = erts_realloc(a_type, (void *) *start, new_size*sizeof(Eterm)); } else { - Eterm* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(Eterm)); + Eterm* new_ptr = erts_alloc(a_type, new_size*sizeof(Eterm)); sys_memcpy(new_ptr, *start, old_size*sizeof(Eterm)); *start = new_ptr; } @@ -204,15 +204,15 @@ erl_grow_stack(Eterm** start, Eterm** sp, Eterm** end) * Helper function for the ESTACK macros defined in global.h. */ void -erl_grow_wstack(UWord** start, UWord** sp, UWord** end) +erl_grow_wstack(ErtsAlcType_t a_type, UWord** start, UWord** sp, UWord** end) { Uint old_size = (*end - *start); Uint new_size = old_size * 2; Uint sp_offs = *sp - *start; if (new_size > 2 * DEF_ESTACK_SIZE) { - *start = erts_realloc(ERTS_ALC_T_ESTACK, (void *) *start, new_size*sizeof(UWord)); + *start = erts_realloc(a_type, (void *) *start, new_size*sizeof(UWord)); } else { - UWord* new_ptr = erts_alloc(ERTS_ALC_T_ESTACK, new_size*sizeof(UWord)); + UWord* new_ptr = erts_alloc(a_type, new_size*sizeof(UWord)); sys_memcpy(new_ptr, *start, old_size*sizeof(UWord)); *start = new_ptr; } @@ -1319,7 +1319,7 @@ make_hash2(Eterm term) { FloatDef ff; GET_DOUBLE(term, ff); -#if defined(WORDS_BIGENDIAN) +#if defined(WORDS_BIGENDIAN) || defined(DOUBLE_MIDDLE_ENDIAN) UINT32_HASH_2(ff.fw[0], ff.fw[1], HCONST_12); #else UINT32_HASH_2(ff.fw[1], ff.fw[0], HCONST_12); @@ -2713,25 +2713,26 @@ tailrecur_ne: case SMALL_FLOAT: GET_DOUBLE(bw, f2); if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { - // Float is within the no loss limit + /* Float is within the no loss limit */ f1.fd = signed_val(aw); j = float_comp(f1.fd, f2.fd); #if ERTS_SIZEOF_ETERM == 8 } else if (f2.fd > (double) (MAX_SMALL + 1)) { - // Float is a positive bignum, i.e. bigger + /* Float is a positive bignum, i.e. bigger */ j = -1; } else if (f2.fd < (double) (MIN_SMALL - 1)) { - // Float is a negative bignum, i.e. smaller + /* Float is a negative bignum, i.e. smaller */ j = 1; - } else { // Float is a Sint but less precise + } else { + /* Float is a Sint but less precise */ j = signed_val(aw) - (Sint) f2.fd; } #else } else { - // If float is positive it is bigger than small + /* If float is positive it is bigger than small */ j = (f2.fd > 0.0) ? -1 : 1; } -#endif // ERTS_SIZEOF_ETERM == 8 +#endif /* ERTS_SIZEOF_ETERM == 8 */ break; case FLOAT_BIG: { @@ -2743,18 +2744,18 @@ tailrecur_ne: GET_DOUBLE(bw, f2); if ((f2.fd < (double) (MAX_SMALL + 1)) && (f2.fd > (double) (MIN_SMALL - 1))) { - // Float is a Sint + /* Float is a Sint */ j = big_sign(aw) ? -1 : 1; } else if (big_arity(aw) > BIG_ARITY_FLOAT_MAX || pow(2.0,(big_arity(aw)-1)*D_EXP) > fabs(f2.fd)) { - // If bignum size shows that it is bigger than the abs float + /* If bignum size shows that it is bigger than the abs float */ j = big_sign(aw) ? -1 : 1; } else if (big_arity(aw) < BIG_ARITY_FLOAT_MAX && (pow(2.0,(big_arity(aw))*D_EXP)-1.0) < fabs(f2.fd)) { - // If bignum size shows that it is smaller than the abs float + /* If bignum size shows that it is smaller than the abs float */ j = f2.fd < 0 ? 1 : -1; } else if (f2.fd < MAX_LOSSLESS_FLOAT && f2.fd > MIN_LOSSLESS_FLOAT) { - // Float is within the no loss limit + /* Float is within the no loss limit */ if (big_to_double(aw, &f1.fd) < 0) { j = big_sign(aw) ? -1 : 1; } else { @@ -2771,25 +2772,26 @@ tailrecur_ne: case FLOAT_SMALL: GET_DOUBLE(aw, f1); if (f1.fd < MAX_LOSSLESS_FLOAT && f1.fd > MIN_LOSSLESS_FLOAT) { - // Float is within the no loss limit + /* Float is within the no loss limit */ f2.fd = signed_val(bw); j = float_comp(f1.fd, f2.fd); #if ERTS_SIZEOF_ETERM == 8 } else if (f1.fd > (double) (MAX_SMALL + 1)) { - // Float is a positive bignum, i.e. bigger + /* Float is a positive bignum, i.e. bigger */ j = 1; } else if (f1.fd < (double) (MIN_SMALL - 1)) { - // Float is a negative bignum, i.e. smaller + /* Float is a negative bignum, i.e. smaller */ j = -1; - } else { // Float is a Sint but less precise it + } else { + /* Float is a Sint but less precise it */ j = (Sint) f1.fd - signed_val(bw); } #else } else { - // If float is positive it is bigger than small + /* If float is positive it is bigger than small */ j = (f1.fd > 0.0) ? 1 : -1; } -#endif // ERTS_SIZEOF_ETERM == 8 +#endif /* ERTS_SIZEOF_ETERM == 8 */ break; default: j = b_tag - a_tag; @@ -2981,7 +2983,7 @@ char* Sint_to_buf(Sint n, struct Sint_buf *buf) */ Eterm -buf_to_intlist(Eterm** hpp, char *buf, size_t len, Eterm tail) +buf_to_intlist(Eterm** hpp, const char *buf, size_t len, Eterm tail) { Eterm* hp = *hpp; size_t i = len; @@ -3464,6 +3466,291 @@ erts_free_read_env(void *value) erts_free(ERTS_ALC_T_TMP, value); } + +typedef struct { + size_t sz; + char *ptr; +} ErtsEmuArg; + +typedef struct { + int argc; + ErtsEmuArg *arg; + size_t no_bytes; +} ErtsEmuArgs; + +ErtsEmuArgs saved_emu_args = {0}; + +void +erts_save_emu_args(int argc, char **argv) +{ +#ifdef DEBUG + char *end_ptr; +#endif + char *ptr; + int i; + size_t arg_sz[100]; + size_t size; + + ASSERT(!saved_emu_args.argc); + + size = sizeof(ErtsEmuArg)*argc; + for (i = 0; i < argc; i++) { + size_t sz = sys_strlen(argv[i]); + if (i < sizeof(arg_sz)/sizeof(arg_sz[0])) + arg_sz[i] = sz; + size += sz+1; + } + ptr = (char *) malloc(size); +#ifdef DEBUG + end_ptr = ptr + size; +#endif + saved_emu_args.arg = (ErtsEmuArg *) ptr; + ptr += sizeof(ErtsEmuArg)*argc; + saved_emu_args.argc = argc; + saved_emu_args.no_bytes = 0; + for (i = 0; i < argc; i++) { + size_t sz; + if (i < sizeof(arg_sz)/sizeof(arg_sz[0])) + sz = arg_sz[i]; + else + sz = sys_strlen(argv[i]); + saved_emu_args.arg[i].ptr = ptr; + saved_emu_args.arg[i].sz = sz; + saved_emu_args.no_bytes += sz; + ptr += sz+1; + sys_strcpy(saved_emu_args.arg[i].ptr, argv[i]); + } + ASSERT(ptr == end_ptr); +} + +Eterm +erts_get_emu_args(Process *c_p) +{ +#ifdef DEBUG + Eterm *end_hp; +#endif + int i; + Uint hsz; + Eterm *hp, res; + + hsz = saved_emu_args.no_bytes*2; + hsz += saved_emu_args.argc*2; + + hp = HAlloc(c_p, hsz); +#ifdef DEBUG + end_hp = hp + hsz; +#endif + res = NIL; + + for (i = saved_emu_args.argc-1; i >= 0; i--) { + Eterm arg = buf_to_intlist(&hp, + saved_emu_args.arg[i].ptr, + saved_emu_args.arg[i].sz, + NIL); + res = CONS(hp, arg, res); + hp += 2; + } + + ASSERT(hp == end_hp); + + return res; +} + + +Eterm +erts_get_ethread_info(Process *c_p) +{ + Uint sz, *szp; + Eterm res, *hp, **hpp, *end_hp = NULL; + + sz = 0; + szp = &sz; + hpp = NULL; + + while (1) { + Eterm tup, list, name; +#if defined(ETHR_NATIVE_ATOMIC32_IMPL) \ + || defined(ETHR_NATIVE_ATOMIC64_IMPL) \ + || defined(ETHR_NATIVE_DW_ATOMIC_IMPL) + char buf[1024]; + int i; + char **str; +#endif + + res = NIL; + +#ifdef ETHR_X86_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "sse2"), +#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + erts_bld_string(hpp, szp, + (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__ + ? "yes" : "no")) +#else + erts_bld_string(hpp, szp, "yes") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, + "x86" +#ifdef ARCH_64 + "_64" +#endif + " OOO"), + erts_bld_string(hpp, szp, +#ifdef ETHR_X86_OUT_OF_ORDER + "yes" +#else + "no" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); +#endif + +#ifdef ETHR_SPARC_V9_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Sparc V9"), + erts_bld_string(hpp, szp, +#if defined(ETHR_SPARC_TSO) + "TSO" +#elif defined(ETHR_SPARC_PSO) + "PSO" +#elif defined(ETHR_SPARC_RMO) + "RMO" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + +#ifdef ETHR_PPC_MEMBAR_H__ + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "lwsync"), + erts_bld_string(hpp, szp, +#if defined(ETHR_PPC_HAVE_LWSYNC) + "yes" +#elif defined(ETHR_PPC_HAVE_NO_LWSYNC) + "no" +#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__) + ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no" +#else + "undefined" +#endif + )); + + res = erts_bld_cons(hpp, szp, tup, res); + +#endif + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native rw-spinlocks"), +#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_RWSPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + tup = erts_bld_tuple(hpp, szp, 2, + erts_bld_string(hpp, szp, "Native spinlocks"), +#ifdef ETHR_NATIVE_SPINLOCK_IMPL + erts_bld_string(hpp, szp, ETHR_NATIVE_SPINLOCK_IMPL) +#else + erts_bld_string(hpp, szp, "no") +#endif + ); + res = erts_bld_cons(hpp, szp, tup, res); + + + list = NIL; +#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL + if (ethr_have_native_dw_atomic()) { + name = erts_bld_string(hpp, szp, ETHR_NATIVE_DW_ATOMIC_IMPL); + str = ethr_native_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + str = ethr_native_su_dw_atomic_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_su_dw_atomic_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } + } + else +#endif + name = erts_bld_string(hpp, szp, "no"); + + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "Double word native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC64_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC64_IMPL); + str = ethr_native_atomic64_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic64_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "64-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + list = NIL; +#ifdef ETHR_NATIVE_ATOMIC32_IMPL + name = erts_bld_string(hpp, szp, ETHR_NATIVE_ATOMIC32_IMPL); + str = ethr_native_atomic32_ops(); + for (i = 0; str[i]; i++) { + erts_snprintf(buf, sizeof(buf), "ethr_native_atomic32_%s()", str[i]); + list = erts_bld_cons(hpp, szp, + erts_bld_string(hpp, szp, buf), + list); + } +#else + name = erts_bld_string(hpp, szp, "no"); +#endif + tup = erts_bld_tuple(hpp, szp, 3, + erts_bld_string(hpp, szp, "32-bit native atomics"), + name, + list); + res = erts_bld_cons(hpp, szp, tup, res); + + if (hpp) { + HRelease(c_p, end_hp, *hpp) + return res; + } + + hp = HAlloc(c_p, sz); + end_hp = hp + sz; + hpp = &hp; + szp = NULL; + } +} + /* * To be used to silence unused result warnings, but do not abuse it. */ @@ -3720,6 +4007,24 @@ erts_smp_ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) } +/* + * A millisecond timestamp without time correction where there's no hrtime + * - for tracing on "long" things... + */ +Uint64 erts_timestamp_millis(void) +{ +#ifdef HAVE_GETHRTIME + return (Uint64) (sys_gethrtime() / 1000000); +#else + Uint64 res; + SysTimeval tv; + sys_gettimeofday(&tv); + res = (Uint64) tv.tv_sec*1000000; + res += (Uint64) tv.tv_usec; + return (res / 1000); +#endif +} + #ifdef DEBUG /* * Handy functions when using a debugger - don't use in the code! diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index 69acfc9dfd..8de578d8b7 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -397,7 +397,6 @@ struct t_pwritev { ErlDrvPort port; ErlDrvPDL q_mtx; size_t size; - size_t free_size; unsigned cnt; unsigned n; struct t_pbuf_spec specs[1]; @@ -462,7 +461,6 @@ struct t_data ErlDrvPort port; ErlDrvPDL q_mtx; size_t size; - size_t free_size; size_t reply_size; } writev; struct t_pwritev pwritev; @@ -544,57 +542,85 @@ static void *ef_safe_realloc(void *op, Uint s) */ /* char EV_CHAR_P(ErlIOVec *ev, int p, int q) */ -#define EV_CHAR_P(ev, p, q) \ - (((char *)(ev)->iov[(q)].iov_base) + (p)) +#define EV_CHAR_P(ev, p, q) \ + (((char *)(ev)->iov[q].iov_base) + (p)) /* int EV_GET_CHAR(ErlIOVec *ev, char *p, int *pp, int *qp) */ -#define EV_GET_CHAR(ev, p, pp, qp) \ - (*(pp)+1 <= (ev)->iov[*(qp)].iov_len \ - ? (*(p) = *EV_CHAR_P(ev, *(pp), *(qp)), \ - *(pp) = ( *(pp)+1 < (ev)->iov[*(qp)].iov_len \ - ? *(pp)+1 \ - : ((*(qp))++, 0)), \ - !0) \ - : 0) +#define EV_GET_CHAR(ev, p, pp, qp) efile_ev_get_char(ev, p ,pp, qp) +static int +efile_ev_get_char(ErlIOVec *ev, char *p, size_t *pp, size_t *qp) { + if (*pp + 1 <= ev->iov[*qp].iov_len) { + *p = *EV_CHAR_P(ev, *pp, *qp); + if (*pp + 1 < ev->iov[*qp].iov_len) + *pp += 1; + else { + *qp += 1; + *pp = 0; + } + return !0; + } + return 0; +} /* Uint32 EV_UINT32(ErlIOVec *ev, int p, int q)*/ -#define EV_UINT32(ev, p, q) \ - ((Uint32) *(((unsigned char *)(ev)->iov[(q)].iov_base) + (p))) +#define EV_UINT32(ev, p, q) \ + ((Uint32) ((unsigned char *)(ev)->iov[q].iov_base)[p]) /* int EV_GET_UINT32(ErlIOVec *ev, Uint32 *p, int *pp, int *qp) */ -#define EV_GET_UINT32(ev, p, pp, qp) \ - (*(pp)+4 <= (ev)->iov[*(qp)].iov_len \ - ? (*(p) = (EV_UINT32(ev, *(pp), *(qp)) << 24) \ - | (EV_UINT32(ev, *(pp)+1, *(qp)) << 16) \ - | (EV_UINT32(ev, *(pp)+2, *(qp)) << 8) \ - | (EV_UINT32(ev, *(pp)+3, *(qp))), \ - *(pp) = ( *(pp)+4 < (ev)->iov[*(qp)].iov_len \ - ? *(pp)+4 \ - : ((*(qp))++, 0)), \ - !0) \ - : 0) +#define EV_GET_UINT32(ev, p, pp, qp) efile_ev_get_uint32(ev, p, pp, qp) +static int +efile_ev_get_uint32(ErlIOVec *ev, Uint32 *p, size_t *pp, size_t *qp) { + if (*pp + 4 <= ev->iov[*qp].iov_len) { + *p = (EV_UINT32(ev, *pp, *qp) << 24) + | (EV_UINT32(ev, *pp + 1, *qp) << 16) + | (EV_UINT32(ev, *pp + 2, *qp) << 8) + | (EV_UINT32(ev, *pp + 3, *qp)); + if (*pp + 4 < ev->iov[*qp].iov_len) + *pp += 4; + else { + *qp += 1; + *pp = 0; + } + return !0; + } + return 0; +} /* Uint64 EV_UINT64(ErlIOVec *ev, int p, int q)*/ -#define EV_UINT64(ev, p, q) \ - ((Uint64) *(((unsigned char *)(ev)->iov[(q)].iov_base) + (p))) - -/* int EV_GET_UINT64(ErlIOVec *ev, Uint32 *p, int *pp, int *qp) */ -#define EV_GET_UINT64(ev, p, pp, qp) \ - (*(pp)+8 <= (ev)->iov[*(qp)].iov_len \ - ? (*(p) = (EV_UINT64(ev, *(pp), *(qp)) << 56) \ - | (EV_UINT64(ev, *(pp)+1, *(qp)) << 48) \ - | (EV_UINT64(ev, *(pp)+2, *(qp)) << 40) \ - | (EV_UINT64(ev, *(pp)+3, *(qp)) << 32) \ - | (EV_UINT64(ev, *(pp)+4, *(qp)) << 24) \ - | (EV_UINT64(ev, *(pp)+5, *(qp)) << 16) \ - | (EV_UINT64(ev, *(pp)+6, *(qp)) << 8) \ - | (EV_UINT64(ev, *(pp)+7, *(qp))), \ - *(pp) = ( *(pp)+8 < (ev)->iov[*(qp)].iov_len \ - ? *(pp)+8 \ - : ((*(qp))++, 0)), \ - !0) \ - : 0) +#define EV_UINT64(ev, p, q) \ + ((Uint64) ((unsigned char *)(ev)->iov[q].iov_base)[p]) + +/* int EV_GET_UINT64(ErlIOVec *ev, Uint64 *p, int *pp, int *qp) */ +#define EV_GET_UINT64(ev, p, pp, qp) efile_ev_get_uint64(ev, p, pp, qp) +static int +efile_ev_get_uint64(ErlIOVec *ev, Uint64 *p, size_t *pp, size_t *qp) { + if (*pp + 8 <= ev->iov[*qp].iov_len) { + *p = (EV_UINT64(ev, *pp, *qp) << 56) + | (EV_UINT64(ev, *pp + 1, *qp) << 48) + | (EV_UINT64(ev, *pp + 2, *qp) << 40) + | (EV_UINT64(ev, *pp + 3, *qp) << 32) + | (EV_UINT64(ev, *pp + 4, *qp) << 24) + | (EV_UINT64(ev, *pp + 5, *qp) << 16) + | (EV_UINT64(ev, *pp + 6, *qp) << 8) + | (EV_UINT64(ev, *pp + 7, *qp)); + if (*pp + 8 < ev->iov[*qp].iov_len) + *pp += 8; + else { + *qp += 1; + *pp = 0; + } + return !0; + } + return 0; +} +/* int EV_GET_SINT64(ErlIOVec *ev, Uint64 *p, int *pp, int *qp) */ +#define EV_GET_SINT64(ev, p, pp, qp) efile_ev_get_sint64(ev, p, pp, qp) +static int +efile_ev_get_sint64(ErlIOVec *ev, Sint64 *p, size_t *pp, size_t *qp) { + Uint64 *tmp = (Uint64*)p; + return EV_GET_UINT64(ev, tmp, pp, qp); +} #if 0 @@ -746,6 +772,7 @@ file_init(void) return 0; } + /********************************************************************* * Driver entry point -> start */ @@ -762,7 +789,7 @@ file_start(ErlDrvPort port, char* command) } desc->fd = FILE_FD_INVALID; desc->port = port; - desc->key = (unsigned int) (UWord) port; + desc->key = driver_async_port_key(port); desc->flags = 0; desc->invoke = NULL; desc->d = NULL; @@ -1112,7 +1139,7 @@ static void invoke_read(void *data) read_size = erts_gzread((gzFile)d->fd, d->c.read.binp->orig_bytes + d->c.read.bin_offset, size); - status = (read_size != -1); + status = (read_size != (size_t) -1); if (!status) { d->errInfo.posix_errno = EIO; } @@ -1186,7 +1213,7 @@ static void invoke_read_line(void *data) d->c.read_line.binp->orig_bytes + d->c.read_line.read_offset + d->c.read_line.read_size, size); - status = (read_size != -1); + status = (read_size != (size_t) -1); if (!status) { d->errInfo.posix_errno = EIO; } @@ -1346,7 +1373,7 @@ static void invoke_preadv(void *data) = efile_pread(&d->errInfo, (int) d->fd, c->offsets[c->cnt] + c->size, - ev->iov[1 + c->cnt].iov_base + c->size, + ((char *)ev->iov[1 + c->cnt].iov_base) + c->size, read_size, &bytes_read))) { bytes_read_so_far += bytes_read; @@ -1532,26 +1559,24 @@ static void invoke_writev(void *data) { } EF_FREE(iov); - d->c.writev.free_size = size; - d->c.writev.size -= size; if (! d->result_ok) { d->again = 0; + MUTEX_LOCK(d->c.writev.q_mtx); + driver_deq(d->c.writev.port, d->c.writev.size); + MUTEX_UNLOCK(d->c.writev.q_mtx); } else { if (! segment) { d->again = 0; } + d->c.writev.size -= size; TRACE_F(("w%lu", (unsigned long)size)); - + MUTEX_LOCK(d->c.writev.q_mtx); + driver_deq(d->c.writev.port, size); + MUTEX_UNLOCK(d->c.writev.q_mtx); } - DTRACE_INVOKE_RETURN(FILE_WRITE); -} -static void free_writev(void *data) { - struct t_data *d = data; - MUTEX_LOCK(d->c.writev.q_mtx); - driver_deq(d->c.writev.port, d->c.writev.size + d->c.writev.free_size); - MUTEX_UNLOCK(d->c.writev.q_mtx); - EF_FREE(d); + + DTRACE_INVOKE_RETURN(FILE_WRITE); } static void invoke_pwd(void *data) @@ -1602,7 +1627,7 @@ static void invoke_pwritev(void *data) { struct t_pwritev *c = &d->c.pwritev; size_t p; int segment; - size_t size, write_size; + size_t size, write_size, written; DTRACE_INVOKE_SETUP(FILE_PWRITEV); segment = d->again && c->size >= 2*FILE_SEGMENT_WRITE; @@ -1622,39 +1647,35 @@ static void invoke_pwritev(void *data) { if (iovlen < 0) goto error; /* Port terminated */ - for (iovcnt = 0, c->free_size = 0; - c->cnt < c->n && iovcnt < iovlen && c->free_size < size; + for (iovcnt = 0, written = 0; + c->cnt < c->n && iovcnt < iovlen && written < size; c->cnt++) { int chop; write_size = c->specs[c->cnt].size; if (iov[iovcnt].iov_len - p < write_size) { - /* Mismatch between pos/size spec and what is queued */ - d->errInfo.posix_errno = EINVAL; - d->result_ok = 0; - d->again = 0; - goto done; + goto error; } - chop = segment && c->free_size + write_size >= 2*FILE_SEGMENT_WRITE; + chop = segment && written + write_size >= 2*FILE_SEGMENT_WRITE; if (chop) { - ASSERT(c->free_size < FILE_SEGMENT_WRITE); + ASSERT(written < FILE_SEGMENT_WRITE); write_size = FILE_SEGMENT_WRITE + FILE_SEGMENT_WRITE/2 - - c->free_size; + - written; } d->result_ok = efile_pwrite(&d->errInfo, (int) d->fd, - iov[iovcnt].iov_base + p, + (char *)(iov[iovcnt].iov_base) + p, write_size, c->specs[c->cnt].offset); if (! d->result_ok) { d->again = 0; - goto done; + goto deq_error; } - c->free_size += write_size; + written += write_size; c->size -= write_size; if (chop) { c->specs[c->cnt].offset += write_size; c->specs[c->cnt].size -= write_size; /* Schedule out (d->again != 0) */ - goto done; + break; } /* Move forward in buffer */ p += write_size; @@ -1676,25 +1697,29 @@ static void invoke_pwritev(void *data) { d->errInfo.posix_errno = EINVAL; d->result_ok = 0; d->again = 0; + deq_error: + MUTEX_LOCK(d->c.writev.q_mtx); + driver_deq(d->c.pwritev.port, c->size); + MUTEX_UNLOCK(d->c.writev.q_mtx); + + goto done; } else { - ASSERT(c->free_size == size); + ASSERT(written == size); d->again = 0; } + } else { + ASSERT(written >= FILE_SEGMENT_WRITE); } + + MUTEX_LOCK(d->c.writev.q_mtx); + driver_deq(d->c.pwritev.port, written); + MUTEX_UNLOCK(d->c.writev.q_mtx); done: EF_FREE(iov); /* Free our copy of the vector, nothing to restore */ + DTRACE_INVOKE_RETURN(FILE_PWRITEV); } -static void free_pwritev(void *data) { - struct t_data *d = data; - - MUTEX_LOCK(d->c.writev.q_mtx); - driver_deq(d->c.pwritev.port, d->c.pwritev.free_size + d->c.pwritev.size); - MUTEX_UNLOCK(d->c.writev.q_mtx); - EF_FREE(d); -} - static void invoke_flstat(void *data) { struct t_data *d = (struct t_data *) data; @@ -2008,21 +2033,8 @@ static void try_free_read_bin(file_descriptor *desc) { static int try_again(file_descriptor *desc, struct t_data *d) { - if (! d->again) { + if (! d->again) return 0; - } - switch (d->command) { - case FILE_WRITE: - MUTEX_LOCK(d->c.writev.q_mtx); - driver_deq(d->c.writev.port, d->c.writev.free_size); - MUTEX_UNLOCK(d->c.writev.q_mtx); - break; - case FILE_PWRITEV: - MUTEX_LOCK(d->c.writev.q_mtx); - driver_deq(d->c.pwritev.port, d->c.pwritev.free_size); - MUTEX_UNLOCK(d->c.writev.q_mtx); - break; - } if (desc->timer_state != timer_idle) { driver_cancel_timer(desc->port); } @@ -2078,10 +2090,9 @@ static struct t_data *async_write(file_descriptor *desc, int *errp, } #endif d->reply = reply; - d->c.writev.free_size = 0; d->c.writev.reply_size = reply_size; d->invoke = invoke_writev; - d->free = free_writev; + d->free = free_data; d->level = 1; cq_enq(desc, d); desc->write_buffered = 0; @@ -2394,7 +2405,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) desc->write_errInfo = d->errInfo; } } - free_writev(data); + free_data(data); break; case FILE_LSEEK: if (d->reply) { @@ -2524,7 +2535,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) } else { reply_Uint(desc, d->c.pwritev.n); } - free_pwritev(data); + free_data(data); break; case FILE_PREADV: if (!d->result_ok) { @@ -2559,7 +2570,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) reply_Sint64(desc, d->c.sendfile.written); desc->sendfile_state = not_sending; free_sendfile(data); - } else if (d->result_ok == 1) { // If we are using select to send the rest of the data + } else if (d->result_ok == 1) { /* If we are using select to send the rest of the data */ desc->sendfile_state = sending; desc->d = d; driver_select(desc->port, (ErlDrvEvent)(long)d->c.sendfile.out_fd, @@ -2581,7 +2592,7 @@ file_async_ready(ErlDrvData e, ErlDrvThreadData data) } DTRACE6(efile_drv_return, sched_i1, sched_i2, sched_utag, command, result_ok, posix_errno); - if (desc->write_buffered != 0 && desc->timer_state == timer_idle) { + if (desc->write_buffered != 0 && desc->timer_state == timer_idle ) { desc->timer_state = timer_write; driver_set_timer(desc->port, desc->write_delay); } @@ -3124,25 +3135,25 @@ file_flush(ErlDrvData e) { /********************************************************************* * Driver entry point -> control + * Only debug functionality... */ static ErlDrvSSizeT file_control(ErlDrvData e, unsigned int command, char* buf, ErlDrvSizeT len, char **rbuf, ErlDrvSizeT rlen) { - /* - * warning: variable ‘desc’ set but not used - * [-Wunused-but-set-variable] - * ... no kidding ... - * - * file_descriptor *desc = (file_descriptor *)e; switch (command) { + case 'K' : + if (rlen < 4) { + *rbuf = EF_ALLOC(4); + } + (*rbuf)[0] = ((desc->key) >> 24) & 0xFF; + (*rbuf)[1] = ((desc->key) >> 16) & 0xFF; + (*rbuf)[2] = ((desc->key) >> 8) & 0xFF; + (*rbuf)[3] = (desc->key) & 0xFF; + return 4; default: return 0; - } - ASSERT(0); - desc = NULL; - */ - return 0; + } } /********************************************************************* @@ -3195,7 +3206,7 @@ static void file_outputv(ErlDrvData e, ErlIOVec *ev) { file_descriptor* desc = (file_descriptor*)e; char command; - int p, q; + size_t p, q; int err; struct t_data *d = NULL; #ifdef USE_VM_PROBES @@ -3623,7 +3634,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { for(i = 0; i < n; i++) { Uint32 sizeH, sizeL; size_t size; - if ( !EV_GET_UINT64(ev, &d->c.pwritev.specs[i].offset, &p, &q) + if ( !EV_GET_SINT64(ev, &d->c.pwritev.specs[i].offset, &p, &q) || !EV_GET_UINT32(ev, &sizeH, &p, &q) || !EV_GET_UINT32(ev, &sizeL, &p, &q)) { /* Misalignment in buffer */ @@ -3651,7 +3662,6 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { #ifdef USE_VM_PROBES dt_i3 = d->c.pwritev.size; #endif - d->c.pwritev.free_size = 0; if (j == 0) { /* Trivial case - nothing to write */ EF_FREE(d); @@ -3675,7 +3685,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { MUTEX_UNLOCK(desc->q_mtx); /* Execute the command */ d->invoke = invoke_pwritev; - d->free = free_pwritev; + d->free = free_data; d->level = 1; cq_enq(desc, d); } @@ -3766,7 +3776,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { for (i = 1; i < 1+n; i++) { Uint32 sizeH, sizeL; size_t size; - if ( !EV_GET_UINT64(ev, &d->c.preadv.offsets[i-1], &p, &q) + if ( !EV_GET_SINT64(ev, &d->c.preadv.offsets[i-1], &p, &q) || !EV_GET_UINT32(ev, &sizeH, &p, &q) || !EV_GET_UINT32(ev, &sizeL, &p, &q)) { reply_posix_error(desc, EINVAL); @@ -3813,7 +3823,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { res_ev->iov[0].iov_base = res_ev->binv[0]->orig_bytes; /* Fill in the number of buffers in the header */ put_int32(0, res_ev->iov[0].iov_base); - put_int32(n, res_ev->iov[0].iov_base+4); + put_int32(n, (char *)(res_ev->iov[0].iov_base) + 4); /**/ res_ev->size = res_ev->iov[0].iov_len; if (n == 0) { @@ -3834,7 +3844,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { Uint32 origin; /* Origin of seek. */ if (ev->size < 1+8+4 - || !EV_GET_UINT64(ev, &offset, &p, &q) + || !EV_GET_SINT64(ev, &offset, &p, &q) || !EV_GET_UINT32(ev, &origin, &p, &q)) { /* Wrong length of buffer to contain offset and origin */ reply_posix_error(desc, EINVAL); @@ -3947,7 +3957,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { goto done; } if (ev->size < 1+1+8+4 - || !EV_GET_UINT64(ev, &hdr_offset, &p, &q) + || !EV_GET_SINT64(ev, &hdr_offset, &p, &q) || !EV_GET_UINT32(ev, &max_size, &p, &q)) { /* Buffer too short to contain * the header offset and max size spec */ @@ -4114,7 +4124,7 @@ file_outputv(ErlDrvData e, ErlIOVec *ev) { } if (hd_len != 0 || tl_len != 0 || flags != 0) { - // We do not allow header, trailers and/or flags right now + /* We do not allow header, trailers and/or flags right now */ reply_posix_error(desc, EINVAL); goto done; } diff --git a/erts/emulator/drivers/common/erl_efile.h b/erts/emulator/drivers/common/erl_efile.h index b29b4f971c..5387f75efc 100644 --- a/erts/emulator/drivers/common/erl_efile.h +++ b/erts/emulator/drivers/common/erl_efile.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2011. All Rights Reserved. + * Copyright Ericsson AB 1997-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -67,7 +67,7 @@ #define FILENAMES_16BIT 1 #endif -// We use sendfilev if it exist on solaris +/* We use sendfilev if it exist on solaris */ #if !defined(HAVE_SENDFILE) && defined(HAVE_SENDFILEV) #define HAVE_SENDFILE #endif @@ -88,20 +88,6 @@ typedef struct _Efile_error { } Efile_error; /* - * This structure contains date and time. - */ - -//typedef struct _Efile_time { -// unsigned year; /* (4 digits). */ -// unsigned month; /* (1..12). */ -// unsigned day; /* (1..31). */ -// unsigned hour; /* (0..23). */ -// unsigned minute; /* (0..59). */ -// unsigned second; /* (0..59). */ -//} Efile_time; - - -/* * Describes what is returned by file:file_info/1. */ diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index fb128fff7d..60db50e80a 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -282,7 +282,7 @@ static BOOL (WINAPI *fpSetHandleInformation)(HANDLE,DWORD,DWORD); static unsigned long zero_value = 0; static unsigned long one_value = 1; -#else +#else /* #ifdef __WIN32__ */ #include <sys/time.h> #ifdef NETDB_H_NEEDS_IN_H @@ -315,9 +315,17 @@ static unsigned long one_value = 1; #include <net/if.h> +#ifdef HAVE_SCHED_H +#include <sched.h> +#endif + +#ifdef HAVE_SETNS_H +#include <setns.h> +#endif + /* SCTP support -- currently for UNIX platforms only: */ #undef HAVE_SCTP -#if (!defined(__WIN32__) && defined(HAVE_SCTP_H)) +#if defined(HAVE_SCTP_H) #include <netinet/sctp.h> @@ -418,7 +426,7 @@ static int (*p_sctp_bindx)(int sd, struct sockaddr *addrs, static int (*p_sctp_peeloff)(int sd, sctp_assoc_t assoc_id) = NULL; #endif -#endif /* SCTP supported */ +#endif /* #if defined(HAVE_SCTP_H) */ #ifndef WANT_NONBLOCKING #define WANT_NONBLOCKING @@ -512,7 +520,7 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) } while(0) -#endif /* __WIN32__ */ +#endif /* #ifdef __WIN32__ #else */ #ifdef HAVE_SOCKLEN_T # define SOCKLEN_T socklen_t @@ -678,8 +686,9 @@ static int my_strncasecmp(const char *s1, const char *s2, size_t n) #define INET_LOPT_UDP_READ_PACKETS 33 /* Number of packets to read */ #define INET_OPT_RAW 34 /* Raw socket options */ #define INET_LOPT_TCP_SEND_TIMEOUT_CLOSE 35 /* auto-close on send timeout or not */ -#define INET_LOPT_TCP_MSGQ_HIWTRMRK 36 /* set local high watermark */ -#define INET_LOPT_TCP_MSGQ_LOWTRMRK 37 /* set local low watermark */ +#define INET_LOPT_MSGQ_HIWTRMRK 36 /* set local msgq high watermark */ +#define INET_LOPT_MSGQ_LOWTRMRK 37 /* set local msgq low watermark */ +#define INET_LOPT_NETNS 38 /* Network namespace pathname */ /* SCTP options: a separate range, from 100: */ #define SCTP_OPT_RTOINFO 100 #define SCTP_OPT_ASSOCINFO 101 @@ -955,6 +964,10 @@ typedef struct { int is_ignored; /* if a fd is ignored by the inet_drv. This flag should be set to true when the fd is used outside of inet_drv. */ +#ifdef HAVE_SETNS + char *netns; /* Socket network namespace name + as full file path */ +#endif } inet_descriptor; @@ -1181,6 +1194,7 @@ static ErlDrvTermData am_dontroute; static ErlDrvTermData am_priority; static ErlDrvTermData am_tos; static ErlDrvTermData am_ipv6_v6only; +static ErlDrvTermData am_netns; #endif /* speical errors for bad ports and sequences */ @@ -3498,6 +3512,7 @@ static void inet_init_sctp(void) { INIT_ATOM(priority); INIT_ATOM(tos); INIT_ATOM(ipv6_v6only); + INIT_ATOM(netns); /* Option names */ INIT_ATOM(sctp_rtoinfo); @@ -3854,8 +3869,10 @@ static void desc_close(inet_descriptor* desc) desc->forced_events = 0; desc->send_would_block = 0; #endif - // We should close the fd here, but the other driver might still - // be selecting on it. + /* + * We should close the fd here, but the other driver might still + * be selecting on it. + */ if (!desc->is_ignored) driver_select(desc->port,(ErlDrvEvent)(long)desc->event, ERL_DRV_USE, 0); @@ -3906,12 +3923,81 @@ static int erl_inet_close(inet_descriptor* desc) static ErlDrvSSizeT inet_ctl_open(inet_descriptor* desc, int domain, int type, char** rbuf, ErlDrvSizeT rsize) { + int save_errno; +#ifdef HAVE_SETNS + int current_ns, new_ns; + current_ns = new_ns = 0; +#endif + save_errno = 0; + if (desc->state != INET_STATE_CLOSED) return ctl_xerror(EXBADSEQ, rbuf, rsize); + +#ifdef HAVE_SETNS + if (desc->netns != NULL) { + /* Temporarily change network namespace for this thread + * while creating the socket + */ + current_ns = open("/proc/self/ns/net", O_RDONLY); + if (current_ns == INVALID_SOCKET) + return ctl_error(sock_errno(), rbuf, rsize); + new_ns = open(desc->netns, O_RDONLY); + if (new_ns == INVALID_SOCKET) { + save_errno = sock_errno(); + while (close(current_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + return ctl_error(save_errno, rbuf, rsize); + } + if (setns(new_ns, CLONE_NEWNET) != 0) { + save_errno = sock_errno(); + while (close(new_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + while (close(current_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + return ctl_error(save_errno, rbuf, rsize); + } + else { + while (close(new_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + } + } +#endif if ((desc->s = sock_open(domain, type, desc->sprotocol)) == INVALID_SOCKET) - return ctl_error(sock_errno(), rbuf, rsize); - if ((desc->event = sock_create_event(desc)) == INVALID_EVENT) - return ctl_error(sock_errno(), rbuf, rsize); + save_errno = sock_errno(); +#ifdef HAVE_SETNS + if (desc->netns != NULL) { + /* Restore network namespace */ + if (setns(current_ns, CLONE_NEWNET) != 0) { + /* XXX Failed to restore network namespace. + * What to do? Tidy up and return an error... + * Note that the thread now might still be in the namespace. + * Can this even happen? Should the emulator be aborted? + */ + if (desc->s != INVALID_SOCKET) + save_errno = sock_errno(); + while (close(desc->s) == INVALID_SOCKET && + sock_errno() == EINTR); + desc->s = INVALID_SOCKET; + while (close(current_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + return ctl_error(save_errno, rbuf, rsize); + } + else { + while (close(current_ns) == INVALID_SOCKET && + sock_errno() == EINTR); + } + } +#endif + if (desc->s == INVALID_SOCKET) + return ctl_error(save_errno, rbuf, rsize); + + if ((desc->event = sock_create_event(desc)) == INVALID_EVENT) { + save_errno = sock_errno(); + while (close(desc->s) == INVALID_SOCKET && + sock_errno() == EINTR); + desc->s = INVALID_SOCKET; + return ctl_error(save_errno, rbuf, rsize); + } SET_NONBLOCKING(desc->s); #ifdef __WIN32__ driver_select(desc->port, desc->event, ERL_DRV_READ, 1); @@ -5474,27 +5560,25 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) } continue; - case INET_LOPT_TCP_MSGQ_HIWTRMRK: - if (desc->stype == SOCK_STREAM) { - ErlDrvSizeT high; - if (ival < ERL_DRV_BUSY_MSGQ_LIM_MIN - || ERL_DRV_BUSY_MSGQ_LIM_MAX < ival) - return -1; - high = (ErlDrvSizeT) ival; - erl_drv_busy_msgq_limits(desc->port, NULL, &high); - } + case INET_LOPT_MSGQ_HIWTRMRK: { + ErlDrvSizeT high; + if (ival < ERL_DRV_BUSY_MSGQ_LIM_MIN + || ERL_DRV_BUSY_MSGQ_LIM_MAX < ival) + return -1; + high = (ErlDrvSizeT) ival; + erl_drv_busy_msgq_limits(desc->port, NULL, &high); continue; + } - case INET_LOPT_TCP_MSGQ_LOWTRMRK: - if (desc->stype == SOCK_STREAM) { - ErlDrvSizeT low; - if (ival < ERL_DRV_BUSY_MSGQ_LIM_MIN - || ERL_DRV_BUSY_MSGQ_LIM_MAX < ival) - return -1; - low = (ErlDrvSizeT) ival; - erl_drv_busy_msgq_limits(desc->port, &low, NULL); - } + case INET_LOPT_MSGQ_LOWTRMRK: { + ErlDrvSizeT low; + if (ival < ERL_DRV_BUSY_MSGQ_LIM_MIN + || ERL_DRV_BUSY_MSGQ_LIM_MAX < ival) + return -1; + low = (ErlDrvSizeT) ival; + erl_drv_busy_msgq_limits(desc->port, &low, NULL); continue; + } case INET_LOPT_TCP_SEND_TIMEOUT: if (desc->stype == SOCK_STREAM) { @@ -5529,6 +5613,20 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) } continue; +#ifdef HAVE_SETNS + case INET_LOPT_NETNS: + /* It is annoying that ival and len are both (signed) int */ + if (ival < 0) return -1; + if (len < ival) return -1; + if (desc->netns != NULL) FREE(desc->netns); + desc->netns = ALLOC(((unsigned int) ival) + 1); + memcpy(desc->netns, ptr, ival); + desc->netns[ival] = '\0'; + ptr += ival; + len -= ival; + continue; +#endif + case INET_OPT_REUSEADDR: #ifdef __WIN32__ continue; /* Bjorn says */ @@ -5858,6 +5956,21 @@ static int sctp_set_opts(inet_descriptor* desc, char* ptr, int len) res = 0; continue; +#ifdef HAVE_SETNS + case INET_LOPT_NETNS: + { + size_t ns_len; + ns_len = get_int32(curr); curr += 4; + CHKLEN(curr, ns_len); + if (desc->netns != NULL) FREE(desc->netns); + desc->netns = ALLOC(ns_len + 1); + memcpy(desc->netns, curr, ns_len); + desc->netns[ns_len] = '\0'; + curr += ns_len; + } + continue; +#endif + /* SCTP options and applicable generic INET options: */ case SCTP_OPT_RTOINFO: @@ -6396,31 +6509,23 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, } continue; - case INET_LOPT_TCP_MSGQ_HIWTRMRK: - if (desc->stype == SOCK_STREAM) { - ErlDrvSizeT high = ERL_DRV_BUSY_MSGQ_READ_ONLY; - *ptr++ = opt; - erl_drv_busy_msgq_limits(desc->port, NULL, &high); - ival = high > INT_MAX ? INT_MAX : (int) high; - put_int32(ival, ptr); - } - else { - TRUNCATE_TO(0,ptr); - } + case INET_LOPT_MSGQ_HIWTRMRK: { + ErlDrvSizeT high = ERL_DRV_BUSY_MSGQ_READ_ONLY; + *ptr++ = opt; + erl_drv_busy_msgq_limits(desc->port, NULL, &high); + ival = high > INT_MAX ? INT_MAX : (int) high; + put_int32(ival, ptr); continue; + } - case INET_LOPT_TCP_MSGQ_LOWTRMRK: - if (desc->stype == SOCK_STREAM) { - ErlDrvSizeT low = ERL_DRV_BUSY_MSGQ_READ_ONLY; - *ptr++ = opt; - erl_drv_busy_msgq_limits(desc->port, &low, NULL); - ival = low > INT_MAX ? INT_MAX : (int) low; - put_int32(ival, ptr); - } - else { - TRUNCATE_TO(0,ptr); - } + case INET_LOPT_MSGQ_LOWTRMRK: { + ErlDrvSizeT low = ERL_DRV_BUSY_MSGQ_READ_ONLY; + *ptr++ = opt; + erl_drv_busy_msgq_limits(desc->port, &low, NULL); + ival = low > INT_MAX ? INT_MAX : (int) low; + put_int32(ival, ptr); continue; + } case INET_LOPT_TCP_SEND_TIMEOUT: if (desc->stype == SOCK_STREAM) { @@ -6462,6 +6567,22 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, } continue; +#ifdef HAVE_SETNS + case INET_LOPT_NETNS: + if (desc->netns != NULL) { + size_t netns_len; + netns_len = strlen(desc->netns); + *ptr++ = opt; + put_int32(netns_len, ptr); + PLACE_FOR(netns_len, ptr); + memcpy(ptr, desc->netns, netns_len); + ptr += netns_len; + } else { + TRUNCATE_TO(0,ptr); + } + continue; +#endif + case INET_OPT_PRIORITY: #ifdef SO_PRIORITY type = SO_PRIORITY; @@ -6745,6 +6866,22 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, break; } +#ifdef HAVE_SETNS + case INET_LOPT_NETNS: + if (desc->netns != NULL) { + PLACE_FOR + (spec, i, + LOAD_ATOM_CNT + LOAD_BUF2BINARY_CNT + LOAD_TUPLE_CNT); + i = LOAD_ATOM (spec, i, am_netns); + i = LOAD_BUF2BINARY + (spec, i, desc->netns, strlen(desc->netns)); + i = LOAD_TUPLE (spec, i, 2); + break; + } + else + continue; /* Ignore */ +#endif + /* SCTP and generic INET options: */ case SCTP_OPT_RTOINFO: @@ -7466,9 +7603,27 @@ static ErlDrvSSizeT inet_subscribe(inet_descriptor* desc, static void inet_stop(inet_descriptor* desc) { erl_inet_close(desc); +#ifdef HAVE_SETNS + if (desc->netns != NULL) + FREE(desc->netns); +#endif FREE(desc); } +static void set_default_msgq_limits(ErlDrvPort port) +{ + ErlDrvSizeT q_high = INET_HIGH_MSGQ_WATERMARK; + ErlDrvSizeT q_low = INET_LOW_MSGQ_WATERMARK; + if (q_low < ERL_DRV_BUSY_MSGQ_LIM_MIN) + q_low = ERL_DRV_BUSY_MSGQ_LIM_MIN; + else if (q_low > ERL_DRV_BUSY_MSGQ_LIM_MAX) + q_low = ERL_DRV_BUSY_MSGQ_LIM_MAX; + if (q_high < ERL_DRV_BUSY_MSGQ_LIM_MIN) + q_high = ERL_DRV_BUSY_MSGQ_LIM_MIN; + else if (q_high > ERL_DRV_BUSY_MSGQ_LIM_MAX) + q_high = ERL_DRV_BUSY_MSGQ_LIM_MAX; + erl_drv_busy_msgq_limits(port, &q_low, &q_high); +} /* Allocate descriptor */ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol) @@ -7531,6 +7686,10 @@ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol) desc->is_ignored = 0; +#ifdef HAVE_SETNS + desc->netns = NULL; +#endif + return (ErlDrvData)desc; } @@ -8089,9 +8248,8 @@ static int tcp_inet_init(void) /* initialize the TCP descriptor */ -static ErlDrvData tcp_inet_start(ErlDrvPort port, char* args) +static ErlDrvData prep_tcp_inet_start(ErlDrvPort port, char* args) { - ErlDrvSizeT q_low, q_high; tcp_descriptor* desc; DEBUGF(("tcp_inet_start(%ld) {\r\n", (long)port)); @@ -8101,17 +8259,6 @@ static ErlDrvData tcp_inet_start(ErlDrvPort port, char* args) return ERL_DRV_ERROR_ERRNO; desc->high = INET_HIGH_WATERMARK; desc->low = INET_LOW_WATERMARK; - q_high = INET_HIGH_MSGQ_WATERMARK; - q_low = INET_LOW_MSGQ_WATERMARK; - if (q_low < ERL_DRV_BUSY_MSGQ_LIM_MIN) - q_low = ERL_DRV_BUSY_MSGQ_LIM_MIN; - else if (q_low > ERL_DRV_BUSY_MSGQ_LIM_MAX) - q_low = ERL_DRV_BUSY_MSGQ_LIM_MAX; - if (q_high < ERL_DRV_BUSY_MSGQ_LIM_MIN) - q_high = ERL_DRV_BUSY_MSGQ_LIM_MIN; - else if (q_high > ERL_DRV_BUSY_MSGQ_LIM_MAX) - q_high = ERL_DRV_BUSY_MSGQ_LIM_MAX; - erl_drv_busy_msgq_limits(port, &q_low, &q_high); desc->send_timeout = INET_INFINITY; desc->send_timeout_close = 0; desc->busy_on_send = 0; @@ -8128,6 +8275,12 @@ static ErlDrvData tcp_inet_start(ErlDrvPort port, char* args) return (ErlDrvData) desc; } +static ErlDrvData tcp_inet_start(ErlDrvPort port, char* args) +{ + ErlDrvData data = prep_tcp_inet_start(port, args); + set_default_msgq_limits(port); + return data; +} /* Copy a descriptor, by creating a new port with same settings * as the descriptor desc. * return NULL on error (SYSTEM_LIMIT no ports avail) @@ -8139,7 +8292,7 @@ static tcp_descriptor* tcp_inet_copy(tcp_descriptor* desc,SOCKET s, ErlDrvPort port = desc->inet.port; tcp_descriptor* copy_desc; - copy_desc = (tcp_descriptor*) tcp_inet_start(port, NULL); + copy_desc = (tcp_descriptor*) prep_tcp_inet_start(port, NULL); /* Setup event if needed */ if ((copy_desc->inet.s = s) != INVALID_SOCKET) { @@ -9862,12 +10015,15 @@ static int should_use_so_bsdcompat(void) * as the descriptor desc. * return NULL on error (ENFILE no ports avail) */ +static ErlDrvData packet_inet_start(ErlDrvPort port, char* args, int protocol); + static udp_descriptor* sctp_inet_copy(udp_descriptor* desc, SOCKET s, int* err) { + ErlDrvSizeT q_low, q_high; ErlDrvPort port = desc->inet.port; udp_descriptor* copy_desc; - copy_desc = (udp_descriptor*) sctp_inet_start(port, NULL); + copy_desc = (udp_descriptor*) packet_inet_start(port, NULL, IPPROTO_SCTP); /* Setup event if needed */ if ((copy_desc->inet.s = s) != INVALID_SOCKET) { @@ -9898,9 +10054,17 @@ static udp_descriptor* sctp_inet_copy(udp_descriptor* desc, SOCKET s, int* err) FREE(copy_desc); return NULL; } + + /* Read busy msgq limits of parent */ + q_low = q_high = ERL_DRV_BUSY_MSGQ_READ_ONLY; + erl_drv_busy_msgq_limits(desc->inet.port, &q_low, &q_high); + /* Write same busy msgq limits to child */ + erl_drv_busy_msgq_limits(port, &q_low, &q_high); + copy_desc->inet.port = port; copy_desc->inet.dport = driver_mk_port(port); *err = 0; + return copy_desc; } #endif @@ -9933,13 +10097,17 @@ static ErlDrvData packet_inet_start(ErlDrvPort port, char* args, int protocol) static ErlDrvData udp_inet_start(ErlDrvPort port, char *args) { - return packet_inet_start(port, args, IPPROTO_UDP); + ErlDrvData data = packet_inet_start(port, args, IPPROTO_UDP); + set_default_msgq_limits(port); + return data; } #ifdef HAVE_SCTP static ErlDrvData sctp_inet_start(ErlDrvPort port, char *args) { - return packet_inet_start(port, args, IPPROTO_SCTP); + ErlDrvData data = packet_inet_start(port, args, IPPROTO_SCTP); + set_default_msgq_limits(port); + return data; } #endif diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c index 60394b610b..3fe5d282dc 100644 --- a/erts/emulator/drivers/common/zlib_drv.c +++ b/erts/emulator/drivers/common/zlib_drv.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2003-2012. All Rights Reserved. + * Copyright Ericsson AB 2003-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -119,7 +119,7 @@ static int zlib_deflate(ZLibData* d, int flush); #if defined(__WIN32__) static int i32(char* buf) #else -static inline int i32(char* buf) +static __inline__ int i32(char* buf) #endif { return (int) ( diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c index 8912d148a5..1e436830e7 100644 --- a/erts/emulator/drivers/unix/ttsl_drv.c +++ b/erts/emulator/drivers/unix/ttsl_drv.c @@ -42,6 +42,9 @@ static ErlDrvData ttysl_start(ErlDrvPort, char*); #include <locale.h> #include <unistd.h> #include <termios.h> +#ifdef HAVE_WCWIDTH +#include <wchar.h> +#endif #ifdef HAVE_FCNTL_H #include <fcntl.h> #endif @@ -95,6 +98,9 @@ static int lpos; /* The current "cursor position" in the line buf */ #define CONTROL_TAG 0x10000000U /* Control character, value in first position */ #define ESCAPED_TAG 0x01000000U /* Escaped character, value in first position */ +#ifdef HAVE_WCWIDTH +#define WIDE_TAG 0x02000000U /* Wide character, value in first position */ +#endif #define TAG_MASK 0xFF000000U #define MAXSIZE (1 << 16) @@ -597,12 +603,20 @@ static int check_buf_size(byte *s, int n) } if (utf8_mode) { /* That is, terminal is UTF8 compliant */ if (ch >= 128 || isprint(ch)) { - DEBUGLOG(("Printable(UTF-8:%d):%d",(pos - opos),ch)); - size++; /* Buffer contains wide characters... */ +#ifdef HAVE_WCWIDTH + int width; +#endif + DEBUGLOG(("Printable(UTF-8:%d):%d",pos,ch)); + size++; +#ifdef HAVE_WCWIDTH + if ((width = wcwidth(ch)) > 1) { + size += width - 1; + } +#endif } else if (ch == '\t') { size += 8; } else { - DEBUGLOG(("Magic(UTF-8:%d):%d",(pos - opos),ch)); + DEBUGLOG(("Magic(UTF-8:%d):%d",pos,ch)); size += 2; } } else { @@ -823,7 +837,7 @@ static int del_chars(int n) r = llen - lpos - l; /* Characters after deleted */ /* Fix up buffer and buffer pointers. */ if (r > 0) - memcpy(lbuf + lpos, lbuf + pos, r * sizeof(Uint32)); + memmove(lbuf + lpos, lbuf + pos, r * sizeof(Uint32)); llen -= l; /* Write out characters after, blank the tail and jump back to lpos. */ write_buf(lbuf + lpos, r); @@ -842,7 +856,7 @@ static int del_chars(int n) move_cursor(lpos, lpos-l); /* Move back */ /* Fix up buffer and buffer pointers. */ if (r > 0) - memcpy(lbuf + pos, lbuf + lpos, r * sizeof(Uint32)); + memmove(lbuf + pos, lbuf + lpos, r * sizeof(Uint32)); lpos -= l; llen -= l; /* Write out characters after, blank the tail and jump back to lpos. */ @@ -868,12 +882,22 @@ static int step_over_chars(int n) end = lbuf + llen; c = lbuf + lpos; for ( ; n > 0 && c < end; --n) { +#ifdef HAVE_WCWIDTH + while (*c & WIDE_TAG) { + c++; + } +#endif c++; while (c < end && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0)) c++; } for ( ; n < 0 && c > beg; n++) { --c; +#ifdef HAVE_WCWIDTH + while (c > beg + 1 && (c[-1] & WIDE_TAG)) { + --c; + } +#endif while (c > beg && (*c & TAG_MASK) && ((*c & ~TAG_MASK) == 0)) --c; } @@ -899,6 +923,15 @@ static int insert_buf(byte *s, int n) ++pos; } if ((utf8_mode && (ch >= 128 || isprint(ch))) || (ch <= 255 && isprint(ch))) { +#ifdef HAVE_WCWIDTH + int width; + if ((width = wcwidth(ch)) > 1) { + while (--width) { + DEBUGLOG(("insert_buf: Wide(UTF-8):%d,%d",width,ch)); + lbuf[lpos++] = (WIDE_TAG | ((Uint32) ch)); + } + } +#endif DEBUGLOG(("insert_buf: Printable(UTF-8):%d",ch)); lbuf[lpos++] = (Uint32) ch; } else if (ch >= 128) { /* not utf8 mode */ @@ -1006,6 +1039,10 @@ static int write_buf(Uint32 *s, int n) if (octbuff != octtmp) { driver_free(octbuff); } +#ifdef HAVE_WCWIDTH + } else if (*s & WIDE_TAG) { + --n; s++; +#endif } else { DEBUGLOG(("Very unexpected character %d",(int) *s)); ++n; diff --git a/erts/emulator/drivers/unix/unix_efile.c b/erts/emulator/drivers/unix/unix_efile.c index 558651fff9..55539b44dd 100644 --- a/erts/emulator/drivers/unix/unix_efile.c +++ b/erts/emulator/drivers/unix/unix_efile.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2012. All Rights Reserved. + * Copyright Ericsson AB 1997-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -629,7 +629,7 @@ efile_writev(Efile_error* errInfo, /* Where to return error codes */ if (w < iov[cnt].iov_len) { /* Adjust the buffer for next write */ iov[cnt].iov_len -= w; - iov[cnt].iov_base += w; + iov[cnt].iov_base = ((char *)iov[cnt].iov_base) + w; w = 0; break; } else { diff --git a/erts/emulator/drivers/win32/ttsl_drv.c b/erts/emulator/drivers/win32/ttsl_drv.c index 1a74d21e99..502cb58dfa 100644 --- a/erts/emulator/drivers/win32/ttsl_drv.c +++ b/erts/emulator/drivers/win32/ttsl_drv.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -414,12 +414,12 @@ static int check_buf_size(byte *s, int n) } if (utf8_mode) { /* That is, terminal is UTF8 compliant */ if (ch >= 128 || isprint(ch)) { - DEBUGLOG(("Printable(UTF-8:%d):%d",(pos - opos),ch)); + DEBUGLOG(("Printable(UTF-8:%d):%d",pos,ch)); size++; /* Buffer contains wide characters... */ } else if (ch == '\t') { size += 8; } else { - DEBUGLOG(("Magic(UTF-8:%d):%d",(pos - opos),ch)); + DEBUGLOG(("Magic(UTF-8:%d):%d",pos,ch)); size += 2; } } else { diff --git a/erts/emulator/drivers/win32/win_efile.c b/erts/emulator/drivers/win32/win_efile.c index 1059fa5c3a..b36a103f8e 100644 --- a/erts/emulator/drivers/win32/win_efile.c +++ b/erts/emulator/drivers/win32/win_efile.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2012. All Rights Reserved. + * Copyright Ericsson AB 1997-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -772,6 +772,7 @@ efile_may_openfile(Efile_error* errInfo, char *name) { DWORD attr; if ((attr = GetFileAttributesW(wname)) == INVALID_FILE_ATTRIBUTES) { + errno = ENOENT; return check_error(-1, errInfo); } diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index ae2c650bd2..1f76268934 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -238,7 +238,7 @@ void hipe_handle_exception(Process *c_p) if (c_p->mbuf) { erts_printf("%s line %u: p==%p, p->mbuf==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf); - //erts_garbage_collect(c_p, 0, NULL, 0); + /* erts_garbage_collect(c_p, 0, NULL, 0); */ } /* @@ -270,7 +270,7 @@ void hipe_handle_exception(Process *c_p) c_p->def_arg_reg[0] = exception_tag[GET_EXC_CLASS(c_p->freason)]; if (c_p->mbuf) { - //erts_printf("%s line %u: p==%p, p->mbuf==%p, p->lastbif==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf, c_p->hipe.lastbif); + /* erts_printf("%s line %u: p==%p, p->mbuf==%p, p->lastbif==%p\n", __FUNCTION__, __LINE__, c_p, c_p->mbuf, c_p->hipe.lastbif); */ erts_garbage_collect(c_p, 0, NULL, 0); } diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index ccb0d2ffb8..ac6b4f70bb 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -25,7 +25,7 @@ #include "hipe_x86_asm.h" /* for NR_ARG_REGS */ /* uncomment to simulate & test what the initial PowerPC port will do */ -//#define SKIP_YOUNGEST_FRAME +/* #define SKIP_YOUNGEST_FRAME */ struct nstack_walk_state { #ifdef SKIP_YOUNGEST_FRAME diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c index 64c0e0da3e..8f997aafab 100644 --- a/erts/emulator/hipe/hipe_x86_signal.c +++ b/erts/emulator/hipe/hipe_x86_signal.c @@ -2,7 +2,7 @@ * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2011. All Rights Reserved. + * Copyright Ericsson AB 2001-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -183,6 +183,7 @@ static void do_init(void) #include <dlfcn.h> static int (*__next_sigaction)(int, const struct sigaction*, struct sigaction*); #define init_done() (__next_sigaction != 0) +extern int _sigaction(int, const struct sigaction*, struct sigaction*); #define __SIGACTION _sigaction static void do_init(void) { diff --git a/erts/emulator/internal_doc/dec.erl b/erts/emulator/internal_doc/dec.erl index 0315f2a52d..bb69e6e81b 100644 --- a/erts/emulator/internal_doc/dec.erl +++ b/erts/emulator/internal_doc/dec.erl @@ -1,7 +1,8 @@ +%% -*- coding: utf-8 -*- %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2010. All Rights Reserved. +%% Copyright Ericsson AB 2000-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -25,10 +26,10 @@ %% {RevList,Translation} %% Where 'RevList' is a reversed list of the denormalized repressentation of %% the character 'Translation'. An example would be the swedish character -%% '�', which would be represented in the file as: +%% 'ö', which would be represented in the file as: %% {[776,111],246}, as the denormalized representation of codepoint 246 %% is [111,776] (i.e an 'o' followed by the "double dot accent character 776), -%% while '�' instead is represented as {[776,97],228}, as the denormalized +%% while 'ä' instead is represented as {[776,97],228}, as the denormalized %% form would be [97,776] (same accent but an 'a' instead). %% The datafile is generated from the table on Apple's developer connection %% http://developer.apple.com/library/mac/#technotes/tn/tn1150table.html diff --git a/erts/emulator/pcre/pcre_exec.c b/erts/emulator/pcre/pcre_exec.c index 3fe13ca32e..26cb219ef1 100644 --- a/erts/emulator/pcre/pcre_exec.c +++ b/erts/emulator/pcre/pcre_exec.c @@ -36,7 +36,7 @@ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ----------------------------------------------------------------------------- */ -//#define ERLANG_DEBUG 1 +/* #define ERLANG_DEBUG 1 */ /* This module contains erts_pcre_exec(), the externally visible function that does pattern matching using an NFA algorithm, trying to mimic Perl as closely as diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 853dd90c34..7035dc77df 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -874,7 +874,7 @@ need2steal(ErtsDrvEventState *state, int mode) static void print_driver_name(erts_dsprintf_buf_t *dsbufp, Eterm id) { - ErtsPortNames *pnp = erts_get_port_names(id); + ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); if (!pnp->name && !pnp->driver_name) erts_dsprintf(dsbufp, "%s ", "<unknown>"); else { @@ -1357,8 +1357,8 @@ bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport, "Bad %s fd in erts_poll()! fd=%d, ", io_str, (int) state->fd); if (is_nil(port)) { - ErtsPortNames *ipnp = erts_get_port_names(inport); - ErtsPortNames *opnp = erts_get_port_names(outport); + ErtsPortNames *ipnp = erts_get_port_names(inport, ERTS_INVALID_ERL_DRV_PORT); + ErtsPortNames *opnp = erts_get_port_names(outport, ERTS_INVALID_ERL_DRV_PORT); erts_dsprintf(dsbufp, "ports=%T/%T, drivers=%s/%s, names=%s/%s\n", is_nil(inport) ? am_undefined : inport, is_nil(outport) ? am_undefined : outport, @@ -1370,7 +1370,7 @@ bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport, erts_free_port_names(opnp); } else { - ErtsPortNames *pnp = erts_get_port_names(port); + ErtsPortNames *pnp = erts_get_port_names(port, ERTS_INVALID_ERL_DRV_PORT); erts_dsprintf(dsbufp, "port=%T, driver=%s, name=%s\n", is_nil(port) ? am_undefined : port, pnp->driver_name ? pnp->driver_name : "<unknown>", @@ -1390,7 +1390,7 @@ bad_fd_in_pollset(ErtsDrvEventState *state, Eterm inport, static void stale_drv_select(Eterm id, ErtsDrvEventState *state, int mode) { - erts_stale_drv_select(id, (ErlDrvEvent) state->fd, mode, 0); + erts_stale_drv_select(id, ERTS_INVALID_ERL_DRV_PORT, (ErlDrvEvent) state->fd, mode, 0); deselect(state, mode); } @@ -1774,7 +1774,7 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters) err = 1; } else { - ErtsPortNames *pnp = erts_get_port_names(id); + ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); erts_printf(" inport=%T inname=%s indrv=%s ", id, pnp->name ? pnp->name : "unknown", @@ -1791,7 +1791,7 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters) err = 1; } else { - ErtsPortNames *pnp = erts_get_port_names(id); + ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); erts_printf(" outport=%T outname=%s outdrv=%s ", id, pnp->name ? pnp->name : "unknown", @@ -1827,7 +1827,7 @@ static void doit_erts_check_io_debug(void *vstate, void *vcounters) err = 1; } else { - ErtsPortNames *pnp = erts_get_port_names(id); + ErtsPortNames *pnp = erts_get_port_names(id, ERTS_INVALID_ERL_DRV_PORT); erts_printf(" port=%T name=%s drv=%s ", id, pnp->name ? pnp->name : "unknown", diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index bd8ba82a5f..2748edba02 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -350,7 +350,7 @@ schedule_cache_check(ErtsMsegAllctr_t *ma) { static ERTS_INLINE void * mmap_align(ErtsMsegAllctr_t *ma, void *addr, size_t length, int prot, int flags, int fd, off_t offset) { - void *p, *q; + char *p, *q; UWord d; p = mmap(addr, length, prot, flags, fd, offset); @@ -366,14 +366,14 @@ mmap_align(ErtsMsegAllctr_t *ma, void *addr, size_t length, int prot, int flags, if ((p = mmap(addr, length + MSEG_ALIGNED_SIZE, prot, flags, fd, offset)) == MAP_FAILED) return MAP_FAILED; - q = (void *)ALIGNED_CEILING(p); - d = q - p; + q = (void *)ALIGNED_CEILING((char *)p); + d = (UWord)(q - p); if (d > 0) munmap(p, d); if (MSEG_ALIGNED_SIZE - d > 0) - munmap((void *) (q + length), MSEG_ALIGNED_SIZE - d); + munmap((void *)(q + length), MSEG_ALIGNED_SIZE - d); return q; } @@ -439,7 +439,7 @@ mseg_destroy(ErtsMsegAllctr_t *ma, MemKind* mk, void *seg, Uint size) { } #if HAVE_MSEG_RECREATE -#if defined(__NetBsd__) +#if defined(__NetBSD__) #define MREMAP_FLAGS (0) #else #define MREMAP_FLAGS (MREMAP_MAYMOVE) @@ -708,7 +708,7 @@ static ERTS_INLINE void *cache_get_segment(MemKind *mk, Uint *size_p, Uint flags /* Use current cache placement for remaining segment space */ - best->seg = seg + size; + best->seg = ((char *)seg) + size; best->size = csize - size; ASSERT((size % GET_PAGE_SIZE) == 0); @@ -1028,7 +1028,7 @@ mseg_realloc(ErtsMsegAllctr_t *ma, ErtsAlcType_t atype, void *seg, sys_memcpy(((char *) new_seg), ((char *) seg), MIN(new_size, old_size)); - mseg_dealloc(ma, atype, seg, old_size, opt); + mseg_dealloc(ma, atype, seg, old_size, flags, opt); } #endif } @@ -1749,45 +1749,42 @@ erts_mseg_late_init(void) #endif /* #if HAVE_ERTS_MSEG */ -unsigned long -erts_mseg_test(unsigned long op, - unsigned long a1, - unsigned long a2, - unsigned long a3) +UWord +erts_mseg_test(UWord op, UWord a1, UWord a2, UWord a3) { switch (op) { #if HAVE_ERTS_MSEG case 0x400: /* Have erts_mseg */ - return (unsigned long) 1; + return (UWord) 1; case 0x401: - return (unsigned long) erts_mseg_alloc(ERTS_ALC_A_INVALID, (Uint *) a1, (Uint) 0); + return (UWord) erts_mseg_alloc(ERTS_ALC_A_INVALID, (Uint *) a1, (Uint) 0); case 0x402: erts_mseg_dealloc(ERTS_ALC_A_INVALID, (void *) a1, (Uint) a2, (Uint) 0); - return (unsigned long) 0; + return (UWord) 0; case 0x403: - return (unsigned long) erts_mseg_realloc(ERTS_ALC_A_INVALID, + return (UWord) erts_mseg_realloc(ERTS_ALC_A_INVALID, (void *) a1, (Uint) a2, (Uint *) a3, (Uint) 0); case 0x404: erts_mseg_clear_cache(); - return (unsigned long) 0; + return (UWord) 0; case 0x405: - return (unsigned long) erts_mseg_no(&erts_mseg_default_opt); + return (UWord) erts_mseg_no(&erts_mseg_default_opt); case 0x406: { ErtsMsegAllctr_t *ma = ERTS_MSEG_ALLCTR_IX(0); - unsigned long res; + UWord res; ERTS_MSEG_LOCK(ma); - res = (unsigned long) tot_cache_size(ma); + res = (UWord) tot_cache_size(ma); ERTS_MSEG_UNLOCK(ma); return res; } #else /* #if HAVE_ERTS_MSEG */ case 0x400: /* Have erts_mseg */ - return (unsigned long) 0; + return (UWord) 0; #endif /* #if HAVE_ERTS_MSEG */ - default: ASSERT(0); return ~((unsigned long) 0); + default: ASSERT(0); return ~((UWord) 0); } } diff --git a/erts/emulator/sys/common/erl_mseg.h b/erts/emulator/sys/common/erl_mseg.h index 127576b850..a1b000f51c 100644 --- a/erts/emulator/sys/common/erl_mseg.h +++ b/erts/emulator/sys/common/erl_mseg.h @@ -109,9 +109,6 @@ Eterm erts_mseg_info(int, int *, void*, int, Uint **, Uint *); #endif /* #if HAVE_ERTS_MSEG */ -unsigned long erts_mseg_test(unsigned long, - unsigned long, - unsigned long, - unsigned long); +UWord erts_mseg_test(UWord, UWord, UWord, UWord); #endif /* #ifndef ERL_MSEG_H_ */ diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index a523d67158..7676d8872a 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2006-2012. All Rights Reserved. + * Copyright Ericsson AB 2006-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -40,6 +40,13 @@ # include "config.h" #endif +#if defined(__DARWIN__) || defined(__APPLE__) && defined(__MACH__) +/* Setting _DARWIN_UNLIMITED_SELECT before including sys/select.h enables + * the version of select() that does not place a limit on the fd_set. + */ +# define _DARWIN_UNLIMITED_SELECT +#endif + #ifndef WANT_NONBLOCKING # define WANT_NONBLOCKING #endif @@ -90,6 +97,52 @@ #define HARD_DEBUG #endif +#ifdef _DARWIN_UNLIMITED_SELECT +typedef struct { + size_t sz; + fd_set* ptr; +}ERTS_fd_set; +# define ERTS_FD_CLR(fd, fds) FD_CLR((fd), (fds)->ptr) +# define ERTS_FD_SET(fd, fds) FD_SET((fd), (fds)->ptr) +# define ERTS_FD_ISSET(fd,fds) FD_ISSET((fd), (fds)->ptr) +# define ERTS_FD_ZERO(fds) memset((fds)->ptr, 0, (fds)->sz) +# define ERTS_FD_SIZE(n) ((((n)+NFDBITS-1)/NFDBITS)*sizeof(fd_mask)) + +static void ERTS_FD_COPY(ERTS_fd_set *src, ERTS_fd_set *dst) +{ + if (dst->sz != src->sz) { + dst->ptr = dst->ptr + ? erts_realloc(ERTS_ALC_T_SELECT_FDS, dst->ptr, src->sz) + : erts_alloc(ERTS_ALC_T_SELECT_FDS, src->sz); + dst->sz = src->sz; + } + memcpy(dst->ptr, src->ptr, src->sz); +} + +static ERTS_INLINE +int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds, + ERTS_fd_set *exceptfds, struct timeval *timeout) +{ + ASSERT(!readfds || readfds->sz >= ERTS_FD_SIZE(nfds)); + ASSERT(!writefds || writefds->sz >= ERTS_FD_SIZE(nfds)); + ASSERT(!exceptfds); + return select(nfds, + (readfds ? readfds->ptr : NULL ), + (writefds ? writefds->ptr : NULL), + NULL, + timeout); +} + +#else /* !_DARWIN_UNLIMITED_SELECT */ +# define ERTS_fd_set fd_set +# define ERTS_FD_CLR FD_CLR +# define ERTS_FD_ISSET FD_ISSET +# define ERTS_FD_SET FD_SET +# define ERTS_FD_ZERO FD_ZERO +# define ERTS_FD_COPY(src,dst) (*(dst) = *(src)) +# define ERTS_SELECT select +#endif + #define ERTS_POLL_USE_BATCH_UPDATE_POLLSET (ERTS_POLL_USE_DEVPOLL \ || ERTS_POLL_USE_KQUEUE) #define ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE \ @@ -244,10 +297,10 @@ struct ErtsPollSet_ { #if ERTS_POLL_USE_FALLBACK int no_select_fds; #endif - fd_set input_fds; - fd_set res_input_fds; - fd_set output_fds; - fd_set res_output_fds; + ERTS_fd_set input_fds; + ERTS_fd_set res_input_fds; + ERTS_fd_set output_fds; + ERTS_fd_set res_output_fds; #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE ErtsPollSetUpdateRequestsBlock update_requests; @@ -624,6 +677,33 @@ grow_poll_fds(ErtsPollSet ps, int min_ix) } #endif +#ifdef _DARWIN_UNLIMITED_SELECT +static void +grow_select_fds(int fd, ERTS_fd_set* fds) +{ + int new_len = ERTS_POLL_EXPORT(erts_poll_get_table_len)(fd + 1); + if (new_len > max_fds) + new_len = max_fds; + new_len = ERTS_FD_SIZE(new_len); + fds->ptr = fds->sz + ? erts_realloc(ERTS_ALC_T_SELECT_FDS, fds->ptr, new_len) + : erts_alloc(ERTS_ALC_T_SELECT_FDS, new_len); + memset((char*)fds->ptr + fds->sz, 0, new_len - fds->sz); + fds->sz = new_len; +} +static ERTS_INLINE void +ensure_select_fds(int fd, ERTS_fd_set* in, ERTS_fd_set* out) +{ + ASSERT(in->sz == out->sz); + if (ERTS_FD_SIZE(fd+1) > in->sz) { + grow_select_fds(fd, in); + grow_select_fds(fd, out); + } +} +#else +# define ensure_select_fds(fd, in, out) do {} while(0) +#endif /* _DARWIN_UNLIMITED_SELECT */ + static void grow_fds_status(ErtsPollSet ps, int min_fd) { @@ -1290,22 +1370,23 @@ static int update_pollset(ErtsPollSet ps, int fd) #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ { ErtsPollEvents events = ps->fds_status[fd].events; + ensure_select_fds(fd, &ps->input_fds, &ps->output_fds); if ((ERTS_POLL_EV_IN & events) != (ERTS_POLL_EV_IN & ps->fds_status[fd].used_events)) { if (ERTS_POLL_EV_IN & events) { - FD_SET(fd, &ps->input_fds); + ERTS_FD_SET(fd, &ps->input_fds); } else { - FD_CLR(fd, &ps->input_fds); + ERTS_FD_CLR(fd, &ps->input_fds); } } if ((ERTS_POLL_EV_OUT & events) != (ERTS_POLL_EV_OUT & ps->fds_status[fd].used_events)) { if (ERTS_POLL_EV_OUT & events) { - FD_SET(fd, &ps->output_fds); + ERTS_FD_SET(fd, &ps->output_fds); } else { - FD_CLR(fd, &ps->output_fds); + ERTS_FD_CLR(fd, &ps->output_fds); } } @@ -1789,7 +1870,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, while (fd < end_fd && res < max_res) { pr[res].events = (ErtsPollEvents) 0; - if (FD_ISSET(fd, &ps->res_input_fds)) { + if (ERTS_FD_ISSET(fd, &ps->res_input_fds)) { #if ERTS_POLL_USE_FALLBACK if (fd == ps->kp_fd) { res += get_kp_results(ps, &pr[res], max_res-res); @@ -1805,7 +1886,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, #endif pr[res].events |= ERTS_POLL_EV_IN; } - if (FD_ISSET(fd, &ps->res_output_fds)) + if (ERTS_FD_ISSET(fd, &ps->res_output_fds)) pr[res].events |= ERTS_POLL_EV_OUT; if (pr[res].events) { pr[res].fd = fd; @@ -1832,24 +1913,23 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, while (fd < end_fd && res < max_res) { if (ps->fds_status[fd].events) { int sres; - fd_set *iset = NULL; - fd_set *oset = NULL; + ERTS_fd_set *iset = NULL; + ERTS_fd_set *oset = NULL; if (ps->fds_status[fd].events & ERTS_POLL_EV_IN) { iset = &ps->res_input_fds; - FD_ZERO(iset); - FD_SET(fd, iset); + ERTS_FD_ZERO(iset); + ERTS_FD_SET(fd, iset); } if (ps->fds_status[fd].events & ERTS_POLL_EV_OUT) { oset = &ps->res_output_fds; - FD_ZERO(oset); - FD_SET(fd, oset); - + ERTS_FD_ZERO(oset); + ERTS_FD_SET(fd, oset); } do { /* Initiate 'tv' each time; select() may modify it */ SysTimeval tv = {0, 0}; - sres = select(ps->max_fd+1, iset, oset, NULL, &tv); + sres = ERTS_SELECT(ps->max_fd+1, iset, oset, NULL, &tv); } while (sres < 0 && errno == EINTR); if (sres < 0) { #if ERTS_POLL_USE_FALLBACK @@ -1873,7 +1953,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, } else if (sres > 0) { pr[res].fd = fd; - if (iset && FD_ISSET(fd, iset)) { + if (iset && ERTS_FD_ISSET(fd, iset)) { #if ERTS_POLL_USE_FALLBACK if (fd == ps->kp_fd) { res += get_kp_results(ps, @@ -1891,7 +1971,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, #endif pr[res].events |= ERTS_POLL_EV_IN; } - if (oset && FD_ISSET(fd, oset)) { + if (oset && ERTS_FD_ISSET(fd, oset)) { pr[res].events |= ERTS_POLL_EV_OUT; } ASSERT(pr[res].events); @@ -1992,14 +2072,14 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res) #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ SysTimeval to = *tv; - ps->res_input_fds = ps->input_fds; - ps->res_output_fds = ps->output_fds; - + ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds); + ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds); + #ifdef ERTS_SMP if (to.tv_sec || to.tv_usec) erts_thr_progress_prepare_wait(NULL); #endif - res = select(ps->max_fd + 1, + res = ERTS_SELECT(ps->max_fd + 1, &ps->res_input_fds, &ps->res_output_fds, NULL, @@ -2027,7 +2107,7 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res) ERTS_POLLSET_LOCK(ps); handle_update_requests(ps); ERTS_POLLSET_UNLOCK(ps); - res = select(ps->max_fd + 1, + res = ERTS_SELECT(ps->max_fd + 1, &ps->res_input_fds, &ps->res_output_fds, NULL, @@ -2233,7 +2313,8 @@ ERTS_POLL_EXPORT(erts_poll_init)(void) max_fds = OPEN_MAX; #endif -#if ERTS_POLL_USE_SELECT && defined(FD_SETSIZE) +#if ERTS_POLL_USE_SELECT && defined(FD_SETSIZE) && \ + !defined(_DARWIN_UNLIMITED_SELECT) if (max_fds > FD_SETSIZE) max_fds = FD_SETSIZE; #endif @@ -2301,10 +2382,21 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #if ERTS_POLL_USE_FALLBACK ps->no_select_fds = 0; #endif - FD_ZERO(&ps->input_fds); - FD_ZERO(&ps->res_input_fds); - FD_ZERO(&ps->output_fds); - FD_ZERO(&ps->res_output_fds); +#ifdef _DARWIN_UNLIMITED_SELECT + ps->input_fds.sz = 0; + ps->input_fds.ptr = NULL; + ps->res_input_fds.sz = 0; + ps->res_input_fds.ptr = NULL; + ps->output_fds.sz = 0; + ps->output_fds.ptr = NULL; + ps->res_output_fds.sz = 0; + ps->res_output_fds.ptr = NULL; +#else + ERTS_FD_ZERO(&ps->input_fds); + ERTS_FD_ZERO(&ps->res_input_fds); + ERTS_FD_ZERO(&ps->output_fds); + ERTS_FD_ZERO(&ps->res_output_fds); +#endif #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE ps->update_requests.next = NULL; @@ -2382,6 +2474,16 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) if (ps->poll_fds) erts_free(ERTS_ALC_T_POLL_FDS, (void *) ps->poll_fds); #elif ERTS_POLL_USE_SELECT +#ifdef _DARWIN_UNLIMITED_SELECT + if (ps->input_fds.ptr) + erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->input_fds.ptr); + if (ps->res_input_fds.ptr) + erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_input_fds.ptr); + if (ps->output_fds.ptr) + erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->output_fds.ptr); + if (ps->res_output_fds.ptr) + erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_output_fds.ptr); +#endif #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE { @@ -2445,6 +2547,10 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #if ERTS_POLL_USE_POLL size += ps->poll_fds_len*sizeof(struct pollfd); #elif ERTS_POLL_USE_SELECT +#ifdef _DARWIN_UNLIMITED_SELECT + size += ps->input_fds.sz + ps->res_input_fds.sz + + ps->output_fds.sz + ps->res_output_fds.sz; +#endif #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c index f2f4de869d..689be98969 100644 --- a/erts/emulator/sys/unix/sys_float.c +++ b/erts/emulator/sys/unix/sys_float.c @@ -152,7 +152,7 @@ static int mask_sse2(void) #if defined(__x86_64__) -static inline int cpu_has_sse2(void) { return 1; } +static ERTS_INLINE int cpu_has_sse2(void) { return 1; } #else /* !__x86_64__ */ @@ -179,7 +179,7 @@ static unsigned int xor_eflags(unsigned int mask) return eax; } -static __inline__ unsigned int cpuid_eax(unsigned int op) +static ERTS_INLINE unsigned int cpuid_eax(unsigned int op) { unsigned int eax, save_ebx; @@ -195,7 +195,7 @@ static __inline__ unsigned int cpuid_eax(unsigned int op) return eax; } -static __inline__ unsigned int cpuid_edx(unsigned int op) +static ERTS_INLINE unsigned int cpuid_edx(unsigned int op) { unsigned int eax, edx, save_ebx; @@ -215,7 +215,7 @@ static __inline__ unsigned int cpuid_edx(unsigned int op) * register on the Intel486 processor to generate alignment * faults. This bit cannot be set on the Intel386 processor. */ -static __inline__ int is_386(void) +static ERTS_INLINE int is_386(void) { return ((xor_eflags(1<<18) >> 18) & 1) == 0; } @@ -223,7 +223,7 @@ static __inline__ int is_386(void) /* Newer x86 processors have a CPUID instruction, as indicated by * the ID bit (#21) in EFLAGS being modifiable. */ -static __inline__ int has_CPUID(void) +static ERTS_INLINE int has_CPUID(void) { return (xor_eflags(1<<21) >> 21) & 1; } @@ -832,6 +832,8 @@ sys_chars_to_double(char* buf, double* fp) return 0; } +#ifdef USE_MATHERR + int matherr(struct exception *exc) { @@ -842,3 +844,5 @@ matherr(struct exception *exc) #endif return 1; } + +#endif diff --git a/erts/emulator/sys/win32/erl_win_dyn_driver.h b/erts/emulator/sys/win32/erl_win_dyn_driver.h index a7c53c904d..b9a9838a36 100644 --- a/erts/emulator/sys/win32/erl_win_dyn_driver.h +++ b/erts/emulator/sys/win32/erl_win_dyn_driver.h @@ -80,6 +80,7 @@ WDD_TYPEDEF(int, erl_drv_output_term, (ErlDrvTermData, ErlDrvTermData*, int)); WDD_TYPEDEF(int, driver_output_term, (ErlDrvPort, ErlDrvTermData*, int)); WDD_TYPEDEF(int, erl_drv_send_term, (ErlDrvTermData, ErlDrvTermData, ErlDrvTermData*, int)); WDD_TYPEDEF(int, driver_send_term, (ErlDrvPort, ErlDrvTermData, ErlDrvTermData*, int)); +WDD_TYPEDEF(unsigned int, driver_async_port_key, (ErlDrvPort)); WDD_TYPEDEF(long, driver_async, (ErlDrvPort,unsigned int*,void (*)(void*),void*,void (*)(void*))); WDD_TYPEDEF(int, driver_async_cancel, (unsigned int)); WDD_TYPEDEF(int, driver_lock_driver, (ErlDrvPort)); @@ -197,6 +198,7 @@ typedef struct { WDD_FTYPE(driver_output_term) *driver_output_term; WDD_FTYPE(erl_drv_send_term) *erl_drv_send_term; WDD_FTYPE(driver_send_term) *driver_send_term; + WDD_FTYPE(driver_async_port_key) *driver_async_port_key; WDD_FTYPE(driver_async) *driver_async; WDD_FTYPE(driver_async_cancel) *driver_async_cancel; WDD_FTYPE(driver_lock_driver) *driver_lock_driver; @@ -308,6 +310,7 @@ extern TWinDynDriverCallbacks WinDynDriverCallbacks; #define driver_output_term (WinDynDriverCallbacks.driver_output_term) #define erl_drv_send_term (WinDynDriverCallbacks.erl_drv_send_term) #define driver_send_term (WinDynDriverCallbacks.driver_send_term) +#define driver_async_port_key (WinDynDriverCallbacks.driver_async_port_key) #define driver_async (WinDynDriverCallbacks.driver_async) #define driver_async_cancel (WinDynDriverCallbacks.driver_async_cancel) #define driver_lock_driver (WinDynDriverCallbacks.driver_lock_driver) @@ -443,6 +446,7 @@ do { \ ((W).driver_output_term) = driver_output_term; \ ((W).erl_drv_send_term) = erl_drv_send_term; \ ((W).driver_send_term) = driver_send_term; \ +((W).driver_async_port_key) = driver_async_port_key; \ ((W).driver_async) = driver_async; \ ((W).driver_async_cancel) = driver_async_cancel; \ ((W).driver_lock_driver) = driver_lock_driver; \ diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 8244a43583..6a1d6b08f4 100755 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -399,11 +399,12 @@ int* pBuild; /* Pointer to build number. */ * Definitions for driver flags. */ -#define DF_OVR_READY 1 /* Overlapped result is ready. */ -#define DF_EXIT_THREAD 2 /* The thread should exit. */ -#define DF_XLAT_CR 4 /* The thread should translate CRs. */ -#define DF_DROP_IF_INVH 8 /* Drop packages instead of crash if +#define DF_OVR_READY 1 /* Overlapped result is ready. */ +#define DF_EXIT_THREAD 2 /* The thread should exit. */ +#define DF_XLAT_CR 4 /* The thread should translate CRs. */ +#define DF_DROP_IF_INVH 8 /* Drop packages instead of crash if invalid handle (stderr) */ +#define DF_THREAD_FLUSHED 16 /* The thread should exit. */ #define OV_BUFFER_PTR(dp) ((LPVOID) ((dp)->ov.Internal)) #define OV_NUM_TO_READ(dp) ((dp)->ov.InternalHigh) @@ -2141,8 +2142,9 @@ threaded_writer(LPVOID param) for (;;) { handle = WaitForMultipleObjects(2, handles, FALSE, INFINITE); - if (aio->flags & DF_EXIT_THREAD) + if (aio->flags & DF_EXIT_THREAD) { break; + } buf = OV_BUFFER_PTR(aio); numToWrite = OV_NUM_TO_READ(aio); @@ -2150,6 +2152,7 @@ threaded_writer(LPVOID param) if (handle == (WAIT_OBJECT_0 + 1) && numToWrite == 0) { SetEvent(aio->flushReplyEvent); + aio->flags |= DF_THREAD_FLUSHED; continue; } @@ -2197,6 +2200,7 @@ threaded_writer(LPVOID param) if (aio->flags & DF_EXIT_THREAD) break; } + aio->flags |= DF_THREAD_FLUSHED; CloseHandle(aio->fd); aio->fd = INVALID_HANDLE_VALUE; unrefer_driver_data(aio->dp); @@ -2340,9 +2344,11 @@ static void fd_stop(ErlDrvData data) (void) driver_select(dp->port_num, (ErlDrvEvent)dp->out.ov.hEvent, ERL_DRV_WRITE, 0); - ASSERT(dp->out.flushEvent); - SetEvent(dp->out.flushEvent); - WaitForSingleObject(dp->out.flushReplyEvent, INFINITE); + do { + ASSERT(dp->out.flushEvent); + SetEvent(dp->out.flushEvent); + } while (WaitForSingleObject(dp->out.flushReplyEvent, 10) == WAIT_TIMEOUT + || !(dp->out.flags & DF_THREAD_FLUSHED)); } } @@ -2433,12 +2439,12 @@ threaded_exiter(LPVOID param) */ i = 0; if (dp->out.thread != (HANDLE) -1) { - dp->out.flags = DF_EXIT_THREAD; + dp->out.flags |= DF_EXIT_THREAD; SetEvent(dp->out.ioAllowed); handles[i++] = dp->out.thread; } if (dp->in.thread != (HANDLE) -1) { - dp->in.flags = DF_EXIT_THREAD; + dp->in.flags |= DF_EXIT_THREAD; SetEvent(dp->in.ioAllowed); handles[i++] = dp->in.thread; } diff --git a/erts/emulator/sys/win32/sys_float.c b/erts/emulator/sys/win32/sys_float.c index fb1ffc3089..0e19746cf5 100644 --- a/erts/emulator/sys/win32/sys_float.c +++ b/erts/emulator/sys/win32/sys_float.c @@ -52,7 +52,7 @@ void erts_thread_disable_fpe(void) int sys_chars_to_double(char *buf, double *fp) { - char *s = buf, *t, *dp; + unsigned char *s = buf, *t, *dp; /* Robert says that something like this is what he really wanted: * (The [.,] radix test is NOT what Robert wanted - it was added later) @@ -120,7 +120,7 @@ sys_chars_to_double(char *buf, double *fp) int sys_double_to_chars_ext(double fp, char *buffer, size_t buffer_size, size_t decimals) { - char *s = buffer; + unsigned char *s = buffer; if (erts_snprintf(buffer, buffer_size, "%.*e", decimals, fp) >= buffer_size) return -1; @@ -133,6 +133,8 @@ sys_double_to_chars_ext(double fp, char *buffer, size_t buffer_size, size_t deci return s-buffer; /* i.e strlen(buffer) */ } +#ifdef USE_MATHERR + int matherr(struct _exception *exc) { @@ -141,6 +143,8 @@ matherr(struct _exception *exc) return 1; } +#endif + static void fpe_exception(int sig) { diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c index 2f2dfc8197..b84c8f85ce 100644 --- a/erts/emulator/sys/win32/sys_time.c +++ b/erts/emulator/sys/win32/sys_time.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1997-2011. All Rights Reserved. + * Copyright Ericsson AB 1997-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -63,6 +63,8 @@ static SysHrTime wrap = 0; static DWORD last_tick_count = 0; +static erts_smp_mtx_t wrap_lock; +static ULONGLONG (WINAPI *pGetTickCount64)(void) = NULL; /* Getting timezone information is a heavy operation, so we want to do this only once */ @@ -77,11 +79,23 @@ static int days_in_month[2][13] = { int sys_init_time(void) { + char kernel_dll_name[] = "kernel32"; + HMODULE module; + + module = GetModuleHandle(kernel_dll_name); + pGetTickCount64 = (module != NULL) ? + (ULONGLONG (WINAPI *)(void)) + GetProcAddress(module,"GetTickCount64") : + NULL; + if(GetTimeZoneInformation(&static_tzi) && static_tzi.StandardDate.wMonth != 0 && static_tzi.DaylightDate.wMonth != 0) { have_static_tzi = 1; } + + erts_smp_mtx_init(&wrap_lock, "sys_gethrtime"); + return 1; } @@ -363,15 +377,39 @@ sys_gettimeofday(SysTimeval *tv) EPOCH_JULIAN_DIFF); } +extern int erts_initialized; SysHrTime sys_gethrtime(void) { - DWORD ticks = (SysHrTime) (GetTickCount() & 0x7FFFFFFF); - if (ticks < (SysHrTime) last_tick_count) { - wrap += LL_LITERAL(1) << 31; + if (pGetTickCount64 != NULL) { + return ((SysHrTime) pGetTickCount64()) * LL_LITERAL(1000000); + } else { + DWORD ticks; + SysHrTime res; + erts_smp_mtx_lock(&wrap_lock); + ticks = (SysHrTime) (GetTickCount() & 0x7FFFFFFF); + if (ticks < (SysHrTime) last_tick_count) { + /* Detect a race that should no longer be here... */ + if ((((SysHrTime) last_tick_count) - ((SysHrTime) ticks)) > 1000) { + wrap += LL_LITERAL(1) << 31; + } else { + /* + * XXX Debug: Violates locking order, remove all this, + * after testing! + */ + erts_dsprintf_buf_t *dsbufp = erts_create_logger_dsbuf(); + erts_dsprintf(dsbufp, "Did not wrap when last_tick %d " + "and tick %d", + last_tick_count, ticks); + erts_send_error_to_logger_nogl(dsbufp); + ticks = last_tick_count; + } + } + last_tick_count = ticks; + res = ((((LONGLONG) ticks) + wrap) * LL_LITERAL(1000000)); + erts_smp_mtx_unlock(&wrap_lock); + return res; } - last_tick_count = ticks; - return ((((LONGLONG) ticks) + wrap) * LL_LITERAL(1000000)); } clock_t diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 9594ab48b1..f02ca3cb98 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -140,7 +140,8 @@ EMAKEFILE=Emakefile TEST_SPEC_FILES= emulator.spec \ emulator.spec.win \ - emulator_bench.spec + emulator_bench.spec \ + emulator_smoke.spec # ---------------------------------------------------- # Release directory specification diff --git a/erts/emulator/test/a_SUITE.erl b/erts/emulator/test/a_SUITE.erl index b541be3df6..195c9c0a5f 100644 --- a/erts/emulator/test/a_SUITE.erl +++ b/erts/emulator/test/a_SUITE.erl @@ -68,6 +68,9 @@ pollset_size(doc) -> pollset_size(suite) -> []; pollset_size(Config) when is_list(Config) -> + %% Ensure inet_gethost_native port program started, in order to + %% allow other suites to use it... + inet_gethost_native:gethostbyname("localhost"), ?line Parent = self(), ?line Go = make_ref(), ?line spawn(fun () -> diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 22b5d93983..801ed0f85a 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2003-2011. All Rights Reserved. +%% Copyright Ericsson AB 2003-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -28,7 +28,8 @@ bucket_index/1, bucket_mask/1, rbtree/1, - mseg_clear_cache/1]). + mseg_clear_cache/1, + cpool/1]). -export([init_per_testcase/2, end_per_testcase/2]). @@ -40,7 +41,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [basic, coalesce, threads, realloc_copy, bucket_index, - bucket_mask, rbtree, mseg_clear_cache]. + bucket_mask, rbtree, mseg_clear_cache, cpool]. groups() -> []. @@ -105,6 +106,10 @@ mseg_clear_cache(suite) -> []; mseg_clear_cache(doc) -> []; mseg_clear_cache(Cfg) -> ?line drv_case(Cfg). +cpool(suite) -> []; +cpool(doc) -> []; +cpool(Cfg) -> ?line drv_case(Cfg). + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% %% Internal functions %% diff --git a/erts/emulator/test/alloc_SUITE_data/Makefile.src b/erts/emulator/test/alloc_SUITE_data/Makefile.src index 035415d73e..1d4286e671 100644 --- a/erts/emulator/test/alloc_SUITE_data/Makefile.src +++ b/erts/emulator/test/alloc_SUITE_data/Makefile.src @@ -23,7 +23,8 @@ TEST_DRVS = basic@dll@ \ bucket_index@dll@ \ bucket_mask@dll@ \ rbtree@dll@ \ - mseg_clear_cache@dll@ + mseg_clear_cache@dll@ \ + cpool@dll@ CC = @CC@ LD = @LD@ diff --git a/erts/emulator/test/alloc_SUITE_data/allocator_test.h b/erts/emulator/test/alloc_SUITE_data/allocator_test.h index c0396ddb61..2d6c5f9dc7 100644 --- a/erts/emulator/test/alloc_SUITE_data/allocator_test.h +++ b/erts/emulator/test/alloc_SUITE_data/allocator_test.h @@ -75,6 +75,15 @@ typedef void* erts_cond; #define PREV_BLK(B) ((Block_t *) ALC_TEST1(0x019, (B))) #define IS_MBC_FIRST_BLK(A,B) ((Ulong) ALC_TEST2(0x01a, (A), (B))) #define UNIT_SZ ((Ulong) ALC_TEST0(0x01b)) +#define BLK_TO_MBC(B) ((Carrier_t *) ALC_TEST1(0x01c, (B))) +#define ADD_MBC(A, C) ((void) ALC_TEST2(0x01d, (A), (C))) +#define REMOVE_MBC(A, C) ((void) ALC_TEST2(0x01e, (A), (C))) +#define ZERO_CRR_SIZE ((Ulong) ALC_TEST0(0x01f)) +#define ZERO_CRR_INIT(A,B) ((Carrier_t *) ALC_TEST2(0x020, (A), (B))) +#define CPOOL_INSERT(A,B) ((Carrier_t *) ALC_TEST2(0x021, (A), (B))) +#define CPOOL_DELETE(A,B) ((Carrier_t *) ALC_TEST2(0x022, (A), (B))) +#define CPOOL_IS_EMPTY(A) ((int) ALC_TEST1(0x023, (A))) +#define CPOOL_IS_IN_POOL(A,B) ((int) ALC_TEST2(0x024, (A), (B))) /* From erl_goodfit_alloc.c */ #define BKT_IX(A, S) ((Ulong) ALC_TEST2(0x100, (A), (S))) @@ -84,15 +93,17 @@ typedef void* erts_cond; /* From erl_bestfit_alloc.c and erl_ao_firstfit_alloc.c */ #define IS_AOBF(A) ((Ulong) ALC_TEST1(RBT_OP(0), (A))) -#define RBT_ROOT(A) ((RBT_t *) ALC_TEST1(RBT_OP(1), (A))) +#define RBT_ROOT(A,SZ) ((RBT_t *) ALC_TEST2(RBT_OP(1), (A), (SZ))) #define RBT_PARENT(T) ((RBT_t *) ALC_TEST1(RBT_OP(2), (T))) #define RBT_LEFT(T) ((RBT_t *) ALC_TEST1(RBT_OP(3), (T))) #define RBT_RIGHT(T) ((RBT_t *) ALC_TEST1(RBT_OP(4), (T))) #define RBT_NEXT(T) ((RBTL_t *) ALC_TEST1(RBT_OP(5), (T))) #define RBT_IS_BLACK(T) ((Ulong) ALC_TEST1(RBT_OP(6), (T))) #define RBT_IS_TREE(T) ((Ulong) ALC_TEST1(RBT_OP(7), (T))) -#define IS_AOFF(A) ((Ulong) ALC_TEST1(RBT_OP(8), (A))) +#define IS_BF_ALGO(A) ((Ulong) ALC_TEST1(RBT_OP(8), (A))) #define RBT_MAX_SZ(T) ((Ulong) ALC_TEST1(RBT_OP(9), (T))) +#define IS_BF(A) ((Ulong) ALC_TEST1(RBT_OP(0xa), (A))) +#define RBT_PREV(T) ((RBTL_t *) ALC_TEST1(RBT_OP(0xb), (T))) /* From erl_mseg.c */ #define HAVE_MSEG() ((int) ALC_TEST0(0x400)) @@ -129,5 +140,6 @@ typedef void* erts_cond; #define THR_CREATE(F, A) ((erts_thread) ALC_TEST2(0xf10, (F), (A))) #define THR_JOIN(T) ((void) ALC_TEST1(0xf11, (T))) #define THR_EXIT(R) ((void) ALC_TEST1(0xf12, (R))) +#define IS_SMP_ENABLED ((int) ALC_TEST0(0xf13)) #endif diff --git a/erts/emulator/test/alloc_SUITE_data/coalesce.c b/erts/emulator/test/alloc_SUITE_data/coalesce.c index 981fa6d43e..9da49a0d14 100644 --- a/erts/emulator/test/alloc_SUITE_data/coalesce.c +++ b/erts/emulator/test/alloc_SUITE_data/coalesce.c @@ -267,7 +267,7 @@ void testcase_run(TestCaseState_t *tcs) { char *argv_org[] = {"-tsmbcs511","-tmmbcs511", "-tsbct512", "-trmbcmt100", "-tas", NULL, NULL}; - char *alg[] = {"af", "gf", "bf", "aobf", "aoff", NULL}; + char *alg[] = {"af", "gf", "bf", "aobf", "aoff", "aoffcbf", "aoffcaobf", NULL}; int i; for (i = 0; alg[i]; i++) { diff --git a/erts/emulator/test/alloc_SUITE_data/cpool.c b/erts/emulator/test/alloc_SUITE_data/cpool.c new file mode 100644 index 0000000000..276ed7be04 --- /dev/null +++ b/erts/emulator/test/alloc_SUITE_data/cpool.c @@ -0,0 +1,157 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2013. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + + +#ifndef __WIN32__ +#include <sys/types.h> +#include <unistd.h> +#include <errno.h> +#endif +#include <stdio.h> +#include <stdarg.h> +#include <string.h> +#include "testcase_driver.h" +#include "allocator_test.h" + +#define FATAL_ASSERT(A) \ + ((void) ((A) \ + ? 1 \ + : (fatal_assert_failed(#A, \ + (char *) __FILE__, \ + __LINE__), \ + 0))) + +static void +fatal_assert_failed(char* expr, char* file, int line) +{ + fflush(stdout); + fprintf(stderr, "%s:%d: Assertion failed: %s\n", + file, line, expr); + fflush(stderr); + abort(); +} + +#define TEST_NO_THREADS 10 +#define TEST_NO_CARRIERS_PER_THREAD 100000 +#define TEST_CARRIERS_OFFSET 5 + +static Allctr_t *alloc = NULL; + +static void stop_allocator(void) +{ + if (alloc) { + STOP_ALC(alloc); + alloc = NULL; + } +} + + +void *thread_func(void *arg); + +char * +testcase_name(void) +{ + return "cpool"; +} + +void +testcase_cleanup(TestCaseState_t *tcs) +{ + stop_allocator(); +} + +void * +thread_func(void *arg) +{ + Carrier_t *crr = (Carrier_t *) arg; + int i; + + for (i = 0; i < (TEST_NO_CARRIERS_PER_THREAD+TEST_CARRIERS_OFFSET); i++) { + int d; + if (i < TEST_NO_CARRIERS_PER_THREAD) { + CPOOL_INSERT(alloc, crr[i]); + if ((i & 0x7) == 0) + FATAL_ASSERT(CPOOL_IS_IN_POOL(alloc, crr[i])); + } + d = i-TEST_CARRIERS_OFFSET; + if (d >= 0) { + CPOOL_DELETE(alloc, crr[d]); + if ((d & 0x7) == 0) + FATAL_ASSERT(!CPOOL_IS_IN_POOL(alloc, crr[d])); + } + } + for (i = 0; i < TEST_NO_CARRIERS_PER_THREAD; i++) + FATAL_ASSERT(!CPOOL_IS_IN_POOL(alloc, crr[i])); + return NULL; +} + +static struct { + erts_thread tid; + Carrier_t *crr[TEST_NO_CARRIERS_PER_THREAD]; +} threads[TEST_NO_THREADS] = {{0}}; + +void +testcase_run(TestCaseState_t *tcs) +{ + int no_threads, t, c; + char *block, *p; + Ulong zcrr_sz; + + if (!IS_SMP_ENABLED) + testcase_skipped(tcs, "No SMP support"); + + alloc = START_ALC("Zero carrier allocator", 1, NULL); + + zcrr_sz = ZERO_CRR_SIZE; + + block = p = ALLOC(alloc, zcrr_sz*TEST_NO_THREADS*TEST_NO_CARRIERS_PER_THREAD); + + ASSERT(tcs, block != NULL); + + for (t = 0; t < TEST_NO_THREADS; t++) { + for (c = 0; c < TEST_NO_CARRIERS_PER_THREAD; c++) { + Carrier_t *crr = (Carrier_t *) p; + p += zcrr_sz; + ZERO_CRR_INIT(alloc, crr); + threads[t].crr[c] = crr; + } + } + + no_threads = 0; + for (t = 0; t < TEST_NO_THREADS; t++) { + threads[t].tid = THR_CREATE(thread_func, (void *) threads[t].crr); + if (threads[t].tid) { + testcase_printf(tcs, "Successfully created thread %d\n", t); + no_threads++; + } + else { + testcase_printf(tcs, "Failed to create thread %d\n", t); + break; + } + } + + for (t = 0; t < no_threads; t++) + THR_JOIN(threads[t].tid); + + FATAL_ASSERT(CPOOL_IS_EMPTY(alloc)); + + FREE(alloc, block); + + ASSERT(tcs, no_threads == TEST_NO_THREADS); +} diff --git a/erts/emulator/test/alloc_SUITE_data/rbtree.c b/erts/emulator/test/alloc_SUITE_data/rbtree.c index 4e7f821baf..49df2f0245 100644 --- a/erts/emulator/test/alloc_SUITE_data/rbtree.c +++ b/erts/emulator/test/alloc_SUITE_data/rbtree.c @@ -85,20 +85,23 @@ print_tree(TestCaseState_t *tcs, RBT_t *root) static RBT_t * check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) { - enum { BF, AOBF, AOFF }type; + enum { BF, AOBF, AOFF } type; int i, max_i; char stk[128]; RBT_t *root, *x, *y, *res; Ulong x_sz, y_sz, is_x_black; long blacks, curr_blacks; + int have_max_sz; res = NULL; - if (IS_AOBF(alc)) type = AOBF; - else if (IS_AOFF(alc)) type = AOFF; - else type = BF; + if (IS_AOBF(alc)) type = AOBF; + else if (IS_BF(alc)) type = BF; + else type = AOFF; - root = RBT_ROOT(alc); + have_max_sz = !IS_BF_ALGO(alc); + + root = RBT_ROOT(alc, size); #ifdef PRINT_TREE print_tree(tcs, root); @@ -186,9 +189,11 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) break; case AOFF: ASSERT(tcs, y < x); - ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x)); break; } + if (have_max_sz) { + ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x)); + } } y = RBT_RIGHT(x); @@ -205,18 +210,22 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) break; case AOFF: ASSERT(tcs, y > x); - ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x)); break; } + if (have_max_sz) { + ASSERT(tcs, RBT_MAX_SZ(y) <= RBT_MAX_SZ(x)); + } } if (type == BF) { Ulong l_sz; - RBTL_t *l = RBT_NEXT(x); + RBTL_t *l, *prev=x; for (l = RBT_NEXT(x); l; l = RBT_NEXT(l)) { l_sz = BLK_SZ(l); ASSERT(tcs, l_sz == x_sz); ASSERT(tcs, !RBT_IS_TREE(l)); + ASSERT(tcs, RBT_PREV(l) == prev); + prev = l; } } @@ -263,17 +272,20 @@ check_tree(TestCaseState_t *tcs, Allctr_t *alc, Ulong size) } static void -do_check(TestCaseState_t *tcs, Allctr_t *a, Ulong size) +do_check(TestCaseState_t *tcs, Allctr_t *a, Ulong size, int ignore_null) { Ulong sz = ((size + 7) / 8)*8; void *tmp; Block_t *x, *y; x = (Block_t *) check_tree(tcs, a, sz); + if (!x && ignore_null) + return; + tmp = ALLOC(a, sz - ABLK_HDR_SZ); ASSERT(tcs, tmp); y = UMEM2BLK(tmp); - if (IS_AOBF(a)) { + if (!IS_BF(a)) { ASSERT(tcs, x == y); } else { @@ -306,7 +318,7 @@ test_it(TestCaseState_t *tcs) blk[i] = NULL; } if (i % (NO_BLOCKS/2) == 0) - do_check(tcs, a, 50); + do_check(tcs, a, 50, 0); } for (i = 0; i < NO_BLOCKS; i++) { @@ -315,7 +327,7 @@ test_it(TestCaseState_t *tcs) blk[i] = NULL; } if (i % (NO_BLOCKS/2) == 0) - do_check(tcs, a, 200); + do_check(tcs, a, 200, 0); } for (i = 0; i < NO_BLOCKS; i++) { @@ -324,20 +336,101 @@ test_it(TestCaseState_t *tcs) blk[i] = NULL; } if (i % (NO_BLOCKS/2) == 0) - do_check(tcs, a, 100); + do_check(tcs, a, 100, 0); } - do_check(tcs, a, 250); + do_check(tcs, a, 250, 0); for (i = 0; i < NO_BLOCKS; i++) { FREE(a, fence[i]); if (i % (NO_BLOCKS/3) == 0) - do_check(tcs, a, 300); + do_check(tcs, a, 300, 0); } - ASSERT(tcs, RBT_ROOT(a)); - ASSERT(tcs, !RBT_LEFT(RBT_ROOT(a))); - ASSERT(tcs, !RBT_RIGHT(RBT_ROOT(a))); + ASSERT(tcs, RBT_ROOT(a,0)); + ASSERT(tcs, !RBT_LEFT(RBT_ROOT(a,0))); + ASSERT(tcs, !RBT_RIGHT(RBT_ROOT(a,0))); +} + + +static int is_single_ablk_in_mbc(Allctr_t* a, void* ptr, void* crr) +{ + Block_t* blk = UMEM2BLK(ptr); + if (crr == BLK_TO_MBC(blk)) { + Block_t* first = MBC_TO_FIRST_BLK(a, crr); + if (blk == first || (IS_FREE_BLK(first) && blk == NXT_BLK(first))) { + Block_t* nxt; + if (IS_LAST_BLK(blk)) { + return 1; + } + nxt = NXT_BLK(blk); + return IS_FREE_BLK(nxt) && IS_LAST_BLK(nxt); + } + } + return 0; +} + +static void +test_carrier_migration(TestCaseState_t *tcs) +{ + int i, j; + Allctr_t* a = ((rbtree_test_data *) tcs->extra)->allocator; + void **blk = ((rbtree_test_data *) tcs->extra)->blk; + void **fence = ((rbtree_test_data *) tcs->extra)->fence; + void *crr, *p, *free_crr; + Ulong min_blk_sz; + + min_blk_sz = MIN_BLK_SZ(a); + + for (i = 0; i < NO_BLOCKS; i++) { + blk[i] = ALLOC(a, min_blk_sz + i % 500); + fence[i] = ALLOC(a, 1); + ASSERT(tcs, blk[i] && fence[i]); + } + + for (j = 0; j < NO_BLOCKS; j += 997) { + crr = BLK_TO_MBC(UMEM2BLK(blk[j])); + REMOVE_MBC(a, crr); + + for (i = 0; i < NO_BLOCKS; i++) { + if (i % 3 == 0) { + if (is_single_ablk_in_mbc(a, blk[i], crr)) { + crr = NULL; /* about to destroy the removed mbc */ + } + FREE(a, blk[i]); + blk[i] = NULL; + } + if (i % (NO_BLOCKS/2) == 0) + do_check(tcs, a, 50, 1); + } + + for (i = 0; i < NO_BLOCKS; i++) { + if (i % 3 == 0) { + ASSERT(tcs, !blk[i]); + blk[i] = ALLOC(a, min_blk_sz + i % 500); + ASSERT(tcs, BLK_TO_MBC(UMEM2BLK(blk[i])) != crr); + } + if (i % (NO_BLOCKS/2) == 0) + do_check(tcs, a, 50, 1); + } + if (crr) { + ADD_MBC(a, crr); + } + } + + for (crr = FIRST_MBC(a); crr; crr = NEXT_C(crr)) { + REMOVE_MBC(a, crr); + } + + p = ALLOC(a, 1); + free_crr = BLK_TO_MBC(UMEM2BLK(p)); + FREE(a, p); + + for (crr = FIRST_MBC(a); crr; crr = NEXT_C(crr)) { + ASSERT(tcs, free_crr != crr); + } + + ASSERT(tcs, !RBT_ROOT(a,0)); } @@ -369,6 +462,8 @@ testcase_run(TestCaseState_t *tcs) char *argv1[] = {"-tasbf", NULL}; char *argv2[] = {"-tasaobf", NULL}; char *argv3[] = {"-tasaoff", NULL}; + char *argv4[] = {"-tasaoffcaobf", NULL}; + char *argv5[] = {"-tasaoffcbf", NULL}; Allctr_t *a; rbtree_test_data *td; @@ -390,7 +485,9 @@ testcase_run(TestCaseState_t *tcs) td->allocator = a = START_ALC("rbtree_bf_", 0, argv1); ASSERT(tcs, a); + ASSERT(tcs, IS_BF_ALGO(a)); ASSERT(tcs, !IS_AOBF(a)); + ASSERT(tcs, IS_BF(a)); test_it(tcs); @@ -407,7 +504,9 @@ testcase_run(TestCaseState_t *tcs) td->allocator = a = START_ALC("rbtree_aobf_", 0, argv2); ASSERT(tcs, a); + ASSERT(tcs, IS_BF_ALGO(a)); ASSERT(tcs, IS_AOBF(a)); + ASSERT(tcs, !IS_BF(a)); test_it(tcs); @@ -424,11 +523,56 @@ testcase_run(TestCaseState_t *tcs) td->allocator = a = START_ALC("rbtree_aoff_", 0, argv3); ASSERT(tcs, a); + ASSERT(tcs, !IS_BF_ALGO(a)); + ASSERT(tcs, !IS_AOBF(a)); + ASSERT(tcs, !IS_BF(a)); test_it(tcs); + test_carrier_migration(tcs); STOP_ALC(a); td->allocator = NULL; testcase_printf(tcs, "Address order first fit test succeeded!\n"); + + /* Address order first fit, aobf within carrier */ + + testcase_printf(tcs, "Starting test of aoffcaobf...\n"); + + current_rbt_type_op_base = AO_FIRSTFIT_OP_BASE; + td->allocator = a = START_ALC("rbtree_aoffcaobf_", 0, argv4); + + ASSERT(tcs, a); + ASSERT(tcs, !IS_BF_ALGO(a)); + ASSERT(tcs, IS_AOBF(a)); + ASSERT(tcs, !IS_BF(a)); + + test_it(tcs); + test_carrier_migration(tcs); + + STOP_ALC(a); + td->allocator = NULL; + + testcase_printf(tcs, "aoffcaobf test succeeded!\n"); + + /* Address order first fit, bf within carrier */ + + testcase_printf(tcs, "Starting test of aoffcbf...\n"); + + current_rbt_type_op_base = AO_FIRSTFIT_OP_BASE; + td->allocator = a = START_ALC("rbtree_aoffcbf_", 0, argv5); + + ASSERT(tcs, a); + ASSERT(tcs, !IS_BF_ALGO(a)); + ASSERT(tcs, !IS_AOBF(a)); + ASSERT(tcs, IS_BF(a)); + + test_it(tcs); + test_carrier_migration(tcs); + + STOP_ALC(a); + td->allocator = NULL; + + testcase_printf(tcs, "aoffcaobf test succeeded!\n"); + } diff --git a/erts/emulator/test/binary_SUITE.erl b/erts/emulator/test/binary_SUITE.erl index babdb3363f..08ab094019 100644 --- a/erts/emulator/test/binary_SUITE.erl +++ b/erts/emulator/test/binary_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -47,7 +47,8 @@ copy_terms/1, conversions/1, deep_lists/1, deep_bitstr_lists/1, bad_list_to_binary/1, bad_binary_to_list/1, t_split_binary/1, bad_split/1, - terms/1, terms_float/1, external_size/1, t_iolist_size/1, + terms/1, terms_float/1, float_middle_endian/1, + external_size/1, t_iolist_size/1, t_hash/1, bad_size/1, bad_term_to_binary/1, @@ -57,10 +58,10 @@ ordering/1,unaligned_order/1,gc_test/1, bit_sized_binary_sizes/1, otp_6817/1,deep/1,obsolete_funs/1,robustness/1,otp_8117/1, - otp_8180/1]). + otp_8180/1, ttb_trap/1]). %% Internal exports. --export([sleeper/0]). +-export([sleeper/0,ttb_loop/2]). suite() -> [{ct_hooks,[ts_install_cth]}, {timetrap,{minutes,2}}]. @@ -69,13 +70,13 @@ all() -> [copy_terms, conversions, deep_lists, deep_bitstr_lists, t_split_binary, bad_split, bad_list_to_binary, bad_binary_to_list, terms, - terms_float, external_size, t_iolist_size, + terms_float, float_middle_endian, external_size, t_iolist_size, bad_binary_to_term_2, safe_binary_to_term2, bad_binary_to_term, bad_terms, t_hash, bad_size, bad_term_to_binary, more_bad_terms, otp_5484, otp_5933, ordering, unaligned_order, gc_test, bit_sized_binary_sizes, otp_6817, otp_8117, deep, - obsolete_funs, robustness, otp_8180]. + obsolete_funs, robustness, otp_8180, ttb_trap]. groups() -> []. @@ -486,6 +487,11 @@ terms_float(Config) when is_list(Config) -> true = Size1 < Size0 end). +float_middle_endian(Config) when is_list(Config) -> + %% Testing for roundtrip is not enough. + ?line <<131,70,63,240,0,0,0,0,0,0>> = term_to_binary(1.0, [{minor_version,1}]), + ?line 1.0 = binary_to_term(<<131,70,63,240,0,0,0,0,0,0>>). + external_size(Config) when is_list(Config) -> %% Build a term whose external size only fits in a big num (on 32-bit CPU). ?line external_size_1(16#11111111111111117777777777777777888889999, 0, 16#FFFFFFF), @@ -1322,6 +1328,38 @@ run_otp_8180(Name) -> end || Bin <- Bins], ok. +%% Test that exit and GC during term_to_binary trap does not crash. +ttb_trap(Config) when is_list(Config)-> + case erlang:system_info(wordsize) of + N when N < 8 -> + {skipped, "Only on 64bit machines"}; + _ -> + do_ttb_trap(5) + end. + +do_ttb_trap(0) -> + ok; +do_ttb_trap(N) -> + Pid = spawn(?MODULE,ttb_loop,[1000,self()]), + receive ok -> ok end, + receive after 100 -> ok end, + erlang:garbage_collect(Pid), + receive after 100 -> ok end, + exit(Pid,kill), + receive after 1 -> ok end, + do_ttb_trap(N-1). + +ttb_loop(N,Pid) -> + Term = lists:duplicate(2000000,2000000), + Pid ! ok, + ttb_loop2(N,Term). +ttb_loop2(0,_T) -> + ok; +ttb_loop2(N,T) -> + apply(erlang,term_to_binary,[T]), + ttb_loop2(N-1,T). + + %% Utilities. make_sub_binary(Bin) when is_binary(Bin) -> diff --git a/erts/emulator/test/bs_construct_SUITE.erl b/erts/emulator/test/bs_construct_SUITE.erl index 123952d01d..1a8724d63b 100644 --- a/erts/emulator/test/bs_construct_SUITE.erl +++ b/erts/emulator/test/bs_construct_SUITE.erl @@ -438,6 +438,8 @@ in_guard(Config) when is_list(Config) -> ?line 1 = in_guard(<<16#74ad:16>>, 16#e95, 5), ?line 2 = in_guard(<<16#3A,16#F7,"hello">>, 16#3AF7, <<"hello">>), ?line 3 = in_guard(<<16#FBCD:14,3.1415/float,3:2>>, 16#FBCD, 3.1415), + ?line 3 = in_guard(<<16#FBCD:14,3/float,3:2>>, 16#FBCD, 3), + ?line 3 = in_guard(<<16#FBCD:14,(2 bsl 226)/float,3:2>>, 16#FBCD, 2 bsl 226), nope = in_guard(<<1>>, 42, b), nope = in_guard(<<1>>, a, b), nope = in_guard(<<1,2>>, 1, 1), diff --git a/erts/emulator/test/bs_match_misc_SUITE.erl b/erts/emulator/test/bs_match_misc_SUITE.erl index 15427661f3..b0904acbe9 100644 --- a/erts/emulator/test/bs_match_misc_SUITE.erl +++ b/erts/emulator/test/bs_match_misc_SUITE.erl @@ -23,7 +23,8 @@ bound_var/1,bound_tail/1,t_float/1,little_float/1,sean/1, kenneth/1,encode_binary/1,native/1,happi/1, size_var/1,wiger/1,x0_context/1,huge_float_field/1, - writable_binary_matched/1,otp_7198/1,unordered_bindings/1]). + writable_binary_matched/1,otp_7198/1,unordered_bindings/1, + float_middle_endian/1]). -include_lib("test_server/include/test_server.hrl"). @@ -33,7 +34,7 @@ all() -> [bound_var, bound_tail, t_float, little_float, sean, kenneth, encode_binary, native, happi, size_var, wiger, x0_context, huge_float_field, writable_binary_matched, - otp_7198, unordered_bindings]. + otp_7198, unordered_bindings, float_middle_endian]. groups() -> []. @@ -92,6 +93,13 @@ t_float(Config) when is_list(Config) -> ok. +float_middle_endian(Config) when is_list(Config) -> + F = 9007199254740990.0, % turns to -NaN when word-swapped + ?line fcmp(F, match_float(<<F:64/float>>, 64, 0)), + ?line fcmp(F, match_float(<<1:1,F:64/float,127:7>>, 64, 1)), + ?line fcmp(F, match_float(<<1:13,F:64/float,127:3>>, 64, 13)), + ok. + fcmp(F1, F2) when (F1 - F2) / F2 < 0.0000001 -> ok. diff --git a/erts/emulator/test/busy_port_SUITE.erl b/erts/emulator/test/busy_port_SUITE.erl index 2c63296b83..4b4af0babe 100644 --- a/erts/emulator/test/busy_port_SUITE.erl +++ b/erts/emulator/test/busy_port_SUITE.erl @@ -170,6 +170,7 @@ send_3(Config) when is_list(Config) -> ?line {Owner,Slave} = get_slave(), ?line ok = erlang:send(Slave, {Owner,{command,"set busy"}}, [nosuspend]), + receive after 100 -> ok end, % ensure command reached port ?line nosuspend = erlang:send(Slave, {Owner,{command,"busy"}}, [nosuspend]), ?line unlock_slave(), @@ -563,6 +564,7 @@ scheduling_delay_busy_nosuspend(Config) -> {2,{call,[{var,1},open_port]}}, {0,{cast,[{var,1},{command,1,100}]}}, {0,{cast,[{var,1},{busy,2}]}}, + {0,{timer,sleep,[200]}}, % ensure reached port {10,{call,[{var,1},{command,3,[nosuspend]}]}}, {0,{timer,sleep,[200]}}, {0,{erlang,port_command,[{var,2},<<$N>>,[force]]}}, diff --git a/erts/emulator/test/code_parallel_load_SUITE.erl b/erts/emulator/test/code_parallel_load_SUITE.erl index aa9e4c96c6..1cfe015ea6 100644 --- a/erts/emulator/test/code_parallel_load_SUITE.erl +++ b/erts/emulator/test/code_parallel_load_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012. All Rights Reserved. +%% Copyright Ericsson AB 2012-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -16,7 +16,6 @@ %% %% %CopyrightEnd% %% -%% Author: Björn-Egil Dahlberg -module(code_parallel_load_SUITE). -export([ diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index dfba7d098f..104bdf8aec 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -2582,10 +2582,9 @@ driver_alloc_size() -> MemInfo -> CS = lists:foldl( fun ({instance, _, L}, Acc) -> - {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L), {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), - [SBMBCS,MBCS,SBCS | Acc] + [MBCS,SBCS | Acc] end, [], MemInfo), diff --git a/erts/emulator/test/driver_SUITE_data/thr_alloc_drv.c b/erts/emulator/test/driver_SUITE_data/thr_alloc_drv.c index 95a6ae9bdf..500725a7cd 100644 --- a/erts/emulator/test/driver_SUITE_data/thr_alloc_drv.c +++ b/erts/emulator/test/driver_SUITE_data/thr_alloc_drv.c @@ -58,7 +58,7 @@ void * test_thread(void *vsize) { int i; - int size = (int) (long) vsize; + int size = (int) (ErlDrvSInt) vsize; void *mem; mem = driver_alloc(size); if (mem) @@ -88,7 +88,7 @@ ErlDrvSSizeT control(ErlDrvData drv_data, unsigned int command, char *buf, res = erl_drv_thread_create("test_thread", &tid, test_thread, - (void *) (long) size, + (void *) (ErlDrvSInt) size, NULL); if (res == 0) { res = erl_drv_thread_join(tid, NULL); diff --git a/erts/emulator/test/efile_SUITE.erl b/erts/emulator/test/efile_SUITE.erl index 65367eab98..f79bb761d1 100644 --- a/erts/emulator/test/efile_SUITE.erl +++ b/erts/emulator/test/efile_SUITE.erl @@ -19,16 +19,16 @@ -module(efile_SUITE). -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2]). --export([iter_max_files/1]). +-export([iter_max_files/1, async_dist/1]). --export([do_iter_max_files/2]). +-export([do_iter_max_files/2, do_async_dist/1]). -include_lib("test_server/include/test_server.hrl"). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [iter_max_files]. + [iter_max_files, async_dist]. groups() -> []. @@ -45,6 +45,84 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. +do_async_dist(Dir) -> + X = 100, + AT = erlang:system_info(thread_pool_size), + Keys = file_keys(Dir,AT*X,[],[]), + Tab = ets:new(x,[ordered_set]), + [ ets:insert(Tab,{N,0}) || N <- lists:seq(0,AT-1) ], + [ ets:update_counter(Tab,(N rem AT),1) || N <- Keys ], + Res = [ V || {_,V} <- ets:tab2list(Tab) ], + ets:delete(Tab), + {Res, sdev(Res)/X}. + +sdev(List) -> + Len = length(List), + Mean = lists:sum(List)/Len, + math:sqrt(lists:sum([ (X - Mean) * (X - Mean) || X <- List ]) / Len). + +file_keys(_,0,FdList,FnList) -> + [ file:close(FD) || FD <- FdList ], + [ file:delete(FN) || FN <- FnList ], + []; +file_keys(Dir,Num,FdList,FnList) -> + Name = "dummy"++integer_to_list(Num), + FN = filename:join([Dir,Name]), + case file:open(FN,[write,raw]) of + {ok,FD} -> + {file_descriptor,prim_file,{Port,_}} = FD, + <<X:32/integer-big>> = + iolist_to_binary(erlang:port_control(Port,$K,[])), + [X | file_keys(Dir,Num-1,[FD|FdList],[FN|FnList])]; + {error,_} -> + % Try freeing up FD's if there are any + case FdList of + [] -> + exit({cannot_open_file,FN}); + _ -> + [ file:close(FD) || FD <- FdList ], + [ file:delete(F) || F <- FnList ], + file_keys(Dir,Num,[],[]) + end + end. + +async_dist(doc) -> + "Check that the distribution of files over async threads is fair"; +async_dist(Config) when is_list(Config) -> + DataDir = ?config(data_dir,Config), + TestFile = filename:join(DataDir, "existing_file"), + Dir = filename:dirname(code:which(?MODULE)), + AsyncSizes = [7,10,100,255,256,64,63,65], + Max = 0.5, + + lists:foreach(fun(Size) -> + {ok,Node} = + test_server:start_node + (test_iter_max_files,slave, + [{args, + "+A "++integer_to_list(Size)++ + " -pa " ++ Dir}]), + {Distr,SD} = rpc:call(Node,?MODULE,do_async_dist, + [DataDir]), + test_server:stop_node(Node), + if + SD > Max -> + io:format("Bad async queue distribution for " + "~p async threads:~n" + " Standard deviation is ~p~n" + " Key distribution:~n ~lp~n", + [Size,SD,Distr]), + exit({bad_async_dist,Size,SD,Distr}); + true -> + io:format("OK async queue distribution for " + "~p async threads:~n" + " Standard deviation is ~p~n" + " Key distribution:~n ~lp~n", + [Size,SD,Distr]), + ok + end + end, AsyncSizes), + ok. %% %% Open as many files as possible. Do this several times and check @@ -98,7 +176,7 @@ open_files(Name) -> ?line case file:open(Name, [read,raw]) of {ok, Fd} -> [Fd| open_files(Name)]; - {error, Reason} -> -% io:format("Error reason: ~p", [Reason]), + {error, _Reason} -> +% io:format("Error reason: ~p", [_Reason]), [] end. diff --git a/erts/emulator/test/emulator_smoke.spec b/erts/emulator/test/emulator_smoke.spec new file mode 100644 index 0000000000..3219aeb823 --- /dev/null +++ b/erts/emulator/test/emulator_smoke.spec @@ -0,0 +1,3 @@ +{suites,"../emulator_test",[smoke_test_SUITE,time_SUITE]}. +{cases,"../emulator_test",crypto_SUITE,[t_md5]}. +{cases,"../emulator_test",float_SUITE,[fpe,cmp_integer]}.
\ No newline at end of file diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl index 8dbc6b6538..b56b7ce525 100644 --- a/erts/emulator/test/match_spec_SUITE.erl +++ b/erts/emulator/test/match_spec_SUITE.erl @@ -22,7 +22,7 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, not_run/1]). -export([test_1/1, test_2/1, test_3/1, bad_match_spec_bin/1, - trace_control_word/1, silent/1, silent_no_ms/1, + trace_control_word/1, silent/1, silent_no_ms/1, silent_test/1, ms_trace2/1, ms_trace3/1, boxed_and_small/1, destructive_in_test_bif/1, guard_exceptions/1, unary_plus/1, unary_minus/1, moving_labels/1]). @@ -55,7 +55,7 @@ all() -> case test_server:is_native(match_spec_SUITE) of false -> [test_1, test_2, test_3, bad_match_spec_bin, - trace_control_word, silent, silent_no_ms, ms_trace2, + trace_control_word, silent, silent_no_ms, silent_test, ms_trace2, ms_trace3, boxed_and_small, destructive_in_test_bif, guard_exceptions, unary_plus, unary_minus, fpe, moving_labels, @@ -501,6 +501,14 @@ silent_no_ms(Config) when is_list(Config) -> {trace,Tracee,return_to,{?MODULE,f3,2}}] end). +silent_test(doc) -> + ["Test that match_spec_test does not activate silent"]; +silent_test(_Config) -> + {flags,[]} = erlang:trace_info(self(),flags), + erlang:match_spec_test([],[{'_',[],[{silent,true}]}],trace), + {flags,[]} = erlang:trace_info(self(),flags). + + ms_trace2(doc) -> ["Test the match spec functions {trace/2}"]; ms_trace2(suite) -> []; diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index dcf58fa474..9a70e8646a 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -1368,10 +1368,9 @@ tmpmem() -> MemInfo -> MSBCS = lists:foldl( fun ({instance, _, L}, Acc) -> - {value,{_,SBMBCS}} = lists:keysearch(sbmbcs, 1, L), {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), - [SBMBCS,MBCS,SBCS | Acc] + [MBCS,SBCS | Acc] end, [], MemInfo), diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index c8f286f629..0c4a9f7e5c 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -40,7 +40,7 @@ typedef struct int ref_cnt; CallInfo* call_history; NifModPrivData* nif_mod; - union { ErlNifResourceType* t; long l; } rt_arr[2]; + union { ErlNifResourceType* t; void* vp; } rt_arr[2]; } PrivData; /* @@ -93,6 +93,23 @@ struct binary_resource { unsigned size; }; +static int get_pointer(ErlNifEnv* env, ERL_NIF_TERM term, void** pp) +{ + ErlNifUInt64 i64; + int r = enif_get_uint64(env, term, &i64); + if (r) { + *pp = (void*)i64; + } + return r; +} + +static ERL_NIF_TERM make_pointer(ErlNifEnv* env, void* p) +{ + ErlNifUInt64 i64 = (ErlNifUInt64) p; + return enif_make_uint64(env, i64); +} + + static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) { PrivData* data = enif_alloc(sizeof(PrivData)); @@ -224,15 +241,15 @@ static ERL_NIF_TERM call_history(ErlNifEnv* env, int argc, const ERL_NIF_TERM ar static ERL_NIF_TERM hold_nif_mod_priv_data(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { PrivData* data = (PrivData*) enif_priv_data(env); - unsigned long ptr_as_ulong; + void* ptr; - if (!enif_get_ulong(env,argv[0],&ptr_as_ulong)) { + if (!get_pointer(env,argv[0],&ptr)) { return enif_make_badarg(env); } if (data->nif_mod != NULL) { NifModPrivData_release(data->nif_mod); } - data->nif_mod = (NifModPrivData*) ptr_as_ulong; + data->nif_mod = (NifModPrivData*) ptr; return enif_make_int(env,++(data->nif_mod->ref_cnt)); } @@ -696,7 +713,7 @@ static ERL_NIF_TERM last_resource_dtor_call(ErlNifEnv* env, int argc, const ERL_ memcpy(enif_make_new_binary(env, resource_dtor_last_sz, &bin), resource_dtor_last_data, resource_dtor_last_sz); ret = enif_make_tuple3(env, - enif_make_long(env, (long)resource_dtor_last), + make_pointer(env, resource_dtor_last), bin, enif_make_int(env, resource_dtor_cnt)); } @@ -717,40 +734,40 @@ static ERL_NIF_TERM get_resource_type(ErlNifEnv* env, int argc, const ERL_NIF_TE if (!enif_get_int(env, argv[0], &ix) || ix >= 2) { return enif_make_badarg(env); } - return enif_make_long(env, data->rt_arr[ix].l); + return make_pointer(env, data->rt_arr[ix].vp); } static ERL_NIF_TERM alloc_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary data_bin; - union { ErlNifResourceType* t; long l; } type; - union { void* p; long l;} data; - if (!enif_get_long(env, argv[0], &type.l) + union { ErlNifResourceType* t; void* vp; } type; + void* data; + if (!get_pointer(env, argv[0], &type.vp) || !enif_inspect_binary(env, argv[1], &data_bin) - || (data.p = enif_alloc_resource(type.t, data_bin.size))==NULL) { + || (data = enif_alloc_resource(type.t, data_bin.size))==NULL) { return enif_make_badarg(env); } - memcpy(data.p, data_bin.data, data_bin.size); - return enif_make_long(env, data.l); + memcpy(data, data_bin.data, data_bin.size); + return make_pointer(env, data); } static ERL_NIF_TERM make_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - union { void* p; long l; } data; - if (!enif_get_long(env, argv[0], &data.l)) { + void* data; + if (!get_pointer(env, argv[0], &data)) { return enif_make_badarg(env); } - return enif_make_resource(env, data.p); + return enif_make_resource(env, data); } static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary data_bin; - union { ErlNifResourceType* t; long l; } type; + union { ErlNifResourceType* t; void* vp; } type; void* data; ERL_NIF_TERM ret; - if (!enif_get_long(env, argv[0], &type.l) + if (!get_pointer(env, argv[0], &type.vp) || !enif_inspect_binary(env, argv[1], &data_bin) || (data = enif_alloc_resource(type.t, data_bin.size))==NULL) { @@ -765,7 +782,7 @@ static ERL_NIF_TERM make_new_resource(ErlNifEnv* env, int argc, const ERL_NIF_TE static ERL_NIF_TERM make_new_resource_binary(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary data_bin; - union { struct binary_resource* p; void* vp; long l; } br; + union { struct binary_resource* p; void* vp; } br; void* buf; ERL_NIF_TERM ret; if (!enif_inspect_binary(env, argv[0], &data_bin) @@ -781,7 +798,7 @@ static ERL_NIF_TERM make_new_resource_binary(ErlNifEnv* env, int argc, const ERL memcpy(br.p->data, data_bin.data, data_bin.size); ret = enif_make_resource_binary(env, br.vp, br.p->data, br.p->size); enif_release_resource(br.p); - return enif_make_tuple2(env, enif_make_long(env,br.l), ret); + return enif_make_tuple2(env, make_pointer(env,br.vp), ret); } static void binary_resource_dtor(ErlNifEnv* env, void* obj) @@ -796,33 +813,33 @@ static void binary_resource_dtor(ErlNifEnv* env, void* obj) static ERL_NIF_TERM get_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary data_bin; - union { ErlNifResourceType* t; long l; } type; - union { void* p; long l; } data; + union { ErlNifResourceType* t; void* vp; } type; + void* data; type.t = NULL; if (enif_is_identical(argv[0], atom_binary_resource_type)) { type.t = binary_resource_type; } else { - enif_get_long(env, argv[0], &type.l); + get_pointer(env, argv[0], &type.vp); } if (type.t == NULL - || !enif_get_resource(env, argv[1], type.t, &data.p)) { + || !enif_get_resource(env, argv[1], type.t, &data)) { return enif_make_badarg(env); } - enif_alloc_binary(enif_sizeof_resource(data.p), &data_bin); - memcpy(data_bin.data, data.p, data_bin.size); - return enif_make_tuple2(env, enif_make_long(env,data.l), + enif_alloc_binary(enif_sizeof_resource(data), &data_bin); + memcpy(data_bin.data, data, data_bin.size); + return enif_make_tuple2(env, make_pointer(env,data), enif_make_binary(env, &data_bin)); } static ERL_NIF_TERM release_resource(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { - union { void* p; long l; } data; - if (!enif_get_long(env, argv[0], &data.l)) { + void* data; + if (!get_pointer(env, argv[0], &data)) { return enif_make_badarg(env); } - enif_release_resource(data.p); + enif_release_resource(data); return enif_make_atom(env,"ok"); } diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl index b92a0e2059..ff8d18eef8 100644 --- a/erts/emulator/test/num_bif_SUITE.erl +++ b/erts/emulator/test/num_bif_SUITE.erl @@ -350,12 +350,34 @@ t_integer_to_string(Config) when is_list(Config) -> (catch erlang:integer_to_list(Value)) end,[atom,1.2,0.0,[$1,[$2]]]), + %% Base-2 integers + test_its("0", 0, 2), + test_its("1", 1, 2), + test_its("110110", 54, 2), + test_its("-1000000", -64, 2), + %% Base-16 integers + test_its("0", 0, 16), + test_its("A", 10, 16), + test_its("D4BE", 54462, 16), + test_its("-D4BE", -54462, 16), + + lists:foreach(fun(Value) -> + {'EXIT', {badarg, _}} = + (catch erlang:integer_to_binary(Value, 8)), + {'EXIT', {badarg, _}} = + (catch erlang:integer_to_list(Value, 8)) + end,[atom,1.2,0.0,[$1,[$2]]]), + ok. test_its(List,Int) -> Int = list_to_integer(List), Int = binary_to_integer(list_to_binary(List)). +test_its(List,Int,Base) -> + Int = list_to_integer(List, Base), + Int = binary_to_integer(list_to_binary(List), Base). + %% Tests binary_to_integer/1. t_string_to_integer(Config) when is_list(Config) -> diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index 13aa0f4c00..fcd4457c34 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2012. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -92,7 +92,7 @@ spawn_driver/1, spawn_executable/1, close_deaf_port/1, unregister_name/1, parallelism_option/1]). --export([]). +-export([do_iter_max_ports/2]). %% Internal exports. -export([tps/3]). @@ -628,16 +628,23 @@ iter_max_ports(Config) when is_list(Config) -> iter_max_ports_test(Config) -> - Dog = test_server:timetrap(test_server:minutes(20)), + Dog = test_server:timetrap(test_server:minutes(30)), PortTest = port_test(Config), Command = lists:concat([PortTest, " -h0 -q"]), Iters = case os:type() of {win32,_} -> 4; _ -> 10 end, - L = do_iter_max_ports(Iters, Command), + %% Run on a different node in order to limit the effect if this test fails. + Dir = filename:dirname(code:which(?MODULE)), + {ok,Node} = test_server:start_node(test_iter_max_socks,slave, + [{args,"+Q 2048 -pa " ++ Dir}]), + L = rpc:call(Node,?MODULE,do_iter_max_ports,[Iters, Command]), + test_server:stop_node(Node), + io:format("Result: ~p",[L]), all_equal(L), + all_equal(L), test_server:timetrap_cancel(Dog), {comment, "Max ports: " ++ integer_to_list(hd(L))}. @@ -670,7 +677,7 @@ close_ports([]) -> ok. open_ports(Name, Settings) -> - test_server:sleep(50), + test_server:sleep(5), case catch open_port(Name, Settings) of P when is_port(P) -> [P| open_ports(Name, Settings)]; diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 863cd2d654..e3aae17df4 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -90,9 +90,19 @@ groups() -> otp_7738_resume]}]. init_per_suite(Config) -> - Config. + A0 = case application:start(sasl) of + ok -> [sasl]; + _ -> [] + end, + A = case application:start(os_mon) of + ok -> [os_mon|A0]; + _ -> A0 + end, + [{started_apps, A}|Config]. end_per_suite(Config) -> + As = ?config(started_apps, Config), + lists:foreach(fun (A) -> application:stop(A) end, As), catch erts_debug:set_internal_state(available_internal_state, false), Config. @@ -102,7 +112,6 @@ init_per_group(_GroupName, Config) -> end_per_group(_GroupName, Config) -> Config. - init_per_testcase(Func, Config) when is_atom(Func), is_list(Config) -> Dog=?t:timetrap(?t:minutes(10)), [{watchdog, Dog},{testcase, Func}|Config]. @@ -1379,6 +1388,9 @@ processes_large_tab(doc) -> processes_large_tab(suite) -> []; processes_large_tab(Config) when is_list(Config) -> + sys_mem_cond_run(2048, fun () -> processes_large_tab_test(Config) end). + +processes_large_tab_test(Config) -> enable_internal_state(), MaxDbgLvl = 20, MinProcTabSize = 2*(1 bsl 15), @@ -1430,6 +1442,9 @@ processes_default_tab(doc) -> processes_default_tab(suite) -> []; processes_default_tab(Config) when is_list(Config) -> + sys_mem_cond_run(1024, fun () -> processes_default_tab_test(Config) end). + +processes_default_tab_test(Config) -> {ok, DefaultNode} = start_node(Config, ""), Res = rpc:call(DefaultNode, ?MODULE, processes_bif_test, []), stop_node(DefaultNode), @@ -1452,7 +1467,7 @@ processes_this_tab(doc) -> processes_this_tab(suite) -> []; processes_this_tab(Config) when is_list(Config) -> - chk_processes_bif_test_res(processes_bif_test()). + sys_mem_cond_run(1024, fun () -> chk_processes_bif_test_res(processes_bif_test()) end). chk_processes_bif_test_res(ok) -> ok; chk_processes_bif_test_res({comment, _} = Comment) -> Comment; @@ -1494,6 +1509,7 @@ processes_bif_cleaner() -> spawn_initial_hangarounds(Cleaner) -> TabSz = erlang:system_info(process_limit), + erts_debug:set_internal_state(next_pid,TabSz), spawn_initial_hangarounds(Cleaner, TabSz, TabSz*2, @@ -1538,14 +1554,21 @@ hangaround(Cleaner, Type) -> spawn_initial_hangarounds(_Cleaner, NP, Max, Len, HAs) when NP > Max -> {Len, HAs}; spawn_initial_hangarounds(Cleaner, NP, Max, Len, HAs) -> - erts_debug:set_internal_state(next_pid,NP), + Skip = 30, HA1 = spawn_opt(?MODULE, hangaround, [Cleaner, initial_hangaround], [{priority, low}]), HA2 = spawn_opt(?MODULE, hangaround, [Cleaner, initial_hangaround], [{priority, normal}]), HA3 = spawn_opt(?MODULE, hangaround, [Cleaner, initial_hangaround], [{priority, high}]), - spawn_initial_hangarounds(Cleaner, NP+30, Max, Len+3, [HA1,HA2,HA3|HAs]). + spawn_drop(Skip), + spawn_initial_hangarounds(Cleaner, NP+Skip, Max, Len+3, [HA1,HA2,HA3|HAs]). + +spawn_drop(N) when N =< 0 -> + ok; +spawn_drop(N) -> + spawn(fun () -> ok end), + spawn_drop(N-1). do_processes(WantReds) -> erts_debug:set_internal_state(reds_left, WantReds), @@ -2087,6 +2110,9 @@ otp_7738_resume(Config) when is_list(Config) -> otp_7738_test(resume). otp_7738_test(Type) -> + sys_mem_cond_run(3072, fun () -> do_otp_7738_test(Type) end). + +do_otp_7738_test(Type) -> T = self(), S = spawn_link(fun () -> receive @@ -2231,3 +2257,31 @@ enable_internal_state() -> true -> true; _ -> erts_debug:set_internal_state(available_internal_state, true) end. + +sys_mem_cond_run(ReqSizeMB, TestFun) when is_integer(ReqSizeMB) -> + case total_memory() of + TotMem when is_integer(TotMem), TotMem >= ReqSizeMB -> + TestFun(); + TotMem when is_integer(TotMem) -> + {skipped, "Not enough memory ("++integer_to_list(TotMem)++" MB)"}; + undefined -> + {skipped, "Could not retrieve memory information"} + end. + + +total_memory() -> + %% Totat memory in MB. + try + MemoryData = memsup:get_system_memory_data(), + case lists:keysearch(total_memory, 1, MemoryData) of + {value, {total_memory, TM}} -> + TM div (1024*1024); + false -> + {value, {system_total_memory, STM}} = + lists:keysearch(system_total_memory, 1, MemoryData), + STM div (1024*1024) + end + catch + _ : _ -> + undefined + end. diff --git a/erts/emulator/test/receive_SUITE.erl b/erts/emulator/test/receive_SUITE.erl index b070e2b986..2e7ac1f50c 100644 --- a/erts/emulator/test/receive_SUITE.erl +++ b/erts/emulator/test/receive_SUITE.erl @@ -59,22 +59,25 @@ end_per_testcase(_Func, Config) -> ?t:timetrap_cancel(Dog). call_with_huge_message_queue(Config) when is_list(Config) -> - ?line Pid = spawn_link(fun echo_loop/0), + Pid = spawn_link(fun echo_loop/0), - ?line {Time,ok} = tc(fun() -> calls(10, Pid) end), + {Time,ok} = tc(fun() -> calls(10, Pid) end), - ?line [self() ! {msg,N} || N <- lists:seq(1, 500000)], + [self() ! {msg,N} || N <- lists:seq(1, 500000)], erlang:garbage_collect(), - ?line {NewTime,ok} = tc(fun() -> calls(10, Pid) end), + {NewTime1,ok} = tc(fun() -> calls(10, Pid) end), + {NewTime2,ok} = tc(fun() -> calls(10, Pid) end), + io:format("Time for empty message queue: ~p", [Time]), - io:format("Time for huge message queue: ~p", [NewTime]), + io:format("Time1 for huge message queue: ~p", [NewTime1]), + io:format("Time2 for huge message queue: ~p", [NewTime2]), - case (NewTime+1) / (Time+1) of + case hd(lists:sort([(NewTime1+1) / (Time+1), (NewTime2+1) / (Time+1)])) of Q when Q < 10 -> ok; Q -> - io:format("Q = ~p", [Q]), - ?line ?t:fail() + io:format("Best Q = ~p", [Q]), + ?t:fail() end, ok. @@ -95,8 +98,8 @@ call(Pid, Msg) -> end. receive_in_between(Config) when is_list(Config) -> - ?line Pid = spawn_link(fun echo_loop/0), - ?line [{ok,{a,b}} = call2(Pid, {a,b}) || _ <- lists:seq(1, 100000)], + Pid = spawn_link(fun echo_loop/0), + [{ok,{a,b}} = call2(Pid, {a,b}) || _ <- lists:seq(1, 100000)], ok. call2(Pid, Msg) -> diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index 8931562828..81539faa09 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -52,6 +52,7 @@ update_cpu_info/1, sct_cmd/1, sbt_cmd/1, + scheduler_threads/1, scheduler_suspend/1, reader_groups/1]). @@ -66,7 +67,7 @@ all() -> equal_with_part_time_max, equal_and_high_with_part_time_max, equal_with_high, equal_with_high_max, bound_process, - {group, scheduler_bind}, scheduler_suspend, + {group, scheduler_bind}, scheduler_threads, scheduler_suspend, reader_groups]. groups() -> @@ -1039,7 +1040,66 @@ sbt_test(Config, CpuTCmd, ClBt, Bt, LP) -> tuple_to_list(SB)), ?line stop_node(Node), ?line ok. - + +scheduler_threads(Config) when is_list(Config) -> + SmpSupport = erlang:system_info(smp_support), + {Sched, SchedOnln, _} = get_sstate(Config, ""), + %% Configure half the number of both the scheduler threads and + %% the scheduler threads online. + {HalfSched, HalfSchedOnln} = case SmpSupport of + false -> {1,1}; + true -> + {Sched div 2, + SchedOnln div 2} + end, + {HalfSched, HalfSchedOnln, _} = get_sstate(Config, "+SP 50:50"), + %% Use +S to configure 4x the number of scheduler threads and + %% 4x the number of scheduler threads online, but alter that + %% setting using +SP to 50% scheduler threads and 25% scheduler + %% threads online. The result should be 2x scheduler threads and + %% 1x scheduler threads online. + TwiceSched = case SmpSupport of + false -> 1; + true -> Sched*2 + end, + FourSched = integer_to_list(Sched*4), + FourSchedOnln = integer_to_list(SchedOnln*4), + CombinedCmd1 = "+S "++FourSched++":"++FourSchedOnln++" +SP50:25", + {TwiceSched, SchedOnln, _} = get_sstate(Config, CombinedCmd1), + %% Now do the same test but with the +S and +SP options in the + %% opposite order, since order shouldn't matter. + CombinedCmd2 = "+SP50:25 +S "++FourSched++":"++FourSchedOnln, + {TwiceSched, SchedOnln, _} = get_sstate(Config, CombinedCmd2), + %% Apply two +SP options to make sure the second overrides the first + TwoCmd = "+SP 25:25 +SP 100:100", + {Sched, SchedOnln, _} = get_sstate(Config, TwoCmd), + %% Configure 50% of scheduler threads online only + {Sched, HalfSchedOnln, _} = get_sstate(Config, "+SP:50"), + %% Configure 2x scheduler threads only + {TwiceSched, SchedOnln, _} = get_sstate(Config, "+SP 200"), + %% Test resetting the scheduler counts + ResetCmd = "+S "++FourSched++":"++FourSchedOnln++" +S 0:0", + {Sched, SchedOnln, _} = get_sstate(Config, ResetCmd), + %% Test negative +S settings, but only for SMP-enabled emulators + case SmpSupport of + false -> ok; + true -> + SchedMinus1 = Sched-1, + SchedOnlnMinus1 = SchedOnln-1, + {SchedMinus1, SchedOnlnMinus1, _} = get_sstate(Config, "+S -1"), + {Sched, SchedOnlnMinus1, _} = get_sstate(Config, "+S :-1"), + {SchedMinus1, SchedOnlnMinus1, _} = get_sstate(Config, "+S -1:-1") + end, + ok. + +get_sstate(Config, Cmd) -> + {ok, Node} = start_node(Config, Cmd), + [SState] = mcall(Node, [fun () -> + erlang:system_info(schedulers_state) + end]), + stop_node(Node), + SState. + scheduler_suspend(Config) when is_list(Config) -> ?line Dog = ?t:timetrap(?t:minutes(5)), ?line lists:foreach(fun (S) -> scheduler_suspend_test(Config, S) end, diff --git a/erts/emulator/test/send_term_SUITE.erl b/erts/emulator/test/send_term_SUITE.erl index ba0ba804ca..b631f55a03 100644 --- a/erts/emulator/test/send_term_SUITE.erl +++ b/erts/emulator/test/send_term_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2005-2011. All Rights Reserved. +%% Copyright Ericsson AB 2005-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -175,10 +175,6 @@ chk_temp_alloc() -> %% Verify that we havn't got anything allocated by temp_alloc lists:foreach( fun ({instance, _, TI}) -> - ?line {value, {sbmbcs, SBMBCInfo}} - = lists:keysearch(sbmbcs, 1, TI), - ?line {value, {blocks, 0, _, _}} - = lists:keysearch(blocks, 1, SBMBCInfo), ?line {value, {mbcs, MBCInfo}} = lists:keysearch(mbcs, 1, TI), ?line {value, {blocks, 0, _, _}} diff --git a/erts/emulator/test/smoke_test_SUITE.erl b/erts/emulator/test/smoke_test_SUITE.erl index 6f5c2080c0..10b7e16a74 100644 --- a/erts/emulator/test/smoke_test_SUITE.erl +++ b/erts/emulator/test/smoke_test_SUITE.erl @@ -26,14 +26,14 @@ init_per_group/2,end_per_group/2, init_per_testcase/2, end_per_testcase/2]). --export([boot_combo/1]). +-export([boot_combo/1, native_atomics/1, jump_table/1]). -define(DEFAULT_TIMEOUT, ?t:minutes(2)). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [boot_combo]. + [boot_combo, native_atomics, jump_table]. groups() -> []. @@ -105,6 +105,41 @@ boot_combo(Config) when is_list(Config) -> end) end. +native_atomics(Config) when is_list(Config) -> + NA32Key = "32-bit native atomics", + NA64Key = "64-bit native atomics", + DWNAKey = "Double word native atomics", + EthreadInfo = erlang:system_info(ethread_info), + ?t:format("~p~n", [EthreadInfo]), + {value,{NA32Key, NA32, _}} = lists:keysearch(NA32Key, 1, EthreadInfo), + {value,{NA64Key, NA64, _}} = lists:keysearch(NA64Key, 1, EthreadInfo), + {value,{DWNAKey, DWNA, _}} = lists:keysearch(DWNAKey, 1, EthreadInfo), + case {erlang:system_info(build_type), erlang:system_info(smp_support), NA32, NA64, DWNA} of + {opt, true, "no", "no", _} -> + ?t:fail(optimized_smp_runtime_without_native_atomics); + {_, false, "no", "no", _} -> + {comment, "No native atomics"}; + _ -> + {comment, + NA32 ++ " 32-bit, " + ++ NA64 ++ " 64-bit, and " + ++ DWNA ++ " double word native atomics"} + end. + +jump_table(Config) when is_list(Config) -> + case erlang:system_info(beam_jump_table) of + true -> + ok; + false -> + case erlang:system_info(build_type) of + opt -> + ?t:fail(optimized_without_beam_jump_table); + BT -> + {comment, "No beam jump table, but build type is " ++ atom_to_list(BT)} + end + end. + + %%% %%% Aux functions -------------------------------------------------------------- %%% diff --git a/erts/emulator/test/system_profile_SUITE.erl b/erts/emulator/test/system_profile_SUITE.erl index ba94a371be..a387c08ef9 100644 --- a/erts/emulator/test/system_profile_SUITE.erl +++ b/erts/emulator/test/system_profile_SUITE.erl @@ -198,7 +198,9 @@ check_multi_scheduling_block(Nodes) -> Pid = start_profiler_process(), undefined = erlang:system_profile(Pid, [scheduler]), {ok, Supervisor} = start_load(Nodes), + wait(600), erlang:system_flag(multi_scheduling, block), + wait(600), erlang:system_flag(multi_scheduling, unblock), {Pid, [scheduler]} = erlang:system_profile(undefined, []), Events = get_profiler_events(), @@ -213,7 +215,6 @@ check_block_system(Nodes) -> Pid = start_profiler_process(), undefined = erlang:system_profile(Pid, [scheduler]), {ok, Supervisor} = start_load(Nodes), - % FIXME: remove wait !! wait(300), undefined = erlang:system_monitor(Dummy, [busy_port]), {Dummy, [busy_port]} = erlang:system_monitor(undefined, []), diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl index 221b65309a..0f513f0dcb 100644 --- a/erts/emulator/test/trace_SUITE.erl +++ b/erts/emulator/test/trace_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1997-2011. All Rights Reserved. +%% Copyright Ericsson AB 1997-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -34,6 +34,7 @@ system_monitor_args/1, more_system_monitor_args/1, system_monitor_long_gc_1/1, system_monitor_long_gc_2/1, system_monitor_large_heap_1/1, system_monitor_large_heap_2/1, + system_monitor_long_schedule/1, bad_flag/1, trace_delivered/1]). -include_lib("test_server/include/test_server.hrl"). @@ -52,6 +53,7 @@ all() -> set_on_first_spawn, system_monitor_args, more_system_monitor_args, system_monitor_long_gc_1, system_monitor_long_gc_2, system_monitor_large_heap_1, + system_monitor_long_schedule, system_monitor_large_heap_2, bad_flag, trace_delivered]. groups() -> @@ -508,6 +510,65 @@ try_l(Val) -> ?line {Self,Comb1} = erlang:system_monitor(undefined), ?line [{large_heap,Val},{long_gc,Arbitrary2}] = lists:sort(Comb1). +monitor_sys(Parent) -> + receive + {monitor,Pid,long_schedule,Data} when is_pid(Pid) -> + io:format("Long schedule of ~w: ~w~n",[Pid,Data]), + Parent ! {Pid,Data}, + monitor_sys(Parent); + {monitor,Port,long_schedule,Data} when is_port(Port) -> + {name,Name} = erlang:port_info(Port,name), + io:format("Long schedule of ~w (~p): ~w~n",[Port,Name,Data]), + Parent ! {Port,Data}, + monitor_sys(Parent); + Other -> + erlang:display(Other) + end. + +start_monitor() -> + Parent = self(), + Mpid = spawn_link(fun() -> monitor_sys(Parent) end), + erlang:system_monitor(Mpid,[{long_schedule,100}]), + erlang:yield(), % Need to be rescheduled for the trace to take + ok. + +system_monitor_long_schedule(suite) -> + []; +system_monitor_long_schedule(doc) -> + ["Tests erlang:system_monitor(Pid, [{long_schedule,Time}])"]; +system_monitor_long_schedule(Config) when is_list(Config) -> + Path = ?config(data_dir, Config), + erl_ddll:start(), + case (catch load_driver(Path, slow_drv)) of + ok -> + do_system_monitor_long_schedule(); + _Error -> + {skip, "Unable to load slow_drv (windows or no usleep()?)"} + end. +do_system_monitor_long_schedule() -> + start_monitor(), + Port = open_port({spawn_driver,slow_drv}, []), + "ok" = erlang:port_control(Port,0,[]), + Self = self(), + receive + {Self,L} when is_list(L) -> + ok + after 1000 -> + ?t:fail(no_trace_of_pid) + end, + "ok" = erlang:port_control(Port,1,[]), + "ok" = erlang:port_control(Port,2,[]), + receive + {Port,LL} when is_list(LL) -> + ok + after 1000 -> + ?t:fail(no_trace_of_port) + end, + port_close(Port), + erlang:system_monitor(undefined), + ok. + + -define(LONG_GC_SLEEP, 670). system_monitor_long_gc_1(suite) -> @@ -1521,3 +1582,11 @@ issue_non_empty_runq_warning(DeadLine, RQLen) -> " Processes info: ~p~n", [DeadLine div 1000, RQLen, self(), PIs]), receive after 1000 -> ok end. + +load_driver(Dir, Driver) -> + case erl_ddll:load_driver(Dir, Driver) of + ok -> ok; + {error, Error} = Res -> + io:format("~s\n", [erl_ddll:format_error(Error)]), + Res + end. diff --git a/erts/emulator/utils/make_compiler_flags b/erts/emulator/utils/make_compiler_flags new file mode 100755 index 0000000000..cebe8cd0c5 --- /dev/null +++ b/erts/emulator/utils/make_compiler_flags @@ -0,0 +1,87 @@ +#!/usr/bin/env perl +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1999-2009. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# +use strict; +use File::Copy; +# This program generates global constants that contains +# config.h, CFLAGS and LDFLAGS + +my $file = ""; +my %constants = (); +my $prev_file = ""; + +while (@ARGV) { + my $d = shift; + if ( $d =~ /^-o$/ ) { + $file = shift or die("-o requires argument"); + open FILE, "<$file" or next; + $prev_file = do { local $/; <FILE> }; + close FILE; + next; + } + if ( $d =~ /^-f/ ) { + my $var = shift or die("-f requires two argument"); + my $value = shift or die("-f requires two argument"); + open FILE, "<$value"; + $value = do { local $/; <FILE> }; + close FILE; + + $value =~ s/\n/\\n\\\n/g; + + $constants{$var} = $value; + } + if ( $d =~ /^-v/ ) { + my $var = shift or die("-v requires two argument"); + my $value = shift; + $constants{$var} = $value; + } +} + +foreach(keys %constants) { + my $value = $constants{$_}; + $value =~ s/"/\\"/g; + $constants{$_} = $value +} + +# Did we want output to a file? +open(my $oldout, ">&STDOUT") or die "Can't dup STDOUT: $!"; +if ( $file ) { + open STDOUT, ">$file.tmp" or die("can't open $file for writing"); +} + +my(@prog) = split('/', $0); +my($prog) = $prog[$#prog]; +print "/* Warning: Do not edit this file.\n"; +print " Auto-generated by '$prog'.*/\n"; + +foreach(keys %constants) { + print "const char* erts_build_flags_$_ = \"$constants{$_}\";\n" +} + +open(STDOUT, ">&", $oldout) or die "Can't dup \$oldout: $!"; + +open FILE, "<$file.tmp"; +my $new_file = do { local $/; <FILE> }; +close FILE; + +if ($new_file ne $prev_file) { + move("$file.tmp","$file"); +} else { + unlink("$file.tmp"); +} diff --git a/erts/emulator/valgrind/suppress.standard b/erts/emulator/valgrind/suppress.standard index beecf1a7b5..a4da31a61d 100644 --- a/erts/emulator/valgrind/suppress.standard +++ b/erts/emulator/valgrind/suppress.standard @@ -174,6 +174,7 @@ obj:*/crypto.valgrind.* { Crypto internal... Memcheck:Cond +... obj:*/libcrypto.* } { @@ -194,6 +195,7 @@ obj:*/crypto.valgrind.* { Crypto internal... Memcheck:Value8 +... obj:*/libcrypto.* } { diff --git a/erts/etc/Makefile b/erts/etc/Makefile index 2b32b8ae50..5b54ef9c3e 100644 --- a/erts/etc/Makefile +++ b/erts/etc/Makefile @@ -18,10 +18,11 @@ # include $(ERL_TOP)/make/target.mk - SUB_DIRECTORIES = common ifeq ($(TARGET),win32) SUB_DIRECTORIES += win32 +else +SUB_DIRECTORIES += unix endif include $(ERL_TOP)/make/otp_subdir.mk diff --git a/erts/etc/common/ct_run.c b/erts/etc/common/ct_run.c index 5e5b612a12..853785dcd1 100644 --- a/erts/etc/common/ct_run.c +++ b/erts/etc/common/ct_run.c @@ -432,18 +432,6 @@ strsave(char* string) return p; } -/* Instead of making sure basename exists, we do our own */ -static char *simple_basename(char *path) -{ - char *ptr; - for (ptr = path; *ptr != '\0'; ++ptr) { - if (*ptr == '/' || *ptr == '\\') { - path = ptr + 1; - } - } - return path; -} - static char* get_default_emulator(char* progname) { diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index 31d9b2e0ad..30560f5a2f 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -64,7 +64,6 @@ static const char plusM_au_allocs[]= { 'u', /* all alloc_util allocators */ 'B', /* binary_alloc */ - 'C', /* sbmbc_alloc */ 'D', /* std_alloc */ 'E', /* ets_alloc */ 'F', /* fix_alloc */ @@ -80,6 +79,7 @@ static const char plusM_au_allocs[]= { static char *plusM_au_alloc_switches[] = { "as", "asbcst", + "acul", "e", "t", "lmbcs", @@ -95,8 +95,6 @@ static char *plusM_au_alloc_switches[] = { "rsbcst", "sbct", "smbcs", - "sbmbcs", - "sbmbct", NULL }; @@ -124,6 +122,7 @@ static char *pluss_val_switches[] = { "bwt", "cl", "ct", + "fwi", "tbt", "wct", "wt", @@ -800,11 +799,11 @@ int main(int argc, char **argv) case 'a': case 'A': case 'b': + case 'e': case 'i': case 'n': case 'P': case 'Q': - case 'S': case 't': case 'T': case 'R': @@ -819,6 +818,19 @@ int main(int argc, char **argv) add_Eargs(argv[i+1]); i++; break; + case 'S': + if (argv[i][2] == 'P') { + if (argv[i][3] != '\0') + goto the_default; + } else if (argv[i][2] != '\0') + goto the_default; + if (i+1 >= argc) + usage(argv[i]); + argv[i][0] = '-'; + add_Eargs(argv[i]); + add_Eargs(argv[i+1]); + i++; + break; case 'B': argv[i][0] = '-'; if (argv[i][2] != '\0') { @@ -1120,7 +1132,9 @@ usage_aux(void) "[+l] [+M<SUBSWITCH> <ARGUMENT>] [+P MAX_PROCS] [+Q MAX_PORTS] " "[+R COMPAT_REL] " "[+r] [+rg READER_GROUPS_LIMIT] [+s SCHEDULER_OPTION] " - "[+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] [+T LEVEL] [+V] [+v] " + "[+S NO_SCHEDULERS:NO_SCHEDULERS_ONLINE] " + "[+SP PERCENTAGE_SCHEDULERS:PERCENTAGE_SCHEDULERS_ONLINE] " + "[+T LEVEL] [+V] [+v] " "[+W<i|w>] [+z MISC_OPTION] [args ...]\n"); exit(1); } diff --git a/erts/etc/unix/Makefile b/erts/etc/unix/Makefile new file mode 100644 index 0000000000..e85d2fab0c --- /dev/null +++ b/erts/etc/unix/Makefile @@ -0,0 +1,46 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2013. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +include $(ERL_TOP)/make/output.mk +include $(ERL_TOP)/make/target.mk + +include $(ERL_TOP)/make/$(TARGET)/otp.mk +include ../../vsn.mk + +opt debug: etc + +.PHONY: etc +etc: etp-commands + +etp-commands: etp-commands.in + sed 's:@ERL_TOP@:${ERL_TOP}:g' etp-commands.in > etp-commands + +.PHONY: docs +docs: + +.PHONY: clean +clean: + +# ---------------------------------------------------- +# Release Target +# ---------------------------------------------------- +include $(ERL_TOP)/make/otp_release_targets.mk + +.PHONY: release_spec +release_spec: etc
\ No newline at end of file diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index cc7d77fd9a..41baa323ed 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -2,7 +2,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2003-2012. All Rights Reserved. +# Copyright Ericsson AB 2003-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -31,6 +31,8 @@ # -debug Run debug compiled emulator # -gdb Run the debug compiled emulator in emacs and gdb. # You have to start beam in gdb using "run". +# -rgdb Run the debug compiled emulator in gdb. +# You have to start beam in gdb using "run". # -break F Run the debug compiled emulator in emacs and gdb and set break. # The session is started, i.e. "run" is already don for you. # -xxgdb FIXME currently disabled @@ -171,8 +173,17 @@ while [ $# -gt 0 ]; do cargs="$cargs -debug" TYPE=.debug ;; + "-frmptr") + shift + cargs="$cargs -frmptr" + TYPE=.frmptr + ;; "-gdb") shift + GDB=egdb + ;; + "-rgdb") + shift GDB=gdb ;; "-break") @@ -183,6 +194,12 @@ while [ $# -gt 0 ]; do ;; "-core") shift + GDB=egdb + core="$1" + shift + ;; + "-rcore") + shift GDB=gdb core="$1" shift @@ -266,6 +283,19 @@ if [ "x$GDB" = "x" ]; then else valgrind_misc_flags="$VALGRIND_MISC_FLAGS" fi + if which taskset > /dev/null && test -e /proc/cpuinfo; then + # We only let valgrind utilize one core with "taskset 1" as it can be very slow + # on multiple cores (especially with async threads). Valgrind only run one pthread + # at a time anyway so there is no point letting it utilize more than one core. + # Use $sched_arg to force all schedulers online to emulate multicore. + taskset1="taskset 1" + ncpu=`cat /proc/cpuinfo | grep -w processor | wc -l` + sched_arg="-S$ncpu:$ncpu" + else + taskset1= + sched_arg= + fi + beam_args=`$EXEC -emu_args_exit ${1+"$@"}` # Time for some argument passing voodoo: @@ -276,10 +306,31 @@ if [ "x$GDB" = "x" ]; then ' set -- $beam_args IFS="$SAVE_IFS" - exec valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $emu_xargs "$@" -pz $PRELOADED + exec $taskset1 valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $sched_arg $emu_xargs "$@" -pz $PRELOADED else exec $EXEC $eeargs $xargs ${1+"$@"} fi +elif [ "x$GDB" = "xgdb" ]; then + case "x$core" in + x) + # Get emu args to use from erlexec... + beam_args=`$EXEC -emu_args_exit ${1+"$@"}` + gdbcmd="--args $EMU_NAME $beam_args" + ;; + x/*) + gdbcmd="$EMU_NAME ${core}" + GDBBP= + ;; + *) + dir=`pwd` + gdbcmd="$EMU_NAME ${dir}/${core}" + GDBBP= + ;; + esac + cmdfile="/tmp/.cerlgdb.$$" + echo "source $ROOTDIR/erts/etc/unix/etp-commands" > $cmdfile + # Fire up gdb in emacs... + exec gdb $GDBBP -x $cmdfile $gdbcmd else if [ "x$EMACS" = "x" ]; then EMACS=emacs diff --git a/erts/etc/unix/etp-commands b/erts/etc/unix/etp-commands.in index f059662271..54ff7b3e3a 100644 --- a/erts/etc/unix/etp-commands +++ b/erts/etc/unix/etp-commands.in @@ -2075,6 +2075,33 @@ document etp-system-info %--------------------------------------------------------------------------- end +define etp-compile-info + printf "--------------- Compile Information ---------------\n" + printf "CFLAGS: %s\n", erts_build_flags_CFLAGS + printf "LDFLAGS: %s\n", erts_build_flags_LDFLAGS + printf "Use etp-config-h-info to dump config.h\n" +end + +document etp-compile-info +%--------------------------------------------------------------------------- +% etp-compile-info +% +% Print information about how the system was compiled +%--------------------------------------------------------------------------- +end + +define etp-config-h-info + printf "%s", erts_build_flags_CONFIG_H +end + +document etp-config-h-info +%--------------------------------------------------------------------------- +% etp-config-h-info +% +% Dump the contents of config.h when the system was compiled +%--------------------------------------------------------------------------- +end + define etp-dictdump # Args: ProcDict* # @@ -2730,6 +2757,10 @@ document etp-run %--------------------------------------------------------------------------- end +define etp-thr + source @ERL_TOP@/erts/etc/unix/etp-thr.py +end + ############################################################################ # Toolbox parameter handling # diff --git a/erts/etc/unix/etp-thr.py b/erts/etc/unix/etp-thr.py new file mode 100644 index 0000000000..64fb858d20 --- /dev/null +++ b/erts/etc/unix/etp-thr.py @@ -0,0 +1,55 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 2013. All Rights Reserved. +# +# The contents of this file are subject to the Erlang Public License, +# Version 1.1, (the "License"); you may not use this file except in +# compliance with the License. You should have received a copy of the +# Erlang Public License along with this software. If not, it can be +# retrieved online at http://www.erlang.org/. +# +# Software distributed under the License is distributed on an "AS IS" +# basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +# the License for the specific language governing rights and limitations +# under the License. +# +# %CopyrightEnd% +# + +def get_thread_name(t): + f = gdb.newest_frame(); + while f: + if f.name() == "async_main": + return "async"; + elif f.name() == "erts_sys_main_thread": + return "main"; + elif f.name() == "signal_dispatcher_thread_func": + return "signal_dispatcher"; + elif f.name() == "sys_msg_dispatcher_func": + return "sys_msg_dispatcher"; + elif f.name() == "child_waiter": + return "child_waiter"; + elif f.name() == "sched_thread_func": + return "scheduler"; + elif f.name() == "aux_thread": + return "aux"; + f = f.older(); + return "unknown"; + + +curr_thread = gdb.selected_thread(); + +for i in gdb.inferiors(): + gdb.write(" Id Thread Name Frame\n"); + for t in i.threads(): + t.switch(); + if curr_thread == t: + gdb.write("*"); + else: + gdb.write(" "); + gdb.write("{0:<3} {1:20} {2}\n".format( + t.num,get_thread_name(t), + gdb.newest_frame().name())); + +curr_thread.switch(); diff --git a/erts/etc/unix/run_erl.c b/erts/etc/unix/run_erl.c index 910be3dce8..b69e31f784 100644 --- a/erts/etc/unix/run_erl.c +++ b/erts/etc/unix/run_erl.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -682,7 +682,7 @@ static void pass_on(pid_t childpid) } } - if (!got_some && wfd && buf[0] == '\022') { + if (!got_some && wfd && buf[0] == '\014') { char wbuf[30]; int wlen = sn_printf(wbuf,sizeof(wbuf),"[run_erl v%u-%u]\n", RUN_ERL_HI_VER, RUN_ERL_LO_VER); @@ -1142,6 +1142,14 @@ static void daemon_init(void) sf_close(i); } + /* Necessary on some platforms */ + + open("/dev/null", O_RDONLY); /* Order is important! */ + open("/dev/null", O_WRONLY); + open("/dev/null", O_WRONLY); + + errno = 0; /* if set by open */ + OPEN_SYSLOG(); run_daemon = 1; } diff --git a/erts/etc/unix/to_erl.c b/erts/etc/unix/to_erl.c index 754b349338..b9e397cbf2 100644 --- a/erts/etc/unix/to_erl.c +++ b/erts/etc/unix/to_erl.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2012. All Rights Reserved. + * Copyright Ericsson AB 1996-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -339,7 +339,7 @@ int main(int argc, char **argv) tty_smode.c_cc[VTIME] =0;/* Note that VTIME is the same as VEOL! */ tty_smode.c_cc[VINTR] =3; - tcsetattr(0, TCSANOW, &tty_smode); + tcsetattr(0, TCSADRAIN, &tty_smode); #ifdef DEBUG show_terminal_settings(&tty_smode); @@ -353,7 +353,7 @@ int main(int argc, char **argv) * at the start of every new to_erl-session. */ - if (write(wfd, "\022", 1) < 0) { + if (write(wfd, "\014", 1) < 0) { fprintf(stderr, "Error in writing ^R to FIFO.\n"); } @@ -484,7 +484,7 @@ int main(int argc, char **argv) * Reset terminal characterstics * XXX */ - tcsetattr(0, TCSANOW, &tty_rmode); + tcsetattr(0, TCSADRAIN, &tty_rmode); return 0; } diff --git a/erts/etc/win32/erlsrv/erlsrv_interactive.c b/erts/etc/win32/erlsrv/erlsrv_interactive.c index 736eabac79..e8d73ae047 100644 --- a/erts/etc/win32/erlsrv/erlsrv_interactive.c +++ b/erts/etc/win32/erlsrv/erlsrv_interactive.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * Copyright Ericsson AB 1998-2013. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -525,7 +525,7 @@ int do_usage(char *arg0){ "list, give option -args as last option on command line " "with\n" "no arguments.\n\n"); - printf("Se Erlang documentation for full description.\n"); + printf("See Erlang documentation for full description.\n"); return 0; } diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index ab728c65fa..38b8e9e9b6 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -361,6 +361,10 @@ extern ethr_runtime_t ethr_runtime__; # endif #endif /* !ETHR_DISABLE_NATIVE_IMPLS */ +#if !defined(ETHR_HAVE_NATIVE_ATOMIC32) && !defined(ETHR_HAVE_NATIVE_ATOMIC64) && !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(ETHR_SMP_REQUIRE_NATIVE_IMPLS) +#error "No native ethread implementation found. If you want to use fallbacks you have to disable native ethread support with configure." +#endif + #include "ethr_atomics.h" /* The atomics API */ #if defined(__GNUC__) @@ -411,6 +415,7 @@ extern ethr_runtime_t ethr_runtime__; #ifdef VALGRIND /* mutex as fallback for spinlock for VALGRIND */ # undef ETHR_HAVE_NATIVE_SPINLOCKS +# undef ETHR_HAVE_NATIVE_RWSPINLOCKS #else # include "ethr_optimized_fallbacks.h" #endif @@ -693,7 +698,7 @@ static ETHR_INLINE int ETHR_INLINE_FUNC_NAME_(ethr_rwlock_destroy)(ethr_rwlock_t *lock) { #ifdef ETHR_HAVE_NATIVE_RWSPINLOCKS - return 0; + return ethr_native_rwlock_destroy(lock); #else return ethr_rwmutex_destroy((ethr_rwmutex *) lock); #endif diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h index fcfdc39441..365a3535cf 100644 --- a/erts/include/internal/gcc/ethread.h +++ b/erts/include/internal/gcc/ethread.h @@ -25,6 +25,9 @@ #ifndef ETHREAD_GCC_H__ #define ETHREAD_GCC_H__ +#if defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32) \ + || defined(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64) + #ifndef ETHR_MEMBAR # include "ethr_membar.h" #endif @@ -46,3 +49,5 @@ #endif #endif + +#endif diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in index 0fc3ac6efc..4f0a5e5202 100644 --- a/erts/lib_src/Makefile.in +++ b/erts/lib_src/Makefile.in @@ -86,6 +86,12 @@ CFLAGS += -DERTS_ENABLE_LOCK_COUNT OMIT_FP=true PRE_LD= else +ifeq ($(TYPE),frmptr) +TYPE_SUFFIX = .frmptr +CFLAGS += -DERTS_FRMPTR +OMIT_OMIT_FP=yes +PRE_LD= +else override TYPE=opt OMIT_FP=true TYPE_SUFFIX= @@ -98,6 +104,7 @@ endif endif endif endif +endif OPSYS=@OPSYS@ sol2CFLAGS= @@ -110,6 +117,7 @@ ultrasparcCFLAGS=-Wa,-xarch=v8plusa ARCHCFLAGS=$($(ARCH)CFLAGS) ifeq ($(OMIT_OMIT_FP),yes) +CFLAGS += -fno-omit-frame-pointer OMIT_FP=false endif diff --git a/erts/preloaded/.gitignore b/erts/preloaded/.gitignore new file mode 100644 index 0000000000..40e4c68638 --- /dev/null +++ b/erts/preloaded/.gitignore @@ -0,0 +1,2 @@ +ebin/erts.app +src/prim_eval.abstr diff --git a/erts/preloaded/Makefile b/erts/preloaded/Makefile index 4235a7fe57..31fdeb96c5 100644 --- a/erts/preloaded/Makefile +++ b/erts/preloaded/Makefile @@ -18,7 +18,6 @@ # include $(ERL_TOP)/make/target.mk - SUB_DIRECTORIES = src include $(ERL_TOP)/make/otp_subdir.mk diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex a4ddae3d5d..f51ccfe8ce 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex 684659cca1..9da4f3cd00 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex 74c08fa4c9..3ab52615ab 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/init.beam b/erts/preloaded/ebin/init.beam Binary files differindex 143d5b18b9..89d3cd6b83 100644 --- a/erts/preloaded/ebin/init.beam +++ b/erts/preloaded/ebin/init.beam diff --git a/erts/preloaded/ebin/otp_ring0.beam b/erts/preloaded/ebin/otp_ring0.beam Binary files differindex a6b2fdb985..733d3fb587 100644 --- a/erts/preloaded/ebin/otp_ring0.beam +++ b/erts/preloaded/ebin/otp_ring0.beam diff --git a/erts/preloaded/ebin/prim_eval.beam b/erts/preloaded/ebin/prim_eval.beam Binary files differnew file mode 100644 index 0000000000..80bd1b3331 --- /dev/null +++ b/erts/preloaded/ebin/prim_eval.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex 9460f1da6f..a73a2f0db1 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/ebin/prim_inet.beam b/erts/preloaded/ebin/prim_inet.beam Binary files differindex 8c47d8b611..b0bef84e68 100644 --- a/erts/preloaded/ebin/prim_inet.beam +++ b/erts/preloaded/ebin/prim_inet.beam diff --git a/erts/preloaded/ebin/prim_zip.beam b/erts/preloaded/ebin/prim_zip.beam Binary files differindex e6f3995b50..562003df0d 100644 --- a/erts/preloaded/ebin/prim_zip.beam +++ b/erts/preloaded/ebin/prim_zip.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex cdbaa43d5d..e4a22ff2c7 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/.gitignore b/erts/preloaded/src/.gitignore new file mode 100644 index 0000000000..e4658fe142 --- /dev/null +++ b/erts/preloaded/src/.gitignore @@ -0,0 +1 @@ +prim_eval.abstr diff --git a/erts/preloaded/src/Makefile b/erts/preloaded/src/Makefile index a224b6a5d4..c1580b1495 100644 --- a/erts/preloaded/src/Makefile +++ b/erts/preloaded/src/Makefile @@ -1,7 +1,7 @@ # # %CopyrightBegin% # -# Copyright Ericsson AB 2008-2012. All Rights Reserved. +# Copyright Ericsson AB 2008-2013. All Rights Reserved. # # The contents of this file are subject to the Erlang Public License, # Version 1.1, (the "License"); you may not use this file except in @@ -32,7 +32,7 @@ STATIC_EBIN=../ebin include $(ERL_TOP)/erts/vsn.mk include $(ERL_TOP)/lib/kernel/vsn.mk -PRE_LOADED_MODULES = \ +PRE_LOADED_ERL_MODULES = \ erl_prim_loader \ init \ prim_file \ @@ -43,14 +43,27 @@ PRE_LOADED_MODULES = \ erlang \ erts_internal +PRE_LOADED_BEAM_MODULES = \ + prim_eval + +PRE_LOADED_MODULES = $(PRE_LOADED_ERL_MODULES) $(PRE_LOADED_BEAM_MODULES) + RELSYSDIR = $(RELEASE_PATH)/lib/erts-$(VSN) # not $(RELEASE_PATH)/erts-$(VSN)/preloaded -ERL_FILES= $(PRE_LOADED_MODULES:%=%.erl) +ERL_FILES= $(PRE_LOADED_ERL_MODULES:%=%.erl) +BEAM_FILES= $(PRE_LOADED_BEAM_MODULES:%=%.S) +STUBS_FILES= $(PRE_LOADED_BEAM_MODULES:%=%.erl) -TARGET_FILES = $(PRE_LOADED_MODULES:%=$(EBIN)/%.$(EMULATOR)) +TARGET_FILES = $(PRE_LOADED_MODULES:%=$(EBIN)/%.$(EMULATOR)) \ + $(APP_TARGET) STATIC_TARGET_FILES = $(PRE_LOADED_MODULES:%=$(STATIC_EBIN)/%.$(EMULATOR)) +APP_FILE= erts.app +APP_SRC= $(APP_FILE).src +APP_TARGET= $(STATIC_EBIN)/$(APP_FILE) + + KERNEL_SRC=$(ERL_TOP)/lib/kernel/src KERNEL_INCLUDE=$(ERL_TOP)/lib/kernel/include STDLIB_INCLUDE=$(ERL_TOP)/lib/stdlib/include @@ -65,14 +78,17 @@ clean: copy: cp *.beam $(STATIC_EBIN) +$(APP_TARGET): $(APP_SRC) $(ERL_TOP)/erts/vsn.mk + $(vsn_verbose)sed -e 's;%VSN%;$(VSN);' $< > $@ + include $(ERL_TOP)/make/otp_release_targets.mk -release_spec: +release_spec: $(APP_TARGET) $(INSTALL_DIR) "$(RELSYSDIR)/src" - $(INSTALL_DATA) $(ERL_FILES) "$(RELSYSDIR)/src" + $(INSTALL_DATA) $(ERL_FILES) $(BEAM_FILES) $(STUBS_FILES) "$(RELSYSDIR)/src" $(INSTALL_DIR) "$(RELSYSDIR)/ebin" - $(INSTALL_DATA) $(STATIC_TARGET_FILES) "$(RELSYSDIR)/ebin" + $(INSTALL_DATA) $(STATIC_TARGET_FILES) $(APP_TARGET) "$(RELSYSDIR)/ebin" release_docs_spec: @@ -80,6 +96,19 @@ release_docs_spec: list_preloaded: @echo $(PRE_LOADED_MODULES) +# +# Combine a BEAM assembly script file a stub Erlang file into a BEAM file. +# See add_abstract_chunk script. +# + +prim_eval.abstr: prim_eval.erl + $(V_ERLC) $(ERL_COMPILE_FLAGS) -o$(dir $@) +dabstr $< + +prim_eval.beam: prim_eval.S prim_eval.abstr + $(gen_verbose) + $(V_at)$(ERLC) $(ERL_COMPILE_FLAGS) $< + $(V_at)escript add_abstract_code $@ prim_eval.abstr || (rm $@; exit 1) + # Include dependencies -- list below added by PaN $(EBIN)/erl_prim_loader.beam: $(KERNEL_SRC)/inet_boot.hrl $(KERNEL_INCLUDE)/file.hrl $(EBIN)/prim_file.beam: $(KERNEL_INCLUDE)/file.hrl diff --git a/erts/preloaded/src/add_abstract_code b/erts/preloaded/src/add_abstract_code new file mode 100644 index 0000000000..e670156d21 --- /dev/null +++ b/erts/preloaded/src/add_abstract_code @@ -0,0 +1,34 @@ +#!/usr/bin/env escript +%% -*- erlang -*- + +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-mode(compile). + +-export([main/1]). + +main([BeamFile,AbstrFile]) -> + {ok,_,Chunks0} = beam_lib:all_chunks(BeamFile), + {ok,Abstr} = file:consult(AbstrFile), + Chunks = lists:keyreplace("Abst", 1, Chunks0, + {"Abst",term_to_binary({raw_abstract_v1,Abstr})}), + {ok,Module} = beam_lib:build_module(Chunks), + ok = file:write_file(BeamFile, Module), + init:stop(). diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 8e4a471a82..a969ef91dc 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -45,7 +45,8 @@ -export([alloc_info/1, alloc_sizes/1]). -export([gather_sched_wall_time_result/1, - await_sched_wall_time_modifications/2]). + await_sched_wall_time_modifications/2, + gather_gc_info_result/1]). -deprecated([hash/2]). @@ -187,6 +188,7 @@ 'busy_port' | 'busy_dist_port' | {'long_gc', non_neg_integer()} | + {'long_schedule', non_neg_integer()} | {'large_heap', non_neg_integer()}. @@ -1785,7 +1787,7 @@ process_flag(_Flag, _Value) -> links | last_calls | memory | - message_que_len | + message_queue_len | messages | min_heap_size | min_bin_vheap_size | @@ -1824,7 +1826,7 @@ process_flag(_Flag, _Value) -> {links, PidsAndPorts :: [pid() | port()]} | {last_calls, false | (Calls :: [mfa()])} | {memory, Size :: non_neg_integer()} | - {message_que_len, MessageQueueLen :: non_neg_integer()} | + {message_queue_len, MessageQueueLen :: non_neg_integer()} | {messages, MessageQueue :: [term()]} | {min_heap_size, MinHeapSize :: non_neg_integer()} | {min_bin_vheap_size, MinBinVHeapSize :: non_neg_integer()} | @@ -1905,11 +1907,11 @@ setelement(_Index, _Tuple1, _Value) -> Function :: atom(), Args :: [term()], Options :: [Option], - Option :: link | monitor | {priority, Level} + Option :: link | monitor + | {priority, Level :: priority_level()} | {fullsweep_after, Number :: non_neg_integer()} | {min_heap_size, Size :: non_neg_integer()} - | {min_bin_vheap_size, VSize :: non_neg_integer()}, - Level :: low | normal | high. + | {min_bin_vheap_size, VSize :: non_neg_integer()}. spawn_opt(_Tuple) -> erlang:nif_error(undefined). @@ -2086,7 +2088,7 @@ tuple_to_list(_Tuple) -> ({allocator_sizes, Alloc}) -> [_] when %% More or less anything Alloc :: atom(); (build_type) -> opt | debug | purify | quantify | purecov | - gcov | valgrind | gprof | lcnt; + gcov | valgrind | gprof | lcnt | frmptr; (c_compiler_used) -> {atom(), term()}; (check_io) -> [_]; (compat_rel) -> integer(); @@ -2244,11 +2246,11 @@ spawn_monitor(M, F, A) -> -spec spawn_opt(Fun, Options) -> pid() | {pid(), reference()} when Fun :: function(), Options :: [Option], - Option :: link | monitor | {priority, Level} + Option :: link | monitor + | {priority, Level :: priority_level()} | {fullsweep_after, Number :: non_neg_integer()} | {min_heap_size, Size :: non_neg_integer()} - | {min_bin_vheap_size, VSize :: non_neg_integer()}, - Level :: low | normal | high. + | {min_bin_vheap_size, VSize :: non_neg_integer()}. spawn_opt(F, O) when erlang:is_function(F) -> spawn_opt(erlang, apply, [F, []], O); spawn_opt({M,F}=MF, O) when erlang:is_atom(M), erlang:is_atom(F) -> @@ -2262,11 +2264,11 @@ spawn_opt(F, O) -> Node :: node(), Fun :: function(), Options :: [Option], - Option :: link | monitor | {priority, Level} + Option :: link | monitor + | {priority, Level :: priority_level()} | {fullsweep_after, Number :: non_neg_integer()} | {min_heap_size, Size :: non_neg_integer()} - | {min_bin_vheap_size, VSize :: non_neg_integer()}, - Level :: low | normal | high. + | {min_bin_vheap_size, VSize :: non_neg_integer()}. spawn_opt(N, F, O) when N =:= erlang:node() -> spawn_opt(F, O); spawn_opt(N, F, O) when erlang:is_function(F) -> @@ -2354,11 +2356,11 @@ spawn_link(N,M,F,A) -> Function :: atom(), Args :: [term()], Options :: [Option], - Option :: link | monitor | {priority, Level} + Option :: link | monitor + | {priority, Level :: priority_level()} | {fullsweep_after, Number :: non_neg_integer()} | {min_heap_size, Size :: non_neg_integer()} - | {min_bin_vheap_size, VSize :: non_neg_integer()}, - Level :: low | normal | high. + | {min_bin_vheap_size, VSize :: non_neg_integer()}. spawn_opt(M, F, A, Opts) -> case catch erlang:spawn_opt({M,F,A,Opts}) of {'EXIT',{Reason,_}} -> @@ -2374,11 +2376,11 @@ spawn_opt(M, F, A, Opts) -> Function :: atom(), Args :: [term()], Options :: [Option], - Option :: link | monitor | {priority, Level} + Option :: link | monitor + | {priority, Level :: priority_level()} | {fullsweep_after, Number :: non_neg_integer()} | {min_heap_size, Size :: non_neg_integer()} - | {min_bin_vheap_size, VSize :: non_neg_integer()}, - Level :: low | normal | high. + | {min_bin_vheap_size, VSize :: non_neg_integer()}. spawn_opt(N, M, F, A, O) when N =:= erlang:node(), erlang:is_atom(M), erlang:is_atom(F), erlang:is_list(A), erlang:is_list(O) -> @@ -2705,26 +2707,14 @@ port_info(Port, Item) -> Port :: port() | atom(), Data :: term(). -port_set_data(Port, Data) -> - case case erts_internal:port_set_data(Port, Data) of - Ref when erlang:is_reference(Ref) -> receive {Ref, Res} -> Res end; - Res -> Res - end of - badarg -> erlang:error(badarg, [Port, Data]); - Result -> Result - end. +port_set_data(_Port, _Data) -> + erlang:nif_error(undefined). -spec erlang:port_get_data(Port) -> term() when Port :: port() | atom(). -port_get_data(Port) -> - case case erts_internal:port_get_data(Port) of - Ref when erlang:is_reference(Ref) -> receive {Ref, Res} -> Res end; - Res -> Res - end of - {ok, Data} -> Data; - Error -> erlang:error(Error, [Port]) - end. +port_get_data(_Port) -> + erlang:nif_error(undefined). %% %% If the emulator wants to perform a distributed command and @@ -2901,22 +2891,23 @@ integer_to_binary(I, Base) when erlang:is_integer(I), erlang:is_integer(Base), Base >= 2, Base =< 1+$Z-$A+10 -> if I < 0 -> - <<"$-",(integer_to_binary(-I, Base, []))/binary>>; + <<$-,(integer_to_binary(-I, Base, <<>>))/binary>>; true -> integer_to_binary(I, Base, <<>>) end; integer_to_binary(I, Base) -> erlang:error(badarg, [I, Base]). -integer_to_binary(0, _Base, R0) -> - R0; integer_to_binary(I0, Base, R0) -> D = I0 rem Base, I1 = I0 div Base, - if D >= 10 -> - integer_to_binary(I1,Base,<<(D-10+$A),R0/binary>>); - true -> - integer_to_binary(I1,Base,<<(D+$0),R0/binary>>) + R1 = if + D >= 10 -> <<(D-10+$A),R0/binary>>; + true -> <<(D+$0),R0/binary>> + end, + if + I1 =:= 0 -> R1; + true -> integer_to_binary(I1, Base, R1) end. %% erlang:flush_monitor_message/2 is for internal use only! @@ -3125,8 +3116,8 @@ max(A, _) -> A. | 'atom' | 'atom_used' | 'binary' | 'code' | 'ets' | 'low' | 'maximum'. --define(CARRIER_ALLOCS, [mseg_alloc, sbmbc_alloc, sbmbc_low_alloc]). --define(LOW_ALLOCS, [sbmbc_low_alloc, ll_low_alloc, std_low_alloc]). +-define(CARRIER_ALLOCS, [mseg_alloc]). +-define(LOW_ALLOCS, [ll_low_alloc, std_low_alloc]). -define(ALL_NEEDED_ALLOCS, (erlang:system_info(alloc_util_allocators) -- ?CARRIER_ALLOCS)). @@ -3293,12 +3284,16 @@ get_blocks_size([{blocks_size, Sz, _, _} | Rest], Acc) -> get_blocks_size(Rest, Acc+Sz); get_blocks_size([{_, _, _, _} | Rest], Acc) -> get_blocks_size(Rest, Acc); +get_blocks_size([{blocks_size, Sz} | Rest], Acc) -> + get_blocks_size(Rest, Acc+Sz); +get_blocks_size([{_, _} | Rest], Acc) -> + get_blocks_size(Rest, Acc); get_blocks_size([], Acc) -> Acc. blocks_size([{Carriers, SizeList} | Rest], Acc) when Carriers == mbcs; - Carriers == sbcs; - Carriers == sbmbcs -> + Carriers == mbcs_pool; + Carriers == sbcs -> blocks_size(Rest, get_blocks_size(SizeList, Acc)); blocks_size([_ | Rest], Acc) -> blocks_size(Rest, Acc); @@ -3317,6 +3312,9 @@ get_fix_proc([], Acc) -> fix_proc([{fix_types, SizeList} | _Rest], Acc) -> get_fix_proc(SizeList, Acc); +fix_proc([{fix_types, Mask, SizeList} | _Rest], Acc) -> + {A, U} = get_fix_proc(SizeList, Acc), + {Mask, A, U}; fix_proc([_ | Rest], Acc) -> fix_proc(Rest, Acc); fix_proc([], Acc) -> @@ -3368,13 +3366,21 @@ au_mem_data(#memory{total = Tot, processes_used = ProcU, system = Sys} = Mem, [{fix_alloc, _, Data} | Rest]) -> - {A, U} = fix_proc(Data, {0, 0}), Sz = blocks_size(Data, 0), - au_mem_data(Mem#memory{total = Tot+Sz, - processes = Proc+A, - processes_used = ProcU+U, - system = Sys+Sz-A}, - Rest); + case fix_proc(Data, {0, 0}) of + {A, U} -> + au_mem_data(Mem#memory{total = Tot+Sz, + processes = Proc+A, + processes_used = ProcU+U, + system = Sys+Sz-A}, + Rest); + {Mask, A, U} -> + au_mem_data(Mem#memory{total = Tot+Sz, + processes = Mask band (Proc+A), + processes_used = Mask band (ProcU+U), + system = Mask band (Sys+Sz-A)}, + Rest) + end; au_mem_data(#memory{total = Tot, system = Sys, low = Low} = Mem, @@ -3392,7 +3398,7 @@ au_mem_data(EMD, []) -> au_mem_data(Allocs) -> Ref = erlang:make_ref(), - erlang:system_info({allocator_sizes, Ref, Allocs}), + erlang:system_info({memory_internal, Ref, Allocs}), receive_emd(Ref). receive_emd(_Ref, EMD, 0) -> @@ -3548,3 +3554,18 @@ sched_wall_time(Ref, N, Acc) -> {Ref, undefined} -> sched_wall_time(Ref, N-1, undefined); {Ref, SWT} -> sched_wall_time(Ref, N-1, [SWT|Acc]) end. + +-spec erlang:gather_gc_info_result(Ref) -> + {number(),number(),0} when Ref :: reference(). + +gather_gc_info_result(Ref) when erlang:is_reference(Ref) -> + gc_info(Ref, erlang:system_info(schedulers), {0,0}). + +gc_info(_Ref, 0, {Colls,Recl}) -> + {Colls,Recl,0}; +gc_info(Ref, N, {OrigColls,OrigRecl}) -> + receive + {Ref, {_,Colls, Recl}} -> + gc_info(Ref, N-1, {Colls+OrigColls,Recl+OrigRecl}) + end. + diff --git a/erts/preloaded/src/erts.app.src b/erts/preloaded/src/erts.app.src new file mode 100644 index 0000000000..fd3e8cb692 --- /dev/null +++ b/erts/preloaded/src/erts.app.src @@ -0,0 +1,41 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +{application, erts, [ + {description, "ERTS CXC 138 10"}, + {vsn, "%VSN%"}, + {modules, [ + %% preloaded + erlang, + erl_prim_loader, + erts_internal, + init, + otp_ring0, + prim_eval, + prim_file, + prim_inet, + prim_zip, + zlib + ]}, + {registered, []}, + {applications, []}, + {env, []}, + {mod, {erts, []}} + ]}. + +%% vim: ft=erlang diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl index f1c83f4518..8a8cd52d64 100644 --- a/erts/preloaded/src/erts_internal.erl +++ b/erts/preloaded/src/erts_internal.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2012. All Rights Reserved. +%% Copyright Ericsson AB 2012-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -31,8 +31,7 @@ -export([await_port_send_result/3]). -export([port_command/3, port_connect/2, port_close/1, - port_control/3, port_call/3, port_set_data/2, port_get_data/1, - port_info/1, port_info/2]). + port_control/3, port_call/3, port_info/1, port_info/2]). %% %% Await result of send to port @@ -86,19 +85,6 @@ port_control(_Port, _Operation, _Data) -> port_call(_Port, _Operation, _Data) -> erlang:nif_error(undefined). --spec erts_internal:port_get_data(P1) -> Result when - P1 :: port() | atom(), - Result :: {ok, term()} | reference() | badarg. -port_get_data(_P1) -> - erlang:nif_error(undefined). - --spec erts_internal:port_set_data(P1, P2) -> Result when - P1 :: port() | atom(), - P2 :: term(), - Result :: true | reference() | badarg. -port_set_data(_P1, _P2) -> - erlang:nif_error(undefined). - -type port_info_1_result_item() :: {registered_name, RegName :: atom()} | {id, Index :: non_neg_integer()} | diff --git a/erts/preloaded/src/prim_eval.S b/erts/preloaded/src/prim_eval.S new file mode 100644 index 0000000000..958a79a1da --- /dev/null +++ b/erts/preloaded/src/prim_eval.S @@ -0,0 +1,70 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +{module, prim_eval}. + +%% This module uses low-level BEAM instructions for the message queue facility +%% to allow erl_eval to evaluate receive expressions correctly. + +{exports, [{'receive',2},{module_info,0},{module_info,1}]}. + +{attributes, []}. + +{labels, 10}. + + +{function, 'receive', 2, 2}. + {label,1}. + {func_info,{atom,prim_eval},{atom,'receive'},2}. + {label,2}. + {allocate,2,2}. + {move,{x,1},{y,0}}. + {move,{x,0},{y,1}}. + {label,3}. + {loop_rec,{f,5},{x,0}}. + {move,{y,1},{x,1}}. + {call_fun,1}. + {test,is_ne_exact,{f,4},[{x,0},{atom,nomatch}]}. + remove_message. + {deallocate,2}. + return. + {label,4}. + {loop_rec_end,{f,3}}. + {label,5}. + {wait_timeout,{f,3},{y,0}}. + timeout. + {move,{atom,timeout},{x,0}}. + {deallocate,2}. + return. + + +{function, module_info, 0, 8}. + {label,6}. + {func_info,{atom,prim_eval},{atom,module_info},0}. + {label,7}. + {move,{atom,prim_eval},{x,0}}. + {call_ext_only,1,{extfunc,erlang,get_module_info,1}}. + + +{function, module_info, 1, 10}. + {label,8}. + {func_info,{atom,prim_eval},{atom,module_info},1}. + {label,9}. + {move,{x,0},{x,1}}. + {move,{atom,prim_eval},{x,0}}. + {call_ext_only,2,{extfunc,erlang,get_module_info,2}}. diff --git a/erts/preloaded/src/prim_eval.erl b/erts/preloaded/src/prim_eval.erl new file mode 100644 index 0000000000..ec5af8c138 --- /dev/null +++ b/erts/preloaded/src/prim_eval.erl @@ -0,0 +1,28 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% +-module(prim_eval). + +%% This module is simply a stub which abstract code gets included in the result +%% of compilation of prim_eval.S, to keep Dialyzer happy. + +-export(['receive'/2]). + +-spec 'receive'(fun((term()) -> nomatch | T), timeout()) -> T. +'receive'(_, _) -> + erlang:nif_error(stub). diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl index 21d23159f0..fa621681f3 100644 --- a/erts/preloaded/src/prim_inet.erl +++ b/erts/preloaded/src/prim_inet.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2012. All Rights Reserved. +%% Copyright Ericsson AB 2000-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -25,7 +25,7 @@ %% Primitive inet_drv interface --export([open/3, fdopen/4, close/1]). +-export([open/3, open/4, fdopen/4, close/1]). -export([bind/3, listen/1, listen/2, peeloff/2]). -export([connect/3, connect/4, async_connect/4]). -export([accept/1, accept/2, async_accept/2]). @@ -64,22 +64,31 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% open(Protocol, Family, Type) -> - open(Protocol, Family, Type, ?INET_REQ_OPEN, []). + open(Protocol, Family, Type, [], ?INET_REQ_OPEN, []). + +open(Protocol, Family, Type, Opts) -> + open(Protocol, Family, Type, Opts, ?INET_REQ_OPEN, []). fdopen(Protocol, Family, Type, Fd) when is_integer(Fd) -> - open(Protocol, Family, Type, ?INET_REQ_FDOPEN, ?int32(Fd)). + open(Protocol, Family, Type, [], ?INET_REQ_FDOPEN, ?int32(Fd)). -open(Protocol, Family, Type, Req, Data) -> +open(Protocol, Family, Type, Opts, Req, Data) -> Drv = protocol2drv(Protocol), AF = enc_family(Family), T = enc_type(Type), try erlang:open_port({spawn_driver,Drv}, [binary]) of S -> - case ctl_cmd(S, Req, [AF,T,Data]) of - {ok,_} -> {ok,S}; - {error,_}=Error -> + case setopts(S, Opts) of + ok -> + case ctl_cmd(S, Req, [AF,T,Data]) of + {ok,_} -> {ok,S}; + {error,_}=E1 -> + close(S), + E1 + end; + {error,_}=E2 -> close(S), - Error + E2 end catch %% The only (?) way to get here is to try to open @@ -151,30 +160,59 @@ shutdown_pend_loop(S, N0) -> %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% close(S) when is_port(S) -> - unlink(S), %% avoid getting {'EXIT', S, Reason} case subscribe(S, [subs_empty_out_q]) of {ok, [{subs_empty_out_q,N}]} when N > 0 -> close_pend_loop(S, N); %% wait for pending output to be sent _ -> - catch erlang:port_close(S), - ok + close_port(S) end. close_pend_loop(S, N) -> receive {empty_out_q,S} -> - catch erlang:port_close(S), ok + close_port(S) after ?INET_CLOSE_TIMEOUT -> case getstat(S, [send_pend]) of {ok, [{send_pend,N1}]} -> - if N1 =:= N -> catch erlang:port_close(S), ok; - true -> close_pend_loop(S, N1) + if + N1 =:= N -> + close_port(S); + true -> + close_pend_loop(S, N1) end; _ -> - catch erlang:port_close(S), ok + close_port(S) end end. - + +close_port(S) -> + case erlang:process_info(self(), trap_exit) of + {trap_exit,true} -> + %% Ensure exit message and consume it + link(S), + %% This is still not a perfect solution. + %% + %% The problem is to close the port and consume any exit + %% message while not knowing if this process traps exit + %% nor if this process has a link to the port. Here we + %% just knows that this process traps exit. + %% + %% If we right here get killed for some reason that exit + %% signal will propagate to the port and onwards to anyone + %% that is linked to the port. E.g when we close a socket + %% that is not ours. + %% + %% The problem can be solved with lists:member on our link + %% list but we deem that as potentially too expensive. We + %% need an is_linked/1 function or guard, or we need + %% a port_close function that can atomically unlink... + catch erlang:port_close(S), + receive {'EXIT',S,_} -> ok end; + {trap_exit,false} -> + catch erlang:port_close(S), + ok + end. + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% %% BIND(insock(), IP, Port) -> {ok, integer()} | {error, Reason} @@ -1072,13 +1110,14 @@ enc_opt(deliver) -> ?INET_LOPT_DELIVER; enc_opt(exit_on_close) -> ?INET_LOPT_EXITONCLOSE; enc_opt(high_watermark) -> ?INET_LOPT_TCP_HIWTRMRK; enc_opt(low_watermark) -> ?INET_LOPT_TCP_LOWTRMRK; -enc_opt(high_msgq_watermark) -> ?INET_LOPT_TCP_MSGQ_HIWTRMRK; -enc_opt(low_msgq_watermark) -> ?INET_LOPT_TCP_MSGQ_LOWTRMRK; +enc_opt(high_msgq_watermark) -> ?INET_LOPT_MSGQ_HIWTRMRK; +enc_opt(low_msgq_watermark) -> ?INET_LOPT_MSGQ_LOWTRMRK; enc_opt(send_timeout) -> ?INET_LOPT_TCP_SEND_TIMEOUT; enc_opt(send_timeout_close) -> ?INET_LOPT_TCP_SEND_TIMEOUT_CLOSE; enc_opt(delay_send) -> ?INET_LOPT_TCP_DELAY_SEND; enc_opt(packet_size) -> ?INET_LOPT_PACKET_SIZE; enc_opt(read_packets) -> ?INET_LOPT_READ_PACKETS; +enc_opt(netns) -> ?INET_LOPT_NETNS; enc_opt(raw) -> ?INET_OPT_RAW; % Names of SCTP opts: enc_opt(sctp_rtoinfo) -> ?SCTP_OPT_RTOINFO; @@ -1128,13 +1167,14 @@ dec_opt(?INET_LOPT_DELIVER) -> deliver; dec_opt(?INET_LOPT_EXITONCLOSE) -> exit_on_close; dec_opt(?INET_LOPT_TCP_HIWTRMRK) -> high_watermark; dec_opt(?INET_LOPT_TCP_LOWTRMRK) -> low_watermark; -dec_opt(?INET_LOPT_TCP_MSGQ_HIWTRMRK) -> high_msgq_watermark; -dec_opt(?INET_LOPT_TCP_MSGQ_LOWTRMRK) -> low_msgq_watermark; +dec_opt(?INET_LOPT_MSGQ_HIWTRMRK) -> high_msgq_watermark; +dec_opt(?INET_LOPT_MSGQ_LOWTRMRK) -> low_msgq_watermark; dec_opt(?INET_LOPT_TCP_SEND_TIMEOUT) -> send_timeout; dec_opt(?INET_LOPT_TCP_SEND_TIMEOUT_CLOSE) -> send_timeout_close; dec_opt(?INET_LOPT_TCP_DELAY_SEND) -> delay_send; dec_opt(?INET_LOPT_PACKET_SIZE) -> packet_size; dec_opt(?INET_LOPT_READ_PACKETS) -> read_packets; +dec_opt(?INET_LOPT_NETNS) -> netns; dec_opt(?INET_OPT_RAW) -> raw; dec_opt(I) when is_integer(I) -> undefined. @@ -1232,6 +1272,7 @@ type_opt_1(send_timeout_close) -> bool; type_opt_1(delay_send) -> bool; type_opt_1(packet_size) -> uint; type_opt_1(read_packets) -> uint; +type_opt_1(netns) -> binary; %% %% SCTP options (to be set). If the type is a record type, the corresponding %% record signature is returned, otherwise, an "elementary" type tag @@ -1458,9 +1499,12 @@ type_value_2({bitenumlist,List,_}, EnumList) -> Ls when is_list(Ls) -> true; false -> false end; -type_value_2(binary,Bin) when is_binary(Bin) -> true; -type_value_2(binary_or_uint,Bin) when is_binary(Bin) -> true; -type_value_2(binary_or_uint,Int) when is_integer(Int), Int >= 0 -> true; +type_value_2(binary,Bin) + when is_binary(Bin), byte_size(Bin) < (1 bsl 32) -> true; +type_value_2(binary_or_uint,Bin) + when is_binary(Bin), byte_size(Bin) < (1 bsl 32) -> true; +type_value_2(binary_or_uint,Int) + when is_integer(Int), Int >= 0 -> true; %% Type-checking of SCTP options type_value_2(sctp_assoc_id, X) when X band 16#ffffffff =:= X -> true; diff --git a/erts/preloaded/src/zlib.erl b/erts/preloaded/src/zlib.erl index 1faae1c1f4..54391bd945 100644 --- a/erts/preloaded/src/zlib.erl +++ b/erts/preloaded/src/zlib.erl @@ -30,6 +30,8 @@ compress/1,uncompress/1,zip/1,unzip/1, gzip/1,gunzip/1]). +-export_type([zstream/0]). + %% flush argument encoding -define(Z_NO_FLUSH, 0). -define(Z_SYNC_FLUSH, 2). diff --git a/erts/test/Makefile b/erts/test/Makefile index 6b409e2f1b..74a5bb1ccc 100644 --- a/erts/test/Makefile +++ b/erts/test/Makefile @@ -79,7 +79,7 @@ release_spec: release_tests_spec: opt $(INSTALL_DIR) "$(RELSYSDIR)" - $(INSTALL_DATA) system.spec system.dynspec \ + $(INSTALL_DATA) system.spec system.dynspec system_smoke.spec \ $(ERL_FILES) $(TARGET_FILES) "$(RELSYSDIR)" chmod -R u+w "$(RELSYSDIR)" tar cf - *_SUITE_data utils | (cd "$(RELSYSDIR)"; tar xf -) diff --git a/erts/test/erlc_SUITE.erl b/erts/test/erlc_SUITE.erl index ab774dbc4f..ed7a43c7e7 100644 --- a/erts/test/erlc_SUITE.erl +++ b/erts/test/erlc_SUITE.erl @@ -23,7 +23,8 @@ -export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, init_per_group/2,end_per_group/2, compile_erl/1, compile_yecc/1, compile_script/1, - compile_mib/1, good_citizen/1, deep_cwd/1, arg_overflow/1]). + compile_mib/1, good_citizen/1, deep_cwd/1, arg_overflow/1, + make_dep_options/1]). -include_lib("test_server/include/test_server.hrl"). @@ -31,7 +32,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [compile_erl, compile_yecc, compile_script, compile_mib, - good_citizen, deep_cwd, arg_overflow]. + good_citizen, deep_cwd, arg_overflow, make_dep_options]. groups() -> []. @@ -255,13 +256,89 @@ erlc() -> Erlc -> "\"" ++ Erlc ++ "\"" end. - + +make_dep_options(Config) -> + {SrcDir,OutDir,Cmd} = get_cmd(Config), + FileName = filename:join(SrcDir, "erl_test_ok.erl"), + + + DepRE = ["/erl_test_ok[.]beam: \\\\$", + "/system_test/erlc_SUITE_data/src/erl_test_ok[.]erl \\\\$", + "/system_test/erlc_SUITE_data/include/erl_test[.]hrl$", + "_OK_"], + + DepRETarget = + ["^target: \\\\$", + "/system_test/erlc_SUITE_data/src/erl_test_ok[.]erl \\\\$", + "/system_test/erlc_SUITE_data/include/erl_test[.]hrl$", + "_OK_"], + + DepREMP = + ["/erl_test_ok[.]beam: \\\\$", + "/system_test/erlc_SUITE_data/src/erl_test_ok[.]erl \\\\$", + "/system_test/erlc_SUITE_data/include/erl_test[.]hrl$", + [], + "/system_test/erlc_SUITE_data/include/erl_test.hrl:$", + "_OK_"], + + DepREMissing = + ["/erl_test_missing_header[.]beam: \\\\$", + "/system_test/erlc_SUITE_data/src/erl_test_missing_header[.]erl \\\\$", + "/system_test/erlc_SUITE_data/include/erl_test[.]hrl \\\\$", + "missing.hrl$", + "_OK_"], + + %% Test plain -M + run(Config, Cmd, FileName, "-M", DepRE), + + %% Test -MF File + DepFile = filename:join(OutDir, "my.deps"), + run(Config, Cmd, FileName, "-MF "++DepFile, ["_OK_"]), + {ok,MFBin} = file:read_file(DepFile), + verify_result(binary_to_list(MFBin)++["_OK_"], DepRE), + + %% Test -MD + run(Config, Cmd, FileName, "-MD", ["_OK_"]), + MDFile = filename:join(OutDir, "erl_test_ok.Pbeam"), + {ok,MFBin} = file:read_file(MDFile), + + %% Test -M -MT Target + run(Config, Cmd, FileName, "-M -MT target", DepRETarget), + + %% Test -MF File -MT Target + TargetDepFile = filename:join(OutDir, "target.deps"), + run(Config, Cmd, FileName, "-MF "++TargetDepFile++" -MT target", + ["_OK_"]), + {ok,TargetBin} = file:read_file(TargetDepFile), + verify_result(binary_to_list(TargetBin)++["_OK_"], DepRETarget), + + %% Test -MD -MT Target + run(Config, Cmd, FileName, "-MD -MT target", ["_OK_"]), + TargetMDFile = filename:join(OutDir, "erl_test_ok.Pbeam"), + {ok,TargetBin} = file:read_file(TargetMDFile), + + %% Test -M -MQ Target. (Note: Passing a $ on the command line + %% portably for Unix and Windows is tricky, so we will just test + %% that MQ works at all.) + run(Config, Cmd, FileName, "-M -MQ target", DepRETarget), + + %% Test -M -MP + run(Config, Cmd, FileName, "-M -MP", DepREMP), + + %% Test -M -MG + MissingHeader = filename:join(SrcDir, "erl_test_missing_header.erl"), + run(Config, Cmd, MissingHeader, "-M -MG", DepREMissing), + ok. + %% Runs a command. run(Config, Cmd0, Name, Options, Expect) -> Cmd = Cmd0 ++ " " ++ Options ++ " " ++ Name, io:format("~s", [Cmd]), Result = run_command(Config, Cmd), + verify_result(Result, Expect). + +verify_result(Result, Expect) -> Messages = split(Result, [], []), io:format("Result: ~p", [Messages]), io:format("Expected: ~p", [Expect]), diff --git a/erts/test/erlc_SUITE_data/src/erl_test_missing_header.erl b/erts/test/erlc_SUITE_data/src/erl_test_missing_header.erl new file mode 100644 index 0000000000..4d6c42c803 --- /dev/null +++ b/erts/test/erlc_SUITE_data/src/erl_test_missing_header.erl @@ -0,0 +1,22 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2013. All Rights Reserved. +%% +%% The contents of this file are subject to the Erlang Public License, +%% Version 1.1, (the "License"); you may not use this file except in +%% compliance with the License. You should have received a copy of the +%% Erlang Public License along with this software. If not, it can be +%% retrieved online at http://www.erlang.org/. +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and limitations +%% under the License. +%% +%% %CopyrightEnd% +%% + +-module(erl_test_missing_header). +-include("erl_test.hrl"). +-include("missing.hrl"). diff --git a/erts/test/nt_SUITE.erl b/erts/test/nt_SUITE.erl index f9bd15a0ce..e440b9e5d9 100644 --- a/erts/test/nt_SUITE.erl +++ b/erts/test/nt_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 1998-2011. All Rights Reserved. +%% Copyright Ericsson AB 1998-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -270,8 +270,11 @@ service_prio(Config) when is_list(Config) -> ?line {ok, OldProcs} = get_current_procs(Config), ?line start_service(Name), ?line {ok, NewProcs} = get_current_procs(Config), + timer:sleep(2000), + ?line {ok, NewProcs2} = get_current_procs(Config), ?line remove_service(Name), ?line Diff = arrived_procs(OldProcs,NewProcs), + io:format("NewProcs ~p~n after sleep~n ~p~n",[Diff, arrived_procs(OldProcs,NewProcs2)]), %% Not really correct, could fail if another heart is %% started at the same time... ?line {value, {"heart.exe",_,"high"}} = diff --git a/erts/test/system_smoke.spec b/erts/test/system_smoke.spec new file mode 100644 index 0000000000..933d1ba22d --- /dev/null +++ b/erts/test/system_smoke.spec @@ -0,0 +1,3 @@ +{suites,"../system_test",[ethread_SUITE]}. +{cases,"../system_test",otp_SUITE,[undefined_functions]}. +{skip_cases,"../system_test",ethread_SUITE,[max_threads],"Skip"}. diff --git a/erts/test/z_SUITE.erl b/erts/test/z_SUITE.erl index 78968ed405..ccf22a9b6b 100644 --- a/erts/test/z_SUITE.erl +++ b/erts/test/z_SUITE.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2008-2012. All Rights Reserved. +%% Copyright Ericsson AB 2008-2013. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -190,8 +190,13 @@ file_inspect(#core_search_conf{file = File}, Core) -> probably_a_core end. -mk_readable(F) -> - catch file:write_file_info(F, #file_info{mode = 8#00444}). +mk_readable(F) -> + try + {ok, Old} = file:read_file_info(F), + file:write_file_info(F, Old#file_info{mode = 8#00444}) + catch + _:_ -> io:format("Failed to \"chmod\" core file ~p\n", [F]) + end. ignore_core(C) -> filelib:is_regular(filename:join([filename:dirname(C), diff --git a/erts/vsn.mk b/erts/vsn.mk index 43ed671cee..e235c50f0b 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -17,8 +17,8 @@ # %CopyrightEnd% # -VSN = 5.10.1.2 -SYSTEM_VSN = R16B +VSN = 5.10.3 +SYSTEM_VSN = R16B02 # Port number 4365 in 4.2 # Port number 4366 in 4.3 |