diff options
Diffstat (limited to 'erts')
90 files changed, 2130 insertions, 1257 deletions
diff --git a/erts/autoconf/configure.vxworks b/erts/autoconf/configure.vxworks index 5be91319ea..a253848403 100755 --- a/erts/autoconf/configure.vxworks +++ b/erts/autoconf/configure.vxworks @@ -113,6 +113,7 @@ CONFIG_FILES="${ERL_TOP}/erts/emulator/$host/Makefile $erlint_dir/$host/eidefs.mk $epmd_dir/$host/Makefile $internal_tools_dir/make/$host/otp.mk + $internal_tools_dir/make/$host/otp_ded.mk $os_mon_dir/$host/Makefile $zlibdir/$host/Makefile $runtime_tools_dir/$host/Makefile diff --git a/erts/autoconf/vxworks/sed.general b/erts/autoconf/vxworks/sed.general index 96a70e4148..a30eb51169 100644 --- a/erts/autoconf/vxworks/sed.general +++ b/erts/autoconf/vxworks/sed.general @@ -43,6 +43,11 @@ s|@LDFLAGS@|| # FIXME: A bit strange to clear out remaining DED_* s|@DED_LDFLAGS@|| s|@DED_CFLAGS@|| +s|@DED_EMU_THR_DEFS@|| +s|@DED_THR_DEFS@|| +s|@DED_SYS_INCLUDE@|| +s|@WERRORFLAGS@|| +s|@DED_STATIC_CFLAGS@|| s|@STATIC_CFLAGS@|| s|@GCCLIB@|libgcc.a| s|@DEFS@|| diff --git a/erts/autoconf/vxworks/sed.vxworks_cpu32 b/erts/autoconf/vxworks/sed.vxworks_cpu32 index 71663676e7..961adc4104 100644 --- a/erts/autoconf/vxworks/sed.vxworks_cpu32 +++ b/erts/autoconf/vxworks/sed.vxworks_cpu32 @@ -28,6 +28,7 @@ s|@host@|vxworks_cpu32| s|@system_type@|vxworks_cpu32| s|@CC@|@TTPREFIX@cc68k| s|@HCC@|gcc| +s|@GCC@|yes| s|@LD@|@TTPREFIX@ld68k| s|@LIBS@|| s|@DED_LD@|@TTPREFIX@ld68k| diff --git a/erts/autoconf/vxworks/sed.vxworks_ppc32 b/erts/autoconf/vxworks/sed.vxworks_ppc32 index 2146e862fd..1e2e760abc 100644 --- a/erts/autoconf/vxworks/sed.vxworks_ppc32 +++ b/erts/autoconf/vxworks/sed.vxworks_ppc32 @@ -33,6 +33,7 @@ s|@system_type@|vxworks_ppc32| s|@ARCH@|ppc32| s|@CC@|GCC_EXEC_PREFIX=@WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/lib/gcc-lib/ @WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/bin/ccppc -mlongcall| s|@HCC@|gcc| +s|@GCC@|yes| s|@LD@|GCC_EXEC_PREFIX=@WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/lib/gcc-lib/ @WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/bin/ldppc| s|@STRIP@|GCC_EXEC_PREFIX=@WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/lib/gcc-lib/ @WIND_BASE@/workbench-2.3/@HOST_TYPE@/bin/stripppc| s|@SYMPREFIX@|| diff --git a/erts/autoconf/vxworks/sed.vxworks_ppc603 b/erts/autoconf/vxworks/sed.vxworks_ppc603 index fca1ba76d9..5beaae4fc9 100644 --- a/erts/autoconf/vxworks/sed.vxworks_ppc603 +++ b/erts/autoconf/vxworks/sed.vxworks_ppc603 @@ -29,6 +29,7 @@ s|@system_type@|vxworks_ppc603| s|@ARCH@|ppc603| s|@CC@|@TTPREFIX@ccppc -mlongcall| s|@HCC@|gcc| +s|@GCC@|yes| s|@LD@|@TTPREFIX@ldppc| s|@STRIP@|@TTPREFIX@stripppc| s|@SYMPREFIX@|| diff --git a/erts/autoconf/vxworks/sed.vxworks_ppc603_nolongcall b/erts/autoconf/vxworks/sed.vxworks_ppc603_nolongcall index 51c589d79a..2c2f2fa372 100644 --- a/erts/autoconf/vxworks/sed.vxworks_ppc603_nolongcall +++ b/erts/autoconf/vxworks/sed.vxworks_ppc603_nolongcall @@ -29,6 +29,7 @@ s|@system_type@|vxworks_ppc603| s|@ARCH@|ppc603| s|@CC@|@TTPREFIX@ccppc| s|@HCC@|gcc| +s|@GCC@|yes| s|@LD@|@TTPREFIX@ldppc| s|@STRIP@|@TTPREFIX@stripppc| s|@SYMPREFIX@|| diff --git a/erts/autoconf/vxworks/sed.vxworks_ppc860 b/erts/autoconf/vxworks/sed.vxworks_ppc860 index 485504e706..71cf887476 100644 --- a/erts/autoconf/vxworks/sed.vxworks_ppc860 +++ b/erts/autoconf/vxworks/sed.vxworks_ppc860 @@ -29,6 +29,7 @@ s|@system_type@|vxworks_ppc860| s|@ARCH@|ppc860| s|@CC@|@TTPREFIX@ccppc -mlongcall| s|@HCC@|gcc| +s|@GCC@|yes| s|@LD@|@TTPREFIX@ldppc| s|@STRIP@|@TTPREFIX@stripppc| s|@SYMPREFIX@|| diff --git a/erts/autoconf/vxworks/sed.vxworks_simlinux b/erts/autoconf/vxworks/sed.vxworks_simlinux index 10cd7bbb82..06b1847602 100644 --- a/erts/autoconf/vxworks/sed.vxworks_simlinux +++ b/erts/autoconf/vxworks/sed.vxworks_simlinux @@ -36,6 +36,7 @@ s|@ARCH@|simlinux| s|@CC@|GCC_EXEC_PREFIX=@WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/lib/gcc-lib/ @WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/bin/ccpentium| s|@HCC@|gcc| +s|@GCC@|yes| s|@LD@|GCC_EXEC_PREFIX=@WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/lib/gcc-lib/ @WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/bin/ldpentium| diff --git a/erts/autoconf/vxworks/sed.vxworks_simso b/erts/autoconf/vxworks/sed.vxworks_simso index cd30f8c2b2..07606cad80 100644 --- a/erts/autoconf/vxworks/sed.vxworks_simso +++ b/erts/autoconf/vxworks/sed.vxworks_simso @@ -36,6 +36,7 @@ s|@ARCH@|simso| s|@CC@|GCC_EXEC_PREFIX=@WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/lib/gcc-lib/ @WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/bin/ccsparc| s|@HCC@|gcc| +s|@GCC@|yes| # Tornado2.2: s|@LD@|@TTPREFIX@ldsimso| s|@LD@|GCC_EXEC_PREFIX=@WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/lib/gcc-lib/ @WIND_BASE@/gnu/3.4.4-vxworks-6.3/@HOST_TYPE@/bin/ldsparc| diff --git a/erts/autoconf/vxworks/sed.vxworks_sparc b/erts/autoconf/vxworks/sed.vxworks_sparc index a3758423e8..59431fddf9 100644 --- a/erts/autoconf/vxworks/sed.vxworks_sparc +++ b/erts/autoconf/vxworks/sed.vxworks_sparc @@ -30,6 +30,7 @@ s/@host@/vxworks_sparc/ s/@system_type@/vxworks_sparc/ s/@CC@/\/home\/gandalf\/bsproj\/tools\/vw-gnu\/solaris.sparc\/bin\/ccsparc/ s/@HCC@/gcc/ +s/@GCC@/yes/ s/@LD@/\/home\/gandalf\/bsproj\/tools\/vw-gnu\/solaris.sparc\/bin\/ldsparc/ s/@DEBUG_FLAGS@/-g/ s/@GCCLIB_PATH@/\/home\/gandalf\/bsproj\/tools\/vw-gnu\/solaris.sparc\/lib\/gcc-lib\/sparc-wrs-vxworks\/cygnus-2.2.3.1\/libgcc.a/ diff --git a/erts/configure.in b/erts/configure.in index 5b604b18e8..c6033187e6 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -2007,29 +2007,6 @@ esac AC_CHECK_DECLS([posix2time, time2posix],,,[#include <time.h>]) -disable_vfork=false -if test "x$EMU_THR_LIB_NAME" != "x"; then - AC_MSG_CHECKING([if vfork is known to hang multithreaded applications]) - case $host_os in - osf*) - AC_MSG_RESULT(yes) - disable_vfork=true;; - *) - AC_MSG_RESULT(no);; - esac -fi - -if test $disable_vfork = false; then - AC_FUNC_VFORK - if test $ac_cv_func_vfork_works = no; then - disable_vfork=true - fi -fi - -if test $disable_vfork = true; then - AC_DEFINE(DISABLE_VFORK, 1, [Define if you want to disable vfork.]) -fi - AC_FUNC_VPRINTF dnl The AC_DEFINEs are necessary for autoheader to work. :-( diff --git a/erts/doc/src/Makefile b/erts/doc/src/Makefile index 3cc29e14b1..21aa3db864 100644 --- a/erts/doc/src/Makefile +++ b/erts/doc/src/Makefile @@ -149,6 +149,7 @@ debug opt: clean: rm -rf $(HTMLDIR)/* + rm -rf $(XMLDIR) rm -f $(MAN1DIR)/* rm -f $(MAN3DIR)/* rm -f $(TOP_PDF_FILE) $(TOP_PDF_FILE:%.pdf=%.fo) diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml index 158f4dc4e8..f29bb7b8f9 100644 --- a/erts/doc/src/absform.xml +++ b/erts/doc/src/absform.xml @@ -801,7 +801,8 @@ <c>{ann_type,LINE,[Rep(A),Rep(T_0)]}</c>.</p> </item> <item> - <p>If T is an atom or integer literal L, then Rep(T) = Rep(L).</p> + <p>If T is an atom, a character, or an integer literal L, + then Rep(T) = Rep(L).</p> </item> <item> <p>If T is a bitstring type <c><<_:M,_:_*N>></c>, diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index a8eff43623..a20b8ee884 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -194,7 +194,7 @@ ok <p>Binaries are sequences of whole bytes. Bitstrings with an arbitrary bit length have no support yet.</p> </item> - <tag>Resource objects</tag> + <tag><marker id="resource_objects"/>Resource objects</tag> <item> <p>The use of resource objects is a safe way to return pointers to native data structures from a NIF. A resource object is @@ -1250,8 +1250,9 @@ typedef struct { <fsummary>Format strings and Erlang terms.</fsummary> <desc> <p>Similar to <c>fprintf</c> but this format string also accepts - <c>"%T"</c>, which formats Erlang terms.</p> - <p>This function was originally intenden for debugging purpose. It is not + <c>"%T"</c>, which formats Erlang terms of type + <seealso marker="#ERL_NIF_TERM"><c>ERL_NIF_TERM</c></seealso>.</p> + <p>This function is primarily intenden for debugging purpose. It is not recommended to print very large terms with <c>%T</c>. The function may change <c>errno</c>, even if successful.</p> </desc> @@ -3191,8 +3192,9 @@ if (retval & ERL_NIF_SELECT_STOP_CALLED) { <fsummary>Format strings and Erlang terms.</fsummary> <desc> <p>Similar to <c>snprintf</c> but this format string also accepts - <c>"%T"</c>, which formats Erlang terms.</p> - <p>This function was originally intenden for debugging purpose. It is not + <c>"%T"</c>, which formats Erlang terms of type + <seealso marker="#ERL_NIF_TERM"><c>ERL_NIF_TERM</c></seealso>.</p> + <p>This function is primarily intenden for debugging purpose. It is not recommended to print very large terms with <c>%T</c>. The function may change <c>errno</c>, even if successful.</p> </desc> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index b39d0e5e23..bd33e35603 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -197,6 +197,15 @@ </desc> </datatype> + <datatype> + <name name="nif_resource"></name> + <desc> + <p>An opaque handle identifing a + <seealso marker="erl_nif#resource_objects">NIF resource object + </seealso>.</p> + </desc> + </datatype> + </datatypes> <funcs> @@ -1742,6 +1751,10 @@ true</pre> <item> <p><c>Pid</c> is the process identifier of the process that originally created the fun.</p> + <p>It might point to the <c>init</c> process if the + <c>Fun</c> was statically allocated when module was + loaded (this optimisation is performed for local + functions that do not capture the enviornment).</p> </item> <tag><c>{index, Index}</c></tag> <item> @@ -3772,13 +3785,6 @@ RealSystem = system + MissedSystem</code> If found, that driver is started. A driver runs in the Erlang work space, which means that it is linked with the Erlang runtime system.</p> - <p>When starting external programs on Solaris, the system - call <c>vfork</c> is used in preference to <c>fork</c> - for performance reasons, although it has a history of - being less robust. If there are problems using - <c>vfork</c>, setting environment variable - <c>ERL_NO_VFORK</c> to any value causes <c>fork</c> - to be used instead.</p> <p>For external programs, <c>PATH</c> is searched (or an equivalent method is used to find programs, depending on the OS). This is done by invoking @@ -5295,10 +5301,10 @@ RealSystem = system + MissedSystem</code> <p><c><anno>MinBinVHeapSize</anno></c> is the minimum binary virtual heap size for the process.</p> </item> - <tag><c>{monitored_by, <anno>Pids</anno>}</c></tag> + <tag><c>{monitored_by, <anno>MonitoredBy</anno>}</c></tag> <item> - <p>A list of process identifiers monitoring the process (with - <c>monitor/2</c>).</p> + <p>A list of identifiers for all the processes, ports and NIF + resources, that are monitoring the process.</p> </item> <tag><c>{monitors, <anno>Monitors</anno>}</c></tag> <item> @@ -6170,7 +6176,7 @@ true</pre> <p>Monitors the new process (like <seealso marker="#monitor/2"><c>monitor/2</c></seealso> does).</p> </item> - <tag><c>{priority, <anno>Level</anno></c></tag> + <tag><c>{priority, <anno>Level</anno>}</c></tag> <item> <p>Sets the priority of the new process. Equivalent to executing <seealso marker="#process_flag_priority"> @@ -7581,7 +7587,7 @@ ok </func> <func> - <name name="system_info" arity="1" clause_i="75"/> + <name name="system_info" arity="1" clause_i="76"/> <fsummary>System info overview.</fsummary> <desc> <p>Returns information about the current system. @@ -7629,6 +7635,7 @@ ok <p> <seealso marker="#system_info_atom_count"><c>atom_count</c></seealso>, <seealso marker="#system_info_atom_limit"><c>atom_limit</c></seealso>, + <seealso marker="#system_info_ets_count"><c>ets_count</c></seealso>, <seealso marker="#system_info_ets_limit"><c>ets_limit</c></seealso>, <seealso marker="#system_info_port_count"><c>port_count</c></seealso>, <seealso marker="#system_info_port_limit"><c>port_limit</c></seealso>, @@ -7872,8 +7879,8 @@ ok <name name="system_info" arity="1" clause_i="12" anchor="system_info_cpu_topology"/> <!-- cpu_topology --> <name name="system_info" arity="1" clause_i="13"/> <!-- {cpu_topology, _} --> - <name name="system_info" arity="1" clause_i="37"/> <!-- logical_processors --> - <name name="system_info" arity="1" clause_i="72"/> <!-- update_cpu_info --> + <name name="system_info" arity="1" clause_i="38"/> <!-- logical_processors --> + <name name="system_info" arity="1" clause_i="73"/> <!-- update_cpu_info --> <fsummary>Information about the CPU topology of the system.</fsummary> <type name="cpu_topology"/> <type name="level_entry"/> @@ -8024,16 +8031,16 @@ ok </func> <func> - <name name="system_info" arity="1" clause_i="30" + <name name="system_info" arity="1" clause_i="31" anchor="system_info_process"/> <!-- fullsweep_after --> - <name name="system_info" arity="1" clause_i="31"/> <!-- garbage_collection --> - <name name="system_info" arity="1" clause_i="32"/> <!-- heap_sizes --> - <name name="system_info" arity="1" clause_i="33"/> <!-- heap_type --> - <name name="system_info" arity="1" clause_i="39"/> <!-- max_heap_size --> - <name name="system_info" arity="1" clause_i="40"/> <!-- message_queue_data --> - <name name="system_info" arity="1" clause_i="41"/> <!-- min_heap_size --> - <name name="system_info" arity="1" clause_i="42"/> <!-- min_bin_vheap_size --> - <name name="system_info" arity="1" clause_i="56"/> <!-- procs --> + <name name="system_info" arity="1" clause_i="32"/> <!-- garbage_collection --> + <name name="system_info" arity="1" clause_i="33"/> <!-- heap_sizes --> + <name name="system_info" arity="1" clause_i="34"/> <!-- heap_type --> + <name name="system_info" arity="1" clause_i="40"/> <!-- max_heap_size --> + <name name="system_info" arity="1" clause_i="41"/> <!-- message_queue_data --> + <name name="system_info" arity="1" clause_i="42"/> <!-- min_heap_size --> + <name name="system_info" arity="1" clause_i="43"/> <!-- min_bin_vheap_size --> + <name name="system_info" arity="1" clause_i="57"/> <!-- procs --> <fsummary>Information about the default process heap settings.</fsummary> <type name="message_queue_data"/> <type name="max_heap_size"/> @@ -8143,14 +8150,14 @@ ok </func> <func> - <name name="system_info" arity="1" clause_i="6" - anchor="system_info_limits"/> <!-- atom_count --> + <name name="system_info" arity="1" clause_i="6" anchor="system_info_limits"/> <!-- atom_count --> <name name="system_info" arity="1" clause_i="7"/> <!-- atom_limit --> - <name name="system_info" arity="1" clause_i="29"/> <!-- ets_limit --> - <name name="system_info" arity="1" clause_i="52"/> <!-- port_count --> - <name name="system_info" arity="1" clause_i="53"/> <!-- port_limit --> - <name name="system_info" arity="1" clause_i="54"/> <!-- process_count --> - <name name="system_info" arity="1" clause_i="55"/> <!-- process_limit --> + <name name="system_info" arity="1" clause_i="29"/> <!-- ets_count --> + <name name="system_info" arity="1" clause_i="30"/> <!-- ets_limit --> + <name name="system_info" arity="1" clause_i="53"/> <!-- port_count --> + <name name="system_info" arity="1" clause_i="54"/> <!-- port_limit --> + <name name="system_info" arity="1" clause_i="55"/> <!-- process_count --> + <name name="system_info" arity="1" clause_i="56"/> <!-- process_limit --> <fsummary>Information about various system limits.</fsummary> <desc> <marker id="system_info_limits"/> @@ -8173,7 +8180,7 @@ ok <c>erl(1)</c>. </p> </item> - <tag><marker id="system_info_ets_count"/> + <tag><marker id="system_info_ets_count"/> <c>ets_count</c></tag> <item> <p>Returns the number of ETS tables currently existing at the @@ -8225,13 +8232,13 @@ ok <func> <name name="system_info" arity="1" clause_i="26" anchor="system_info_time"/> <!-- end_time --> - <name name="system_info" arity="1" clause_i="49"/> <!-- os_monotonic_time_source --> - <name name="system_info" arity="1" clause_i="50"/> <!-- os_system_time_source --> - <name name="system_info" arity="1" clause_i="62"/> <!-- start_time --> - <name name="system_info" arity="1" clause_i="67"/> <!-- time_correction --> - <name name="system_info" arity="1" clause_i="68"/> <!-- time_offset --> - <name name="system_info" arity="1" clause_i="69"/> <!-- time_warp_mode --> - <name name="system_info" arity="1" clause_i="70"/> <!-- tolerant_timeofday --> + <name name="system_info" arity="1" clause_i="50"/> <!-- os_monotonic_time_source --> + <name name="system_info" arity="1" clause_i="51"/> <!-- os_system_time_source --> + <name name="system_info" arity="1" clause_i="63"/> <!-- start_time --> + <name name="system_info" arity="1" clause_i="68"/> <!-- time_correction --> + <name name="system_info" arity="1" clause_i="69"/> <!-- time_offset --> + <name name="system_info" arity="1" clause_i="70"/> <!-- time_warp_mode --> + <name name="system_info" arity="1" clause_i="71"/> <!-- tolerant_timeofday --> <fsummary>Information about system time.</fsummary> <desc> <marker id="system_info_time_tags"/> @@ -8455,16 +8462,16 @@ ok anchor="system_info_scheduler"/> <!-- dirty_cpu_schedulers --> <name name="system_info" arity="1" clause_i="18"/> <!-- dirty_cpu_schedulers_online --> <name name="system_info" arity="1" clause_i="19"/> <!-- dirty_io_schedulers --> - <name name="system_info" arity="1" clause_i="44"/> <!-- multi_scheduling --> - <name name="system_info" arity="1" clause_i="45"/> <!-- multi_scheduling_blockers --> - <name name="system_info" arity="1" clause_i="47"/> <!-- normal_multi_scheduling_blockers --> - <name name="system_info" arity="1" clause_i="57"/> <!-- scheduler_bind_type --> - <name name="system_info" arity="1" clause_i="58"/> <!-- scheduler_bindings --> - <name name="system_info" arity="1" clause_i="59"/> <!-- scheduler_id --> - <name name="system_info" arity="1" clause_i="60"/> <!-- schedulers --> - <name name="system_info" arity="1" clause_i="61"/> <!-- smp_support --> - <name name="system_info" arity="1" clause_i="65"/> <!-- threads --> - <name name="system_info" arity="1" clause_i="66"/> <!-- thread_pool_size --> + <name name="system_info" arity="1" clause_i="45"/> <!-- multi_scheduling --> + <name name="system_info" arity="1" clause_i="46"/> <!-- multi_scheduling_blockers --> + <name name="system_info" arity="1" clause_i="49"/> <!-- normal_multi_scheduling_blockers --> + <name name="system_info" arity="1" clause_i="58"/> <!-- scheduler_bind_type --> + <name name="system_info" arity="1" clause_i="59"/> <!-- scheduler_bindings --> + <name name="system_info" arity="1" clause_i="60"/> <!-- scheduler_id --> + <name name="system_info" arity="1" clause_i="61"/> <!-- schedulers --> + <name name="system_info" arity="1" clause_i="62"/> <!-- smp_support --> + <name name="system_info" arity="1" clause_i="66"/> <!-- threads --> + <name name="system_info" arity="1" clause_i="67"/> <!-- thread_pool_size --> <fsummary>Information about system schedulers.</fsummary> <desc> <marker id="system_info_scheduler_tags"/> @@ -8851,53 +8858,54 @@ ok <!-- <name name="system_info" arity="1" clause_i="26"/> end_time --> <!-- <name name="system_info" arity="1" clause_i="27"/> elib_malloc --> <!-- <name name="system_info" arity="1" clause_i="28"/> eager_check_io, removed --> - <!-- <name name="system_info" arity="1" clause_i="29"/> ets_limit --> - <!-- <name name="system_info" arity="1" clause_i="30"/> fullsweep_after --> - <!-- <name name="system_info" arity="1" clause_i="31"/> garbage_collection --> - <!-- <name name="system_info" arity="1" clause_i="32"/> heap_sizes --> - <!-- <name name="system_info" arity="1" clause_i="33"/> heap_type --> - <name name="system_info" arity="1" clause_i="34"/> <!-- info --> - <name name="system_info" arity="1" clause_i="35"/> <!-- kernel_poll --> - <name name="system_info" arity="1" clause_i="36"/> <!-- loaded --> - <!-- <name name="system_info" arity="1" clause_i="37"/> logical_processors --> - <name name="system_info" arity="1" clause_i="38"/> <!-- machine --> - <!-- <name name="system_info" arity="1" clause_i="39"/> max_heap_size --> - <!-- <name name="system_info" arity="1" clause_i="40"/> message_queue_data --> - <!-- <name name="system_info" arity="1" clause_i="41"/> min_heap_size --> - <!-- <name name="system_info" arity="1" clause_i="42"/> min_bin_vheap_size --> - <name name="system_info" arity="1" clause_i="43"/> <!-- modified_timing_level --> - <!-- <name name="system_info" arity="1" clause_i="44"/> multi_scheduling --> - <!-- <name name="system_info" arity="1" clause_i="45"/> multi_scheduling_blockers --> - <name name="system_info" arity="1" clause_i="46"/> <!-- nif_version --> - <!-- n<name name="system_info" arity="1" clause_i="47"/> ormal_multi_scheduling_blockers --> - <name name="system_info" arity="1" clause_i="48"/> <!-- otp_release --> - <!-- <name name="system_info" arity="1" clause_i="49"/> os_monotonic_time_source --> - <!-- <name name="system_info" arity="1" clause_i="50"/> os_system_time_source --> - <name name="system_info" arity="1" clause_i="51"/> <!-- port_parallelism --> - <!-- <name name="system_info" arity="1" clause_i="52"/> port_count --> - <!-- <name name="system_info" arity="1" clause_i="53"/> port_limit --> - <!-- <name name="system_info" arity="1" clause_i="54"/> process_count --> - <!-- <name name="system_info" arity="1" clause_i="55"/> process_limit --> - <!-- <name name="system_info" arity="1" clause_i="56"/> procs --> - <!-- <name name="system_info" arity="1" clause_i="57"/> scheduler_bind_type --> - <!-- <name name="system_info" arity="1" clause_i="58"/> scheduler_bindings --> - <!-- <name name="system_info" arity="1" clause_i="59"/> scheduler_id --> - <!-- <name name="system_info" arity="1" clause_i="60"/> schedulers --> - <!-- <name name="system_info" arity="1" clause_i="61"/> smp_support --> - <!-- <name name="system_info" arity="1" clause_i="62"/> start_time --> - <name name="system_info" arity="1" clause_i="63"/> <!-- system_version --> - <name name="system_info" arity="1" clause_i="64"/> <!-- system_architecture --> - <!-- <name name="system_info" arity="1" clause_i="65"/> threads --> - <!-- <name name="system_info" arity="1" clause_i="66"/> thread_pool_size --> - <!-- <name name="system_info" arity="1" clause_i="67"/> time_correction --> - <!-- <name name="system_info" arity="1" clause_i="68"/> time_offset --> - <!-- <name name="system_info" arity="1" clause_i="69"/> time_warp_mode --> - <!-- <name name="system_info" arity="1" clause_i="70"/> tolerant_timeofday --> - <name name="system_info" arity="1" clause_i="71"/> <!-- trace_control_word --> - <!-- <name name="system_info" arity="1" clause_i="72"/> update_cpu_info --> - <name name="system_info" arity="1" clause_i="73"/> <!-- version --> - <name name="system_info" arity="1" clause_i="74"/> <!-- wordsize --> - <!-- <name name="system_info" arity="1" clause_i="75"/> overview --> + <!-- <name name="system_info" arity="1" clause_i="29"/> ets_count --> + <!-- <name name="system_info" arity="1" clause_i="30"/> ets_limit --> + <!-- <name name="system_info" arity="1" clause_i="31"/> fullsweep_after --> + <!-- <name name="system_info" arity="1" clause_i="32"/> garbage_collection --> + <!-- <name name="system_info" arity="1" clause_i="33"/> heap_sizes --> + <!-- <name name="system_info" arity="1" clause_i="34"/> heap_type --> + <name name="system_info" arity="1" clause_i="35"/> <!-- info --> + <name name="system_info" arity="1" clause_i="36"/> <!-- kernel_poll --> + <name name="system_info" arity="1" clause_i="37"/> <!-- loaded --> + <!-- <name name="system_info" arity="1" clause_i="38"/> logical_processors --> + <name name="system_info" arity="1" clause_i="39"/> <!-- machine --> + <!-- <name name="system_info" arity="1" clause_i="40"/> max_heap_size --> + <!-- <name name="system_info" arity="1" clause_i="41"/> message_queue_data --> + <!-- <name name="system_info" arity="1" clause_i="42"/> min_heap_size --> + <!-- <name name="system_info" arity="1" clause_i="43"/> min_bin_vheap_size --> + <name name="system_info" arity="1" clause_i="44"/> <!-- modified_timing_level --> + <!-- <name name="system_info" arity="1" clause_i="45"/> multi_scheduling --> + <!-- <name name="system_info" arity="1" clause_i="46"/> multi_scheduling_blockers --> + <name name="system_info" arity="1" clause_i="47"/> <!-- nif_version --> + <!-- n<name name="system_info" arity="1" clause_i="48"/> ormal_multi_scheduling_blockers --> + <name name="system_info" arity="1" clause_i="49"/> <!-- otp_release --> + <!-- <name name="system_info" arity="1" clause_i="50"/> os_monotonic_time_source --> + <!-- <name name="system_info" arity="1" clause_i="51"/> os_system_time_source --> + <name name="system_info" arity="1" clause_i="52"/> <!-- port_parallelism --> + <!-- <name name="system_info" arity="1" clause_i="53"/> port_count --> + <!-- <name name="system_info" arity="1" clause_i="54"/> port_limit --> + <!-- <name name="system_info" arity="1" clause_i="55"/> process_count --> + <!-- <name name="system_info" arity="1" clause_i="56"/> process_limit --> + <!-- <name name="system_info" arity="1" clause_i="57"/> procs --> + <!-- <name name="system_info" arity="1" clause_i="58"/> scheduler_bind_type --> + <!-- <name name="system_info" arity="1" clause_i="59"/> scheduler_bindings --> + <!-- <name name="system_info" arity="1" clause_i="60"/> scheduler_id --> + <!-- <name name="system_info" arity="1" clause_i="61"/> schedulers --> + <!-- <name name="system_info" arity="1" clause_i="62"/> smp_support --> + <!-- <name name="system_info" arity="1" clause_i="63"/> start_time --> + <name name="system_info" arity="1" clause_i="64"/> <!-- system_version --> + <name name="system_info" arity="1" clause_i="65"/> <!-- system_architecture --> + <!-- <name name="system_info" arity="1" clause_i="66"/> threads --> + <!-- <name name="system_info" arity="1" clause_i="67"/> thread_pool_size --> + <!-- <name name="system_info" arity="1" clause_i="68"/> time_correction --> + <!-- <name name="system_info" arity="1" clause_i="69"/> time_offset --> + <!-- <name name="system_info" arity="1" clause_i="70"/> time_warp_mode --> + <!-- <name name="system_info" arity="1" clause_i="71"/> tolerant_timeofday --> + <name name="system_info" arity="1" clause_i="72"/> <!-- trace_control_word --> + <!-- <name name="system_info" arity="1" clause_i="73"/> update_cpu_info --> + <name name="system_info" arity="1" clause_i="74"/> <!-- version --> + <name name="system_info" arity="1" clause_i="75"/> <!-- wordsize --> + <!-- <name name="system_info" arity="1" clause_i="76"/> overview --> <fsummary>Information about the system.</fsummary> <desc> <marker id="system_info_misc_tags"/> diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index a094217959..962bc9a244 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -487,11 +487,10 @@ utilization value used. Once a carrier is abandoned, no new allocations are made in it. When an allocator instance gets an increased multiblock carrier need, it first tries to fetch an - abandoned carrier from an allocator instance of the same - allocator type. If no abandoned carrier can be fetched, it - creates a new empty carrier. When an abandoned carrier has been - fetched, it will function as an ordinary carrier. This feature has - special requirements on the + abandoned carrier from another allocator instance. If no abandoned + carrier can be fetched, it creates a new empty carrier. When an + abandoned carrier has been fetched, it will function as an ordinary + carrier. This feature has special requirements on the <seealso marker="#M_as">allocation strategy</seealso> used. Only the strategies <c>aoff</c>, <c>aoffcbf</c>, <c>aoffcaobf</c>, <c>ageffcaoff</c>m, <c>ageffcbf</c> and <c>ageffcaobf</c> @@ -584,7 +583,7 @@ carriers are decided in section <seealso marker="#mseg_mbc_sizes"> The alloc_util Framework</seealso>. On - 32-bit Unix style OS this limit cannot be set > 128 MB.</p> + 32-bit Unix style OS this limit cannot be set > 64 MB.</p> </item> <tag><marker id="M_mbcgs"/><c><![CDATA[+M<S>mbcgs <ratio>]]></c></tag> <item> diff --git a/erts/doc/src/notes.xml b/erts/doc/src/notes.xml index ed0946d6ba..fa02c9dd21 100644 --- a/erts/doc/src/notes.xml +++ b/erts/doc/src/notes.xml @@ -31,6 +31,129 @@ </header> <p>This document describes the changes made to the ERTS application.</p> +<section><title>Erts 10.0.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug which caused an emulator crash when + <c>enif_send()</c> was called by a NIF that executed on a + dirty scheduler. The bug was either triggered when the + NIF called <c>enif_send()</c> without a message + environment, or when the process executing the NIF was + <c>send</c> traced.</p> + <p> + Own Id: OTP-15223</p> + </item> + <item> + <p> + Fixed a bug causing some Erlang references to be + inconsistently ordered. This could for example cause + failure to look up certain elements with references as + keys in search data structures. This bug was introduced + in R13B02.</p> + <p> + Thanks to Simon Cornish for finding the bug and supplying + a fix.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15225</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 10.0.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a bug that prevented the <c>noshell</c> option + from working correctly on Mac OS X and BSD.</p> + <p> + Own Id: OTP-15169</p> + </item> + <item> + <p>Fixed a crash when matching directly against a literal + map using a single key that had been saved on the + stack.</p> + <p> + Own Id: OTP-15184</p> + </item> + <item> + <p>Fix node crash when passing a bad time option to + <c>file:read_file_info/2</c>.</p> + <p> + Own Id: OTP-15196</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 10.0.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a scheduler bug that caused normal schedulers to + run dirty code.</p> + <p> + Own Id: OTP-15154</p> + </item> + <item> + <p> + Fixed a bug in <c>erlang:trace_info/2</c> which caused + the emulator to crash when a bad argument was passed. The + bug was introduced in ERTS version 10.0.</p> + <p> + Own Id: OTP-15183 Aux Id: ERL-670 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 10.0.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a rare bug that could cause processes to be + scheduled after they had been freed.</p> + <p> + Own Id: OTP-15067 Aux Id: ERL-573 </p> + </item> + <item> + <p>Fixed a race condition in the inet driver that could + cause receive to hang when the emulator was compiled with + gcc 8.</p> + <p> + Own Id: OTP-15158 Aux Id: ERL-654 </p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 10.0.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>The keys used in <c>os:getenv</c> and <c>os:putenv</c> + are case-insensitive again on Windows.</p> + <p> + Own Id: OTP-15147 Aux Id: ERL-644 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 10.0</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -967,6 +1090,81 @@ </section> +<section><title>Erts 9.3.3.3</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p> + Fixed a bug which caused an emulator crash when + <c>enif_send()</c> was called by a NIF that executed on a + dirty scheduler. The bug was either triggered when the + NIF called <c>enif_send()</c> without a message + environment, or when the process executing the NIF was + <c>send</c> traced.</p> + <p> + Own Id: OTP-15223</p> + </item> + <item> + <p> + Fixed a bug causing some Erlang references to be + inconsistently ordered. This could for example cause + failure to look up certain elements with references as + keys in search data structures. This bug was introduced + in R13B02.</p> + <p> + Thanks to Simon Cornish for finding the bug and supplying + a fix.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15225</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.3.3.2</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a race condition in the inet driver that could + cause receive to hang when the emulator was compiled with + gcc 8.</p> + <p> + Own Id: OTP-15158 Aux Id: ERL-654 </p> + </item> + <item> + <p> + Fix bug in generation of erl_crash.dump, which could + cause VM to crash.</p> + <p> + Bug exist since erts-9.2 (OTP-20.2).</p> + <p> + Own Id: OTP-15181</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 9.3.3.1</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a rare bug that could cause processes to be + scheduled after they had been freed.</p> + <p> + Own Id: OTP-15067 Aux Id: ERL-573 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 9.3.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -2394,6 +2592,63 @@ </section> +<section><title>Erts 8.3.5.5</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a race condition in the inet driver that could + cause receive to hang when the emulator was compiled with + gcc 8.</p> + <p> + Own Id: OTP-15158 Aux Id: ERL-654 </p> + </item> + <item> + <p> + Fixed a bug causing some Erlang references to be + inconsistently ordered. This could for example cause + failure to look up certain elements with references as + keys in search data structures. This bug was introduced + in R13B02.</p> + <p> + Thanks to Simon Cornish for finding the bug and supplying + a fix.</p> + <p> + *** POTENTIAL INCOMPATIBILITY ***</p> + <p> + Own Id: OTP-15225</p> + </item> + </list> + </section> + +</section> + +<section><title>Erts 8.3.5.4</title> + + <section><title>Fixed Bugs and Malfunctions</title> + <list> + <item> + <p>Fixed a bug in file closure on Unix; close(2) was + retried on EINTR which could cause a different (recently + opened) file to be closed as well.</p> + <p> + Own Id: OTP-14775</p> + </item> + <item> + <p> + A race-condition when tearing down a connection with + active node monitors could cause the runtime system to + crash.</p> + <p> + This bug was introduced in ERTS version 8.0 (OTP 19.0).</p> + <p> + Own Id: OTP-14781 Aux Id: OTP-13047 </p> + </item> + </list> + </section> + +</section> + <section><title>Erts 8.3.5.3</title> <section><title>Fixed Bugs and Malfunctions</title> @@ -9804,7 +10059,7 @@ you use erlang:halt/2 with an integer first argument and an option list containing {flush,false} as the second argument. Note that now is flushing not dependant of the - exit code, and you can not only flush async threads + exit code, and you cannot only flush async threads operations which we deemed as a strange behaviour anyway. </p> <p>Also, erlang:halt/1,2 has gotten a new feature: If the diff --git a/erts/doc/src/time_correction.xml b/erts/doc/src/time_correction.xml index 77e7a40529..53b555387c 100644 --- a/erts/doc/src/time_correction.xml +++ b/erts/doc/src/time_correction.xml @@ -940,7 +940,7 @@ EventTag = {Time, UMI}</code> </item> <item> <seealso marker="erlang#system_info_os_system_time_source"> - <c>erlang:system_info(os_system_time_source)</c></seealso>) + <c>erlang:system_info(os_system_time_source)</c></seealso> </item> </list> diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index a0dbd9ec7b..d221e6aea6 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1753,6 +1753,7 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) if (literals) { ErtsLiteralAreaRef *ref; + ErtsMessage *mp; ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, sizeof(ErtsLiteralAreaRef)); ref->literal_area = literals; @@ -1767,10 +1768,12 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) release_literal_areas.last = ref; } erts_mtx_unlock(&release_literal_areas.mtx); + mp = erts_alloc_message(0, NULL); + ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_proc_message(BIF_P, erts_literal_area_collector, 0, - erts_alloc_message(0, NULL), + mp, am_copy_literals); } diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 6d3b99c43e..9633de2021 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -667,7 +667,7 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) ap++; break; case 'l': /* fr(N) */ - erts_print(to, to_arg, "fr(%d)", loader_reg_index(ap[0])); + erts_print(to, to_arg, "fr(%d)", ap[0] / sizeof(FloatDef)); ap++; break; default: @@ -1191,6 +1191,7 @@ dirty_send_message(Process *c_p, Eterm to, Eterm tag) mp = erts_alloc_message_heap(rp, &rp_locks, 3, &hp, &ohp); msg = TUPLE2(hp, tag, c_p->common.id); + ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_proc_message(c_p, rp, rp_locks, mp, msg); if (rp == real_c_p) diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index ab5920a67e..aa61a2d7f9 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -3061,12 +3061,14 @@ erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p Uint need; flatmap_t *old_mp, *mp; Eterm res; + Eterm* old_hp; Eterm* hp; Eterm* E; Eterm* old_keys; Eterm* old_vals; Eterm new_key; Eterm map; + int changed = 0; n /= 2; /* Number of values to be updated */ ASSERT(n > 0); @@ -3133,6 +3135,7 @@ erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p * Update map, keeping the old key tuple. */ + old_hp = p->htop; hp = p->htop; E = p->stop; @@ -3155,20 +3158,26 @@ erts_gc_update_map_exact(Process* p, Eterm* reg, Uint live, Uint n, Eterm* new_p /* Not same keys */ *hp++ = *old_vals; } else { - GET_TERM(new_p[1], *hp); - hp++; - n--; + GET_TERM(new_p[1], *hp); + if(*hp != *old_vals) changed = 1; + hp++; + n--; if (n == 0) { - /* - * All updates done. Copy remaining values - * and return the result. - */ - for (i++, old_vals++; i < num_old; i++) { - *hp++ = *old_vals++; - } - ASSERT(hp == p->htop + need); - p->htop = hp; - return res; + /* + * All updates done. Copy remaining values + * if any changed or return the original one. + */ + if(changed) { + for (i++, old_vals++; i < num_old; i++) { + *hp++ = *old_vals++; + } + ASSERT(hp == p->htop + need); + p->htop = hp; + return res; + } else { + p->htop = old_hp; + return map; + } } else { new_p += 2; GET_TERM(*new_p, new_key); diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index e61199a8fd..50cbb37f3e 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -4245,21 +4245,55 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) { ErlFunEntry* fe; GenOp* op; + Uint arity, num_free; if (idx.val >= stp->num_lambdas) { - stp->lambda_error = "missing or short chunk 'FunT'"; - fe = 0; + stp->lambda_error = "missing or short chunk 'FunT'"; + fe = 0; + num_free = 0; + arity = 0; } else { - fe = stp->lambdas[idx.val].fe; + fe = stp->lambdas[idx.val].fe; + num_free = stp->lambdas[idx.val].num_free; + arity = fe->arity; } NEW_GENOP(stp, op); - op->op = genop_i_make_fun_2; - op->arity = 2; - op->a[0].type = TAG_u; - op->a[0].val = (BeamInstr) fe; - op->a[1].type = TAG_u; - op->a[1].val = stp->lambdas[idx.val].num_free; + + /* + * It's possible this is called before init process is started, + * skip the optimisation in such case. + */ + if (num_free == 0 && erts_init_process_id != ERTS_INVALID_PID) { + Uint lit; + Eterm* hp; + ErlFunThing* funp; + + lit = new_literal(stp, &hp, ERL_FUN_SIZE); + funp = (ErlFunThing *) hp; + erts_refc_inc(&fe->refc, 2); + funp->thing_word = HEADER_FUN; + funp->next = NULL; + funp->fe = fe; + funp->num_free = 0; + funp->creator = erts_init_process_id; + funp->arity = arity; + + op->op = genop_move_2; + op->arity = 2; + op->a[0].type = TAG_q; + op->a[0].val = lit; + op->a[1].type = TAG_x; + op->a[1].val = 0; + } else { + op->op = genop_i_make_fun_2; + op->arity = 2; + op->a[0].type = TAG_u; + op->a[0].val = (BeamInstr) fe; + op->a[1].type = TAG_u; + op->a[1].val = num_free; + } + op->next = NULL; return op; } diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index f0c9496341..9f3153724a 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -35,10 +35,8 @@ typedef struct { /* * Used for crash dumping of literals. The size of erts_dump_lit_areas is - * always twice the number of active ranges (to allow for literals in both - * current and old code). + * always at least the number of active ranges. */ - ErtsLiteralArea** erts_dump_lit_areas; Uint erts_dump_num_lit_areas; @@ -180,8 +178,8 @@ erts_end_staging_ranges(int commit) (erts_aint_t) (r[dst].modules + r[dst].n / 2)); - if (r[dst].allocated * 2 > erts_dump_num_lit_areas) { - erts_dump_num_lit_areas *= 2; + if (r[dst].allocated > erts_dump_num_lit_areas) { + erts_dump_num_lit_areas = r[dst].allocated * 2; erts_dump_lit_areas = (ErtsLiteralArea **) erts_realloc(ERTS_ALC_T_CRASH_DUMP, (void *) erts_dump_lit_areas, diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 97e1ee1286..56ac072449 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -1803,6 +1803,7 @@ ebif_bang_2(BIF_ALIST_2) #define SEND_INTERNAL_ERROR (-6) #define SEND_AWAIT_RESULT (-7) #define SEND_YIELD_CONTINUE (-8) +#define SEND_SYSTEM_LIMIT (-9) static Sint remote_send(Process *p, DistEntry *dep, @@ -1842,6 +1843,8 @@ static Sint remote_send(Process *p, DistEntry *dep, res = SEND_YIELD_RETURN; else if (code == ERTS_DSIG_SEND_CONTINUE) res = SEND_YIELD_CONTINUE; + else if (code == ERTS_DSIG_SEND_TOO_LRG) + res = SEND_SYSTEM_LIMIT; else res = 0; break; @@ -2063,7 +2066,7 @@ do_send(Process *p, Eterm to, Eterm msg, Eterm *refp, ErtsSendContext *ctx) if (p == rp) rp_locks |= ERTS_PROC_LOCK_MAIN; /* send to local process */ - erts_send_message(p, rp, &rp_locks, msg, 0); + erts_send_message(p, rp, &rp_locks, msg); erts_proc_unlock(rp, p == rp ? (rp_locks & ~ERTS_PROC_LOCK_MAIN) @@ -2162,6 +2165,9 @@ BIF_RETTYPE send_3(BIF_ALIST_3) case SEND_BADARG: ERTS_BIF_PREP_ERROR(retval, p, BADARG); break; + case SEND_SYSTEM_LIMIT: + ERTS_BIF_PREP_ERROR(retval, p, SYSTEM_LIMIT); + break; case SEND_USER_ERROR: ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR); break; @@ -2218,6 +2224,10 @@ static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1) BUMP_ALL_REDS(BIF_P); BIF_TRAP1(&dsend_continue_trap_export, BIF_P, BIF_ARG_1); } + case ERTS_DSIG_SEND_TOO_LRG: { /*SEND_SYSTEM_LIMIT*/ + erts_set_gc_state(BIF_P, 1); + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + } default: erts_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result); break; @@ -2275,6 +2285,9 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) case SEND_BADARG: ERTS_BIF_PREP_ERROR(retval, p, BADARG); break; + case SEND_SYSTEM_LIMIT: + ERTS_BIF_PREP_ERROR(retval, p, SYSTEM_LIMIT); + break; case SEND_USER_ERROR: ERTS_BIF_PREP_ERROR(retval, p, EXC_ERROR); break; diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 7548924178..a770524221 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -629,7 +629,6 @@ bif maps:from_list/1 bif maps:is_key/2 bif maps:keys/1 bif maps:merge/2 -bif maps:new/0 bif maps:put/3 bif maps:remove/2 bif maps:update/3 diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 7556205063..a1ad75708c 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -42,7 +42,7 @@ typedef Uint16 ErtsHalfDigit; #undef BIG_HAVE_DOUBLE_DIGIT typedef Uint32 ErtsHalfDigit; #else -#error "can not determine machine size" +#error "cannot determine machine size" #endif typedef Uint dsize_t; /* Vector size type */ diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 9ff52c92b8..81531f6cc8 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -108,6 +108,7 @@ process_killer(void) erts_exit(0, ""); switch(j) { case 'k': + ASSERT(erts_init_process_id != ERTS_INVALID_PID); /* Send a 'kill' exit signal from init process */ erts_proc_sig_send_exit(NULL, erts_init_process_id, rp->common.id, am_kill, NIL, diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 70474898b2..db594a23a0 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1200,21 +1200,8 @@ erts_dsig_send_group_leader(ErtsDSigData *dsdp, Eterm leader, Eterm remote) #include <valgrind/valgrind.h> #include <valgrind/memcheck.h> -#ifndef HAVE_VALGRIND_PRINTF_XML -#define VALGRIND_PRINTF_XML VALGRIND_PRINTF -#endif - # define PURIFY_MSG(msg) \ - do { \ - char buf__[1]; size_t bufsz__ = sizeof(buf__); \ - if (erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf__, &bufsz__) >= 0) { \ - VALGRIND_PRINTF_XML("<erlang_error_log>" \ - "%s, line %d: %s</erlang_error_log>\n", \ - __FILE__, __LINE__, msg); \ - } else { \ - VALGRIND_PRINTF("%s, line %d: %s", __FILE__, __LINE__, msg); \ - } \ - } while (0) + VALGRIND_PRINTF("%s, line %d: %s", __FILE__, __LINE__, msg) #else # define PURIFY_MSG(msg) #endif @@ -1937,6 +1924,12 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) ASSERT(ctx->obuf->ext_endp <= &ctx->obuf->data[0] + ctx->data_size); ctx->data_size = ctx->obuf->ext_endp - ctx->obuf->extp; + if (ctx->data_size > (Uint) INT_MAX) { + free_dist_obuf(ctx->obuf); + ctx->obuf = NULL; + retval = ERTS_DSIG_SEND_TOO_LRG; + goto done; + } ctx->obuf->hopefull_flags = ctx->u.ec.hopefull_flags; /* @@ -3685,6 +3678,7 @@ int erts_auto_connect(DistEntry* dep, Process *proc, ErtsProcLocks proc_locks) dhandle = erts_build_dhandle(&hp, ohp, dep); msg = TUPLE4(hp, am_auto_connect, dep->sysname, make_small(conn_id), dhandle); + ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_proc_message(proc, net_kernel, nk_locks, mp, msg); erts_proc_unlock(net_kernel, nk_locks); } diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index dda2029a4c..55204eb83d 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -376,6 +376,7 @@ typedef struct { #define ERTS_DSIG_SEND_OK 0 #define ERTS_DSIG_SEND_YIELD 1 #define ERTS_DSIG_SEND_CONTINUE 2 +#define ERTS_DSIG_SEND_TOO_LRG 3 extern int erts_dsig_send_link(ErtsDSigData *, Eterm, Eterm); extern int erts_dsig_send_msg(Eterm, Eterm, ErtsSendContext*); diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 575d6ca867..36c46fd7aa 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -64,9 +64,6 @@ # error "Too many schedulers; cannot create that many pref alloc instances" #endif -#define ERTS_ALC_FIX_TYPE_IX(T) \ - (ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE) - #define ERTS_ALC_DEFAULT_MAX_THR_PREF ERTS_MAX_NO_OF_SCHEDULERS #if defined(SMALL_MEMORY) || defined(PURIFY) || defined(VALGRIND) @@ -114,7 +111,7 @@ typedef union { char align_afa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AFAllctr_t))]; AOFFAllctr_t aoffa; char align_aoffa[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(AOFFAllctr_t))]; -} ErtsAllocatorState_t; +} ErtsAllocatorState_t erts_align_attribute(ERTS_CACHE_LINE_SIZE); static ErtsAllocatorState_t std_alloc_state; static ErtsAllocatorState_t ll_alloc_state; @@ -156,20 +153,13 @@ ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(aireq, ErtsAlcType_t erts_fix_core_allocator_ix; -enum allctr_type { - GOODFIT, - BESTFIT, - AFIT, - FIRSTFIT -}; - struct au_init { int enable; int thr_spec; int disable_allowed; int thr_spec_allowed; int carrier_migration_allowed; - enum allctr_type atype; + ErtsAlcStrat_t astrat; struct { AllctrInit_t util; GFAllctrInit_t gf; @@ -219,7 +209,9 @@ typedef struct { struct au_init test_alloc; } erts_alc_hndl_args_init_t; -#define ERTS_AU_INIT__ {0, 0, 1, 1, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} +#define ERTS_AU_INIT__ {0, 0, 1, 1, 1, \ + ERTS_ALC_S_GOODFIT, DEFAULT_ALLCTR_INIT, \ + {1,1,1,1}} #define SET_DEFAULT_ALLOC_OPTS(IP) \ do { \ @@ -233,7 +225,7 @@ set_default_sl_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = GOODFIT; + ip->astrat = ERTS_ALC_S_GOODFIT; ip->init.util.name_prefix = "sl_"; ip->init.util.alloc_no = ERTS_ALC_A_SHORT_LIVED; #ifndef SMALL_MEMORY @@ -252,7 +244,7 @@ set_default_std_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.util.name_prefix = "std_"; ip->init.util.alloc_no = ERTS_ALC_A_STANDARD; #ifndef SMALL_MEMORY @@ -270,7 +262,7 @@ set_default_ll_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 0; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; @@ -299,7 +291,7 @@ set_default_literal_alloc_opts(struct au_init *ip) ip->disable_allowed = 0; ip->thr_spec_allowed = 0; ip->carrier_migration_allowed = 0; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; @@ -349,7 +341,7 @@ set_default_exec_alloc_opts(struct au_init *ip) ip->disable_allowed = 0; ip->thr_spec_allowed = 0; ip->carrier_migration_allowed = 0; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.bf.ao = 1; ip->init.util.ramv = 0; ip->init.util.mmsbc = 0; @@ -378,7 +370,7 @@ set_default_temp_alloc_opts(struct au_init *ip) ip->thr_spec = 1; ip->disable_allowed = 0; ip->carrier_migration_allowed = 0; - ip->atype = AFIT; + ip->astrat = ERTS_ALC_S_AFIT; ip->init.util.name_prefix = "temp_"; ip->init.util.alloc_no = ERTS_ALC_A_TEMPORARY; #ifndef SMALL_MEMORY @@ -397,7 +389,7 @@ set_default_eheap_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = GOODFIT; + ip->astrat = ERTS_ALC_S_GOODFIT; ip->init.util.name_prefix = "eheap_"; ip->init.util.alloc_no = ERTS_ALC_A_EHEAP; #ifndef SMALL_MEMORY @@ -416,7 +408,7 @@ set_default_binary_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.util.name_prefix = "binary_"; ip->init.util.alloc_no = ERTS_ALC_A_BINARY; #ifndef SMALL_MEMORY @@ -435,7 +427,7 @@ set_default_ets_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.util.name_prefix = "ets_"; ip->init.util.alloc_no = ERTS_ALC_A_ETS; #ifndef SMALL_MEMORY @@ -453,7 +445,7 @@ set_default_driver_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.util.name_prefix = "driver_"; ip->init.util.alloc_no = ERTS_ALC_A_DRIVER; #ifndef SMALL_MEMORY @@ -473,7 +465,7 @@ set_default_fix_alloc_opts(struct au_init *ip, SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = AU_ALLOC_DEFAULT_ENABLE(1); ip->thr_spec = 1; - ip->atype = BESTFIT; + ip->astrat = ERTS_ALC_S_BESTFIT; ip->init.bf.ao = 1; ip->init.util.name_prefix = "fix_"; ip->init.util.fix_type_size = fix_type_sizes; @@ -493,7 +485,7 @@ set_default_test_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = 0; /* Disabled by default */ ip->thr_spec = -1 * erts_no_schedulers; - ip->atype = FIRSTFIT; + ip->astrat = ERTS_ALC_S_FIRSTFIT; ip->init.aoff.crr_order = FF_AOFF; ip->init.aoff.blk_order = FF_BF; ip->init.util.name_prefix = "test_"; @@ -552,8 +544,8 @@ start_au_allocator(ErtsAlcType_t alctr_n, static void refuse_af_strategy(struct au_init *init) { - if (init->atype == AFIT) - init->atype = GOODFIT; + if (init->astrat == ERTS_ALC_S_AFIT) + init->astrat = ERTS_ALC_S_GOODFIT; } #ifdef HARD_DEBUG @@ -576,7 +568,10 @@ static void adjust_fix_alloc_sizes(UWord extra_block_size) for (i=0; i < tspec->size; i++) { Allctr_t* allctr = tspec->allctr[i]; for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) { - allctr->fix[j].type_size += extra_block_size; + size_t size = allctr->fix[j].type_size; + size = MAX(size + extra_block_size, + sizeof(ErtsAllctrDDBlock_t)); + allctr->fix[j].type_size = size; } } } @@ -584,8 +579,11 @@ static void adjust_fix_alloc_sizes(UWord extra_block_size) { Allctr_t* allctr = erts_allctrs_info[ERTS_ALC_A_FIXED_SIZE].extra; for (j=0; j < ERTS_ALC_NO_FIXED_SIZES; ++j) { - allctr->fix[j].type_size += extra_block_size; - } + size_t size = allctr->fix[j].type_size; + size = MAX(size + extra_block_size, + sizeof(ErtsAllctrDDBlock_t)); + allctr->fix[j].type_size = size; + } } } } @@ -597,7 +595,7 @@ strategy_support_carrier_migration(struct au_init *auip) * Currently only aoff* and ageff* support carrier * migration, i.e, type AOFIRSTFIT. */ - return auip->atype == FIRSTFIT; + return auip->astrat == ERTS_ALC_S_FIRSTFIT; } static ERTS_INLINE void @@ -612,7 +610,7 @@ adjust_carrier_migration_support(struct au_init *auip) */ if (!strategy_support_carrier_migration(auip)) { /* Default to aoffcbf */ - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AOFF; auip->init.aoff.blk_order = FF_BF; } @@ -1018,7 +1016,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, int i; int size = 1; void *as0; - enum allctr_type atype; + ErtsAlcStrat_t astrat; ErtsAllocatorFunctions_t *af = &erts_allctrs[alctr_n]; ErtsAllocatorInfo_t *ai = &erts_allctrs_info[alctr_n]; ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[alctr_n]; @@ -1077,7 +1075,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, for (i = 0; i < size; i++) { Allctr_t *as; - atype = init->atype; + astrat = init->astrat; if (!init->thr_spec) as0 = state; @@ -1094,8 +1092,8 @@ start_au_allocator(ErtsAlcType_t alctr_n, if (i != 0) init->init.util.ts = 0; else { - if (atype == AFIT) - atype = GOODFIT; + if (astrat == ERTS_ALC_S_AFIT) + astrat = ERTS_ALC_S_GOODFIT; init->init.util.ts = 1; } init->init.util.tspec = init->thr_spec + 1; @@ -1109,25 +1107,26 @@ start_au_allocator(ErtsAlcType_t alctr_n, (((char *) fix_lists) + fix_list_size)); } + init->init.util.alloc_strat = astrat; init->init.util.ix = i; - switch (atype) { - case GOODFIT: + switch (astrat) { + case ERTS_ALC_S_GOODFIT: as = erts_gfalc_start((GFAllctr_t *) as0, &init->init.gf, &init->init.util); break; - case BESTFIT: + case ERTS_ALC_S_BESTFIT: as = erts_bfalc_start((BFAllctr_t *) as0, &init->init.bf, &init->init.util); break; - case AFIT: + case ERTS_ALC_S_AFIT: as = erts_afalc_start((AFAllctr_t *) as0, &init->init.af, &init->init.util); break; - case FIRSTFIT: + case ERTS_ALC_S_FIRSTFIT: as = erts_aoffalc_start((AOFFAllctr_t *) as0, &init->init.aoff, &init->init.util); @@ -1363,51 +1362,59 @@ handle_au_arg(struct au_init *auip, else if(has_prefix("as", sub_param)) { char *alg = get_value(sub_param + 2, argv, ip); if (sys_strcmp("bf", alg) == 0) { - auip->atype = BESTFIT; + auip->astrat = ERTS_ALC_S_BESTFIT; auip->init.bf.ao = 0; } else if (sys_strcmp("aobf", alg) == 0) { - auip->atype = BESTFIT; + auip->astrat = ERTS_ALC_S_BESTFIT; auip->init.bf.ao = 1; } else if (sys_strcmp("gf", alg) == 0) { - auip->atype = GOODFIT; + auip->astrat = ERTS_ALC_S_GOODFIT; } else if (sys_strcmp("af", alg) == 0) { - auip->atype = AFIT; + auip->astrat = ERTS_ALC_S_AFIT; } else if (sys_strcmp("aoff", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AOFF; auip->init.aoff.blk_order = FF_AOFF; } else if (sys_strcmp("aoffcbf", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AOFF; auip->init.aoff.blk_order = FF_BF; } else if (sys_strcmp("aoffcaobf", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AOFF; auip->init.aoff.blk_order = FF_AOBF; } else if (sys_strcmp("ageffcaoff", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AGEFF; auip->init.aoff.blk_order = FF_AOFF; } else if (sys_strcmp("ageffcbf", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AGEFF; auip->init.aoff.blk_order = FF_BF; } else if (sys_strcmp("ageffcaobf", alg) == 0) { - auip->atype = FIRSTFIT; + auip->astrat = ERTS_ALC_S_FIRSTFIT; auip->init.aoff.crr_order = FF_AGEFF; auip->init.aoff.blk_order = FF_AOBF; } else { - bad_value(param, sub_param + 1, alg); + if (auip->init.util.alloc_no == ERTS_ALC_A_TEST + && sys_strcmp("chaosff", alg) == 0) { + auip->astrat = ERTS_ALC_S_FIRSTFIT; + auip->init.aoff.crr_order = FF_CHAOS; + auip->init.aoff.blk_order = FF_CHAOS; + } + else { + bad_value(param, sub_param + 1, alg); + } } if (!strategy_support_carrier_migration(auip)) auip->init.util.acul = 0; @@ -2030,33 +2037,55 @@ erts_realloc_n_enomem(ErtsAlcType_t n, void *ptr, Uint size) } static ERTS_INLINE UWord -alcu_size(ErtsAlcType_t ai, ErtsAlcUFixInfo_t *fi, int fisz) +alcu_size(ErtsAlcType_t alloc_no, ErtsAlcUFixInfo_t *fi, int fisz) { - UWord res = 0; + UWord res; + int ai; - ASSERT(erts_allctrs_info[ai].enabled); - ASSERT(erts_allctrs_info[ai].alloc_util); + if (!erts_allctrs_info[alloc_no].thr_spec) { + AllctrSize_t size; + Allctr_t *allctr; - if (!erts_allctrs_info[ai].thr_spec) { - Allctr_t *allctr = erts_allctrs_info[ai].extra; - AllctrSize_t asize; - erts_alcu_current_size(allctr, &asize, fi, fisz); - res += asize.blocks; + allctr = erts_allctrs_info[alloc_no].extra; + erts_alcu_current_size(allctr, &size, fi, fisz); + + return size.blocks; } - else { - ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ai]; - int i; - ASSERT(tspec->enabled); + res = 0; - for (i = tspec->size - 1; i >= 0; i--) { - Allctr_t *allctr = tspec->allctr[i]; - AllctrSize_t asize; - if (allctr) { - erts_alcu_current_size(allctr, &asize, fi, fisz); - res += asize.blocks; - } - } + /* Thread-specific allocators can migrate carriers across types, so we have + * to visit every allocator type to gather information on blocks that were + * allocated by us. */ + for (ai = ERTS_ALC_A_MIN; ai < ERTS_ALC_A_MAX; ai++) { + ErtsAllocatorThrSpec_t *tspec; + Allctr_t *allctr; + int i; + + if (!erts_allctrs_info[ai].thr_spec) { + continue; + } + + tspec = &erts_allctr_thr_spec[ai]; + ASSERT(tspec->enabled); + + for (i = tspec->size - 1; i >= 0; i--) { + allctr = tspec->allctr[i]; + + if (allctr) { + AllctrSize_t size; + + if (ai == alloc_no) { + erts_alcu_current_size(allctr, &size, fi, fisz); + } else { + erts_alcu_foreign_size(allctr, alloc_no, &size); + } + + ASSERT(((SWord)size.blocks) >= 0); + + res += size.blocks; + } + } } return res; @@ -2400,6 +2429,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg) } if (want_tot_or_sys) { + ASSERT(size.total >= size.processes); size.system = size.total - size.processes; } @@ -3459,29 +3489,29 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) switch (op) { case 0xf00: if (((Allctr_t *) a1)->thread_safe) - return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_alloc_ts(ERTS_ALC_T_TEST, (void *) a1, (Uint) a2); else - return (UWord) erts_alcu_alloc(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_alloc(ERTS_ALC_T_TEST, (void *) a1, (Uint) a2); case 0xf01: if (((Allctr_t *) a1)->thread_safe) - return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_realloc_ts(ERTS_ALC_T_TEST, (void *) a1, (void *) a2, (Uint) a3); else - return (UWord) erts_alcu_realloc(ERTS_ALC_T_UNDEF, + return (UWord) erts_alcu_realloc(ERTS_ALC_T_TEST, (void *) a1, (void *) a2, (Uint) a3); case 0xf02: if (((Allctr_t *) a1)->thread_safe) - erts_alcu_free_ts(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2); + erts_alcu_free_ts(ERTS_ALC_T_TEST, (void *) a1, (void *) a2); else - erts_alcu_free(ERTS_ALC_T_UNDEF, (void *) a1, (void *) a2); + erts_alcu_free(ERTS_ALC_T_TEST, (void *) a1, (void *) a2); return 0; case 0xf03: { Allctr_t *allctr; @@ -3489,8 +3519,10 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) SET_DEFAULT_ALLOC_OPTS(&init); init.enable = 1; - init.atype = GOODFIT; + init.astrat = ERTS_ALC_S_GOODFIT; init.init.util.name_prefix = (char *) a1; + init.init.util.alloc_no = ERTS_ALC_A_TEST; + init.init.util.alloc_strat = init.astrat; init.init.util.ts = 1; if ((char **) a3) { char **argv = (char **) a3; @@ -3504,31 +3536,31 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) } } - switch (init.atype) { - case GOODFIT: + switch (init.astrat) { + case ERTS_ALC_S_GOODFIT: allctr = erts_gfalc_start((GFAllctr_t *) - erts_alloc(ERTS_ALC_T_UNDEF, + erts_alloc(ERTS_ALC_T_TEST, sizeof(GFAllctr_t)), &init.init.gf, &init.init.util); break; - case BESTFIT: + case ERTS_ALC_S_BESTFIT: allctr = erts_bfalc_start((BFAllctr_t *) - erts_alloc(ERTS_ALC_T_UNDEF, + erts_alloc(ERTS_ALC_T_TEST, sizeof(BFAllctr_t)), &init.init.bf, &init.init.util); break; - case AFIT: + case ERTS_ALC_S_AFIT: allctr = erts_afalc_start((AFAllctr_t *) - erts_alloc(ERTS_ALC_T_UNDEF, + erts_alloc(ERTS_ALC_T_TEST, sizeof(AFAllctr_t)), &init.init.af, &init.init.util); break; - case FIRSTFIT: + case ERTS_ALC_S_FIRSTFIT: allctr = erts_aoffalc_start((AOFFAllctr_t *) - erts_alloc(ERTS_ALC_T_UNDEF, + erts_alloc(ERTS_ALC_T_TEST, sizeof(AOFFAllctr_t)), &init.init.aoff, &init.init.util); @@ -3544,7 +3576,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) } case 0xf04: erts_alcu_stop((Allctr_t *) a1); - erts_free(ERTS_ALC_T_UNDEF, (void *) a1); + erts_free(ERTS_ALC_T_TEST, (void *) a1); break; case 0xf05: return (UWord) 1; case 0xf06: return (UWord) ((Allctr_t *) a1)->thread_safe; @@ -3554,7 +3586,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) case 0xf07: return (UWord) ((Allctr_t *) a1)->thread_safe; #endif case 0xf08: { - ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_mutex)); + ethr_mutex *mtx = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_mutex)); if (ethr_mutex_init(mtx) != 0) ERTS_ALC_TEST_ABORT; return (UWord) mtx; @@ -3563,7 +3595,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) ethr_mutex *mtx = (ethr_mutex *) a1; if (ethr_mutex_destroy(mtx) != 0) ERTS_ALC_TEST_ABORT; - erts_free(ERTS_ALC_T_UNDEF, (void *) mtx); + erts_free(ERTS_ALC_T_TEST, (void *) mtx); break; } case 0xf0a: @@ -3573,7 +3605,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) ethr_mutex_unlock((ethr_mutex *) a1); break; case 0xf0c: { - ethr_cond *cnd = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_cond)); + ethr_cond *cnd = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_cond)); if (ethr_cond_init(cnd) != 0) ERTS_ALC_TEST_ABORT; return (UWord) cnd; @@ -3582,7 +3614,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) ethr_cond *cnd = (ethr_cond *) a1; if (ethr_cond_destroy(cnd) != 0) ERTS_ALC_TEST_ABORT; - erts_free(ERTS_ALC_T_UNDEF, (void *) cnd); + erts_free(ERTS_ALC_T_TEST, (void *) cnd); break; } case 0xf0e: @@ -3596,7 +3628,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) break; } case 0xf10: { - ethr_tid *tid = erts_alloc(ERTS_ALC_T_UNDEF, sizeof(ethr_tid)); + ethr_tid *tid = erts_alloc(ERTS_ALC_T_TEST, sizeof(ethr_tid)); if (ethr_thr_create(tid, (void * (*)(void *)) a1, (void *) a2, @@ -3608,7 +3640,7 @@ UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) ethr_tid *tid = (ethr_tid *) a1; if (ethr_thr_join(*tid, NULL) != 0) ERTS_ALC_TEST_ABORT; - erts_free(ERTS_ALC_T_UNDEF, (void *) tid); + erts_free(ERTS_ALC_T_TEST, (void *) tid); break; } case 0xf12: @@ -3960,9 +3992,10 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1]; static void * -debug_alloc(ErtsAlcType_t n, void *extra, Uint size) +debug_alloc(ErtsAlcType_t type, void *extra, Uint size) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; + ErtsAlcType_t n; Uint dsize; void *res; @@ -3970,9 +4003,11 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size) erts_hdbg_chk_blks(); #endif + n = ERTS_ALC_T2N(type); + ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; - res = (*real_af->alloc)(n, real_af->extra, dsize); + res = (*real_af->alloc)(type, real_af->extra, dsize); res = set_memory_fence(res, size, n); @@ -3986,14 +4021,17 @@ debug_alloc(ErtsAlcType_t n, void *extra, Uint size) static void * -debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) +debug_realloc(ErtsAlcType_t type, void *extra, void *ptr, Uint size) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; + ErtsAlcType_t n; Uint dsize; Uint old_size; void *dptr; void *res; + n = ERTS_ALC_T2N(type); + ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dsize = size + FENCE_SZ; @@ -4008,7 +4046,7 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) 0xf, sizeof(Uint) + old_size - size); - res = (*real_af->realloc)(n, real_af->extra, dptr, dsize); + res = (*real_af->realloc)(type, real_af->extra, dptr, dsize); res = set_memory_fence(res, size, n); @@ -4021,12 +4059,16 @@ debug_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size) } static void -debug_free(ErtsAlcType_t n, void *extra, void *ptr) +debug_free(ErtsAlcType_t type, void *extra, void *ptr) { ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; + ErtsAlcType_t n; void *dptr; Uint size; - int free_pattern = n; + int free_pattern; + + n = ERTS_ALC_T2N(type); + free_pattern = n; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); @@ -4041,7 +4083,7 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) #endif sys_memset((void *) dptr, free_pattern, size + FENCE_SZ); - (*real_af->free)(n, real_af->extra, dptr); + (*real_af->free)(type, real_af->extra, dptr); #ifdef PRINT_OPS fprintf(stderr, "free(%s, 0x%lx)\r\n", ERTS_ALC_N2TD(n), (Uint) ptr); diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index fcb58ff58a..c13cf3f5b0 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -26,10 +26,23 @@ #define ERL_THR_PROGRESS_TSD_TYPE_ONLY #include "erl_thr_progress.h" #undef ERL_THR_PROGRESS_TSD_TYPE_ONLY -#include "erl_alloc_util.h" #include "erl_threads.h" #include "erl_mmap.h" +typedef enum { + ERTS_ALC_S_INVALID = 0, + + ERTS_ALC_S_GOODFIT, + ERTS_ALC_S_BESTFIT, + ERTS_ALC_S_AFIT, + ERTS_ALC_S_FIRSTFIT, + + ERTS_ALC_S_MIN = ERTS_ALC_S_GOODFIT, + ERTS_ALC_S_MAX = ERTS_ALC_S_FIRSTFIT +} ErtsAlcStrat_t; + +#include "erl_alloc_util.h" + #ifdef DEBUG # undef ERTS_ALC_WANT_INLINE # define ERTS_ALC_WANT_INLINE 0 @@ -52,6 +65,14 @@ #define ERTS_ALC_NO_FIXED_SIZES \ (ERTS_ALC_N_MAX_A_FIXED_SIZE - ERTS_ALC_N_MIN_A_FIXED_SIZE + 1) +#define ERTS_ALC_IS_FIX_TYPE(T) \ + (ERTS_ALC_T2N(T) >= ERTS_ALC_N_MIN_A_FIXED_SIZE && \ + ERTS_ALC_T2N(T) <= ERTS_ALC_N_MAX_A_FIXED_SIZE) + +#define ERTS_ALC_FIX_TYPE_IX(T) \ + (ASSERT(ERTS_ALC_IS_FIX_TYPE(T)), \ + ERTS_ALC_T2N((T)) - ERTS_ALC_N_MIN_A_FIXED_SIZE) + void erts_sys_alloc_init(void); void *erts_sys_alloc(ErtsAlcType_t, void *, Uint); void *erts_sys_realloc(ErtsAlcType_t, void *, void *, Uint); @@ -228,7 +249,7 @@ void *erts_alloc(ErtsAlcType_t type, Uint size) void *res; ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, size); if (!res) @@ -243,7 +264,7 @@ void *erts_realloc(ErtsAlcType_t type, void *ptr, Uint size) void *res; ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr, size); @@ -258,7 +279,7 @@ void erts_free(ErtsAlcType_t type, void *ptr) { ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); (*erts_allctrs[ERTS_ALC_T2A(type)].free)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr); ERTS_MSACC_POP_STATE_X(); @@ -271,7 +292,7 @@ void *erts_alloc_fnf(ErtsAlcType_t type, Uint size) void *res; ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].alloc)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, size); ERTS_MSACC_POP_STATE_X(); @@ -285,7 +306,7 @@ void *erts_realloc_fnf(ErtsAlcType_t type, void *ptr, Uint size) void *res; ERTS_MSACC_PUSH_AND_SET_STATE_X(ERTS_MSACC_STATE_ALLOC); res = (*erts_allctrs[ERTS_ALC_T2A(type)].realloc)( - ERTS_ALC_T2N(type), + type, erts_allctrs[ERTS_ALC_T2A(type)].extra, ptr, size); diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 5409b89bab..f1e99820af 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -30,10 +30,10 @@ # name space). # * Types, allocators, classes, and descriptions have different name # spaces. -# * The type, allocator, and class names INVALID are reserved and can -# not be used. +# * The type, allocator, and class names INVALID are reserved and +# cannot be used. # * The descriptions invalid_allocator, invalid_class, and invalid_type -# are reserved and can not be used. +# are reserved and cannot be used. # * Declarations can be done conditionally by use of a # +if <boolean_variable> # diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index a5740a08cf..b7a8b9c2d0 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -96,7 +96,6 @@ static int initialized = 0; #define MBC_REALLOC_ALWAYS_MOVES #endif - /* alloc_util global parameters */ static Uint sys_alloc_carrier_size; #if HAVE_ERTS_MSEG @@ -113,32 +112,38 @@ static int allow_sys_alloc_carriers; #define DEC_CC(CC) ((CC)--) -/* Multi block carrier (MBC) memory layout in R16: +/* Multi block carrier (MBC) memory layout in OTP 22: Empty MBC: -[Carrier_t|pad|Block_t L0T|fhdr| free... ] +[Carrier_t|pad|Block_t L0T0|fhdr| free... ] MBC after allocating first block: -[Carrier_t|pad|Block_t 000| udata |pad|Block_t L0T|fhdr| free... ] +[Carrier_t|pad|Block_t 0000| udata |pad|Block_t L0T0|fhdr| free... ] MBC after allocating second block: -[Carrier_t|pad|Block_t 000| udata |pad|Block_t 000| udata |pad|Block_t L0T|fhdr| free... ] +[Carrier_t|pad|Block_t 0000| udata |pad|Block_t 0000| udata |pad|Block_t L0T0|fhdr| free... ] MBC after deallocating first block: -[Carrier_t|pad|Block_t 00T|fhdr| free |FreeBlkFtr_t|Block_t 0P0| udata |pad|Block_t L0T|fhdr| free... ] +[Carrier_t|pad|Block_t 00T0|fhdr| free |FreeBlkFtr_t|Block_t 0P00| udata |pad|Block_t L0T0|fhdr| free... ] +MBC after allocating first block, with allocation tagging enabled: +[Carrier_t|pad|Block_t 000A| udata |atag|pad|Block_t L0T0|fhdr| free... ] udata = Allocated user data + atag = A tag with basic metadata about this allocation pad = Padding to ensure correct alignment for user data fhdr = Allocator specific header to keep track of free block free = Unused free memory T = This block is free (THIS_FREE_BLK_HDR_FLG) P = Previous block is free (PREV_FREE_BLK_HDR_FLG) L = Last block in carrier (LAST_BLK_HDR_FLG) + A = Block has an allocation tag footer, only valid for allocated blocks + (ATAG_BLK_HDR_FLG) */ /* Single block carrier (SBC): -[Carrier_t|pad|Block_t 111| udata... ] +[Carrier_t|pad|Block_t 1110| udata... ] +[Carrier_t|pad|Block_t 111A| udata | atag] */ /* Allocation tags ... @@ -154,20 +159,20 @@ MBC after deallocating first block: typedef UWord alcu_atag_t; -#define MAKE_ATAG(IdAtom, Type) \ - (ASSERT((Type) >= ERTS_ALC_N_MIN && (Type) <= ERTS_ALC_N_MAX), \ +#define MAKE_ATAG(IdAtom, TypeNum) \ + (ASSERT((TypeNum) >= ERTS_ALC_N_MIN && (TypeNum) <= ERTS_ALC_N_MAX), \ ASSERT(atom_val(IdAtom) <= MAX_ATAG_ATOM_ID), \ - (atom_val(IdAtom) << ERTS_ALC_N_BITS) | (Type)) + (atom_val(IdAtom) << ERTS_ALC_N_BITS) | (TypeNum)) #define ATAG_ID(AT) (make_atom((AT) >> ERTS_ALC_N_BITS)) #define ATAG_TYPE(AT) ((AT) & ERTS_ALC_N_MASK) #define MAX_ATAG_ATOM_ID (ERTS_UWORD_MAX >> ERTS_ALC_N_BITS) -#define DBG_IS_VALID_ATAG(Allocator, AT) \ +#define DBG_IS_VALID_ATAG(AT) \ (ATAG_TYPE(AT) >= ERTS_ALC_N_MIN && \ ATAG_TYPE(AT) <= ERTS_ALC_N_MAX && \ - (Allocator)->alloc_no == ERTS_ALC_T2A(ERTS_ALC_N2T(ATAG_TYPE(AT)))) + ATAG_ID(AT) <= MAX_ATAG_ATOM_ID) /* Blocks ... */ @@ -182,10 +187,15 @@ typedef UWord alcu_atag_t; #endif #define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t)) +#define BLK_HAS_ATAG(B) \ + (!!((B)->bhdr & ATAG_BLK_HDR_FLG)) + #define GET_BLK_ATAG(B) \ - (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1]) + (ASSERT(BLK_HAS_ATAG(B)), \ + ((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1]) #define SET_BLK_ATAG(B, T) \ - (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1] = (T)) + ((B)->bhdr |= ATAG_BLK_HDR_FLG, \ + ((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1] = (T)) #define BLK_ATAG_SZ(AP) ((AP)->atags ? sizeof(alcu_atag_t) : 0) @@ -203,13 +213,13 @@ typedef UWord alcu_atag_t; (((FreeBlkFtr_t *) (((char *) (B)) + (SZ)))[-1] = (SZ)) #define SET_MBC_ABLK_SZ(B, SZ) \ - (ASSERT(((SZ) & FLG_MASK) == 0), \ + (ASSERT(((SZ) & BLK_FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~MBC_ABLK_SZ_MASK) | (SZ)) #define SET_MBC_FBLK_SZ(B, SZ) \ - (ASSERT(((SZ) & FLG_MASK) == 0), \ + (ASSERT(((SZ) & BLK_FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~MBC_FBLK_SZ_MASK) | (SZ)) #define SET_SBC_BLK_SZ(B, SZ) \ - (ASSERT(((SZ) & FLG_MASK) == 0), \ + (ASSERT(((SZ) & BLK_FLG_MASK) == 0), \ (B)->bhdr = (((B)->bhdr) & ~SBC_BLK_SZ_MASK) | (SZ)) #define SET_PREV_BLK_FREE(AP,B) \ (ASSERT(!IS_MBC_FIRST_BLK(AP,B)), \ @@ -235,12 +245,12 @@ typedef UWord alcu_atag_t; # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & ~MBC_ABLK_SZ_MASK) == 0), \ - ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ + ASSERT(!((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ (B)->bhdr = ((Sz) | (F) | (BLK_CARRIER_OFFSET(B,C) << MBC_ABLK_OFFSET_SHIFT))) # define SET_MBC_FBLK_HDR(B, Sz, F, C) \ (ASSERT(((Sz) & ~MBC_FBLK_SZ_MASK) == 0), \ - ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + ASSERT(((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->u.carrier = (C)) @@ -257,8 +267,8 @@ typedef UWord alcu_atag_t; # define SET_BLK_FREE(B) \ (ASSERT(!IS_PREV_BLK_FREE(B)), \ (B)->u.carrier = ABLK_TO_MBC(B), \ - (B)->bhdr |= THIS_FREE_BLK_HDR_FLG, \ - (B)->bhdr &= (MBC_ABLK_SZ_MASK|FLG_MASK)) + (B)->bhdr &= (MBC_ABLK_SZ_MASK|LAST_BLK_HDR_FLG), \ + (B)->bhdr |= THIS_FREE_BLK_HDR_FLG) # define SET_BLK_ALLOCED(B) \ (ASSERT(((B)->bhdr & (MBC_ABLK_OFFSET_MASK|THIS_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ @@ -270,15 +280,16 @@ typedef UWord alcu_atag_t; # define MBC_SZ_MAX_LIMIT ((UWord)~0) # define SET_MBC_ABLK_HDR(B, Sz, F, C) \ - (ASSERT(((Sz) & FLG_MASK) == 0), \ - ASSERT(!((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ - ASSERT((UWord)(F) < SBC_BLK_HDR_FLG), \ + (ASSERT(((Sz) & BLK_FLG_MASK) == 0), \ + ASSERT(((F) & ~BLK_FLG_MASK) == 0), \ + ASSERT(!((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG))), \ (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) # define SET_MBC_FBLK_HDR(B, Sz, F, C) \ - (ASSERT(((Sz) & FLG_MASK) == 0), \ - ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ + (ASSERT(((Sz) & BLK_FLG_MASK) == 0), \ + ASSERT(((F) & ~BLK_FLG_MASK) == 0), \ + ASSERT(((UWord)(F) & (~BLK_FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->carrier = (C)) @@ -297,7 +308,7 @@ typedef UWord alcu_atag_t; #endif /* !MBC_ABLK_OFFSET_BITS */ #define SET_SBC_BLK_HDR(B, Sz) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (B)->bhdr = ((Sz) | (SBC_BLK_HDR_FLG))) + (ASSERT(((Sz) & BLK_FLG_MASK) == 0), (B)->bhdr = ((Sz) | (SBC_BLK_HDR_FLG))) #define BLK_UMEM_SZ(B) \ @@ -320,7 +331,7 @@ typedef UWord alcu_atag_t; #define GET_PREV_FREE_BLK_HDR_FLG(B) \ ((B)->bhdr & PREV_FREE_BLK_HDR_FLG) #define GET_BLK_HDR_FLGS(B) \ - ((B)->bhdr & FLG_MASK) + ((B)->bhdr & BLK_FLG_MASK) #define NXT_BLK(B) \ (ASSERT(IS_MBC_BLK(B)), \ @@ -419,7 +430,7 @@ do { \ #define SCH_SBC SBC_CARRIER_HDR_FLAG #define SET_CARRIER_HDR(C, Sz, F, AP) \ - (ASSERT(((Sz) & FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \ + (ASSERT(((Sz) & CRR_FLG_MASK) == 0), (C)->chdr = ((Sz) | (F)), \ erts_atomic_init_nob(&(C)->allctr, (erts_aint_t) (AP))) #define BLK_TO_SBC(B) \ @@ -444,8 +455,8 @@ do { \ (!IS_SB_CARRIER((C))) #define SET_CARRIER_SZ(C, SZ) \ - (ASSERT(((SZ) & FLG_MASK) == 0), \ - ((C)->chdr = ((C)->chdr & FLG_MASK) | (SZ))) + (ASSERT(((SZ) & CRR_FLG_MASK) == 0), \ + ((C)->chdr = ((C)->chdr & CRR_FLG_MASK) | (SZ))) #define CFLG_SBC (1 << 0) #define CFLG_MBC (1 << 1) @@ -575,10 +586,12 @@ do { \ STAT_MSEG_MBC_ALLOC((AP), csz__); \ else \ STAT_SYS_ALLOC_MBC_ALLOC((AP), csz__); \ - (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks; \ + set_new_allctr_abandon_limit(AP); \ + (AP)->mbcs.blocks.curr.no += (CRR)->cpool.blocks[(AP)->alloc_no]; \ if ((AP)->mbcs.blocks.max.no < (AP)->mbcs.blocks.curr.no) \ (AP)->mbcs.blocks.max.no = (AP)->mbcs.blocks.curr.no; \ - (AP)->mbcs.blocks.curr.size += (CRR)->cpool.blocks_size; \ + (AP)->mbcs.blocks.curr.size += \ + (CRR)->cpool.blocks_size[(AP)->alloc_no]; \ if ((AP)->mbcs.blocks.max.size < (AP)->mbcs.blocks.curr.size) \ (AP)->mbcs.blocks.max.size = (AP)->mbcs.blocks.curr.size; \ } while (0) @@ -601,25 +614,33 @@ do { \ DEBUG_CHECK_CARRIER_NO_SZ((AP)); \ } while (0) -#define STAT_MBC_ABANDON(AP, CRR) \ -do { \ - UWord csz__ = CARRIER_SZ((CRR)); \ - if (IS_MSEG_CARRIER((CRR))) \ - STAT_MSEG_MBC_FREE((AP), csz__); \ - else \ - STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ - ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \ - >= (CRR)->cpool.blocks); \ - (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks; \ - ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \ - >= (CRR)->cpool.blocks_size); \ - (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size; \ +#define STAT_MBC_FREE(AP, CRR) \ +do { \ + UWord csz__ = CARRIER_SZ((CRR)); \ + if (IS_MSEG_CARRIER((CRR))) { \ + STAT_MSEG_MBC_FREE((AP), csz__); \ + } else { \ + STAT_SYS_ALLOC_MBC_FREE((AP), csz__); \ + } \ + set_new_allctr_abandon_limit(AP); \ } while (0) -#define STAT_MBC_BLK_ALLOC_CRR(CRR, BSZ) \ +#define STAT_MBC_ABANDON(AP, CRR) \ +do { \ + STAT_MBC_FREE(AP, CRR); \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.no \ + >= (CRR)->cpool.blocks[(AP)->alloc_no]); \ + (AP)->mbcs.blocks.curr.no -= (CRR)->cpool.blocks[(AP)->alloc_no]; \ + ERTS_ALC_CPOOL_ASSERT((AP)->mbcs.blocks.curr.size \ + >= (CRR)->cpool.blocks_size[(AP)->alloc_no]); \ + (AP)->mbcs.blocks.curr.size -= (CRR)->cpool.blocks_size[(AP)->alloc_no]; \ +} while (0) + +#define STAT_MBC_BLK_ALLOC_CRR(AP, CRR, BSZ) \ do { \ - (CRR)->cpool.blocks++; \ - (CRR)->cpool.blocks_size += (BSZ); \ + (CRR)->cpool.blocks[(AP)->alloc_no]++; \ + (CRR)->cpool.blocks_size[(AP)->alloc_no] += (BSZ); \ + (CRR)->cpool.total_blocks_size += (BSZ); \ } while (0) #define STAT_MBC_BLK_ALLOC(AP, CRR, BSZ, FLGS) \ @@ -631,50 +652,67 @@ do { \ cstats__->blocks.curr.size += (BSZ); \ if (cstats__->blocks.max.size < cstats__->blocks.curr.size) \ cstats__->blocks.max.size = cstats__->blocks.curr.size; \ - STAT_MBC_BLK_ALLOC_CRR((CRR), (BSZ)); \ + STAT_MBC_BLK_ALLOC_CRR((AP), (CRR), (BSZ)); \ } while (0) static ERTS_INLINE int stat_cpool_mbc_blk_free(Allctr_t *allctr, + ErtsAlcType_t type, Carrier_t *crr, Carrier_t **busy_pcrr_pp, UWord blksz) { + Allctr_t *orig_allctr; + int alloc_no; - ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks > 0); - crr->cpool.blocks--; - ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size >= blksz); - crr->cpool.blocks_size -= blksz; + alloc_no = ERTS_ALC_T2A(type); - if (!busy_pcrr_pp || !*busy_pcrr_pp) - return 0; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks[alloc_no] > 0); + crr->cpool.blocks[alloc_no]--; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size[alloc_no] >= blksz); + crr->cpool.blocks_size[alloc_no] -= blksz; + ERTS_ALC_CPOOL_ASSERT(crr->cpool.total_blocks_size >= blksz); + crr->cpool.total_blocks_size -= blksz; - ERTS_ALC_CPOOL_ASSERT(crr == *busy_pcrr_pp); + if (allctr->alloc_no == alloc_no && (!busy_pcrr_pp || !*busy_pcrr_pp)) { + /* This is a local block, so we should not update the pool + * statistics. */ + return 0; + } + + /* This is either a foreign block that's been fetched from the pool, or any + * block that's in the pool. The carrier's owner keeps the statistics for + * both pooled and foreign blocks. */ + + orig_allctr = crr->cpool.orig_allctr; + + ERTS_ALC_CPOOL_ASSERT(alloc_no != allctr->alloc_no || + (crr == *busy_pcrr_pp && allctr == orig_allctr)); #ifdef ERTS_ALC_CPOOL_DEBUG ERTS_ALC_CPOOL_ASSERT( - erts_atomic_dec_read_nob(&allctr->cpool.stat.no_blocks) >= 0); + erts_atomic_dec_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]) >= 0); ERTS_ALC_CPOOL_ASSERT( - erts_atomic_add_read_nob(&allctr->cpool.stat.blocks_size, + erts_atomic_add_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], -((erts_aint_t) blksz)) >= 0); #else - erts_atomic_dec_nob(&allctr->cpool.stat.no_blocks); - erts_atomic_add_nob(&allctr->cpool.stat.blocks_size, + erts_atomic_dec_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]); + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], -((erts_aint_t) blksz)); #endif return 1; } -#define STAT_MBC_BLK_FREE(AP, CRR, BPCRRPP, BSZ, FLGS) \ -do { \ - if (!stat_cpool_mbc_blk_free((AP), (CRR), (BPCRRPP), (BSZ))) { \ - CarriersStats_t *cstats__ = &(AP)->mbcs; \ - ASSERT(cstats__->blocks.curr.no > 0); \ - cstats__->blocks.curr.no--; \ - ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ - cstats__->blocks.curr.size -= (BSZ); \ - } \ +#define STAT_MBC_BLK_FREE(AP, TYPE, CRR, BPCRRPP, BSZ, FLGS) \ +do { \ + if (!stat_cpool_mbc_blk_free((AP), (TYPE), (CRR), (BPCRRPP), (BSZ))) { \ + CarriersStats_t *cstats__ = &(AP)->mbcs; \ + ASSERT(cstats__->blocks.curr.no > 0); \ + cstats__->blocks.curr.no--; \ + ASSERT(cstats__->blocks.curr.size >= (BSZ)); \ + cstats__->blocks.curr.size -= (BSZ); \ + } \ } while (0) /* Debug stuff... */ @@ -721,8 +759,8 @@ static void make_name_atoms(Allctr_t *allctr); static Block_t *create_carrier(Allctr_t *, Uint, UWord); static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **); -static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp); -static void dealloc_block(Allctr_t *, void *, ErtsAlcFixList_t *, int); +static void mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp); +static void dealloc_block(Allctr_t *, ErtsAlcType_t, Uint32, void *, ErtsAlcFixList_t *); static alcu_atag_t determine_alloc_tag(Allctr_t *allocator, ErtsAlcType_t type) { @@ -764,14 +802,14 @@ static alcu_atag_t determine_alloc_tag(Allctr_t *allocator, ErtsAlcType_t type) } } - return MAKE_ATAG(id, type); + return MAKE_ATAG(id, ERTS_ALC_T2N(type)); } static void set_alloc_tag(Allctr_t *allocator, void *p, alcu_atag_t tag) { Block_t *block; - ASSERT(DBG_IS_VALID_ATAG(allocator, tag)); + ASSERT(DBG_IS_VALID_ATAG(tag)); ASSERT(allocator->atags && p); (void)allocator; @@ -1308,28 +1346,9 @@ chk_fix_list(Allctr_t *allctr, ErtsAlcFixList_t *fix, int ix, int before) #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) #endif +static ERTS_INLINE Allctr_t *get_pref_allctr(void *extra); static void *mbc_alloc(Allctr_t *allctr, Uint size); -typedef struct { - ErtsAllctrDDBlock_t ddblock__; /* must be first */ - ErtsAlcType_t fix_type; -} ErtsAllctrFixDDBlock_t; - -#define ERTS_ALC_FIX_NO_UNUSE (((ErtsAlcType_t) 1) << ERTS_ALC_N_BITS) - -static ERTS_INLINE void -dealloc_fix_block(Allctr_t *allctr, - ErtsAlcType_t type, - void *ptr, - ErtsAlcFixList_t *fix, - int dec_cc_on_redirect) -{ - /* May be redirected... */ - ASSERT((type & ERTS_ALC_FIX_NO_UNUSE) == 0); - ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type | ERTS_ALC_FIX_NO_UNUSE; - dealloc_block(allctr, ptr, fix, dec_cc_on_redirect); -} - static ERTS_INLINE void sched_fix_shrink(Allctr_t *allctr, int on) { @@ -1371,7 +1390,7 @@ fix_cpool_check_shrink(Allctr_t *allctr, if (fix->u.cpool.min_list_size > fix->list_size) fix->u.cpool.min_list_size = fix->list_size; - dealloc_fix_block(allctr, type, p, fix, 0); + dealloc_block(allctr, type, DEALLOC_FLG_FIX_SHRINK, p, fix); } } } @@ -1382,11 +1401,9 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) void *res; ErtsAlcFixList_t *fix; - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; - ASSERT(size == fix->type_size); + fix = &allctr->fix[ERTS_ALC_FIX_TYPE_IX(type)]; + ASSERT(type == fix->type && size == fix->type_size); + ASSERT(size >= sizeof(ErtsAllctrDDBlock_t)); res = fix->list; if (res) { @@ -1415,21 +1432,39 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) static ERTS_INLINE void fix_cpool_free(Allctr_t *allctr, ErtsAlcType_t type, + Uint32 flags, void *p, - Carrier_t **busy_pcrr_pp, - int unuse) + Carrier_t **busy_pcrr_pp) { ErtsAlcFixList_t *fix; + Allctr_t *fix_allctr; + + /* If this isn't a fix allocator we need to update the fix list of our + * neighboring fix_alloc to keep the statistics consistent. */ + if (!allctr->fix) { + ErtsAllocatorThrSpec_t *tspec = &erts_allctr_thr_spec[ERTS_ALC_A_FIXED_SIZE]; + fix_allctr = get_pref_allctr(tspec); + ASSERT(!fix_allctr->thread_safe); + ASSERT(allctr != fix_allctr); + } + else { + fix_allctr = allctr; + } - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(fix_allctr)); + ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); - fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + fix = &fix_allctr->fix[ERTS_ALC_FIX_TYPE_IX(type)]; + ASSERT(type == fix->type); - if (unuse) - fix->u.cpool.used--; + if (!(flags & DEALLOC_FLG_FIX_SHRINK)) { + fix->u.cpool.used--; + } - if ((!busy_pcrr_pp || !*busy_pcrr_pp) + /* We don't want foreign blocks to be long-lived, so we skip recycling if + * allctr != fix_allctr. */ + if (allctr == fix_allctr + && (!busy_pcrr_pp || !*busy_pcrr_pp) && !fix->u.cpool.shrink_list && fix->list_size < ERTS_ALCU_FIX_MAX_LIST_SZ) { *((void **) p) = fix->list; @@ -1442,7 +1477,7 @@ fix_cpool_free(Allctr_t *allctr, if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); fix->u.cpool.allocated--; fix_cpool_check_shrink(allctr, type, fix, busy_pcrr_pp); } @@ -1469,7 +1504,7 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) fix->u.cpool.shrink_list = fix->u.cpool.min_list_size; fix->u.cpool.min_list_size = fix->list_size; } - type = (ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE); + type = ERTS_ALC_N2T((ErtsAlcType_t) (ix + ERTS_ALC_N_MIN_A_FIXED_SIZE)); for (o = 0; o < ERTS_ALC_FIX_MAX_SHRINK_OPS || flush; o++) { void *ptr; @@ -1483,7 +1518,7 @@ fix_cpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) fix->list = *((void **) ptr); fix->list_size--; fix->u.cpool.shrink_list--; - dealloc_fix_block(allctr, type, ptr, fix, 0); + dealloc_block(allctr, type, DEALLOC_FLG_FIX_SHRINK, ptr, fix); } if (fix->u.cpool.min_list_size > fix->list_size) fix->u.cpool.min_list_size = fix->list_size; @@ -1509,11 +1544,9 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) ErtsAlcFixList_t *fix; void *res; - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; - ASSERT(size == fix->type_size); + fix = &allctr->fix[ERTS_ALC_FIX_TYPE_IX(type)]; + ASSERT(type == fix->type && size == fix->type_size); + ASSERT(size >= sizeof(ErtsAllctrDDBlock_t)); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); fix->u.nocpool.used++; @@ -1530,7 +1563,7 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, NULL); + mbc_free(allctr, type, p, NULL); fix->u.nocpool.allocated--; } ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); @@ -1565,10 +1598,8 @@ fix_nocpool_free(Allctr_t *allctr, Block_t *blk; ErtsAlcFixList_t *fix; - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE <= type - && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); - - fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + fix = &allctr->fix[ERTS_ALC_T2N(type) - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(fix->type == type); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); fix->u.nocpool.used--; @@ -1587,7 +1618,7 @@ fix_nocpool_free(Allctr_t *allctr, if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, NULL); + mbc_free(allctr, type, p, NULL); p = fix->list; fix->list = *((void **) p); fix->list_size--; @@ -1598,7 +1629,7 @@ fix_nocpool_free(Allctr_t *allctr, if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, NULL); + mbc_free(allctr, type, p, NULL); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); } @@ -1639,7 +1670,7 @@ fix_nocpool_alloc_shrink(Allctr_t *allctr, erts_aint32_t flgs) ptr = fix->list; fix->list = *((void **) ptr); fix->list_size--; - dealloc_block(allctr, ptr, NULL, 0); + dealloc_block(allctr, fix->type, 0, ptr, NULL); fix->u.nocpool.allocated--; } if (fix->list_size != 0) { @@ -1681,6 +1712,7 @@ dealloc_mbc(Allctr_t *allctr, Carrier_t *crr) } +static UWord allctr_abandon_limit(Allctr_t *allctr); static void set_new_allctr_abandon_limit(Allctr_t*); static void abandon_carrier(Allctr_t*, Carrier_t*); static void poolify_my_carrier(Allctr_t*, Carrier_t*); @@ -1802,7 +1834,7 @@ get_used_allctr(Allctr_t *pref_allctr, int pref_lock, void *p, UWord *sizep, static void init_dd_queue(ErtsAllctrDDQueue_t *ddq) { - erts_atomic_init_nob(&ddq->tail.data.marker.atmc_next, ERTS_AINT_NULL); + erts_atomic_init_nob(&ddq->tail.data.marker.u.atmc_next, ERTS_AINT_NULL); erts_atomic_init_nob(&ddq->tail.data.last, (erts_aint_t) &ddq->tail.data.marker); erts_atomic_init_nob(&ddq->tail.data.um_refc[0], 0); @@ -1823,17 +1855,17 @@ ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) erts_aint_t itmp; ErtsAllctrDDBlock_t *enq, *this = ptr; - erts_atomic_init_nob(&this->atmc_next, ERTS_AINT_NULL); + erts_atomic_init_nob(&this->u.atmc_next, ERTS_AINT_NULL); /* Enqueue at end of list... */ enq = (ErtsAllctrDDBlock_t *) erts_atomic_read_nob(&ddq->tail.data.last); - itmp = erts_atomic_cmpxchg_relb(&enq->atmc_next, + itmp = erts_atomic_cmpxchg_relb(&enq->u.atmc_next, (erts_aint_t) this, ERTS_AINT_NULL); if (itmp == ERTS_AINT_NULL) { /* We are required to move last pointer */ #ifdef DEBUG - ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->atmc_next)); + ASSERT(ERTS_AINT_NULL == erts_atomic_read_nob(&this->u.atmc_next)); ASSERT(((erts_aint_t) enq) == erts_atomic_xchg_relb(&ddq->tail.data.last, (erts_aint_t) this)); @@ -1851,8 +1883,8 @@ ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) while (1) { erts_aint_t itmp2; - erts_atomic_set_nob(&this->atmc_next, itmp); - itmp2 = erts_atomic_cmpxchg_relb(&enq->atmc_next, + erts_atomic_set_nob(&this->u.atmc_next, itmp); + itmp2 = erts_atomic_cmpxchg_relb(&enq->u.atmc_next, (erts_aint_t) this, itmp); if (itmp == itmp2) @@ -1861,7 +1893,7 @@ ddq_managed_thread_enqueue(ErtsAllctrDDQueue_t *ddq, void *ptr, int cinit) itmp = itmp2; else { enq = (ErtsAllctrDDBlock_t *) itmp2; - itmp = erts_atomic_read_acqb(&enq->atmc_next); + itmp = erts_atomic_read_acqb(&enq->u.atmc_next); ASSERT(itmp != ERTS_AINT_NULL); } i++; @@ -1877,8 +1909,8 @@ check_insert_marker(ErtsAllctrDDQueue_t *ddq, erts_aint_t ilast) erts_aint_t itmp; ErtsAllctrDDBlock_t *last = (ErtsAllctrDDBlock_t *) ilast; - erts_atomic_init_nob(&ddq->tail.data.marker.atmc_next, ERTS_AINT_NULL); - itmp = erts_atomic_cmpxchg_relb(&last->atmc_next, + erts_atomic_init_nob(&ddq->tail.data.marker.u.atmc_next, ERTS_AINT_NULL); + itmp = erts_atomic_cmpxchg_relb(&last->u.atmc_next, (erts_aint_t) &ddq->tail.data.marker, ERTS_AINT_NULL); if (itmp == ERTS_AINT_NULL) { @@ -1929,7 +1961,7 @@ ddq_dequeue(ErtsAllctrDDQueue_t *ddq) ASSERT(ddq->head.used_marker); ddq->head.used_marker = 0; blk = ((ErtsAllctrDDBlock_t *) - erts_atomic_read_nob(&blk->atmc_next)); + erts_atomic_read_nob(&blk->u.atmc_next)); if (blk == ddq->head.unref_end) { ddq->head.first = blk; return NULL; @@ -1937,7 +1969,7 @@ ddq_dequeue(ErtsAllctrDDQueue_t *ddq) } ddq->head.first = ((ErtsAllctrDDBlock_t *) - erts_atomic_read_nob(&blk->atmc_next)); + erts_atomic_read_nob(&blk->u.atmc_next)); ASSERT(ddq->head.first); @@ -1999,19 +2031,13 @@ check_pending_dealloc_carrier(Allctr_t *allctr, int *need_more_work); static void -handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) +handle_delayed_fix_dealloc(Allctr_t *allctr, ErtsAlcType_t type, Uint32 flags, + void *ptr) { - ErtsAlcType_t type; - - type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; - - ASSERT(ERTS_ALC_N_MIN_A_FIXED_SIZE - <= (type & ~ERTS_ALC_FIX_NO_UNUSE)); - ASSERT((type & ~ERTS_ALC_FIX_NO_UNUSE) - <= ERTS_ALC_N_MAX_A_FIXED_SIZE); + ASSERT(ERTS_ALC_IS_FIX_TYPE(type)); if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - fix_nocpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), ptr); + fix_nocpool_free(allctr, type, ptr); else { Block_t *blk = UMEM2BLK(ptr); Carrier_t *busy_pcrr_p; @@ -2026,20 +2052,24 @@ handle_delayed_fix_dealloc(Allctr_t *allctr, void *ptr) NULL, &busy_pcrr_p); if (used_allctr == allctr) { doit: - fix_cpool_free(allctr, (type & ~ERTS_ALC_FIX_NO_UNUSE), - ptr, &busy_pcrr_p, - !(type & ERTS_ALC_FIX_NO_UNUSE)); + fix_cpool_free(allctr, type, flags, ptr, &busy_pcrr_p); clear_busy_pool_carrier(allctr, busy_pcrr_p); } else { /* Carrier migrated; need to redirect block to new owner... */ - int cinit = used_allctr->dd.ix - allctr->dd.ix; + ErtsAllctrDDBlock_t *dd_block; + int cinit; + + dd_block = (ErtsAllctrDDBlock_t*)ptr; + dd_block->flags = flags; + dd_block->type = type; ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); DEC_CC(allctr->calls.this_free); - ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type = type; + cinit = used_allctr->dd.ix - allctr->dd.ix; + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(used_allctr->ix); } @@ -2063,7 +2093,6 @@ handle_delayed_dealloc(Allctr_t *allctr, int need_mr_wrk = 0; int have_checked_incoming = 0; int ops = 0; - ErtsAlcFixList_t *fix; int res; ErtsAllctrDDQueue_t *ddq; @@ -2072,8 +2101,6 @@ handle_delayed_dealloc(Allctr_t *allctr, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); - fix = allctr->fix; - ddq = &allctr->dd.q; res = 0; @@ -2162,16 +2189,27 @@ handle_delayed_dealloc(Allctr_t *allctr, } } else { + ErtsAllctrDDBlock_t *dd_block; + ErtsAlcType_t type; + Uint32 flags; + + dd_block = (ErtsAllctrDDBlock_t*)ptr; + flags = dd_block->flags; + type = dd_block->type; + + flags |= DEALLOC_FLG_REDIRECTED; + ASSERT(IS_SBC_BLK(blk) || (ABLK_TO_MBC(blk) != ErtsContainerStruct(blk, Carrier_t, cpool.homecoming_dd.blk))); INC_CC(allctr->calls.this_free); - if (fix) - handle_delayed_fix_dealloc(allctr, ptr); - else - dealloc_block(allctr, ptr, NULL, 1); + if (ERTS_ALC_IS_FIX_TYPE(type)) { + handle_delayed_fix_dealloc(allctr, type, flags, ptr); + } else { + dealloc_block(allctr, type, flags, ptr, NULL); + } } } @@ -2199,8 +2237,10 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, void *ptr, int cinit) { - if (allctr->fix) - ((ErtsAllctrFixDDBlock_t*) ptr)->fix_type = type; + ErtsAllctrDDBlock_t *dd_block = ((ErtsAllctrDDBlock_t*)ptr); + + dd_block->type = type; + dd_block->flags = 0; if (ddq_enqueue(&allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(allctr->ix); @@ -2230,10 +2270,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) return; - allctr->cpool.check_limit_count--; - if (--allctr->cpool.check_limit_count <= 0) - set_new_allctr_abandon_limit(allctr); - + ASSERT(allctr->cpool.abandon_limit == allctr_abandon_limit(allctr)); ASSERT(erts_thr_progress_is_managed_thread()); if (allctr->cpool.disable_abandon) @@ -2251,7 +2288,7 @@ check_abandon_carrier(Allctr_t *allctr, Block_t *fblk, Carrier_t **busy_pcrr_pp) if (allctr->main_carrier == crr) return; - if (crr->cpool.blocks_size > crr->cpool.abandon_limit) + if (crr->cpool.total_blocks_size > crr->cpool.abandon_limit) return; if (crr->cpool.thr_prgr != ERTS_THR_PRGR_INVALID @@ -2287,24 +2324,26 @@ erts_alcu_check_delayed_dealloc(Allctr_t *allctr, ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) static void -dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_redirect) +dealloc_block(Allctr_t *allctr, ErtsAlcType_t type, Uint32 flags, void *ptr, + ErtsAlcFixList_t *fix) { Block_t *blk = UMEM2BLK(ptr); + ASSERT(!fix || type == fix->type); + ERTS_LC_ASSERT(!allctr->thread_safe || erts_lc_mtx_is_locked(&allctr->mutex)); if (IS_SBC_BLK(blk)) { destroy_carrier(allctr, blk, NULL); if (fix && ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { - ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; - if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + if (!(flags & DEALLOC_FLG_FIX_SHRINK)) fix->u.cpool.used--; fix->u.cpool.allocated--; } } else if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - mbc_free(allctr, ptr, NULL); + mbc_free(allctr, type, ptr, NULL); else { Carrier_t *busy_pcrr_p; Allctr_t *used_allctr; @@ -2313,22 +2352,29 @@ dealloc_block(Allctr_t *allctr, void *ptr, ErtsAlcFixList_t *fix, int dec_cc_on_ NULL, &busy_pcrr_p); if (used_allctr == allctr) { if (fix) { - ErtsAlcType_t type = ((ErtsAllctrFixDDBlock_t *) ptr)->fix_type; - if (!(type & ERTS_ALC_FIX_NO_UNUSE)) + if (!(flags & DEALLOC_FLG_FIX_SHRINK)) fix->u.cpool.used--; fix->u.cpool.allocated--; } - mbc_free(allctr, ptr, &busy_pcrr_p); + mbc_free(allctr, type, ptr, &busy_pcrr_p); clear_busy_pool_carrier(allctr, busy_pcrr_p); } else { /* Carrier migrated; need to redirect block to new owner... */ - int cinit = used_allctr->dd.ix - allctr->dd.ix; + ErtsAllctrDDBlock_t *dd_block; + int cinit; + + dd_block = (ErtsAllctrDDBlock_t*)ptr; + dd_block->flags = flags; + dd_block->type = type; ERTS_ALC_CPOOL_ASSERT(!busy_pcrr_p); - if (dec_cc_on_redirect) + if (flags & DEALLOC_FLG_REDIRECTED) DEC_CC(allctr->calls.this_free); + + cinit = used_allctr->dd.ix - allctr->dd.ix; + if (ddq_enqueue(&used_allctr->dd.q, ptr, cinit)) erts_alloc_notify_delayed_dealloc(used_allctr->ix); } @@ -2495,7 +2541,7 @@ mbc_alloc(Allctr_t *allctr, Uint size) } static void -mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) +mbc_free(Allctr_t *allctr, ErtsAlcType_t type, void *p, Carrier_t **busy_pcrr_pp) { Uint is_first_blk; Uint is_last_blk; @@ -2517,7 +2563,8 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) crr = ABLK_TO_MBC(blk); ERTS_ALC_CPOOL_FREE_OP(allctr); - STAT_MBC_BLK_FREE(allctr, crr, busy_pcrr_pp, blk_sz, alcu_flgs); + + STAT_MBC_BLK_FREE(allctr, type, crr, busy_pcrr_pp, blk_sz, alcu_flgs); is_first_blk = IS_MBC_FIRST_ABLK(allctr, blk); is_last_blk = IS_LAST_BLK(blk); @@ -2586,8 +2633,8 @@ mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp) } static void * -mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, - Carrier_t **busy_pcrr_pp) +mbc_realloc(Allctr_t *allctr, ErtsAlcType_t type, void *p, Uint size, + Uint32 alcu_flgs, Carrier_t **busy_pcrr_pp) { void *new_p; Uint old_blk_sz; @@ -2625,7 +2672,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, new_blk = UMEM2BLK(new_p); ASSERT(!(IS_MBC_BLK(new_blk) && ABLK_TO_MBC(new_blk) == *busy_pcrr_pp)); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); return new_p; } @@ -2702,7 +2749,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, crr = ABLK_TO_MBC(blk); ERTS_ALC_CPOOL_REALLOC_OP(allctr); - STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz, alcu_flgs); STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(MBC_BLK_SZ(blk) >= allctr->min_block_size); @@ -2806,7 +2853,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, } ERTS_ALC_CPOOL_REALLOC_OP(allctr); - STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz, alcu_flgs); STAT_MBC_BLK_ALLOC(allctr, crr, blk_sz, alcu_flgs); ASSERT(IS_ALLOCED_BLK(blk)); @@ -2867,7 +2914,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, if (!new_p) return NULL; sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); return new_p; @@ -2897,7 +2944,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, 1); new_p = BLK2UMEM(new_blk); sys_memcpy(new_p, p, MIN(size, old_blk_sz - ABLK_HDR_SZ)); - mbc_free(allctr, p, NULL); + mbc_free(allctr, type, p, NULL); return new_p; } else { @@ -2954,7 +3001,7 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, 0); ERTS_ALC_CPOOL_FREE_OP(allctr); - STAT_MBC_BLK_FREE(allctr, crr, NULL, old_blk_sz, alcu_flgs); + STAT_MBC_BLK_FREE(allctr, type, crr, NULL, old_blk_sz, alcu_flgs); return new_p; } @@ -2965,7 +3012,6 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, #define ERTS_ALC_MAX_DEALLOC_CARRIER 10 #define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 100 -#define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 #define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0) @@ -2992,14 +3038,11 @@ typedef union { # error "Carrier pool implementation assumes ERTS_ALC_A_MIN > ERTS_ALC_A_INVALID" #endif -/* - * The pool is only allowed to be manipulated by managed - * threads except in the alloc_SUITE:cpool case. In this - * test case carrier_pool[ERTS_ALC_A_INVALID] will be - * used. - */ +/* The pools are only allowed to be manipulated by managed threads except in + * the alloc_SUITE:cpool test, where only test_carrier_pool is used. */ -static ErtsAlcCrrPool_t carrier_pool[ERTS_ALC_A_MAX+1] erts_align_attribute(ERTS_CACHE_LINE_SIZE); +static ErtsAlcCrrPool_t firstfit_carrier_pool; +static ErtsAlcCrrPool_t test_carrier_pool; #define ERTS_ALC_CPOOL_MAX_BACKOFF (1 << 8) @@ -3020,12 +3063,12 @@ backoff(int n) static int cpool_dbg_is_in_pool(Allctr_t *allctr, Carrier_t *crr) { - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; ErtsAlcCPoolData_t *cpdp = sentinel; Carrier_t *tmp_crr; while (1) { - cpdp = (ErtsAlcCPoolData_t *) (erts_atomic_read_ddrb(&cpdp->next) & ~FLG_MASK); + cpdp = (ErtsAlcCPoolData_t *) (erts_atomic_read_ddrb(&cpdp->next) & ~CRR_FLG_MASK); if (cpdp == sentinel) return 0; tmp_crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); @@ -3037,7 +3080,7 @@ cpool_dbg_is_in_pool(Allctr_t *allctr, Carrier_t *crr) static int cpool_is_empty(Allctr_t *allctr) { - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; return ((erts_atomic_read_rb(&sentinel->next) == (erts_aint_t) sentinel) && (erts_atomic_read_rb(&sentinel->prev) == (erts_aint_t) sentinel)); } @@ -3127,16 +3170,31 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) { ErtsAlcCPoolData_t *cpd1p, *cpd2p; erts_aint_t val; - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; Allctr_t *orig_allctr = crr->cpool.orig_allctr; - ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_TEST /* testcase */ || erts_thr_progress_is_managed_thread()); - erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size, - (erts_aint_t) crr->cpool.blocks_size); - erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks, - (erts_aint_t) crr->cpool.blocks); + { + int alloc_no = allctr->alloc_no; + + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no]) >= 0 && + crr->cpool.blocks_size[alloc_no] >= 0); + + ERTS_ALC_CPOOL_ASSERT( + erts_atomic_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no]) >= 0 && + crr->cpool.blocks[alloc_no] >= 0); + + /* We only modify the counter for our current type since the others are + * conceptually still in the pool. */ + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], + ((erts_aint_t) crr->cpool.blocks_size[alloc_no])); + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no], + ((erts_aint_t) crr->cpool.blocks[alloc_no])); + } + erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, (erts_aint_t) CARRIER_SZ(crr)); erts_atomic_inc_nob(&orig_allctr->cpool.stat.no_carriers); @@ -3209,10 +3267,10 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) ErtsAlcCPoolData_t *cpd1p, *cpd2p; erts_aint_t val; #ifdef ERTS_ALC_CPOOL_DEBUG - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *sentinel = allctr->cpool.sentinel; #endif - ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_TEST /* testcase */ || erts_thr_progress_is_managed_thread()); ERTS_ALC_CPOOL_ASSERT(sentinel != &crr->cpool); @@ -3288,28 +3346,43 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) crr->cpool.thr_prgr = erts_thr_progress_later(NULL); - erts_atomic_add_nob(&prev_allctr->cpool.stat.blocks_size, - -((erts_aint_t) crr->cpool.blocks_size)); - erts_atomic_add_nob(&prev_allctr->cpool.stat.no_blocks, - -((erts_aint_t) crr->cpool.blocks)); - erts_atomic_add_nob(&prev_allctr->cpool.stat.carriers_size, + { + Allctr_t *orig_allctr = crr->cpool.orig_allctr; + int alloc_no = allctr->alloc_no; + + ERTS_ALC_CPOOL_ASSERT(orig_allctr == prev_allctr); + + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks_size[alloc_no] <= + erts_atomic_read_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no])); + + ERTS_ALC_CPOOL_ASSERT(crr->cpool.blocks[alloc_no] <= + erts_atomic_read_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no])); + + /* We only modify the counters for our current type since the others + * were, conceptually, never taken out of the pool. */ + erts_atomic_add_nob(&orig_allctr->cpool.stat.blocks_size[alloc_no], + -((erts_aint_t) crr->cpool.blocks_size[alloc_no])); + erts_atomic_add_nob(&orig_allctr->cpool.stat.no_blocks[alloc_no], + -((erts_aint_t) crr->cpool.blocks[alloc_no])); + + erts_atomic_add_nob(&orig_allctr->cpool.stat.carriers_size, -((erts_aint_t) CARRIER_SZ(crr))); - erts_atomic_dec_wb(&prev_allctr->cpool.stat.no_carriers); + erts_atomic_dec_wb(&orig_allctr->cpool.stat.no_carriers); + } } static Carrier_t * cpool_fetch(Allctr_t *allctr, UWord size) { - enum { IGNORANT, HAS_SEEN_SENTINEL, THE_LAST_ONE } loop_state; - int i; + int i, seen_sentinel; Carrier_t *crr; Carrier_t *reinsert_crr = NULL; ErtsAlcCPoolData_t *cpdp; ErtsAlcCPoolData_t *cpool_entrance = NULL; ErtsAlcCPoolData_t *sentinel; - ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ + ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_TEST /* testcase */ || erts_thr_progress_is_managed_thread()); i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; @@ -3411,48 +3484,39 @@ cpool_fetch(Allctr_t *allctr, UWord size) /* * Finally search the shared pool and try employ foreign carriers */ - sentinel = &carrier_pool[allctr->alloc_no].sentinel; + sentinel = allctr->cpool.sentinel; if (cpool_entrance) { /* * We saw a pooled carried above, use it as entrance into the pool */ - cpdp = cpool_entrance; } else { /* - * No pooled carried seen above. Start search at cpool sentinel, + * No pooled carrier seen above. Start search at cpool sentinel, * but begin by passing one element before trying to fetch. * This in order to avoid contention with threads inserting elements. */ - cpool_entrance = sentinel; - cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev)); - if (cpdp == sentinel) + cpool_entrance = cpool_aint2cpd(cpool_read(&sentinel->prev)); + if (cpool_entrance == sentinel) goto check_dc_list; } - loop_state = IGNORANT; + cpdp = cpool_entrance; + seen_sentinel = 0; do { erts_aint_t exp; cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); - if (cpdp == cpool_entrance) { - if (cpool_entrance == sentinel) { - cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); - if (cpdp == sentinel) - break; - } - loop_state = THE_LAST_ONE; - } - else if (cpdp == sentinel) { - if (loop_state == HAS_SEEN_SENTINEL) { + if (cpdp == sentinel) { + if (seen_sentinel) { /* We been here before. cpool_entrance must have been removed */ INC_CC(allctr->cpool.stat.entrance_removed); break; } - cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); - if (cpdp == sentinel) - break; - loop_state = HAS_SEEN_SENTINEL; + seen_sentinel = 1; + continue; } + ASSERT(cpdp != cpool_entrance || seen_sentinel); + crr = ErtsContainerStruct(cpdp, Carrier_t, cpool); exp = erts_atomic_read_rb(&crr->allctr); @@ -3485,7 +3549,7 @@ cpool_fetch(Allctr_t *allctr, UWord size) INC_CC(allctr->cpool.stat.fail_shared); return NULL; } - }while (loop_state != THE_LAST_ONE); + }while (cpdp != cpool_entrance); check_dc_list: /* Last; check our own pending dealloc carrier list... */ @@ -3664,8 +3728,9 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) crr->cpool.orig_allctr = allctr; crr->cpool.thr_prgr = ERTS_THR_PRGR_INVALID; erts_atomic_init_nob(&crr->cpool.max_size, 0); - crr->cpool.blocks = 0; - crr->cpool.blocks_size = 0; + sys_memset(&crr->cpool.blocks_size, 0, sizeof(crr->cpool.blocks_size)); + sys_memset(&crr->cpool.blocks, 0, sizeof(crr->cpool.blocks)); + crr->cpool.total_blocks_size = 0; if (!ERTS_ALC_IS_CPOOL_ENABLED(allctr)) crr->cpool.abandon_limit = 0; else { @@ -3680,14 +3745,14 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) crr->cpool.state = ERTS_MBC_IS_HOME; } -static void -set_new_allctr_abandon_limit(Allctr_t *allctr) + + +static UWord +allctr_abandon_limit(Allctr_t *allctr) { UWord limit; UWord csz; - allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; - csz = allctr->mbcs.curr.norm.mseg.size; csz += allctr->mbcs.curr.norm.sys_alloc.size; @@ -3697,7 +3762,13 @@ set_new_allctr_abandon_limit(Allctr_t *allctr) else limit = (csz/100)*allctr->cpool.util_limit; - allctr->cpool.abandon_limit = limit; + return limit; +} + +static void ERTS_INLINE +set_new_allctr_abandon_limit(Allctr_t *allctr) +{ + allctr->cpool.abandon_limit = allctr_abandon_limit(allctr); } static void @@ -3709,7 +3780,6 @@ abandon_carrier(Allctr_t *allctr, Carrier_t *crr) unlink_carrier(&allctr->mbc_list, crr); allctr->remove_mbc(allctr, crr); - set_new_allctr_abandon_limit(allctr); cpool_insert(allctr, crr); @@ -3762,7 +3832,8 @@ poolify_my_carrier(Allctr_t *allctr, Carrier_t *crr) } static void -cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp) +cpool_read_stat(Allctr_t *allctr, int alloc_no, + UWord *nocp, UWord *cszp, UWord *nobp, UWord *bszp) { int i; UWord noc = 0, csz = 0, nob = 0, bsz = 0; @@ -3782,10 +3853,10 @@ cpool_read_stat(Allctr_t *allctr, UWord *nocp, UWord *cszp, UWord *nobp, UWord * ? erts_atomic_read_nob(&allctr->cpool.stat.carriers_size) : 0); tnob = (UWord) (nobp - ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks) + ? erts_atomic_read_nob(&allctr->cpool.stat.no_blocks[alloc_no]) : 0); tbsz = (UWord) (bszp - ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size) + ? erts_atomic_read_nob(&allctr->cpool.stat.blocks_size[alloc_no]) : 0); if (tnoc == noc && tcsz == csz && tnob == nob && tbsz == bsz) break; @@ -4040,6 +4111,7 @@ create_carrier(Allctr_t *allctr, Uint umem_sz, UWord flags) #if HAVE_ERTS_MSEG mbc_final_touch: #endif + set_new_allctr_abandon_limit(allctr); blk = MBC_TO_FIRST_BLK(allctr, crr); @@ -4258,7 +4330,6 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) else { ASSERT(IS_MBC_FIRST_FBLK(allctr, blk)); crr = FIRST_BLK_TO_MBC(allctr, blk); - crr_sz = CARRIER_SZ(crr); #ifdef DEBUG if (!allctr->stopped) { @@ -4290,15 +4361,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) else { unlink_carrier(&allctr->mbc_list, crr); -#if HAVE_ERTS_MSEG - if (IS_MSEG_CARRIER(crr)) { - ASSERT(crr_sz % ERTS_SACRR_UNIT_SZ == 0); - STAT_MSEG_MBC_FREE(allctr, crr_sz); - } - else -#endif - STAT_SYS_ALLOC_MBC_FREE(allctr, crr_sz); - + STAT_MBC_FREE(allctr, crr); if (allctr->remove_mbc) allctr->remove_mbc(allctr, crr); } @@ -4312,7 +4375,7 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) LTTNG5(carrier_destroy, ERTS_ALC_A2AD(allctr->alloc_no), allctr->ix, - crr_sz, + CARRIER_SZ(crr), mbc_stats, sbc_stats); } @@ -4390,6 +4453,8 @@ static struct { Eterm blocks_size; Eterm blocks; + Eterm foreign_blocks; + Eterm calls; Eterm sys_alloc; Eterm sys_free; @@ -4490,6 +4555,7 @@ init_atoms(Allctr_t *allctr) AM_INIT(carriers); AM_INIT(blocks_size); AM_INIT(blocks); + AM_INIT(foreign_blocks); AM_INIT(calls); AM_INIT(sys_alloc); @@ -4625,7 +4691,6 @@ sz_info_fix(Allctr_t *allctr, ErtsAlcFixList_t *fix = &allctr->fix[ix]; UWord alloced = fix->type_size * fix->u.cpool.allocated; UWord used = fix->type_size * fix->u.cpool.used; - ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; if (print_to_p) { fmtfn_t to = *print_to_p; @@ -4633,14 +4698,14 @@ sz_info_fix(Allctr_t *allctr, erts_print(to, arg, "fix type internal: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(n), + (char *) ERTS_ALC_T2TD(fix->type), alloced, used); } if (hpp || szp) { add_3tup(hpp, szp, &res, - alloc_type_atoms[n], + alloc_type_atoms[ERTS_ALC_T2N(fix->type)], bld_unstable_uint(hpp, szp, alloced), bld_unstable_uint(hpp, szp, used)); } @@ -4653,7 +4718,6 @@ sz_info_fix(Allctr_t *allctr, ErtsAlcFixList_t *fix = &allctr->fix[ix]; UWord alloced = fix->type_size * fix->u.nocpool.allocated; UWord used = fix->type_size*fix->u.nocpool.used; - ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix; if (print_to_p) { fmtfn_t to = *print_to_p; @@ -4661,14 +4725,14 @@ sz_info_fix(Allctr_t *allctr, erts_print(to, arg, "fix type: %s %bpu %bpu\n", - (char *) ERTS_ALC_N2TD(n), + (char *) ERTS_ALC_T2TD(fix->type), alloced, used); } if (hpp || szp) { add_3tup(hpp, szp, &res, - alloc_type_atoms[n], + alloc_type_atoms[ERTS_ALC_T2N(fix->type)], bld_unstable_uint(hpp, szp, alloced), bld_unstable_uint(hpp, szp, used)); } @@ -4741,9 +4805,9 @@ info_cpool(Allctr_t *allctr, noc = csz = nob = bsz = ~0; if (print_to_p || hpp) { if (sz_only) - cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + cpool_read_stat(allctr, allctr->alloc_no, NULL, &csz, NULL, &bsz); else - cpool_read_stat(allctr, &noc, &csz, &nob, &bsz); + cpool_read_stat(allctr, allctr->alloc_no, &noc, &csz, &nob, &bsz); } if (print_to_p) { @@ -4758,6 +4822,10 @@ info_cpool(Allctr_t *allctr, } if (hpp || szp) { + Eterm foreign_blocks; + int i; + + foreign_blocks = NIL; res = NIL; if (!sz_only) { @@ -4804,22 +4872,61 @@ info_cpool(Allctr_t *allctr, add_3tup(hpp, szp, &res, am.entrance_removed, bld_unstable_uint(hpp, szp, ERTS_ALC_CC_GIGA_VAL(allctr->cpool.stat.entrance_removed)), bld_unstable_uint(hpp, szp, ERTS_ALC_CC_VAL(allctr->cpool.stat.entrance_removed))); + } add_2tup(hpp, szp, &res, am.carriers_size, bld_unstable_uint(hpp, szp, csz)); - } - if (!sz_only) - add_2tup(hpp, szp, &res, - am.carriers, - bld_unstable_uint(hpp, szp, noc)); + + if (!sz_only) { + add_2tup(hpp, szp, &res, + am.carriers, + bld_unstable_uint(hpp, szp, noc)); + } + add_2tup(hpp, szp, &res, am.blocks_size, bld_unstable_uint(hpp, szp, bsz)); - if (!sz_only) + + if (!sz_only) { add_2tup(hpp, szp, &res, am.blocks, bld_unstable_uint(hpp, szp, nob)); + } + + for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + const char *name_str; + Eterm name, info; + + if (i == allctr->alloc_no) { + continue; + } + + cpool_read_stat(allctr, i, NULL, NULL, &nob, &bsz); + + if (bsz == 0 && (nob == 0 || sz_only)) { + continue; + } + + name_str = ERTS_ALC_A2AD(i); + info = NIL; + + add_2tup(hpp, szp, &info, + am.blocks_size, + bld_unstable_uint(hpp, szp, bsz)); + + if (!sz_only) { + add_2tup(hpp, szp, &info, + am.blocks, + bld_unstable_uint(hpp, szp, nob)); + } + + name = am_atom_put(name_str, sys_strlen(name_str)); + + add_2tup(hpp, szp, &foreign_blocks, name, info); + } + + add_2tup(hpp, szp, &res, am.foreign_blocks, foreign_blocks); } return res; @@ -5455,6 +5562,19 @@ erts_alcu_info(Allctr_t *allctr, return res; } +void +erts_alcu_foreign_size(Allctr_t *allctr, ErtsAlcType_t alloc_no, AllctrSize_t *size) +{ + if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { + UWord csz, bsz; + cpool_read_stat(allctr, alloc_no, NULL, &csz, NULL, &bsz); + size->carriers = csz; + size->blocks = bsz; + } else { + size->carriers = 0; + size->blocks = 0; + } +} void erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *fi, int fisz) @@ -5473,7 +5593,7 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t * if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { UWord csz, bsz; - cpool_read_stat(allctr, NULL, &csz, NULL, &bsz); + cpool_read_stat(allctr, allctr->alloc_no, NULL, &csz, NULL, &bsz); size->blocks += bsz; size->carriers += csz; } @@ -5518,6 +5638,11 @@ do_erts_alcu_alloc(ErtsAlcType_t type, Allctr_t *allctr, Uint size) ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); + /* Reject sizes that can't fit into the header word. */ + if (size > ~BLK_FLG_MASK) { + return NULL; + } + #if ALLOC_ZERO_EQ_NULL if (!size) return NULL; @@ -5684,12 +5809,11 @@ do_erts_alcu_free(ErtsAlcType_t type, Allctr_t *allctr, void *p, ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr); if (p) { - INC_CC(allctr->calls.this_free); - if (allctr->fix) { + if (ERTS_ALC_IS_FIX_TYPE(type)) { if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) - fix_cpool_free(allctr, type, p, busy_pcrr_pp, 1); + fix_cpool_free(allctr, type, 0, p, busy_pcrr_pp); else fix_nocpool_free(allctr, type, p); } @@ -5698,7 +5822,7 @@ do_erts_alcu_free(ErtsAlcType_t type, Allctr_t *allctr, void *p, if (IS_SBC_BLK(blk)) destroy_carrier(allctr, blk, NULL); else - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); } } } @@ -5800,6 +5924,11 @@ do_erts_alcu_realloc(ErtsAlcType_t type, return res; } + /* Reject sizes that can't fit into the header word. */ + if (size > ~BLK_FLG_MASK) { + return NULL; + } + #if ALLOC_ZERO_EQ_NULL if (!size) { ASSERT(p); @@ -5816,7 +5945,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, if (size < allctr->sbc_threshold) { if (IS_MBC_BLK(blk)) - res = mbc_realloc(allctr, p, size, alcu_flgs, busy_pcrr_pp); + res = mbc_realloc(allctr, type, p, size, alcu_flgs, busy_pcrr_pp); else { Uint used_sz = SBC_HEADER_SIZE + ABLK_HDR_SZ + size; Uint crr_sz; @@ -5875,7 +6004,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type, sys_memcpy((void *) res, (void *) p, MIN(MBC_ABLK_SZ(blk) - ABLK_HDR_SZ, size)); - mbc_free(allctr, p, busy_pcrr_pp); + mbc_free(allctr, type, p, busy_pcrr_pp); } else res = NULL; @@ -6243,6 +6372,7 @@ int erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) { /* erts_alcu_start assumes that allctr has been zeroed */ + int i; if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) { erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", @@ -6266,6 +6396,11 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->ix = init->ix; allctr->alloc_no = init->alloc_no; + allctr->alloc_strat = init->alloc_strat; + + ASSERT(allctr->alloc_no >= ERTS_ALC_A_MIN && + allctr->alloc_no <= ERTS_ALC_A_MAX); + if (allctr->alloc_no < ERTS_ALC_A_MIN || ERTS_ALC_A_MAX < allctr->alloc_no) allctr->alloc_no = ERTS_ALC_A_INVALID; @@ -6318,8 +6453,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) + sizeof(FreeBlkFtr_t)); if (init->tpref) { Uint sz = ABLK_HDR_SZ; - sz += (init->fix ? - sizeof(ErtsAllctrFixDDBlock_t) : sizeof(ErtsAllctrDDBlock_t)); + sz += sizeof(ErtsAllctrDDBlock_t); sz = UNIT_CEILING(sz); if (sz > allctr->min_block_size) allctr->min_block_size = sz; @@ -6330,15 +6464,23 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; allctr->cpool.disable_abandon = 0; - erts_atomic_init_nob(&allctr->cpool.stat.blocks_size, 0); - erts_atomic_init_nob(&allctr->cpool.stat.no_blocks, 0); + for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { + erts_atomic_init_nob(&allctr->cpool.stat.blocks_size[i], 0); + erts_atomic_init_nob(&allctr->cpool.stat.no_blocks[i], 0); + } erts_atomic_init_nob(&allctr->cpool.stat.carriers_size, 0); erts_atomic_init_nob(&allctr->cpool.stat.no_carriers, 0); - allctr->cpool.check_limit_count = ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT; if (!init->ts && init->acul && init->acnl) { allctr->cpool.util_limit = init->acul; allctr->cpool.in_pool_limit = init->acnl; allctr->cpool.fblk_min_limit = init->acfml; + + if (allctr->alloc_strat == ERTS_ALC_S_FIRSTFIT) { + allctr->cpool.sentinel = &firstfit_carrier_pool.sentinel; + } + else if (allctr->alloc_no != ERTS_ALC_A_TEST) { + ERTS_INTERNAL_ERROR("Impossible carrier migration config."); + } } else { allctr->cpool.util_limit = 0; @@ -6346,6 +6488,12 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->cpool.fblk_min_limit = 0; } + /* The invasive tests don't really care whether the pool is enabled or not, + * so we need to set this unconditionally for this allocator type. */ + if (allctr->alloc_no == ERTS_ALC_A_TEST) { + allctr->cpool.sentinel = &test_carrier_pool.sentinel; + } + allctr->sbc_threshold = adjust_sbct(allctr, init->sbct); #if HAVE_ERTS_MSEG @@ -6457,9 +6605,9 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->fix_shrink_scheduled = 0; for (i = 0; i < ERTS_ALC_NO_FIXED_SIZES; i++) { allctr->fix[i].type_size = init->fix_type_size[i]; + allctr->fix[i].type = ERTS_ALC_N2T(i + ERTS_ALC_N_MIN_A_FIXED_SIZE); allctr->fix[i].list_size = 0; allctr->fix[i].list = NULL; - ASSERT(allctr->fix[i].type_size >= sizeof(ErtsAllctrFixDDBlock_t)); if (ERTS_ALC_IS_CPOOL_ENABLED(allctr)) { allctr->fix[i].u.cpool.min_list_size = 0; allctr->fix[i].u.cpool.shrink_list = 0; @@ -6508,12 +6656,16 @@ erts_alcu_stop(Allctr_t *allctr) void erts_alcu_init(AlcUInit_t *init) { - int i; - for (i = 0; i <= ERTS_ALC_A_MAX; i++) { - ErtsAlcCPoolData_t *sentinel = &carrier_pool[i].sentinel; - erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); - erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); - } + ErtsAlcCPoolData_t *sentinel; + + sentinel = &firstfit_carrier_pool.sentinel; + erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); + erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); + + sentinel = &test_carrier_pool.sentinel; + erts_atomic_init_nob(&sentinel->next, (erts_aint_t) sentinel); + erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); + ERTS_CT_ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); @@ -6695,7 +6847,7 @@ static int blockscan_cpool_yielding(blockscan_t *state) { ErtsAlcCPoolData_t *sentinel, *cursor; - sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel; + sentinel = (state->allocator)->cpool.sentinel; cursor = blockscan_restore_cpool_cursor(state); if (ERTS_PROC_IS_EXITING(state->process)) { @@ -6827,11 +6979,8 @@ static int blockscan_sweep_mbcs(blockscan_t *state) static int blockscan_sweep_cpool(blockscan_t *state) { if (state->current_op != blockscan_sweep_cpool) { - ErtsAlcCPoolData_t *sentinel; - SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_MBC, state->allocator); - sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel; - state->cpool_cursor = sentinel; + state->cpool_cursor = (state->allocator)->cpool.sentinel; } state->current_op = blockscan_sweep_cpool; @@ -7115,11 +7264,14 @@ static int gather_ahist_scan(Allctr_t *allocator, alcu_atag_t tag; block = SBC2BLK(allocator, carrier); - tag = GET_BLK_ATAG(block); - ASSERT(DBG_IS_VALID_ATAG(allocator, tag)); + if (BLK_HAS_ATAG(block)) { + tag = GET_BLK_ATAG(block); - gather_ahist_update(state, tag, SBC_BLK_SZ(block)); + ASSERT(DBG_IS_VALID_ATAG(tag)); + + gather_ahist_update(state, tag, SBC_BLK_SZ(block)); + } } else { UWord scanned_bytes = MBC_HEADER_SIZE(allocator); @@ -7130,10 +7282,10 @@ static int gather_ahist_scan(Allctr_t *allocator, while (1) { UWord block_size = MBC_BLK_SZ(block); - if (IS_ALLOCED_BLK(block)) { + if (IS_ALLOCED_BLK(block) && BLK_HAS_ATAG(block)) { alcu_atag_t tag = GET_BLK_ATAG(block); - ASSERT(DBG_IS_VALID_ATAG(allocator, tag)); + ASSERT(DBG_IS_VALID_ATAG(tag)); gather_ahist_update(state, tag, block_size); } @@ -7293,8 +7445,6 @@ int erts_alcu_gather_alloc_histograms(Process *p, int allocator_num, sched_id, &allocator)) { return 0; - } else if (!allocator->atags) { - return 0; } ensure_atoms_initialized(allocator); diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index f26ace1534..9ab8589bf3 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -24,6 +24,7 @@ #define ERTS_ALCU_VSN_STR "3.0" #include "erl_alloc_types.h" +#include "erl_alloc.h" #define ERL_THREADS_EMU_INTERNAL__ #include "erl_threads.h" @@ -44,6 +45,7 @@ typedef struct { typedef struct { char *name_prefix; ErtsAlcType_t alloc_no; + ErtsAlcStrat_t alloc_strat; int force; int ix; int ts; @@ -101,6 +103,7 @@ typedef struct { #define ERTS_DEFAULT_ALLCTR_INIT { \ NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ + ERTS_ALC_S_INVALID, /* (number) alloc_strat: allocator strategy */\ 0, /* (bool) force: force enabled */\ 0, /* (number) ix: instance index */\ 1, /* (bool) ts: thread safe */\ @@ -138,6 +141,7 @@ typedef struct { #define ERTS_DEFAULT_ALLCTR_INIT { \ NULL, \ ERTS_ALC_A_INVALID, /* (number) alloc_no: allocator number */\ + ERTS_ALC_S_INVALID, /* (number) alloc_strat: allocator strategy */\ 0, /* (bool) force: force enabled */\ 0, /* (number) ix: instance index */\ 1, /* (bool) ts: thread safe */\ @@ -188,6 +192,7 @@ Eterm erts_alcu_info(Allctr_t *, int, int, fmtfn_t *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); +void erts_alcu_foreign_size(Allctr_t *, ErtsAlcType_t, AllctrSize_t *); void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *); erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); @@ -286,10 +291,18 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); #define UNIT_FLOOR(X) ((X) & UNIT_MASK) #define UNIT_CEILING(X) UNIT_FLOOR((X) + INV_UNIT_MASK) -#define FLG_MASK INV_UNIT_MASK -#define SBC_BLK_SZ_MASK UNIT_MASK -#define MBC_FBLK_SZ_MASK UNIT_MASK -#define CARRIER_SZ_MASK UNIT_MASK +/* We store flags in the bits that no one will ever use. Generally these are + * the bits below the alignment size, but for blocks we also steal the highest + * bit since the header's a size and no one can expect to be able to allocate + * objects that large. */ +#define HIGHEST_WORD_BIT (((UWord) 1) << (sizeof(UWord) * CHAR_BIT - 1)) + +#define BLK_FLG_MASK (INV_UNIT_MASK | HIGHEST_WORD_BIT) +#define SBC_BLK_SZ_MASK (~BLK_FLG_MASK) +#define MBC_FBLK_SZ_MASK (~BLK_FLG_MASK) + +#define CRR_FLG_MASK INV_UNIT_MASK +#define CRR_SZ_MASK UNIT_MASK #if ERTS_HAVE_MSEG_SUPER_ALIGNED \ || (!HAVE_ERTS_MSEG && ERTS_HAVE_ERTS_SYS_ALIGNED_ALLOC) @@ -299,9 +312,9 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); # define ERTS_SUPER_ALIGN_BITS 18 # endif # ifdef ARCH_64 -# define MBC_ABLK_OFFSET_BITS 24 +# define MBC_ABLK_OFFSET_BITS 23 # else -# define MBC_ABLK_OFFSET_BITS 9 +# define MBC_ABLK_OFFSET_BITS 8 /* Affects hard limits for sbct and lmbcs documented in erts_alloc.xml */ # endif # define ERTS_SACRR_UNIT_SHIFT ERTS_SUPER_ALIGN_BITS @@ -322,18 +335,17 @@ void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp); #if MBC_ABLK_OFFSET_BITS # define MBC_ABLK_OFFSET_SHIFT (sizeof(UWord)*8 - MBC_ABLK_OFFSET_BITS) -# define MBC_ABLK_OFFSET_MASK (~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) -# define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~FLG_MASK) +# define MBC_ABLK_OFFSET_MASK ((~((UWord)0) << MBC_ABLK_OFFSET_SHIFT) & ~BLK_FLG_MASK) +# define MBC_ABLK_SZ_MASK (~MBC_ABLK_OFFSET_MASK & ~BLK_FLG_MASK) #else -# define MBC_ABLK_SZ_MASK (~FLG_MASK) +# define MBC_ABLK_SZ_MASK (~BLK_FLG_MASK) #endif #define MBC_ABLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_ABLK_SZ_MASK) #define MBC_FBLK_SZ(B) (ASSERT(!is_sbc_blk(B)), (B)->bhdr & MBC_FBLK_SZ_MASK) #define SBC_BLK_SZ(B) (ASSERT(is_sbc_blk(B)), (B)->bhdr & SBC_BLK_SZ_MASK) -#define CARRIER_SZ(C) \ - ((C)->chdr & CARRIER_SZ_MASK) +#define CARRIER_SZ(C) ((C)->chdr & CRR_SZ_MASK) typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; @@ -351,12 +363,20 @@ typedef struct { #endif } Block_t; -typedef union ErtsAllctrDDBlock_t_ ErtsAllctrDDBlock_t; +typedef struct ErtsAllctrDDBlock__ { + union { + struct ErtsAllctrDDBlock__ *ptr_next; + erts_atomic_t atmc_next; + } u; + ErtsAlcType_t type; + Uint32 flags; +} ErtsAllctrDDBlock_t; -union ErtsAllctrDDBlock_t_ { - erts_atomic_t atmc_next; - ErtsAllctrDDBlock_t *ptr_next; -}; +/* Deallocation was caused by shrinking a fix-list, so usage statistics has + * already been updated. */ +#define DEALLOC_FLG_FIX_SHRINK (1 << 0) +/* Deallocation was redirected to another instance. */ +#define DEALLOC_FLG_REDIRECTED (1 << 1) typedef struct { Block_t blk; @@ -365,11 +385,10 @@ typedef struct { #endif } ErtsFakeDDBlock_t; - - #define THIS_FREE_BLK_HDR_FLG (((UWord) 1) << 0) #define PREV_FREE_BLK_HDR_FLG (((UWord) 1) << 1) #define LAST_BLK_HDR_FLG (((UWord) 1) << 2) +#define ATAG_BLK_HDR_FLG HIGHEST_WORD_BIT #define SBC_BLK_HDR_FLG /* Special flag combo for (allocated) SBC blocks */\ (THIS_FREE_BLK_HDR_FLG | PREV_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) @@ -381,9 +400,9 @@ typedef struct { #define HOMECOMING_MBC_BLK_HDR (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG) #define IS_FREE_LAST_MBC_BLK(B) \ - (((B)->bhdr & FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) + (((B)->bhdr & BLK_FLG_MASK) == (THIS_FREE_BLK_HDR_FLG | LAST_BLK_HDR_FLG)) -#define IS_SBC_BLK(B) (((B)->bhdr & FLG_MASK) == SBC_BLK_HDR_FLG) +#define IS_SBC_BLK(B) (((B)->bhdr & SBC_BLK_HDR_FLG) == SBC_BLK_HDR_FLG) #define IS_MBC_BLK(B) (!IS_SBC_BLK((B))) #define IS_FREE_BLK(B) (ASSERT(IS_MBC_BLK(B)), \ (B)->bhdr & THIS_FREE_BLK_HDR_FLG) @@ -394,7 +413,8 @@ typedef struct { # define ABLK_TO_MBC(B) \ (ASSERT(IS_MBC_BLK(B) && !IS_FREE_BLK(B)), \ (Carrier_t*)((ERTS_SACRR_UNIT_FLOOR((UWord)(B)) - \ - (((B)->bhdr >> MBC_ABLK_OFFSET_SHIFT) << ERTS_SACRR_UNIT_SHIFT)))) + ((((B)->bhdr & ~BLK_FLG_MASK) >> MBC_ABLK_OFFSET_SHIFT) \ + << ERTS_SACRR_UNIT_SHIFT)))) # define BLK_TO_MBC(B) (IS_FREE_BLK(B) ? FBLK_TO_MBC(B) : ABLK_TO_MBC(B)) #else # define FBLK_TO_MBC(B) ((B)->carrier) @@ -433,8 +453,9 @@ typedef struct { ErtsThrPrgrVal thr_prgr; erts_atomic_t max_size; UWord abandon_limit; - UWord blocks; - UWord blocks_size; + UWord blocks[ERTS_ALC_A_MAX + 1]; + UWord blocks_size[ERTS_ALC_A_MAX + 1]; + UWord total_blocks_size; enum { ERTS_MBC_IS_HOME, ERTS_MBC_WAS_POOLED, @@ -452,7 +473,7 @@ struct Carrier_t_ { }; #define ERTS_ALC_CARRIER_TO_ALLCTR(C) \ - ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~FLG_MASK)) + ((Allctr_t *) (erts_atomic_read_nob(&(C)->allctr) & ~CRR_FLG_MASK)) typedef struct { Carrier_t *first; @@ -530,7 +551,6 @@ typedef struct { } head; } ErtsAllctrDDQueue_t; - typedef struct { size_t type_size; SWord list_size; @@ -549,6 +569,7 @@ typedef struct { UWord used; } cpool; } u; + ErtsAlcType_t type; } ErtsAlcFixList_t; struct Allctr_t_ { @@ -569,6 +590,9 @@ struct Allctr_t_ { /* Allocator number */ ErtsAlcType_t alloc_no; + /* Allocator strategy */ + ErtsAlcStrat_t alloc_strat; + /* Instance index */ int ix; @@ -617,6 +641,9 @@ struct Allctr_t_ { AOFF_RBTree_t* pooled_tree; CarrierList_t dc_list; + /* the sentinel of the cpool we're attached to */ + ErtsAlcCPoolData_t *sentinel; + UWord abandon_limit; int disable_abandon; int check_limit_count; @@ -624,8 +651,8 @@ struct Allctr_t_ { UWord in_pool_limit; /* acnl */ UWord fblk_min_limit; /* acmfl */ struct { - erts_atomic_t blocks_size; - erts_atomic_t no_blocks; + erts_atomic_t blocks_size[ERTS_ALC_A_MAX + 1]; + erts_atomic_t no_blocks[ERTS_ALC_A_MAX + 1]; erts_atomic_t carriers_size; erts_atomic_t no_carriers; CallCounter_t fail_pooled; diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 3f0ab33597..0e3e4c890a 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -107,9 +107,11 @@ typedef struct AOFF_Carrier_t_ AOFF_Carrier_t; struct AOFF_Carrier_t_ { Carrier_t crr; - AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ - AOFF_RBTree_t* root; /* Root of my block tree */ + AOFF_RBTree_t rbt_node; /* My node in the carrier tree */ + AOFF_RBTree_t* root; /* Root of my block tree */ + enum AOFFSortOrder blk_order; }; + #define RBT_NODE_TO_MBC(PTR) ErtsContainerStruct((PTR), AOFF_Carrier_t, rbt_node) /* @@ -281,15 +283,28 @@ erts_aoffalc_start(AOFFAllctr_t *alc, sys_memcpy((void *) alc, (void *) &zero.allctr, sizeof(AOFFAllctr_t)); + if (aoffinit->blk_order == FF_CHAOS) { + const enum AOFFSortOrder orders[3] = {FF_AOFF, FF_AOBF, FF_BF}; + int index = init->ix % (sizeof(orders) / sizeof(orders[0])); + + ASSERT(init->alloc_no == ERTS_ALC_A_TEST); + aoffinit->blk_order = orders[index]; + } + + if (aoffinit->crr_order == FF_CHAOS) { + const enum AOFFSortOrder orders[2] = {FF_AGEFF, FF_AOFF}; + int index = init->ix % (sizeof(orders) / sizeof(orders[0])); + + ASSERT(init->alloc_no == ERTS_ALC_A_TEST); + aoffinit->crr_order = orders[index]; + } + alc->blk_order = aoffinit->blk_order; alc->crr_order = aoffinit->crr_order; allctr->mbc_header_size = sizeof(AOFF_Carrier_t); allctr->min_mbc_size = MIN_MBC_SZ; allctr->min_mbc_first_free_size = MIN_MBC_FIRST_FREE_SZ; - allctr->min_block_size = (aoffinit->blk_order == FF_BF - ? (offsetof(AOFF_RBTree_t, u.next) - + ErtsSizeofMember(AOFF_RBTree_t, u.next)) - : offsetof(AOFF_RBTree_t, u)); + allctr->min_block_size = sizeof(AOFF_RBTree_t); allctr->vsn_str = ERTS_ALC_AOFF_ALLOC_VSN_STR; @@ -512,14 +527,15 @@ tree_insert_fixup(AOFF_RBTree_t** root, AOFF_RBTree_t *blk) static void aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) { - AOFFAllctr_t* alc = (AOFFAllctr_t*)allctr; AOFF_RBTree_t* del = (AOFF_RBTree_t*)blk; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) FBLK_TO_MBC(&del->hdr); + (void)allctr; + ASSERT(crr->rbt_node.hdr.bhdr == crr->root->max_sz); - HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, crr->blk_order, crr->root, 0); - if (alc->blk_order == FF_BF) { + if (crr->blk_order == FF_BF) { ASSERT(del->flags & IS_BF_FLG); if (IS_LIST_ELEM(del)) { /* Remove from list */ @@ -540,14 +556,14 @@ aoff_unlink_free_block(Allctr_t *allctr, Block_t *blk) replace(&crr->root, (AOFF_RBTree_t*)del, LIST_NEXT(del)); - HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, crr->blk_order, crr->root, 0); return; } } rbt_delete(&crr->root, (AOFF_RBTree_t*)del); - HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, 0); + HARD_CHECK_TREE(&crr->crr, crr->blk_order, crr->root, 0); /* Update the carrier tree with a potentially new (lower) max_sz */ @@ -737,17 +753,18 @@ rbt_delete(AOFF_RBTree_t** root, AOFF_RBTree_t* del) static void aoff_link_free_block(Allctr_t *allctr, Block_t *block) { - AOFFAllctr_t* alc = (AOFFAllctr_t*) allctr; AOFF_RBTree_t *blk = (AOFF_RBTree_t *) block; AOFF_RBTree_t *crr_node; AOFF_Carrier_t *blk_crr = (AOFF_Carrier_t*) FBLK_TO_MBC(block); Uint blk_sz = AOFF_BLK_SZ(blk); + (void)allctr; + ASSERT(allctr == ERTS_ALC_CARRIER_TO_ALLCTR(&blk_crr->crr)); ASSERT(blk_crr->rbt_node.hdr.bhdr == (blk_crr->root ? blk_crr->root->max_sz : 0)); - HARD_CHECK_TREE(&blk_crr->crr, alc->blk_order, blk_crr->root, 0); + HARD_CHECK_TREE(&blk_crr->crr, blk_crr->blk_order, blk_crr->root, 0); - rbt_insert(alc->blk_order, &blk_crr->root, blk); + rbt_insert(blk_crr->blk_order, &blk_crr->root, blk); /* * Update carrier tree with a potentially new (larger) max_sz @@ -891,7 +908,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, /* Get block within carrier tree */ #ifdef HARD_DEBUG - dbg_blk = HARD_CHECK_TREE(&crr->crr, alc->blk_order, crr->root, size); + dbg_blk = HARD_CHECK_TREE(&crr->crr, crr->blk_order, crr->root, size); #endif blk = rbt_search(crr->root, size); @@ -904,7 +921,7 @@ aoff_get_free_block(Allctr_t *allctr, Uint size, if (!blk) return NULL; - if (cand_blk && cmp_cand_blk(alc->blk_order, cand_blk, blk) < 0) { + if (cand_blk && cmp_cand_blk(crr->blk_order, cand_blk, blk) < 0) { return NULL; /* cand_blk was better */ } @@ -927,21 +944,28 @@ static void aoff_creating_mbc(Allctr_t *allctr, Carrier_t *carrier) AOFFAllctr_t *alc = (AOFFAllctr_t *) allctr; AOFF_Carrier_t *crr = (AOFF_Carrier_t*) carrier; AOFF_RBTree_t **root = &alc->mbc_root; + Sint64 bt = get_birth_time(); HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); crr->rbt_node.hdr.bhdr = 0; - if (alc->crr_order == FF_AGEFF || IS_DEBUG) { - Sint64 bt = get_birth_time(); - crr->rbt_node.u.birth_time = bt; - crr->crr.cpool.pooled.u.birth_time = bt; - } + + /* While birth time is only used for FF_AGEFF, we have to set it for all + * types as we can be migrated to an instance that uses it and we don't + * want to mess its order up. */ + crr->rbt_node.u.birth_time = bt; + crr->crr.cpool.pooled.u.birth_time = bt; + rbt_insert(alc->crr_order, root, &crr->rbt_node); /* aoff_link_free_block will add free block later */ crr->root = NULL; HARD_CHECK_TREE(NULL, alc->crr_order, *root, 0); + + /* When a carrier has been migrated, its block order may differ from that + * of the allocator it's been migrated to. */ + crr->blk_order = alc->blk_order; } #define IS_CRR_IN_TREE(CRR,ROOT) \ diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.h b/erts/emulator/beam/erl_ao_firstfit_alloc.h index 68df9e0a49..9c9b98da86 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.h +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.h @@ -32,7 +32,12 @@ enum AOFFSortOrder { FF_AGEFF = 0, /* carrier trees only */ FF_AOFF = 1, FF_AOBF = 2, /* block trees only */ - FF_BF = 3 /* block trees only */ + FF_BF = 3, /* block trees only */ + + FF_CHAOS = -1 /* A test-specific sort order that picks any of the above + * after instance id. Used to test that carriers created + * under one order will work fine after being migrated + * to another. */ }; typedef struct { diff --git a/erts/emulator/beam/erl_bif_binary.c b/erts/emulator/beam/erl_bif_binary.c index a2610bf2e1..ede317aca3 100644 --- a/erts/emulator/beam/erl_bif_binary.c +++ b/erts/emulator/beam/erl_bif_binary.c @@ -796,6 +796,7 @@ static void compute_goodshifts(BMData *bmd) } #define BM_LOOP_FACTOR 10 /* Should we have a higher value? */ +#define MC_LOOP_FACTOR 8 static void bm_init_find_first_match(BinaryFindContext *ctx) { @@ -819,13 +820,38 @@ static BFReturn bm_find_first_match(BinaryFindContext *ctx, byte *haystack) Sint i; Sint j = state->pos; register Uint reds = *reductions; + byte *pos_pointer; + Sint needle_last = blen - 1; + Sint mem_read = len - needle_last - j; - while (j <= len - blen) { + if (mem_read <= 0) { + return BF_NOT_FOUND; + } + mem_read = MIN(mem_read, reds * MC_LOOP_FACTOR); + ASSERT(mem_read > 0); + + pos_pointer = memchr(&haystack[j + needle_last], needle[needle_last], mem_read); + if (pos_pointer == NULL) { + reds -= mem_read / MC_LOOP_FACTOR; + j += mem_read; + } else { + reds -= (pos_pointer - &haystack[j]) / MC_LOOP_FACTOR; + j = pos_pointer - haystack - needle_last; + } + + // Ensure we have at least one reduction before entering the loop + ++reds; + + for(;;) { + if (j > len - blen) { + *reductions = reds; + return BF_NOT_FOUND; + } if (--reds == 0) { state->pos = j; return BF_RESTART; } - for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) + for (i = needle_last; i >= 0 && needle[i] == haystack[i + j]; --i) ; if (i < 0) { /* found */ *reductions = reds; @@ -835,8 +861,6 @@ static BFReturn bm_find_first_match(BinaryFindContext *ctx, byte *haystack) } j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); } - *reductions = reds; - return BF_NOT_FOUND; } static void bm_init_find_all(BinaryFindContext *ctx) @@ -875,14 +899,38 @@ static BFReturn bm_find_all_non_overlapping(BinaryFindContext *ctx, byte *haysta Sint *gs = bmd->goodshift; Sint *bs = bmd->badshift; byte *needle = bmd->x; - Sint i; + Sint i = -1; /* Use memchr on start and on every match */ Sint j = state->pos; Uint m = state->m; Uint allocated = state->allocated; FindallData *out = state->out; register Uint reds = *reductions; + byte *pos_pointer; + Sint needle_last = blen - 1; + Sint mem_read; - while (j <= len - blen) { + for(;;) { + if (i < 0) { + mem_read = len - needle_last - j; + if(mem_read <= 0) { + goto done; + } + mem_read = MIN(mem_read, reds * MC_LOOP_FACTOR); + ASSERT(mem_read > 0); + pos_pointer = memchr(&haystack[j + needle_last], needle[needle_last], mem_read); + if (pos_pointer == NULL) { + reds -= mem_read / MC_LOOP_FACTOR; + j += mem_read; + } else { + reds -= (pos_pointer - &haystack[j]) / MC_LOOP_FACTOR; + j = pos_pointer - haystack - needle_last; + } + // Ensure we have at least one reduction when resuming the loop + ++reds; + } + if (j > len - blen) { + goto done; + } if (--reds == 0) { state->pos = j; state->m = m; @@ -890,7 +938,7 @@ static BFReturn bm_find_all_non_overlapping(BinaryFindContext *ctx, byte *haysta state->out = out; return BF_RESTART; } - for (i = blen - 1; i >= 0 && needle[i] == haystack[i + j]; --i) + for (i = needle_last; i >= 0 && needle[i] == haystack[i + j]; --i) ; if (i < 0) { /* found */ if (m >= allocated) { @@ -912,6 +960,7 @@ static BFReturn bm_find_all_non_overlapping(BinaryFindContext *ctx, byte *haysta j += MAX(gs[i],bs[haystack[i+j]] - blen + 1 + i); } } + done: state->m = m; state->out = out; *reductions = reds; @@ -931,6 +980,7 @@ static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp) Eterm t, b, comp_term = NIL; Uint characters; Uint words; + Uint size; characters = 0; words = 0; @@ -946,11 +996,12 @@ static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp) if (binary_bitsize(b) != 0) { goto badarg; } - if (binary_size(b) == 0) { + size = binary_size(b); + if (size == 0) { goto badarg; } ++words; - characters += binary_size(b); + characters += size; } if (is_not_nil(t)) { goto badarg; diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 4cda0948a0..639aee29dc 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -829,7 +829,7 @@ BIF_RETTYPE erl_ddll_format_error_int_1(BIF_ALIST_1) "cannot be loaded/unloaded"; break; case am_permanent: - errstring = "DDLL driver is permanent an can not be unloaded/loaded"; + errstring = "DDLL driver is permanent an cannot be unloaded/loaded"; break; case am_not_loaded: errstring = "DDLL driver is not loaded"; diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 5789fa8e71..7fada0d548 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -2093,17 +2093,6 @@ current_stacktrace(ErtsHeapFactory *hfact, Process* rp, return res; } -#if defined(VALGRIND) -static int check_if_xml(void) -{ - char buf[1]; - size_t bufsz = sizeof(buf); - return erts_sys_explicit_8bit_getenv("VALGRIND_LOG_XML", buf, &bufsz) >= 0; -} -#else -#define check_if_xml() 0 -#endif - /* * This function takes care of calls to erlang:system_info/1 when the argument * is a tuple. @@ -2200,15 +2189,9 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ #endif } else if (is_list(*tp)) { #if defined(PURIFY) -#define ERTS_ERROR_CHECKER_PRINTF purify_printf -#define ERTS_ERROR_CHECKER_PRINTF_XML purify_printf +# define ERTS_ERROR_CHECKER_PRINTF purify_printf #elif defined(VALGRIND) -#define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF -# ifndef HAVE_VALGRIND_PRINTF_XML -# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF -# else -# define ERTS_ERROR_CHECKER_PRINTF_XML VALGRIND_PRINTF_XML -# endif +# define ERTS_ERROR_CHECKER_PRINTF VALGRIND_PRINTF #endif ErlDrvSizeT buf_size = 8*1024; /* Try with 8KB first */ char *buf = erts_alloc(ERTS_ALC_T_TMP, buf_size); @@ -2224,12 +2207,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ ASSERT(r == buf_size - 1); } buf[buf_size - 1 - r] = '\0'; - if (check_if_xml()) { - ERTS_ERROR_CHECKER_PRINTF_XML("<erlang_info_log>" - "%s</erlang_info_log>\n", buf); - } else { - ERTS_ERROR_CHECKER_PRINTF("%s\n", buf); - } + ERTS_ERROR_CHECKER_PRINTF("%s\n", buf); erts_free(ERTS_ALC_T_TMP, (void *) buf); BIF_RET(am_true); #undef ERTS_ERROR_CHECKER_PRINTF diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 9861483bf0..711e62c795 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -810,7 +810,7 @@ Eterm trace_info_2(BIF_ALIST_2) } erts_release_code_write_permission(); - if (is_internal_ref(res)) + if (is_value(res) && is_internal_ref(res)) BIF_TRAP1(erts_await_result, BIF_P, res); BIF_RET(res); diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 36d83d93f4..c009a3bde8 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3647,6 +3647,7 @@ send_ets_transfer_message(Process *c_p, Process *proc, hd_copy = copy_struct(heir_data, hd_sz, &hp, ohp); sender = c_p->common.id; msg = TUPLE4(hp, am_ETS_TRANSFER, tid, sender, hd_copy); + ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_proc_message(c_p, proc, *locks, mp, msg); } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 788718ab09..2eb874b005 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -3126,7 +3126,7 @@ static int partly_bound_can_match_lesser(Eterm partly_bound_1, if (ret) erts_fprintf(stderr," can match lesser than "); else - erts_fprintf(stderr," can not match lesser than "); + erts_fprintf(stderr," cannot match lesser than "); erts_fprintf(stderr,"%T\n",partly_bound_2); #endif return ret; @@ -3144,7 +3144,7 @@ static int partly_bound_can_match_greater(Eterm partly_bound_1, if (ret) erts_fprintf(stderr," can match greater than "); else - erts_fprintf(stderr," can not match greater than "); + erts_fprintf(stderr," cannot match greater than "); erts_fprintf(stderr,"%T\n",partly_bound_2); #endif return ret; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 57c6c10c7f..5da7b43b9e 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -128,7 +128,7 @@ const Eterm etp_hole_marker = 0; static int modified_sched_thread_suggested_stack_size = 0; -Eterm erts_init_process_id; +Eterm erts_init_process_id = ERTS_INVALID_PID; /* * Note about VxWorks: All variables must be initialized by executable code, @@ -2258,6 +2258,7 @@ erl_start(int argc, char **argv) erts_init_process_id = erl_first_process_otp("otp_ring0", NULL, 0, boot_argc, boot_argv); + ASSERT(erts_init_process_id != ERTS_INVALID_PID); { /* diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 89d95a73cf..0d47b16e0b 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -532,7 +532,7 @@ ERTS_GLB_INLINE void lcnt_dec_lock_state__(ethr_atomic_t *l_state) { ethr_sint_t state = ethr_atomic_dec_read_acqb(l_state); - /* We can not assume that state is >= -1 here; unlock and unacquire might + /* We cannot assume that state is >= -1 here; unlock and unacquire might * bring it below -1 and race to increment it back. */ if(state < 0) { diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index cba17d3e6a..3d6c9eb43f 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -1505,25 +1505,6 @@ int hashmap_key_hash_cmp(Eterm* ap, Eterm* bp) return ap ? -1 : 1; } -/* maps:new/0 */ - -BIF_RETTYPE maps_new_0(BIF_ALIST_0) { - Eterm* hp; - Eterm tup; - flatmap_t *mp; - - hp = HAlloc(BIF_P, (MAP_HEADER_FLATMAP_SZ + 1)); - tup = make_tuple(hp); - *hp++ = make_arityval(0); - - mp = (flatmap_t*)hp; - mp->thing_word = MAP_HEADER_FLATMAP; - mp->size = 0; - mp->keys = tup; - - BIF_RET(make_flatmap(mp)); -} - /* maps:put/3 */ BIF_RETTYPE maps_put_3(BIF_ALIST_3) { @@ -1707,11 +1688,16 @@ int erts_maps_update(Process *p, Eterm key, Eterm value, Eterm map, Eterm *res) return 0; found_key: - *hp++ = value; - vs++; - if (++i < n) - sys_memcpy(hp, vs, (n - i)*sizeof(Eterm)); - *res = make_flatmap(shp); + if(*vs == value) { + HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); + *res = map; + } else { + *hp++ = value; + vs++; + if (++i < n) + sys_memcpy(hp, vs, (n - i)*sizeof(Eterm)); + *res = make_flatmap(shp); + } return 1; } @@ -1767,9 +1753,7 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { if (is_immed(key)) { for( i = 0; i < n; i ++) { if (ks[i] == key) { - *hp++ = value; - vs++; - c = 1; + goto found_key; } else { *hp++ = *vs++; } @@ -1777,18 +1761,13 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { } else { for( i = 0; i < n; i ++) { if (EQ(ks[i], key)) { - *hp++ = value; - vs++; - c = 1; + goto found_key; } else { *hp++ = *vs++; } } } - if (c) - return res; - /* the map will grow */ if (n >= MAP_SMALL_MAP_LIMIT) { @@ -1843,6 +1822,18 @@ Eterm erts_maps_put(Process *p, Eterm key, Eterm value, Eterm map) { */ *shp = make_pos_bignum_header(0); return res; + +found_key: + if(*vs == value) { + HRelease(p, shp + MAP_HEADER_FLATMAP_SZ + n, shp); + return map; + } else { + *hp++ = value; + vs++; + if (++i < n) + sys_memcpy(hp, vs, (n - i)*sizeof(Eterm)); + return res; + } } ASSERT(is_hashmap(map)); diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index 507cc989d2..a3274d7443 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -409,6 +409,11 @@ ErtsMessage* prepend_pending_sig_maybe(Process* sender, Process* receiver, * * @brief Send one message from *NOT* a local process. * + * seq_trace does not work with this type of messages + * to it is set to am_undefined which means that the + * receiving process will not remove the seq_trace token + * when it gets this message. + * */ void erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks, @@ -417,11 +422,19 @@ erts_queue_message(Process* receiver, ErtsProcLocks receiver_locks, ASSERT(is_not_internal_pid(from)); ERL_MESSAGE_TERM(mp) = msg; ERL_MESSAGE_FROM(mp) = from; + ERL_MESSAGE_TOKEN(mp) = am_undefined; queue_messages(receiver, receiver_locks, mp, &mp->next, 1); } /** * @brief Send one message from a local process. + * + * It is up to the caller of this function to set the + * correct seq_trace. The general rule of thumb is that + * it should be set to am_undefined if the message + * cannot be traced using seq_trace, if it can be + * traced it should be set to the trace token. It should + * very rarely be explicitly set to NIL! */ void erts_queue_proc_message(Process* sender, @@ -584,8 +597,7 @@ void erts_send_message(Process* sender, Process* receiver, ErtsProcLocks *receiver_locks, - Eterm message, - unsigned flags) + Eterm message) { Uint msize; ErtsMessage* mp; @@ -619,7 +631,7 @@ erts_send_message(Process* sender, receiver_state = erts_atomic32_read_nob(&receiver->state); - if (SEQ_TRACE_TOKEN(sender) != NIL && !(flags & ERTS_SND_FLG_NO_SEQ_TRACE)) { + if (SEQ_TRACE_TOKEN(sender) != NIL) { Eterm* hp; Eterm stoken = SEQ_TRACE_TOKEN(sender); Uint seq_trace_size = 0; diff --git a/erts/emulator/beam/erl_message.h b/erts/emulator/beam/erl_message.h index d120111634..b2550814fd 100644 --- a/erts/emulator/beam/erl_message.h +++ b/erts/emulator/beam/erl_message.h @@ -405,8 +405,6 @@ typedef struct erl_trace_message_queue__ { #define SAVE_MESSAGE(p) \ (p)->sig_qs.save = &(*(p)->sig_qs.save)->next -#define ERTS_SND_FLG_NO_SEQ_TRACE (((unsigned) 1) << 0) - #define ERTS_HEAP_FRAG_SIZE(DATA_WORDS) \ (sizeof(ErlHeapFragment) - sizeof(Eterm) + (DATA_WORDS)*sizeof(Eterm)) @@ -429,7 +427,7 @@ typedef struct erl_trace_message_queue__ { do { \ (MP)->next = NULL; \ ERL_MESSAGE_TERM(MP) = THE_NON_VALUE; \ - ERL_MESSAGE_TOKEN(MP) = NIL; \ + ERL_MESSAGE_TOKEN(MP) = THE_NON_VALUE; \ ERL_MESSAGE_FROM(MP) = NIL; \ ERL_MESSAGE_DT_UTAG_INIT(MP); \ MP->data.attached = NULL; \ @@ -446,7 +444,7 @@ void erts_queue_proc_message(Process* from,Process* to, ErtsProcLocks,ErtsMessag void erts_queue_proc_messages(Process* from, Process* to, ErtsProcLocks, ErtsMessage*, ErtsMessage**, Uint); void erts_deliver_exit_message(Eterm, Process*, ErtsProcLocks *, Eterm, Eterm); -void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm, unsigned); +void erts_send_message(Process*, Process*, ErtsProcLocks*, Eterm); void erts_link_mbuf_to_proc(Process *proc, ErlHeapFragment *bp); Uint erts_msg_attached_data_size_aux(ErtsMessage *msg); diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 0fbf0eb03a..7339aa8874 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -453,8 +453,18 @@ static void full_flush_env(ErlNifEnv* env) static void full_cache_env(ErlNifEnv* env) { - if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) + if (env->proc->static_flags & ERTS_STC_FLG_SHADOW_PROC) { erts_cache_dirty_shadow_proc(env->proc); + /* + * If shadow proc had heap fragments when flushed + * those have now been moved to the real proc. + * Ensure heap pointers do not point into a heap + * fragment on real proc... + */ + ASSERT(!env->proc->mbuf); + env->hp_end = HEAP_LIMIT(env->proc); + env->hp = HEAP_TOP(env->proc); + } cache_env(env); } @@ -744,8 +754,58 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, rp_locks = ERTS_PROC_LOCK_MAIN; if (menv) { + Eterm token = c_p ? SEQ_TRACE_TOKEN(c_p) : am_undefined; + if (token != NIL && token != am_undefined) { + /* This code is copied from erts_send_message */ + Eterm stoken = SEQ_TRACE_TOKEN(c_p); +#ifdef USE_VM_PROBES + DTRACE_CHARBUF(sender_name, 64); + DTRACE_CHARBUF(receiver_name, 64); + Sint tok_label = 0; + Sint tok_lastcnt = 0; + Sint tok_serial = 0; + Eterm utag = NIL; + *sender_name = *receiver_name = '\0'; + if (DTRACE_ENABLED(message_send)) { + erts_snprintf(sender_name, sizeof(DTRACE_CHARBUF_NAME(sender_name)), + "%T", c_p->common.id); + erts_snprintf(receiver_name, sizeof(DTRACE_CHARBUF_NAME(receiver_name)), + "%T", rp->common.id); + } +#endif + if (have_seqtrace(stoken)) { + seq_trace_update_send(c_p); + seq_trace_output(stoken, msg, SEQ_TRACE_SEND, + rp->common.id, c_p); + } +#ifdef USE_VM_PROBES + if (!(DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING)) { + stoken = NIL; + } +#endif + token = enif_make_copy(msg_env, stoken); + +#ifdef USE_VM_PROBES + if (DT_UTAG_FLAGS(c_p) & DT_UTAG_SPREADING) { + if (is_immed(DT_UTAG(c_p))) + utag = DT_UTAG(c_p); + else + utag = enif_make_copy(msg_env, DT_UTAG(c_p)); + } + if (DTRACE_ENABLED(message_send)) { + if (have_seqtrace(stoken)) { + tok_label = SEQ_TRACE_T_DTRACE_LABEL(stoken); + tok_lastcnt = signed_val(SEQ_TRACE_T_LASTCNT(stoken)); + tok_serial = signed_val(SEQ_TRACE_T_SERIAL(stoken)); + } + DTRACE6(message_send, sender_name, receiver_name, + size_object(msg), tok_label, tok_lastcnt, tok_serial); + } +#endif + } flush_env(msg_env); mp = erts_alloc_message(0, NULL); + ERL_MESSAGE_TOKEN(mp) = token; mp->data.heap_frag = menv->env.heap_frag; ASSERT(mp->data.heap_frag == MBUF(&menv->phony_proc)); if (mp->data.heap_frag != NULL) { @@ -783,6 +843,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ohp = &bp->off_heap; } } + ERL_MESSAGE_TOKEN(mp) = am_undefined; msg = copy_struct_litopt(msg, sz, &hp, ohp, &litarea); } @@ -826,6 +887,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ERL_MESSAGE_TERM(mp) = msg; ERL_MESSAGE_FROM(mp) = from; + ERL_MESSAGE_TOKEN(mp) = am_undefined; if (!msgq) { msgq = erts_alloc(ERTS_ALC_T_TRACE_MSG_QUEUE, @@ -978,7 +1040,7 @@ ERL_NIF_TERM enif_make_copy(ErlNifEnv* dst_env, ERL_NIF_TERM src_term) Eterm* hp; /* * No preserved sharing allowed as long as literals are also preserved. - * Process independent environment can not be reached by purge. + * Process independent environment cannot be reached by purge. */ sz = size_object(src_term); hp = alloc_heap(dst_env, sz); diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index d6d22677e7..f343e984f7 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -52,8 +52,8 @@ #define ERTS_SIG_Q_OP_MAX 13 -#define ERTS_SIG_Q_OP_EXIT 0 -#define ERTS_SIG_Q_OP_EXIT_LINKED 1 +#define ERTS_SIG_Q_OP_EXIT 0 /* Exit signal due to bif call */ +#define ERTS_SIG_Q_OP_EXIT_LINKED 1 /* Exit signal due to link break*/ #define ERTS_SIG_Q_OP_MONITOR_DOWN 2 #define ERTS_SIG_Q_OP_MONITOR 3 #define ERTS_SIG_Q_OP_DEMONITOR 4 @@ -1167,10 +1167,7 @@ erts_proc_sig_send_persistent_monitor_msg(Uint16 type, Eterm key, ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_PERSISTENT_MON_MSG, type, 0); ERL_MESSAGE_FROM(mp) = from; - ERL_MESSAGE_TOKEN(mp) = NIL; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; -#endif + ERL_MESSAGE_TOKEN(mp) = am_undefined; if (!proc_queue_signal(NULL, to, (ErtsSignal *) mp, ERTS_SIG_Q_OP_PERSISTENT_MON_MSG)) { @@ -1564,11 +1561,6 @@ erts_proc_sig_send_is_alive_request(Process *c_p, Eterm to, Eterm ref) ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_IS_ALIVE, ERTS_SIG_Q_TYPE_UNDEFINED, 0); - ERL_MESSAGE_TOKEN(mp) = NIL; - ERL_MESSAGE_FROM(mp) = am_system; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; -#endif if (proc_queue_signal(c_p, to, (ErtsSignal *) mp, ERTS_SIG_Q_OP_IS_ALIVE)) (void) maybe_elevate_sig_handling_prio(c_p, to); @@ -1672,11 +1664,6 @@ erts_proc_sig_send_sync_suspend(Process *c_p, Eterm to, Eterm tag, Eterm reply) ERL_MESSAGE_TERM(mp) = ERTS_PROC_SIG_MAKE_TAG(ERTS_SIG_Q_OP_SYNC_SUSPEND, ERTS_SIG_Q_TYPE_UNDEFINED, 0); - ERL_MESSAGE_TOKEN(mp) = NIL; - ERL_MESSAGE_FROM(mp) = am_system; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; -#endif if (proc_queue_signal(c_p, to, (ErtsSignal *) mp, ERTS_SIG_Q_OP_SYNC_SUSPEND)) (void) maybe_elevate_sig_handling_prio(c_p, to); @@ -1790,7 +1777,7 @@ handle_rpc(Process *c_p, ErtsProcSigRPC *rpc, int cnt, int limit, int *yieldp) msg = TUPLE2(hp, ref, res); mp->hfrag.next = bp; - + ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_proc_message(c_p, rp, 0, mp, msg); } @@ -2155,10 +2142,7 @@ handle_exit_signal(Process *c_p, ErtsSigRecvTracing *tracing, pid = STORE_NC(&hp, ohp, from); ERL_MESSAGE_TERM(mp) = TUPLE3(hp, am_EXIT, pid, reason); - ERL_MESSAGE_TOKEN(mp) = NIL; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; -#endif + ERL_MESSAGE_TOKEN(mp) = am_undefined; if (is_immed(pid)) ERL_MESSAGE_FROM(mp) = pid; else { @@ -2346,10 +2330,7 @@ convert_to_down_message(Process *c_p, type, from, reason); hp += 6; - ERL_MESSAGE_TOKEN(mp) = NIL; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; -#endif + ERL_MESSAGE_TOKEN(mp) = am_undefined; /* Replace original signal with the exit message... */ convert_to_msg(c_p, sig, mp, next_nm_sig); @@ -2397,10 +2378,7 @@ convert_to_nodedown_messages(Process *c_p, ERL_MESSAGE_TERM(mp) = TUPLE2(hp, am_nodedown, node); ERL_MESSAGE_FROM(mp) = am_system; - ERL_MESSAGE_TOKEN(mp) = NIL; -#ifdef USE_VM_PROBES - ERL_MESSAGE_DT_UTAG(mp) = NIL; -#endif + ERL_MESSAGE_TOKEN(mp) = am_undefined; mp->next = nd_first; nd_first = mp; if (!nd_last) @@ -2830,6 +2808,7 @@ handle_process_info(Process *c_p, ErtsSigRecvTracing *tracing, if (is_alive) erts_factory_trim_and_close(&hfact, &msg, 1); + ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_proc_message(c_p, rp, locks, mp, msg); if (!is_alive && locks) @@ -2931,6 +2910,7 @@ sync_suspend_reply(Process *c_p, ErtsMessage *mp, erts_aint32_t state) tp[2] = ssusp->async ? am_not_suspended : am_internal_error; } } + ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_proc_message(c_p, rp, 0, mp, ssusp->message); } } @@ -3146,8 +3126,8 @@ erts_proc_sig_handle_incoming(Process *c_p, erts_aint32_t *statep, erts_monitor_tree_delete(&ERTS_P_MONITORS(c_p), &mdp->origin); omon = &mdp->origin; + remove_nm_sig(c_p, sig, next_nm_sig); } - remove_nm_sig(c_p, sig, next_nm_sig); break; default: ERTS_INTERNAL_ERROR("invalid monitor type"); diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 8253ec4f40..9386f79b56 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -3761,6 +3761,8 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep) ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER; state = erts_atomic32_read_nob(&p->state); + ASSERT(state & ERTS_PSFLG_IN_RUNQ); + if (statep) *statep = state; @@ -3768,8 +3770,7 @@ dequeue_process(ErtsRunQueue *runq, int prio_q, erts_aint32_t *statep) rqi = &runq->procs.prio_info[prio]; - if (p) - unqueue_process(runq, rpq, rqi, prio, NULL, p); + unqueue_process(runq, rpq, rqi, prio, NULL, p); return p; } @@ -4088,7 +4089,7 @@ evacuate_run_queue(ErtsRunQueue *rq, erts_runq_unlock(to_rq); smp_notify_inc_runq(to_rq); - erts_runq_lock(to_rq); + erts_runq_lock(rq); } if (rq->ports.start) { @@ -4157,22 +4158,17 @@ evacuate_run_queue(ErtsRunQueue *rq, free_proxy_proc(proc); else { erts_aint32_t clr_bits; -#ifdef DEBUG - erts_aint32_t old; -#endif clr_bits = ERTS_PSFLG_IN_RUNQ; clr_bits |= qbit << ERTS_PSFLGS_IN_PRQ_MASK_OFFSET; -#ifdef DEBUG - old = -#else - (void) -#endif - erts_atomic32_read_band_mb(&proc->state, - ~clr_bits); - ASSERT((old & clr_bits) == clr_bits); + state = erts_atomic32_read_band_mb(&proc->state, ~clr_bits); + ASSERT((state & clr_bits) == clr_bits); + if (state & ERTS_PSFLG_FREE) { + /* free and not queued by proxy */ + erts_proc_dec_refc(proc); + } } goto handle_next_proc; @@ -6208,13 +6204,14 @@ fin_dirty_enq_s_change(Process *p, /* Already enqueue by someone else... */ if (pstruct_reserved) { /* We reserved process struct for enqueue; clear it... */ -#ifdef DEBUG - erts_aint32_t old = -#else - (void) -#endif - erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ); - ASSERT(old & ERTS_PSFLG_IN_RUNQ); + erts_aint32_t state; + + state = erts_atomic32_read_band_nob(&p->state, ~ERTS_PSFLG_IN_RUNQ); + ASSERT(state & ERTS_PSFLG_IN_RUNQ); + + if (state & ERTS_PSFLG_FREE) { + erts_proc_dec_refc(p); + } } return 0; } @@ -6407,8 +6404,9 @@ schedule_out_process(ErtsRunQueue *c_rq, erts_aint32_t state, Process *p, == ERTS_PSFLG_ACTIVE)); n &= ~running_flgs; - if ((a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)) - || (a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE) { + if ((!!(a & (ERTS_PSFLG_ACTIVE_SYS|ERTS_PSFLG_DIRTY_ACTIVE_SYS)) + | ((a & (ERTS_PSFLG_ACTIVE|ERTS_PSFLG_SUSPENDED)) == ERTS_PSFLG_ACTIVE)) + & !(a & ERTS_PSFLG_FREE)) { enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); } a = erts_atomic32_cmpxchg_mb(&p->state, n, e); @@ -6655,62 +6653,72 @@ erts_schedule_process(Process *p, erts_aint32_t state, ErtsProcLocks locks) schedule_process(p, state, locks); } +/* Enqueues the given sys task on the process and schedules it. The task may be + * NULL if only scheduling is desired. */ static ERTS_INLINE erts_aint32_t -active_sys_enqueue(Process *p, erts_aint32_t state, - erts_aint32_t enable_flags, int status_locked) -{ - /* - * This function may or may not be called with status locke held. - * It always returns without the status lock held! - */ - unsigned int prof_runnable_procs = erts_system_profile_flags.runnable_procs; - erts_aint32_t n, a = state, enq_prio = -1; - int slocked = status_locked; +active_sys_enqueue(Process *p, ErtsProcSysTask *sys_task, + erts_aint32_t task_prio, erts_aint32_t enable_flags, + erts_aint32_t state, erts_aint32_t *fail_state_p) +{ + int runnable_procs = erts_system_profile_flags.runnable_procs; + erts_aint32_t n, a, enq_prio, fail_state; + int already_scheduled; + int status_locked; int enqueue; /* < 0 -> use proxy */ - /* Status lock prevents out of order "runnable proc" trace msgs */ - ERTS_LC_ASSERT(slocked || !(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); - ERTS_LC_ASSERT(!slocked || (ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); - - if (!prof_runnable_procs) { - if (slocked) { - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - slocked = 0; - } - } - else { - if (!slocked) { - erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); - slocked = !0; - } - } + enable_flags |= ERTS_PSFLG_ACTIVE_SYS; + fail_state = *fail_state_p; + already_scheduled = 0; + status_locked = 0; + enq_prio = -1; + a = state; + ERTS_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); + ASSERT(fail_state & (ERTS_PSFLG_EXITING | ERTS_PSFLG_FREE)); + ASSERT(!(fail_state & enable_flags)); ASSERT(!(state & ERTS_PSFLG_PROXY)); + /* When runnable_procs is enabled, we need to take the status lock to + * prevent trace messages from being sent in the wrong order. The lock must + * be held over the call to add2runq. + * + * Otherwise, we only need to take it when we're enqueuing a task and can + * safely release it before add2runq. */ + if (sys_task || runnable_procs) { + erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); + status_locked = 1; + } + while (1) { erts_aint32_t e; n = e = a; - if (a & ERTS_PSFLG_FREE) - goto cleanup; /* We don't want to schedule free processes... */ + if (a & fail_state) { + *fail_state_p = a & fail_state; + goto cleanup; + } enqueue = ERTS_ENQUEUE_NOT; - n |= enable_flags; - n |= ERTS_PSFLG_ACTIVE_SYS; + n |= enable_flags; + if (!(a & (ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS | ERTS_PSFLG_DIRTY_RUNNING - | ERTS_PSFLG_DIRTY_RUNNING_SYS))) + | ERTS_PSFLG_DIRTY_RUNNING_SYS))) { enqueue = check_enqueue_in_prio_queue(p, &enq_prio, &n, a); + } + a = erts_atomic32_cmpxchg_mb(&p->state, n, e); - if (a == e) + if (a == e) { break; - if (a == n && enqueue == ERTS_ENQUEUE_NOT) - goto cleanup; + } + else if (a == n && enqueue == ERTS_ENQUEUE_NOT) { + already_scheduled = 1; + break; + } } - if (prof_runnable_procs) { - + if (!already_scheduled && runnable_procs) { if (!(a & (ERTS_PSFLG_ACTIVE_SYS | ERTS_PSFLG_RUNNING | ERTS_PSFLG_RUNNING_SYS @@ -6720,19 +6728,56 @@ active_sys_enqueue(Process *p, erts_aint32_t state, /* We activated a prevously inactive process */ profile_runnable_proc(p, am_active); } - - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - slocked = 0; } - add2runq(enqueue, enq_prio, p, n, NULL); + if (sys_task) { + ErtsProcSysTaskQs *stqs = p->sys_task_qs; -cleanup: + if (!stqs) { + sys_task->next = sys_task->prev = sys_task; - if (slocked) - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + stqs = proc_sys_task_queues_alloc(); - ERTS_LC_ASSERT(!(ERTS_PROC_LOCK_STATUS & erts_proc_lc_my_proc_locks(p))); + stqs->qmask = 1 << task_prio; + stqs->ncount = 0; + stqs->q[PRIORITY_MAX] = NULL; + stqs->q[PRIORITY_HIGH] = NULL; + stqs->q[PRIORITY_NORMAL] = NULL; + stqs->q[PRIORITY_LOW] = NULL; + stqs->q[task_prio] = sys_task; + + p->sys_task_qs = stqs; + } + else { + if (!stqs->q[task_prio]) { + sys_task->next = sys_task->prev = sys_task; + + stqs->q[task_prio] = sys_task; + stqs->qmask |= 1 << task_prio; + } + else { + sys_task->next = stqs->q[task_prio]; + sys_task->prev = stqs->q[task_prio]->prev; + sys_task->next->prev = sys_task; + sys_task->prev->next = sys_task; + ASSERT(stqs->qmask & (1 << task_prio)); + } + } + } + + if (status_locked && !runnable_procs) { + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + status_locked = 0; + } + + if (!already_scheduled) { + add2runq(enqueue, enq_prio, p, n, NULL); + } + +cleanup: + if (status_locked) { + erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } return n; } @@ -6740,103 +6785,41 @@ cleanup: erts_aint32_t erts_proc_sys_schedule(Process *p, erts_aint32_t state, erts_aint32_t enable_flag) { - /* We are not allowed to call this function with status lock held... */ - return active_sys_enqueue(p, state, enable_flag, 0); + erts_aint32_t fail_state = ERTS_PSFLG_FREE; + + return active_sys_enqueue(p, NULL, 0, enable_flag, state, &fail_state); } static int schedule_process_sys_task(Process *p, erts_aint32_t prio, ErtsProcSysTask *st, erts_aint32_t *fail_state_p) { - int res; - int locked; - ErtsProcSysTaskQs *stqs, *free_stqs; erts_aint32_t fail_state, state; - fail_state = *fail_state_p; - - res = 1; /* prepare for success */ - st->next = st->prev = st; /* Prep for empty prio queue */ + /* Elevate priority if needed. */ state = erts_atomic32_read_nob(&p->state); - locked = 0; - free_stqs = NULL; - if (state & ERTS_PSFLG_SYS_TASKS) - stqs = NULL; - else { - alloc_qs: - stqs = proc_sys_task_queues_alloc(); - stqs->qmask = 1 << prio; - stqs->ncount = 0; - stqs->q[PRIORITY_MAX] = NULL; - stqs->q[PRIORITY_HIGH] = NULL; - stqs->q[PRIORITY_NORMAL] = NULL; - stqs->q[PRIORITY_LOW] = NULL; - stqs->q[prio] = st; - } - - if (!locked) { - locked = 1; - erts_proc_lock(p, ERTS_PROC_LOCK_STATUS); - - state = erts_atomic32_read_nob(&p->state); - if (state & fail_state) { - erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); - *fail_state_p = (state & fail_state); - free_stqs = stqs; - res = 0; - goto cleanup; - } - } - - if (!p->sys_task_qs) { - if (stqs) - p->sys_task_qs = stqs; - else - goto alloc_qs; - } - else { - free_stqs = stqs; - stqs = p->sys_task_qs; - if (!stqs->q[prio]) { - stqs->q[prio] = st; - stqs->qmask |= 1 << prio; - } - else { - st->next = stqs->q[prio]; - st->prev = stqs->q[prio]->prev; - st->next->prev = st; - st->prev->next = st; - ASSERT(stqs->qmask & (1 << prio)); - } - } - if (ERTS_PSFLGS_GET_ACT_PRIO(state) > prio) { - erts_aint32_t n, a, e; - /* Need to elevate actual prio */ + erts_aint32_t n, a, e; - a = state; - do { - if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) { - n = a; - break; - } - n = e = a; - n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; - n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); - a = erts_atomic32_cmpxchg_nob(&p->state, n, e); - } while (a != e); - state = n; - } - - /* active_sys_enqueue() always return with status lock unlocked */ - (void) active_sys_enqueue(p, state, ERTS_PSFLG_SYS_TASKS, locked); + a = state; + do { + if (ERTS_PSFLGS_GET_ACT_PRIO(a) <= prio) { + n = a; + break; + } + n = e = a; + n &= ~ERTS_PSFLGS_ACT_PRIO_MASK; + n |= (prio << ERTS_PSFLGS_ACT_PRIO_OFFSET); + a = erts_atomic32_cmpxchg_nob(&p->state, n, e); + } while (a != e); -cleanup: + state = n; + } - if (free_stqs) - proc_sys_task_queues_free(free_stqs); + fail_state = *fail_state_p; - return res; + return !(active_sys_enqueue(p, st, prio, ERTS_PSFLG_SYS_TASKS, + state, fail_state_p) & fail_state); } static ERTS_INLINE int @@ -9563,6 +9546,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) } else if (state & ERTS_PSFLG_FREE) { /* free and not queued by proxy */ + ASSERT(state & ERTS_PSFLG_IN_RUNQ); erts_proc_dec_refc(p); } if (!is_normal_sched) @@ -9622,7 +9606,7 @@ Process *erts_schedule(ErtsSchedulerData *esdp, Process *p, int calls) ASSERT(!p->scheduler_data); p->scheduler_data = esdp; if ((!!(state & ERTS_PSFLGS_DIRTY_WORK)) - & (!(state & ERTS_PSFLG_ACTIVE_SYS))) { + & (!(state & ERTS_PSFLG_RUNNING_SYS))) { /* Migrate to dirty scheduler... */ sunlock_sched_out_proc: erts_proc_unlock(p, ERTS_PROC_LOCK_STATUS); @@ -9890,6 +9874,7 @@ notify_sys_task_executed(Process *c_p, ErtsProcSysTask *st, ASSERT(hp_start + hsz == hp); #endif + ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_proc_message(c_p, rp, rp_locks, mp, msg); if (c_p == rp) diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index a60e117bab..8d20ccdf90 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -293,7 +293,7 @@ typedef enum { * highest index... * * Remember to update description in erts_pre_init_process() - * when adding new flags... + * and etp-commands when adding new flags... */ typedef enum { diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index 243db4c734..706530023b 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -963,12 +963,16 @@ dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area) } erts_putc(to, to_arg, '\n'); } - } else if (is_export_header(w)) { + } else if (is_export_header(w) || is_fun_header(w)) { dump_externally(to, to_arg, term); erts_putc(to, to_arg, '\n'); } size = 1 + header_arity(w); switch (w & _HEADER_SUBTAG_MASK) { + case FUN_SUBTAG: + ASSERT(((ErlFunThing*)(htop))->num_free == 0); + size += 1; + break; case MAP_SUBTAG: if (is_flatmap_header(w)) { size += 1 + flatmap_get_size(htop); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 2446b3c074..133ab485d9 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -3287,7 +3287,6 @@ static void deliver_read_message(Port* prt, erts_aint32_t state, Eterm to, if (trace_send) trace_port_send(prt, to, tuple, 1); - ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); if (rp_locks) erts_proc_unlock(rp, rp_locks); @@ -3459,7 +3458,6 @@ deliver_vec_message(Port* prt, /* Port */ if (IS_TRACED_FL(prt, F_TRACE_SEND)) trace_port_send(prt, to, tuple, 1); - ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); erts_proc_unlock(rp, rp_locks); if (!scheduler) @@ -5382,7 +5380,6 @@ void driver_report_exit(ErlDrvPort ix, int status) if (IS_TRACED_FL(prt, F_TRACE_SEND)) trace_port_send(prt, pid, tuple, 1); - ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_message(rp, rp_locks, mp, tuple, prt->common.id); erts_proc_unlock(rp, rp_locks); @@ -5988,8 +5985,6 @@ driver_deliver_term(Port *prt, Eterm to, ErlDrvTermData* data, int len) from = prt->common.id; } - /* send message */ - ERL_MESSAGE_TOKEN(factory.message) = am_undefined; erts_queue_message(rp, rp_locks, factory.message, mess, from); } else if (res == -2) { diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 88ede3bb60..c51e4ef784 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -1421,16 +1421,13 @@ get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \ i_get_map_elements f? s I -i_get_map_element Fail Src=xy Key=y Dst => \ - move Key x | i_get_map_element Fail Src x Dst - i_get_map_element_hash Fail Src=c Key Hash Dst => \ move Src x | i_get_map_element_hash Fail x Key Hash Dst i_get_map_element_hash f? xy c I xy i_get_map_element Fail Src=c Key Dst => \ move Src x | i_get_map_element Fail x Key Dst -i_get_map_element f? xy x xy +i_get_map_element f? xy xy xy # # Convert the plus operations to a generic plus instruction. diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 19b1312ee3..996757ef43 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1569,7 +1569,7 @@ make_hash2(Eterm term) * MUST BE USED AS INPUT FOR THE HASH. Two different terms must always have a * chance of hashing different when salted: hash([Salt|A]) vs hash([Salt|B]). * - * This is why we can not use cached hash values for atoms for example. + * This is why we cannot use cached hash values for atoms for example. * */ @@ -3122,7 +3122,7 @@ tailrecur_ne: ASSERT(alen == blen); for (i = (Sint) alen - 1; i >= 0; i--) if (anum[i] != bnum[i]) - RETURN_NEQ((Sint32) (anum[i] - bnum[i])); + RETURN_NEQ(anum[i] < bnum[i] ? -1 : 1); goto pop_next; case (_TAG_HEADER_EXTERNAL_REF >> _TAG_PRIMARY_SIZE): if (is_internal_ref(b)) { diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index 6fd3bb9fbf..91381bd60d 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -1034,6 +1034,7 @@ typedef struct { inet_async_op* oph; /* queue head or NULL */ inet_async_op* opt; /* queue tail or NULL */ inet_async_op op_queue[INET_MAX_ASYNC]; /* call queue */ + int op_ref; /* queue reference generator */ int active; /* 0 = passive, 1 = active, 2 = active once */ Sint16 active_count; /* counter for {active,N} */ @@ -1299,8 +1300,7 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event); /* convert descriptor pointer to inet_descriptor pointer */ #define INETP(d) (&(d)->inet) -static int async_ref = 0; /* async reference id generator */ -#define NEW_ASYNC_ID() ((async_ref++) & 0xffff) +#define NEW_ASYNC_ID(desc) ((desc)->op_ref++ & 0xffff) /* check for transition from active to passive */ #define INET_CHECK_ACTIVE_TO_PASSIVE(inet) \ @@ -1549,6 +1549,8 @@ static void *realloc_wrapper(void *current, ErlDrvSizeT size){ # define LOAD_ASSOC_ID LOAD_UINT # define LOAD_ASSOC_ID_CNT LOAD_UINT_CNT # define SCTP_ANC_BUFF_SIZE INET_DEF_BUFFER/2 /* XXX: not very good... */ +#else +# define IS_SCTP(desc) 0 #endif #ifdef HAVE_UDP @@ -1785,6 +1787,7 @@ static void release_buffer(ErlDrvBinary* buf) #ifdef HAVE_UDP static ErlDrvBinary* realloc_buffer(ErlDrvBinary* buf, ErlDrvSizeT newsz) { + DEBUGF(("realloc_buffer: %ld -> %ld\r\n", (buf==NULL) ? 0 : buf->orig_size, newsz)); return driver_realloc_binary(buf, newsz); } #endif @@ -1955,7 +1958,7 @@ static void enq_multi_op(tcp_descriptor *desc, char *buf, int req, ErlDrvTermData caller, MultiTimerData *timeout, ErlDrvMonitor *monitorp) { - int id = NEW_ASYNC_ID(); + int id = NEW_ASYNC_ID(INETP(desc)); enq_old_multi_op(desc,id,req,caller,timeout,monitorp); if (buf != NULL) put_int16(id, buf); @@ -2024,7 +2027,7 @@ static int remove_multi_op(tcp_descriptor *desc, int *id_p, int *req_p, static int enq_async_w_tmo(inet_descriptor* desc, char* buf, int req, unsigned timeout, ErlDrvMonitor *monitorp) { - int id = NEW_ASYNC_ID(); + int id = NEW_ASYNC_ID(desc); inet_async_op* opp; if ((opp = desc->oph) == NULL) /* queue empty */ @@ -4390,7 +4393,7 @@ static void desc_close_read(inet_descriptor* desc) { if (desc->s != INVALID_SOCKET) { #ifdef __WIN32__ - /* This call can not be right??? + /* This call cannot be right??? * We want to turn off read events but keep any write events. * But on windows driver_select(...,READ,1) is only used as a * way to hook into the pollset. sock_select is used to control @@ -6435,7 +6438,12 @@ static int inet_set_opts(inet_descriptor* desc, char* ptr, int len) (long)desc->port, desc->s, res)); if (type == SO_RCVBUF) { /* make sure we have desc->bufsz >= SO_RCVBUF */ - if (ival > desc->bufsz) + if (ival > (1 << 16) && desc->stype == SOCK_DGRAM && !IS_SCTP(desc)) + /* For UDP we don't want to automatically + set the buffer size to be larger than + the theoretical max MTU */ + desc->bufsz = 1 << 16; + else if (ival > desc->bufsz) desc->bufsz = ival; } } @@ -8461,6 +8469,7 @@ static ErlDrvData inet_start(ErlDrvPort port, int size, int protocol) desc->delimiter = '\n'; /* line delimiting char */ desc->oph = NULL; desc->opt = NULL; + desc->op_ref = 0; desc->peer_ptr = NULL; desc->name_ptr = NULL; @@ -12041,15 +12050,11 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event) sys_memzero((char *) &other, sizeof(other)); /* udesc->i_buf is only kept between SCTP fragments */ - if (udesc->i_buf == NULL) { - udesc->i_bufsz = desc->bufsz + len; - if ((udesc->i_buf = alloc_buffer(udesc->i_bufsz)) == NULL) - return packet_error(udesc, ENOMEM); - /* pointer to message start */ - udesc->i_ptr = udesc->i_buf->orig_bytes + len; - } else { - ErlDrvBinary* tmp; +#ifdef HAVE_SCTP + if (udesc->i_buf != NULL) { + ErlDrvBinary* tmp; int bufsz; + ASSERT(IS_SCTP(desc)); bufsz = desc->bufsz + (udesc->i_ptr - udesc->i_buf->orig_bytes); if ((tmp = realloc_buffer(udesc->i_buf, bufsz)) == NULL) { release_buffer(udesc->i_buf); @@ -12061,6 +12066,15 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event) udesc->i_buf = tmp; udesc->i_bufsz = bufsz; } + } else +#endif + { + ASSERT(udesc->i_buf == NULL); + udesc->i_bufsz = desc->bufsz + len; + if ((udesc->i_buf = alloc_buffer(udesc->i_bufsz)) == NULL) + return packet_error(udesc, ENOMEM); + /* pointer to message start */ + udesc->i_ptr = udesc->i_buf->orig_bytes + len; } /* Note: On Windows NT, recvfrom() fails if the socket is connected. */ @@ -12119,6 +12133,14 @@ static int packet_inet_input(udp_descriptor* udesc, HANDLE event) ) { sock_select(desc,FD_READ,1); } +#ifdef HAVE_SCTP + if (!short_recv) { +#endif + release_buffer(udesc->i_buf); + udesc->i_buf = NULL; +#ifdef HAVE_SCTP + } +#endif return count; /* strange, not ready */ } diff --git a/erts/emulator/drivers/unix/ttsl_drv.c b/erts/emulator/drivers/unix/ttsl_drv.c index 28c6cc0f94..11bb4373d8 100644 --- a/erts/emulator/drivers/unix/ttsl_drv.c +++ b/erts/emulator/drivers/unix/ttsl_drv.c @@ -31,7 +31,7 @@ static int ttysl_init(void); static ErlDrvData ttysl_start(ErlDrvPort, char*); -#ifdef HAVE_TERMCAP /* else make an empty driver that can not be opened */ +#ifdef HAVE_TERMCAP /* else make an empty driver that cannot be opened */ #ifndef WANT_NONBLOCKING #define WANT_NONBLOCKING diff --git a/erts/emulator/internal_doc/CarrierMigration.md b/erts/emulator/internal_doc/CarrierMigration.md index 3a796d11b7..bb3d8aac28 100644 --- a/erts/emulator/internal_doc/CarrierMigration.md +++ b/erts/emulator/internal_doc/CarrierMigration.md @@ -34,8 +34,7 @@ Solution -------- In order to prevent scenarios like this we've implemented support for -migration of multi-block carriers between allocator instances of the -same type. +migration of multi-block carriers between allocator instances. ### Management of Free Blocks ### @@ -130,10 +129,6 @@ threads may have references to it via the pool. ### Migration ### -There exists one pool for each allocator type enabling migration of -carriers between scheduler specific allocator instances of the same -allocator type. - Each allocator instance keeps track of the current utilization of its multi-block carriers. When the total utilization falls below the "abandon carrier utilization limit" it starts to inspect the utilization of the @@ -208,8 +203,8 @@ limited. We only inspect a limited number of carriers. If none of those carriers had a free block large enough to satisfy the allocation request, the search will fail. A carrier in the pool can also be BUSY if another thread is currently doing block deallocation work on the -carrier. A BUSY carrier will also be skipped by the search as it can -not satisfy the request. The pool is lock-free and we do not want to +carrier. A BUSY carrier will also be skipped by the search as it cannot +satisfy the request. The pool is lock-free and we do not want to block, waiting for the other thread to finish. ### The bad cluster problem ### @@ -287,11 +282,3 @@ reduced using the `aoffcbf` strategy. A trade off between memory consumption and performance is however inevitable, and it is up to the user to decide what is most important. -Further work ------------- - -It would be quite easy to extend this to allow migration of multi-block -carriers between all allocator types. More or less the only obstacle -is maintenance of the statistics information. - - diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index 3e77dce1cd..9f115706dc 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -1493,7 +1493,6 @@ send_event_tuple(struct erts_nif_select_event* e, ErtsResource* resource, } tuple = TUPLE4(hp, am_select, resource_term, ref_term, event_atom); - ERL_MESSAGE_TOKEN(mp) = am_undefined; erts_queue_message(rp, rp_locks, mp, tuple, am_system); if (rp_locks) diff --git a/erts/emulator/sys/common/erl_osenv.c b/erts/emulator/sys/common/erl_osenv.c index 9f54d1dff0..487ff87116 100644 --- a/erts/emulator/sys/common/erl_osenv.c +++ b/erts/emulator/sys/common/erl_osenv.c @@ -75,7 +75,15 @@ static int compare_env_keys(const erts_osenv_data_t a, const erts_osenv_data_t b #include "erl_rbtree.h" static int compare_env_keys(const erts_osenv_data_t a, const erts_osenv_data_t b) { - int relation = sys_memcmp(a.data, b.data, MIN(a.length, b.length)); + int relation; + +#ifdef __WIN32__ + /* Environment variables are case-insensitive on Windows. */ + relation = _wcsnicmp((const WCHAR*)a.data, (const WCHAR*)b.data, + MIN(a.length, b.length) / sizeof(WCHAR)); +#else + relation = sys_memcmp(a.data, b.data, MIN(a.length, b.length)); +#endif if(relation != 0) { return relation; diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index 70b5532af9..b4d1575ee5 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -803,6 +803,23 @@ update_pollset(ErtsPollSet *ps, int fd, ErtsPollOp op, ErtsPollEvents events) struct kevent evts[2]; struct timespec ts = {0, 0}; + if (op == ERTS_POLL_OP_ADD) { + /* This is a hack to make the "noshell" option work; kqueue can poll + * these fds but will not report EV_EOF, so we return NVAL to use the + * fallback instead. + * + * This may be common to all pipes but we have no way to tell whether + * an fd is a pipe or not. */ + switch (fd) { + case STDIN_FILENO: + case STDOUT_FILENO: + case STDERR_FILENO: + return ERTS_POLL_EV_NVAL; + default: + break; + } + } + #if defined(EV_DISPATCH) && !defined(__OpenBSD__) /* If we have EV_DISPATCH we use it, unless we are on OpenBSD as the behavior of EV_EOF seems to be edge triggered there and we need it diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 189ca083d7..36579ffdb4 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -452,9 +452,9 @@ prepare_crash_dump(int secs) envsz = sizeof(env); i = erts_sys_explicit_8bit_getenv("ERL_CRASH_DUMP_NICE", env, &envsz); - if (i >= 0) { + if (i != 0) { int nice_val; - nice_val = i != 1 ? 0 : atoi(env); + nice_val = (i != 1) ? 0 : atoi(env); if (nice_val > 39) { nice_val = 39; } diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c index 872c3a80b1..816bdea9c5 100644 --- a/erts/emulator/sys/unix/sys_drivers.c +++ b/erts/emulator/sys/unix/sys_drivers.c @@ -685,7 +685,7 @@ static ErlDrvData spawn_start(ErlDrvPort port_num, char* name, /* we send the request to do the fork */ if ((res = writev(ofd[1], io_vector, iov_len > MAXIOV ? MAXIOV : iov_len)) < 0) { - if (errno == ERRNO_BLOCK) { + if (errno == ERRNO_BLOCK || errno == EINTR) { res = 0; } else { int err = errno; @@ -1000,7 +1000,7 @@ static void clear_fd_data(ErtsSysFdData *fdd) static void nbio_stop_fd(ErlDrvPort prt, ErtsSysFdData *fdd) { - driver_select(prt, abs(fdd->fd), DO_READ|DO_WRITE, 0); + driver_select(prt, abs(fdd->fd), ERL_DRV_USE_NO_CALLBACK|DO_READ|DO_WRITE, 0); clear_fd_data(fdd); SET_BLOCKING(abs(fdd->fd)); @@ -1257,6 +1257,8 @@ static int port_inp_failure(ErtsSysDriverData *dd, int res) } driver_failure_eof(dd->port_num); } else if (dd->ifd) { + if (dd->alive == -1) + errno = dd->status; erl_drv_init_ack(dd->port_num, ERL_DRV_ERROR_ERRNO); } else { driver_failure_posix(dd->port_num, err); @@ -1287,10 +1289,10 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd) int res; if((res = read(ready_fd, &proto, sizeof(proto))) <= 0) { + if (res < 0 && (errno == ERRNO_BLOCK || errno == EINTR)) + return; /* hmm, child setup seems to have closed the pipe too early... we close the port as there is not much else we can do */ - if (res < 0 && errno == ERRNO_BLOCK) - return; driver_select(port_num, ready_fd, ERL_DRV_READ, 0); if (res == 0) errno = EPIPE; @@ -1424,7 +1426,7 @@ static void ready_input(ErlDrvData e, ErlDrvEvent ready_fd) continue; } else { /* The last message we got was split */ - char *buf = erts_alloc_fnf(ERTS_ALC_T_FD_ENTRY_BUF, h); + char *buf = erts_alloc_fnf(ERTS_ALC_T_FD_ENTRY_BUF, h); if (!buf) { errno = ENOMEM; port_inp_failure(dd, -1); @@ -1670,15 +1672,37 @@ static void forker_stop(ErlDrvData e) the port has been closed by the user. */ } +static ErlDrvSizeT forker_deq(ErlDrvPort port_num, ErtsSysForkerProto *proto) +{ + close(proto->u.start.fds[0]); + close(proto->u.start.fds[1]); + if (proto->u.start.fds[1] != proto->u.start.fds[2]) + close(proto->u.start.fds[2]); + + return driver_deq(port_num, sizeof(*proto)); +} + +static void forker_sigchld(Eterm port_id, int error) +{ + ErtsSysForkerProto *proto = erts_alloc(ERTS_ALC_T_DRV_CTRL_DATA, sizeof(*proto)); + proto->action = ErtsSysForkerProtoAction_SigChld; + proto->u.sigchld.error_number = error; + proto->u.sigchld.port_id = port_id; + + /* ideally this would be a port_command call, but as command is + already used by the spawn_driver, we use control instead. + Note that when using erl_drv_port_control it is an asynchronous + control. */ + erl_drv_port_control(port_id, 'S', (char*)proto, sizeof(*proto)); +} + static void forker_ready_input(ErlDrvData e, ErlDrvEvent fd) { int res; - ErtsSysForkerProto *proto; + ErtsSysForkerProto proto; - proto = erts_alloc(ERTS_ALC_T_DRV_CTRL_DATA, sizeof(*proto)); - - if ((res = read(fd, proto, sizeof(*proto))) < 0) { - if (errno == ERRNO_BLOCK) + if ((res = read(fd, &proto, sizeof(proto))) < 0) { + if (errno == ERRNO_BLOCK || errno == EINTR) return; erts_exit(ERTS_DUMP_EXIT, "Failed to read from erl_child_setup: %d\n", errno); } @@ -1686,10 +1710,10 @@ static void forker_ready_input(ErlDrvData e, ErlDrvEvent fd) if (res == 0) erts_exit(ERTS_DUMP_EXIT, "erl_child_setup closed\n"); - ASSERT(res == sizeof(*proto)); + ASSERT(res == sizeof(proto)); #ifdef FORKER_PROTO_START_ACK - if (proto->action == ErtsSysForkerProtoAction_StartAck) { + if (proto.action == ErtsSysForkerProtoAction_StartAck) { /* Ideally we would like to not have to ack each Start command being sent over the uds, but it would seem that some operating systems (only observed on FreeBSD) @@ -1699,28 +1723,15 @@ static void forker_ready_input(ErlDrvData e, ErlDrvEvent fd) ErlDrvPort port_num = (ErlDrvPort)e; int vlen; SysIOVec *iov = driver_peekq(port_num, &vlen); - ErtsSysForkerProto *proto = (ErtsSysForkerProto *)iov[0].iov_base; - - close(proto->u.start.fds[0]); - close(proto->u.start.fds[1]); - if (proto->u.start.fds[1] != proto->u.start.fds[2]) - close(proto->u.start.fds[2]); + ErtsSysForkerProto *qproto = (ErtsSysForkerProto *)iov[0].iov_base; - driver_deq(port_num, sizeof(*proto)); - - if (driver_sizeq(port_num) > 0) + if (forker_deq(port_num, qproto)) driver_select(port_num, forker_fd, ERL_DRV_WRITE|ERL_DRV_USE, 1); } else #endif { - ASSERT(proto->action == ErtsSysForkerProtoAction_SigChld); - - /* ideally this would be a port_command call, but as command is - already used by the spawn_driver, we use control instead. - Note that when using erl_drv_port_control it is an asynchronous - control. */ - erl_drv_port_control(proto->u.sigchld.port_id, 'S', - (char*)proto, sizeof(*proto)); + ASSERT(proto.action == ErtsSysForkerProtoAction_SigChld); + forker_sigchld(proto.u.sigchld.port_id, proto.u.sigchld.error_number); } } @@ -1730,7 +1741,8 @@ static void forker_ready_output(ErlDrvData e, ErlDrvEvent fd) ErlDrvPort port_num = (ErlDrvPort)e; #ifndef FORKER_PROTO_START_ACK - while (driver_sizeq(port_num) > 0) { + int loops = 10; + while (driver_sizeq(port_num) > 0 && --loops) { #endif int vlen; SysIOVec *iov = driver_peekq(port_num, &vlen); @@ -1738,20 +1750,24 @@ static void forker_ready_output(ErlDrvData e, ErlDrvEvent fd) ASSERT(iov[0].iov_len >= (sizeof(*proto))); if (sys_uds_write(forker_fd, (char*)proto, sizeof(*proto), proto->u.start.fds, 3, 0) < 0) { - if (errno == ERRNO_BLOCK) + if (errno == ERRNO_BLOCK || errno == EINTR) { return; - erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno); + } else if (errno == EMFILE) { + forker_sigchld(proto->u.start.port_id, errno); + if (forker_deq(port_num, proto) == 0) + driver_select(port_num, forker_fd, ERL_DRV_WRITE, 0); + return; + } else { + erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno); + } } #ifndef FORKER_PROTO_START_ACK - close(proto->u.start.fds[0]); - close(proto->u.start.fds[1]); - if (proto->u.start.fds[1] != proto->u.start.fds[2]) - close(proto->u.start.fds[2]); - driver_deq(port_num, sizeof(*proto)); + if (forker_deq(port_num, proto) == 0) + driver_select(port_num, forker_fd, ERL_DRV_WRITE, 0); } -#endif - +#else driver_select(port_num, forker_fd, ERL_DRV_WRITE, 0); +#endif } static ErlDrvSSizeT forker_control(ErlDrvData e, unsigned int cmd, char *buf, @@ -1777,20 +1793,21 @@ static ErlDrvSSizeT forker_control(ErlDrvData e, unsigned int cmd, char *buf, if ((res = sys_uds_write(forker_fd, (char*)proto, sizeof(*proto), proto->u.start.fds, 3, 0)) < 0) { - if (errno == ERRNO_BLOCK) { + if (errno == ERRNO_BLOCK || errno == EINTR) { driver_select(port_num, forker_fd, ERL_DRV_WRITE|ERL_DRV_USE, 1); return 0; + } else if (errno == EMFILE) { + forker_sigchld(proto->u.start.port_id, errno); + forker_deq(port_num, proto); + return 0; + } else { + erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno); } - erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno); } #ifndef FORKER_PROTO_START_ACK ASSERT(res == sizeof(*proto)); - close(proto->u.start.fds[0]); - close(proto->u.start.fds[1]); - if (proto->u.start.fds[1] != proto->u.start.fds[2]) - close(proto->u.start.fds[2]); - driver_deq(port_num, sizeof(*proto)); + forker_deq(port_num, proto); #endif return 0; diff --git a/erts/emulator/sys/unix/sys_uds.c b/erts/emulator/sys/unix/sys_uds.c index c328fd00bb..39a4866065 100644 --- a/erts/emulator/sys/unix/sys_uds.c +++ b/erts/emulator/sys/unix/sys_uds.c @@ -132,7 +132,7 @@ sys_uds_writev(int fd, struct iovec *iov, size_t iov_len, struct msghdr msg; struct cmsghdr *cmsg = NULL; - int res, i; + int res, i, error; /* initialize socket message */ memset(&msg, 0, sizeof(struct msghdr)); @@ -173,11 +173,22 @@ sys_uds_writev(int fd, struct iovec *iov, size_t iov_len, res = sendmsg(fd, &msg, flags); +#ifdef ETOOMANYREFS + /* Linux may give ETOOMANYREFS when there are too many fds in transit. + We map this to EMFILE as bsd and other use this error code and we want + the behaviour to be the same on all OSs */ + if (errno == ETOOMANYREFS) + errno = EMFILE; +#endif + error = errno; + if (iov_len > MAXIOV) free(iov[0].iov_base); free(msg.msg_control); + errno = error; + return res; } diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index fd4c745c3b..fcf5a0d533 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -362,11 +362,11 @@ is_io_ready(ErtsPollSet *ps) } static ERTS_INLINE void -woke_up(ErtsPollSet *ps) +woke_up(ErtsPollSet *ps, int waketype) { if (erts_atomic32_read_nob(&ps->wakeup_state) == ERTS_POLL_NOT_WOKEN) erts_atomic32_cmpxchg_nob(&ps->wakeup_state, - ERTS_POLL_WOKEN_TIMEDOUT, + waketype, ERTS_POLL_NOT_WOKEN); #ifdef DEBUG { @@ -960,12 +960,12 @@ static int cancel_driver_select(ErtsPollSet *ps, HANDLE event) void erts_poll_interrupt(ErtsPollSet *ps, int set /* bool */) { - HARDTRACEF(("In erts_poll_interrupt(%d)",set)); + HARDTRACEF(("In erts_poll_interrupt(%p, %d)",ps,set)); if (!set) reset_interrupt(ps); else set_interrupt(ps); - HARDTRACEF(("Out erts_poll_interrupt(%d)",set)); + HARDTRACEF(("Out erts_poll_interrupt(%p, %d)",ps,set)); } @@ -1051,19 +1051,20 @@ int erts_poll_wait(ErtsPollSet *ps, if (!erts_atomic32_read_nob(&break_waiter_state)) { HANDLE harr[2] = {ps->event_io_ready, break_happened_event}; - int num_h = 2; + int num_h = 2, handle; ERTS_MSACC_PUSH_STATE(); HARDDEBUGF(("Start waiting %d [%d]",num_h, (int) timeout)); ERTS_POLLSET_UNLOCK(ps); erts_thr_progress_prepare_wait(NULL); ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_SLEEP); - WaitForMultipleObjects(num_h, harr, FALSE, timeout); + handle = WaitForMultipleObjects(num_h, harr, FALSE, timeout); erts_thr_progress_finalize_wait(NULL); ERTS_MSACC_POP_STATE(); ERTS_POLLSET_LOCK(ps); HARDDEBUGF(("Stop waiting %d [%d]",num_h, (int) timeout)); - woke_up(ps); + if (handle == WAIT_OBJECT_0) + woke_up(ps, ERTS_POLL_WOKEN_TIMEDOUT); } ERTS_UNSET_BREAK_REQUESTED; @@ -1075,7 +1076,10 @@ int erts_poll_wait(ErtsPollSet *ps, erts_mtx_unlock(&break_waiter_lock); switch (break_state) { case BREAK_WAITER_GOT_BREAK: + woke_up(ps, ERTS_POLL_WOKEN_INTR); ERTS_SET_BREAK_REQUESTED; + /* Wake aux thread to get handle break */ + erts_aux_thread_poke(); break; case BREAK_WAITER_GOT_HALT: erts_exit(0,""); diff --git a/erts/emulator/test/alloc_SUITE.erl b/erts/emulator/test/alloc_SUITE.erl index 343afe85e6..4e0243c1cd 100644 --- a/erts/emulator/test/alloc_SUITE.erl +++ b/erts/emulator/test/alloc_SUITE.erl @@ -71,7 +71,8 @@ migration(Cfg) -> %% Disable driver_alloc to avoid recursive alloc_util calls %% through enif_mutex_create() in my_creating_mbc(). drv_case(Cfg, concurrent, "+MZe true +MRe false"), - drv_case(Cfg, concurrent, "+MZe true +MRe false +MZas ageffcbf"). + drv_case(Cfg, concurrent, "+MZe true +MRe false +MZas ageffcbf"), + drv_case(Cfg, concurrent, "+MZe true +MRe false +MZas chaosff"). erts_mmap(Config) when is_list(Config) -> case {os:type(), mmsc_flags()} of diff --git a/erts/emulator/test/bif_SUITE.erl b/erts/emulator/test/bif_SUITE.erl index 32bfcd5520..9e7bcd5255 100644 --- a/erts/emulator/test/bif_SUITE.erl +++ b/erts/emulator/test/bif_SUITE.erl @@ -36,7 +36,8 @@ error_stacktrace_during_call_trace/1, group_leader_prio/1, group_leader_prio_dirty/1, is_process_alive/1, - process_info_blast/1]). + process_info_blast/1, + os_env_case_sensitivity/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -51,7 +52,7 @@ all() -> erl_crash_dump_bytes, min_max, erlang_halt, is_builtin, error_stacktrace, error_stacktrace_during_call_trace, group_leader_prio, group_leader_prio_dirty, - is_process_alive, process_info_blast]. + is_process_alive, process_info_blast, os_env_case_sensitivity]. %% Uses erlang:display to test that erts_printf does not do deep recursion display(Config) when is_list(Config) -> @@ -443,6 +444,17 @@ os_env_long(Min, Max, Value) -> true = os:unsetenv(EnvVar), os_env_long(Min+1, Max, Value). +os_env_case_sensitivity(Config) when is_list(Config) -> + %% The keys in os:getenv/putenv must be case-insensitive on Windows, and + %% case-sensitive elsewhere. + true = os:putenv("os_env_gurka", "gaffel"), + Expected = case os:type() of + {win32, _} -> "gaffel"; + _ -> false + end, + Expected = os:getenv("OS_ENV_GURKA"), + ok. + %% Test that string:to_integer does not Halloc in wrong order. otp_7526(Config) when is_list(Config) -> ok = test_7526(256). diff --git a/erts/emulator/test/call_trace_SUITE.erl b/erts/emulator/test/call_trace_SUITE.erl index d19f7f81ad..742592f88e 100644 --- a/erts/emulator/test/call_trace_SUITE.erl +++ b/erts/emulator/test/call_trace_SUITE.erl @@ -1395,7 +1395,7 @@ seq(M, N, R) when M =< N -> seq(M, N-1, [N|R]); seq(_, _, R) -> R. -%% lists:reverse can not be called since it is traced +%% lists:reverse cannot be called since it is traced reverse(L) -> reverse(L, []). %% diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 9c6dc3ff83..7e690fd870 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -332,6 +332,7 @@ constant_pools(Config) when is_list(Config) -> A = literals:a(), B = literals:b(), C = literals:huge_bignum(), + D = literals:funs(), process_flag(trap_exit, true), Self = self(), @@ -345,7 +346,7 @@ constant_pools(Config) when is_list(Config) -> true = erlang:purge_module(literals), NoOldHeap ! done, receive - {'EXIT',NoOldHeap,{A,B,C}} -> + {'EXIT',NoOldHeap,{A,B,C,D}} -> ok; Other -> ct:fail({unexpected,Other}) @@ -362,7 +363,7 @@ constant_pools(Config) when is_list(Config) -> erlang:purge_module(literals), OldHeap ! done, receive - {'EXIT',OldHeap,{A,B,C,[1,2,3|_]=Seq}} when length(Seq) =:= 16 -> + {'EXIT',OldHeap,{A,B,C,D,[1,2,3|_]=Seq}} when length(Seq) =:= 16 -> ok end, @@ -390,7 +391,7 @@ constant_pools(Config) when is_list(Config) -> {'DOWN', Mon, process, Hib, Reason} -> {undef, [{no_module, no_function, - [{A,B,C,[1,2,3|_]=Seq}], _}]} = Reason, + [{A,B,C,D,[1,2,3|_]=Seq}], _}]} = Reason, 16 = length(Seq) end, HeapSz = TotHeapSz, %% Ensure restored to hibernated state... @@ -400,7 +401,9 @@ constant_pools(Config) when is_list(Config) -> no_old_heap(Parent) -> A = literals:a(), B = literals:b(), - Res = {A,B,literals:huge_bignum()}, + C = literals:huge_bignum(), + D = literals:funs(), + Res = {A,B,C,D}, Parent ! go, receive done -> @@ -410,7 +413,9 @@ no_old_heap(Parent) -> old_heap(Parent) -> A = literals:a(), B = literals:b(), - Res = {A,B,literals:huge_bignum(),lists:seq(1, 16)}, + C = literals:huge_bignum(), + D = literals:funs(), + Res = {A,B,C,D,lists:seq(1, 16)}, create_old_heap(), Parent ! go, receive @@ -421,7 +426,9 @@ old_heap(Parent) -> hibernated(Parent) -> A = literals:a(), B = literals:b(), - Res = {A,B,literals:huge_bignum(),lists:seq(1, 16)}, + C = literals:huge_bignum(), + D = literals:funs(), + Res = {A,B,C,D,lists:seq(1, 16)}, Parent ! go, erlang:hibernate(no_module, no_function, [Res]). @@ -755,7 +762,8 @@ t_copy_literals_frags(Config) when is_list(Config) -> 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15, 0, 1, 2, 3, 4, 5, 6, 7, - 8, 9,10,11,12,13,14,15>>}]), + 8, 9,10,11,12,13,14,15>>}, + {f, fun ?MODULE:all/0}]), {module, ?mod} = erlang:load_module(?mod, Bin), N = 6000, @@ -796,6 +804,7 @@ literal_receiver() -> C = ?mod:c(), D = ?mod:d(), E = ?mod:e(), + F = ?mod:f(), literal_receiver(); {Pid, sender_confirm} -> io:format("sender confirm ~w~n", [Pid]), @@ -811,7 +820,8 @@ literal_sender(N, Recv) -> ?mod:b(), ?mod:c(), ?mod:d(), - ?mod:e()]}, + ?mod:e(), + ?mod:f()]}, literal_sender(N - 1, Recv). literal_switcher() -> diff --git a/erts/emulator/test/code_SUITE_data/literals.erl b/erts/emulator/test/code_SUITE_data/literals.erl index 7c3b0ebe73..13c8b412b0 100644 --- a/erts/emulator/test/code_SUITE_data/literals.erl +++ b/erts/emulator/test/code_SUITE_data/literals.erl @@ -19,7 +19,8 @@ %% -module(literals). --export([a/0,b/0,huge_bignum/0,binary/0,unused_binaries/0,bits/0]). +-export([a/0,b/0,huge_bignum/0,funs/0, + binary/0,unused_binaries/0,bits/0]). -export([msg1/0,msg2/0,msg3/0,msg4/0,msg5/0]). a() -> @@ -108,3 +109,8 @@ msg2() -> {"hello","world"}. msg3() -> <<"halloj">>. msg4() -> #{ 1=> "hello", b => "world"}. msg5() -> {1,2,3,4,5,6}. + +funs() -> + %% Literal funs (in a non-literal list). + [fun ?MODULE:a/0, + fun() -> ok end]. %No environment. diff --git a/erts/emulator/test/driver_SUITE.erl b/erts/emulator/test/driver_SUITE.erl index 7aff3a6ea1..9ffb484eb4 100644 --- a/erts/emulator/test/driver_SUITE.erl +++ b/erts/emulator/test/driver_SUITE.erl @@ -81,6 +81,7 @@ thr_msg_blast/1, consume_timeslice/1, env/1, + poll_pipe/1, z_test/1]). -export([bin_prefix/2]). @@ -168,6 +169,7 @@ all() -> %% Keep a_test first and z_test last... thr_msg_blast, consume_timeslice, env, + poll_pipe, z_test]. groups() -> @@ -2643,24 +2645,7 @@ wait_deallocations() -> driver_alloc_size() -> wait_deallocations(), - case erlang:system_info({allocator_sizes, driver_alloc}) of - false -> - undefined; - MemInfo -> - CS = lists:foldl( - fun ({instance, _, L}, Acc) -> - {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), - {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), - [MBCS,SBCS | Acc] - end, - [], - MemInfo), - lists:foldl( - fun(L, Sz0) -> - {value,{_,Sz,_,_}} = lists:keysearch(blocks_size, 1, L), - Sz0+Sz - end, 0, CS) - end. + erts_debug:alloc_blocks_size(driver_alloc). rpc(Config, Fun) -> case proplists:get_value(node, Config) of @@ -2693,3 +2678,25 @@ rpc(Config, Fun) -> ct:fail(Other) end end. + +poll_pipe(Config) when is_list(Config) -> + %% ERL-647; we wouldn't see any events on EOF when polling a pipe using + %% kqueue(2). + case os:type() of + {unix, _} -> + Command = "erl -noshell -eval " + "'\"DATA\n\" = io:get_line(\"\")," + "eof = io:get_line(\"\")," + "halt()' <<< 'DATA'", + Ref = make_ref(), + Self = self(), + Pid = spawn(fun() -> os:cmd(Command), Self ! Ref end), + receive + Ref -> ok + after 5000 -> + exit(Pid, kill), + ct:fail("Stuck reading from stdin.") + end; + _ -> + {skipped, "Unix-only test"} + end. diff --git a/erts/emulator/test/erts_debug_SUITE.erl b/erts/emulator/test/erts_debug_SUITE.erl index 6aa7a445b5..f39dbedd8f 100644 --- a/erts/emulator/test/erts_debug_SUITE.erl +++ b/erts/emulator/test/erts_debug_SUITE.erl @@ -22,8 +22,10 @@ -include_lib("common_test/include/ct.hrl"). -export([all/0, suite/0, - test_size/1,flat_size_big/1,df/1,term_type/1, - instructions/1, stack_check/1]). + test_size/1,flat_size_big/1,df/1,term_type/1, + instructions/1, stack_check/1, alloc_blocks_size/1]). + +-export([do_alloc_blocks_size/0]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -31,7 +33,7 @@ suite() -> all() -> [test_size, flat_size_big, df, instructions, term_type, - stack_check]. + stack_check, alloc_blocks_size]. test_size(Config) when is_list(Config) -> ConsCell1 = id([a|b]), @@ -210,5 +212,28 @@ instructions(Config) when is_list(Config) -> _ = [list_to_atom(I) || I <- Is], ok. +alloc_blocks_size(Config) when is_list(Config) -> + F = fun(Args) -> + Node = start_slave(Args), + ok = rpc:call(Node, ?MODULE, do_alloc_blocks_size, []), + true = test_server:stop_node(Node) + end, + F("+Meamax"), + F("+Meamin"), + F(""), + ok. + +do_alloc_blocks_size() -> + _ = erts_debug:alloc_blocks_size(binary_alloc), + ok. + +start_slave(Args) -> + Name = ?MODULE_STRING ++ "_slave", + Pa = filename:dirname(code:which(?MODULE)), + {ok, Node} = test_server:start_node(list_to_atom(Name), + slave, + [{args, "-pa " ++ Pa ++ " " ++ Args}]), + Node. + id(I) -> I. diff --git a/erts/emulator/test/fun_SUITE.erl b/erts/emulator/test/fun_SUITE.erl index 73fe9b0d8f..f8a879182e 100644 --- a/erts/emulator/test/fun_SUITE.erl +++ b/erts/emulator/test/fun_SUITE.erl @@ -576,7 +576,7 @@ refc_dist(Config) when is_list(Config) -> process_flag(trap_exit, true), Pid = spawn_link(Node, fun() -> receive Fun when is_function(Fun) -> - 2 = fun_refc(Fun), + 3 = fun_refc(Fun), exit({normal,Fun}) end end), F = fun() -> 42 end, @@ -598,7 +598,7 @@ refc_dist_send(Node, F) -> Pid = spawn_link(Node, fun() -> receive {To,Fun} when is_function(Fun) -> wait_until(fun () -> - 2 =:= fun_refc(Fun) + 3 =:= fun_refc(Fun) end), To ! Fun end @@ -626,7 +626,7 @@ refc_dist_reg_send(Node, F) -> Me ! Ref, receive {Me,Fun} when is_function(Fun) -> - 2 = fun_refc(Fun), + 3 = fun_refc(Fun), Me ! Fun end end), @@ -806,11 +806,13 @@ verify_not_undef(Fun, Tag) -> ct:fail("tag ~w not defined in fun_info", [Tag]); {Tag,_} -> ok end. - + id(X) -> X. spawn_call(Node, AFun) -> + Parent = self(), + Init = erlang:whereis(init), Pid = spawn_link(Node, fun() -> receive @@ -821,8 +823,10 @@ spawn_call(Node, AFun) -> _ -> lists:seq(0, Arity-1) end, Res = apply(Fun, Args), - {pid,Creator} = erlang:fun_info(Fun, pid), - Creator ! {result,Res} + case erlang:fun_info(Fun, pid) of + {pid,Init} -> Parent ! {result,Res}; + {pid,Creator} -> Creator ! {result,Res} + end end end), Pid ! {AFun,AFun,AFun}, diff --git a/erts/emulator/test/map_SUITE.erl b/erts/emulator/test/map_SUITE.erl index f93c637650..d0a6763fe5 100644 --- a/erts/emulator/test/map_SUITE.erl +++ b/erts/emulator/test/map_SUITE.erl @@ -3080,8 +3080,19 @@ y_regs(Config) when is_list(Config) -> true = is_map(Map2) andalso is_map(Map4), + gurka = y_regs_literal(0), + gaffel = y_regs_literal(1), + ok. +y_regs_literal(Key) when is_integer(Key) -> + %% Forces the key to be placed in a Y register. + lists:seq(1, 2), + case is_map_key(Key, #{ 0 => 0 }) of + true -> gurka; + false -> gaffel + end. + y_regs_update(Map0, Val0) -> Val1 = {t,Val0}, K1 = id({key,1}), diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 7c85cf2259..a2f3489943 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -45,8 +45,9 @@ api_macros/1, from_array/1, iolist_as_binary/1, resource/1, resource_binary/1, resource_takeover/1, - threading/1, send/1, send2/1, send3/1, send_threaded/1, neg/1, - is_checks/1, + threading/1, send/1, send2/1, send3/1, send_threaded/1, + send_trace/1, send_seq_trace/1, + neg/1, is_checks/1, get_length/1, make_atom/1, make_string/1, reverse_list_test/1, otp_9828/1, otp_9668/1, consume_timeslice/1, nif_schedule/1, @@ -1789,6 +1790,59 @@ send(Config) when is_list(Config) -> {ok,0} = send_list_seq(7, DeadPid), ok. + +%% Test tracing of enif_send +send_trace(Config) when is_list(Config) -> + ensure_lib_loaded(Config), + + Papa = self(), + N = 1500, + List = lists:seq(1,N), + + Tracer = spawn_link(fun F() -> receive get -> Papa ! receive_any(), F() end end), + + erlang:trace(self(), true, [send,'receive',{tracer,Tracer}]), + {ok,1} = send_list_seq(N, self()), + List = receive_any(), + timeout = receive_any(0), + Tracer ! get, + {trace,Papa,send,List,Papa} = receive_any(), + Tracer ! get, + {trace,Papa,'receive',List} = receive_any(). + +%% Test that seq_trace works with nif trace +send_seq_trace(Config) when is_list(Config) -> + ensure_lib_loaded(Config), + + Papa = self(), + N = 1500, + List = lists:seq(1,N), + Label = make_ref(), + + Tracer = spawn_link(fun F() -> receive get -> Papa ! receive_any(), F() end end), + + seq_trace:set_system_tracer(Tracer), + seq_trace:set_token(label,Label), + seq_trace:set_token(send,true), + seq_trace:set_token('receive',true), + + {ok,1} = send_list_seq(N, self()), + List = receive_any(), + timeout = receive_any(0), + {ok,1} = send_list_seq(N, self()), + List = receive_any(), + timeout = receive_any(0), + + Tracer ! get, + {seq_trace,Label,{send,{0,1},Papa,Papa,List}} = receive_any(), + Tracer ! get, + {seq_trace,Label,{'receive',{0,1},Papa,Papa,List}} = receive_any(), + Tracer ! get, + {seq_trace,Label,{send,{1,2},Papa,Papa,List}} = receive_any(), + Tracer ! get, + {seq_trace,Label,{'receive',{1,2},Papa,Papa,List}} = receive_any(). + + %% More NIF message sending send2(Config) when is_list(Config) -> ensure_lib_loaded(Config), diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 585c5a1871..f4b1d885fe 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -59,6 +59,7 @@ no_priority_inversion2/1, system_task_blast/1, system_task_on_suspended/1, + system_task_failed_enqueue/1, gc_request_when_gc_disabled/1, gc_request_blast_when_gc_disabled/1]). -export([prio_server/2, prio_client/2, init/1, handle_event/2]). @@ -106,7 +107,7 @@ groups() -> otp_7738_resume]}, {system_task, [], [no_priority_inversion, no_priority_inversion2, - system_task_blast, system_task_on_suspended, + system_task_blast, system_task_on_suspended, system_task_failed_enqueue, gc_request_when_gc_disabled, gc_request_blast_when_gc_disabled]}]. init_per_suite(Config) -> @@ -2232,8 +2233,8 @@ processes_term_proc_list(Config) when is_list(Config) -> %% We have to run this test case with +S1 since instrument:allocations() %% will report a free()'d block as present until it's actually deallocated %% by its employer. - Run("+MSe true +MSatags false +S1"), - Run("+MSe true +MSatags true +S1"), + Run("+MSe true +Muatags false +S1"), + Run("+MSe true +Muatags true +S1"), ok. @@ -2241,10 +2242,12 @@ processes_term_proc_list(Config) when is_list(Config) -> chk_term_proc_list(?LINE, MC, XB)). chk_term_proc_list(Line, MustChk, ExpectBlks) -> - Allocs = instrument:allocations(#{ allocator_types => [sl_alloc] }), + Allocs = instrument:allocations(), case {MustChk, Allocs} of {false, {error, not_enabled}} -> not_enabled; + {false, {ok, {_Shift, _Unscanned, ByOrigin}}} when ByOrigin =:= #{} -> + not_enabled; {_, {ok, {_Shift, _Unscanned, ByOrigin}}} -> ByType = maps:get(system, ByOrigin, #{}), Hist = maps:get(ptab_list_deleted_el, ByType, {}), @@ -2625,6 +2628,57 @@ system_task_on_suspended(Config) when is_list(Config) -> ok end. +%% When a system task couldn't be enqueued due to the process being in an +%% incompatible state, it would linger in the system task list and get executed +%% anyway the next time the process was scheduled. This would result in a +%% double-free at best. +%% +%% This test continuously purges modules while other processes run dirty code, +%% which will provoke this error as ERTS_PSTT_CPC can't be enqueued while a +%% process is running dirty code. +system_task_failed_enqueue(Config) when is_list(Config) -> + case erlang:system_info(dirty_cpu_schedulers) of + N when N > 0 -> + system_task_failed_enqueue_1(Config); + _ -> + {skipped, "No dirty scheduler support"} + end. + +system_task_failed_enqueue_1(Config) -> + Priv = proplists:get_value(priv_dir, Config), + + Purgers = [spawn_link(fun() -> purge_loop(Priv, Id) end) + || Id <- lists:seq(1, erlang:system_info(schedulers))], + Hogs = [spawn_link(fun() -> dirty_loop() end) + || _ <- lists:seq(1, erlang:system_info(dirty_cpu_schedulers))], + + ct:sleep(5000), + + [begin + unlink(Pid), + exit(Pid, kill) + end || Pid <- (Purgers ++ Hogs)], + + ok. + +purge_loop(PrivDir, Id) -> + Mod = "failed_enq_" ++ integer_to_list(Id), + Path = PrivDir ++ "/" ++ Mod, + file:write_file(Path ++ ".erl", + "-module('" ++ Mod ++ "').\n" ++ + "-export([t/0]).\n" ++ + "t() -> ok."), + purge_loop_1(Path). +purge_loop_1(Path) -> + {ok, Mod} = compile:file(Path, []), + erlang:delete_module(Mod), + erts_code_purger:purge(Mod), + purge_loop_1(Path). + +dirty_loop() -> + ok = erts_debug:dirty_cpu(reschedule, 10000), + dirty_loop(). + gc_request_when_gc_disabled(Config) when is_list(Config) -> AIS = erts_debug:set_internal_state(available_internal_state, true), gc_request_when_gc_disabled_do(ref), diff --git a/erts/emulator/test/ref_SUITE.erl b/erts/emulator/test/ref_SUITE.erl index 5f519d522e..74df857c65 100644 --- a/erts/emulator/test/ref_SUITE.erl +++ b/erts/emulator/test/ref_SUITE.erl @@ -22,6 +22,7 @@ -export([all/0, suite/0]). -export([wrap_1/1]). +-export([compare_list/1, compare_ets/1]). -export([loop_ref/1]). @@ -32,7 +33,7 @@ suite() -> {timetrap, {minutes, 2}}]. all() -> - [wrap_1]. + [wrap_1, compare_list, compare_ets]. %% Check that refs don't wrap around easily. wrap_1(Config) when is_list(Config) -> @@ -53,3 +54,28 @@ loop_ref(Parent) -> loop_ref(R, R, _) -> ok; loop_ref(R0, _, N) -> loop_ref(R0, make_ref(), N+1). + +%% Check that ref ordering works +compare_list(Config) when is_list(Config) -> + %% Although this test uses external refs, it would apply the same to plain refs + ExtRef1 = <<131,114,0,3,100,0,3,110,64,98,3, 0,0,173,156, 0,216,0,4, 0,0,0,0>>, + ExtRef2 = <<131,114,0,3,100,0,3,110,64,98,3, 0,1,31,27, 129,4,0,1, 0,0,0,0>>, + + Ref1 = binary_to_term(ExtRef1), %% #Ref<[email protected]> + Ref2 = binary_to_term(ExtRef2), %% #Ref<[email protected]> + OrderedList = [Ref1, Ref2], + OrderedList = lists:sort(OrderedList), + ok. + +%% This is the scarier case since it makes terms "invisible" in ets or Mnesia +%% (the underlying fault cause is the same as compare_list/1) +compare_ets(Config) when is_list(Config) -> + W2s = [610350147,899574699,2994196869,686384822,2397690439, 923302211], + ExtRefBase = <<131,114,0,3,100,0,3,110,64,98,3>>, + ExtRefs = [<<ExtRefBase/binary, 1:32, W2:32, 0:32>> || W2 <- W2s], + Refs = [binary_to_term(Bin) || Bin <- ExtRefs], + + Ets = ets:new(refbug, [ordered_set]), + ets:insert(Ets, [{Ref,Ref} || Ref <- Refs]), + 0 = length([R || R <- ets:tab2list(Ets), ets:lookup(Ets, element(1,R)) == []]), + ok. diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index 21ab6b378a..8ea2d88ec4 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -457,11 +457,16 @@ cmp_memory(MWs, Str) -> %% Total, processes, processes_used, and system will seldom %% give us exactly the same result since the two readings %% aren't taken atomically. + %% + %% Torerance is scaled according to the number of schedulers + %% to match spawn_mem_workers. + + Tolerance = 1.05 + 0.01 * erlang:system_info(schedulers_online), - cmp_memory(total, EM, EDM, 1.05), - cmp_memory(processes, EM, EDM, 1.05), - cmp_memory(processes_used, EM, EDM, 1.05), - cmp_memory(system, EM, EDM, 1.05), + cmp_memory(total, EM, EDM, Tolerance), + cmp_memory(processes, EM, EDM, Tolerance), + cmp_memory(processes_used, EM, EDM, Tolerance), + cmp_memory(system, EM, EDM, Tolerance), ok. diff --git a/erts/emulator/test/timer_bif_SUITE.erl b/erts/emulator/test/timer_bif_SUITE.erl index fc11a04a31..15fe13c8c0 100644 --- a/erts/emulator/test/timer_bif_SUITE.erl +++ b/erts/emulator/test/timer_bif_SUITE.erl @@ -361,7 +361,7 @@ evil_timers(Config) when is_list(Config) -> %% %% 1. A timer started with erlang:start_timer(Time, Receiver, Msg), %% where Msg is a composite term, expires, and the receivers main - %% lock *can not* be acquired immediately (typically when the + %% lock *cannot* be acquired immediately (typically when the %% receiver *is* running). %% %% The wrap tuple ({timeout, TRef, Msg}) will in this case @@ -372,7 +372,7 @@ evil_timers(Config) when is_list(Config) -> RecvTimeOutMsgs0 = evil_recv_timeouts(200), %% 2. A timer started with erlang:start_timer(Time, Receiver, Msg), %% where Msg is an immediate term, expires, and the receivers main - %% lock *can not* be acquired immediately (typically when the + %% lock *cannot* be acquired immediately (typically when the %% receiver *is* running). %% %% The wrap tuple will in this case be allocated in a new diff --git a/erts/emulator/test/trace_SUITE.erl b/erts/emulator/test/trace_SUITE.erl index 138aefb29c..c2d5cd7023 100644 --- a/erts/emulator/test/trace_SUITE.erl +++ b/erts/emulator/test/trace_SUITE.erl @@ -38,7 +38,8 @@ system_monitor_long_gc_1/1, system_monitor_long_gc_2/1, system_monitor_large_heap_1/1, system_monitor_large_heap_2/1, system_monitor_long_schedule/1, - bad_flag/1, trace_delivered/1, trap_exit_self_receive/1]). + bad_flag/1, trace_delivered/1, trap_exit_self_receive/1, + trace_info_badarg/1, erl_704/1]). -include_lib("common_test/include/ct.hrl"). @@ -62,7 +63,7 @@ all() -> system_monitor_long_gc_2, system_monitor_large_heap_1, system_monitor_long_schedule, system_monitor_large_heap_2, bad_flag, trace_delivered, - trap_exit_self_receive]. + trap_exit_self_receive, trace_info_badarg, erl_704]. init_per_testcase(_Case, Config) -> [{receiver,spawn(fun receiver/0)}|Config]. @@ -1734,6 +1735,25 @@ trap_exit_self_receive(Config) -> receive_nothing(), ok. +trace_info_badarg(Config) when is_list(Config) -> + catch erlang:trace_info({a,b,c},d), + ok. + +%% An incoming suspend monitor down wasn't handled +%% correct when the local monitor half had been +%% removed with an emulator crash as result. +erl_704(Config) -> + erl_704_test(100). + +erl_704_test(0) -> + ok; +erl_704_test(N) -> + P = spawn(fun () -> receive infinity -> ok end end), + erlang:suspend_process(P), + exit(P, kill), + (catch erlang:resume_process(P)), + erl_704_test(N-1). + drop_trace_until_down(Proc, Mon) -> drop_trace_until_down(Proc, Mon, false, 0, 0). diff --git a/erts/etc/unix/cerl.src b/erts/etc/unix/cerl.src index 3572fdd954..2e034513b0 100644 --- a/erts/etc/unix/cerl.src +++ b/erts/etc/unix/cerl.src @@ -69,15 +69,6 @@ cxargs_add() { done } -eeargs= -eeargs_add() { - while [ $# -gt 0 ]; do - cargs="$cargs $1" - eeargs="$eeargs $1" - shift - done -} - core= GDB= @@ -97,8 +88,6 @@ TARGET=%TARGET% PROGNAME=$ROOTDIR/bin/cerl EMU=beam -PRELOADED=$ROOTDIR/erts/preloaded/ebin - while [ $# -gt 0 ]; do case "$1" in @@ -255,7 +244,7 @@ EXEC=$BINDIR/erlexec PROGNAME="$PROGNAME$cargs" EMU="$EMU$TYPE" -EMU_NAME=`$EXEC -emu_name_exit $eeargs` +EMU_NAME=`$EXEC -emu_name_exit` if [ $skip_erlexec = yes ]; then emu_xargs=`echo $xargs | sed "s|+|-|g"` @@ -269,21 +258,20 @@ if [ $skip_erlexec = yes ]; then ' set -- $beam_args IFS="$SAVE_IFS" -else - xargs="$xargs -pz $PRELOADED --" fi if [ "x$GDB" = "x" ]; then if [ $run_valgrind = yes ]; then valversion=`valgrind --version` valmajor=`echo $valversion | sed 's,[a-z]*\-\([0-9]*\).*,\1,'` valminor=`echo $valversion | sed 's,[a-z]*\-[0-9]*.\([0-9]*\).*,\1,'` + valint=`echo "$valmajor * 1000 + $valminor" | bc` if [ "x$VALGRIND_LOG_XML" = "x" ]; then valgrind_xml= log_file_prefix="--log-file=" else export VALGRIND_LOG_XML valgrind_xml="--xml=yes" - if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then + if [ $valint -gt 3004 ]; then log_file_prefix="--xml-file=" else log_file_prefix="--log-file=" @@ -292,7 +280,7 @@ if [ "x$GDB" = "x" ]; then if [ "x$VALGRIND_LOG_DIR" = "x" ]; then valgrind_log= else - if [ $valmajor -gt 2 -a $valminor -gt 4 ]; then + if [ $valint -gt 3004 ]; then valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log.$$" else valgrind_log="$log_file_prefix$VALGRIND_LOG_DIR/$VALGRIND_LOGFILE_PREFIX$VALGRIND_LOGFILE_INFIX$EMU_NAME.log" @@ -316,12 +304,12 @@ if [ "x$GDB" = "x" ]; then sched_arg= fi - exec $taskset1 valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $sched_arg $emu_xargs "$@" -pz $PRELOADED + exec $taskset1 valgrind $valgrind_xml $valgrind_log $valgrind_misc_flags $BINDIR/$EMU_NAME $sched_arg $emu_xargs "$@" elif [ $run_rr = yes ]; then - exec rr record --ignore-nested $BINDIR/$EMU_NAME $emu_xargs "$@" -pz $PRELOADED + exec rr record --ignore-nested $BINDIR/$EMU_NAME $emu_xargs "$@" else - exec $EXEC $eeargs $xargs ${1+"$@"} + exec $EXEC $xargs ${1+"$@"} fi elif [ "x$GDB" = "xgdb" ]; then case "x$core" in diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 5b89621733..b12a205ba7 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -2734,25 +2734,37 @@ define etp-aux-work-flags printf " fix-alloc-lower-lim" end if ($arg0 & 0x10) - printf " async-ready" + printf " later-op" end if ($arg0 & 0x20) - printf " async-ready-clean" + printf " canceled-timers" end if ($arg0 & 0x40) - printf " misc-work-thr-prgr" + printf " canceled-timers-thr-prgr" end if ($arg0 & 0x80) - printf " misc-work" + printf " async-ready" end if ($arg0 & 0x100) - printf " check-children" + printf " async-ready-clean" end if ($arg0 & 0x200) - printf " set-tmo" + printf " misc-thr-prgr" end if ($arg0 & 0x400) - printf " mseg-cached-check" + printf " misc" + end + if ($arg0 & 0x800) + printf " set-tmo" + end + if ($arg0 & 0x1000) + printf " mseg-cache-check" + end + if ($arg0 & 0x2000) + printf " yield" + end + if ($arg0 & 0x1000) + printf " reap-ports" end if ($arg0 & ~0x7ff) printf " GARBAGE" diff --git a/erts/include/internal/gcc/ethr_membar.h b/erts/include/internal/gcc/ethr_membar.h index 07960ce040..5cbda5582d 100644 --- a/erts/include/internal/gcc/ethr_membar.h +++ b/erts/include/internal/gcc/ethr_membar.h @@ -96,7 +96,7 @@ * issue an aquire memory barrier and an __atomic * builtin memory acess with the __ATOMIC_RELEASE * memory model must at least issue a release memory - * barrier. Otherwise the two can not be paired. + * barrier. Otherwise the two cannot be paired. * 4. All __atomic builtins accessing memory using the * __ATOMIC_CONSUME builtin can be used for the same * reason __ATOMIC_ACQUIRE can be used. The ethread diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex bd8cc7d7e0..7a4167b0e9 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex f375c05c99..3316e4348c 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index f4f31b1e4b..261b731900 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -86,15 +86,15 @@ | 'nano_seconds'. -opaque prepared_code() :: reference(). - -export_type([prepared_code/0]). --opaque dist_handle() :: atom(). +-opaque nif_resource() :: reference(). +-export_type([nif_resource/0]). +-opaque dist_handle() :: atom(). -export_type([dist_handle/0]). -type iovec() :: [binary()]. - -export_type([iovec/0]). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -2316,7 +2316,7 @@ process_flag(_Flag, _Value) -> {min_heap_size, MinHeapSize :: non_neg_integer()} | {min_bin_vheap_size, MinBinVHeapSize :: non_neg_integer()} | {max_heap_size, MaxHeapSize :: max_heap_size()} | - {monitored_by, Pids :: [pid()]} | + {monitored_by, MonitoredBy :: [pid() | port() | nif_resource()]} | {monitors, Monitors :: [{process | port, Pid :: pid() | port() | {RegName :: atom(), Node :: node()}}]} | @@ -2681,11 +2681,12 @@ tuple_to_list(_Tuple) -> (dist_ctrl) -> {Node :: node(), ControllingEntity :: port() | pid()}; (driver_version) -> string(); - (dynamic_trace) -> none | dtrace | systemtap; + (dynamic_trace) -> none | dtrace | systemtap; (dynamic_trace_probes) -> boolean(); (end_time) -> non_neg_integer(); (elib_malloc) -> false; (eager_check_io) -> boolean(); + (ets_count) -> pos_integer(); (ets_limit) -> pos_integer(); (fullsweep_after) -> {fullsweep_after, non_neg_integer()}; (garbage_collection) -> [{atom(), integer()}]; @@ -3653,90 +3654,28 @@ memory() -> -spec erlang:memory(Type :: memory_type()) -> non_neg_integer(); (TypeList :: [memory_type()]) -> [{memory_type(), non_neg_integer()}]. memory(Type) when erlang:is_atom(Type) -> - {AA, ALCU, ChkSup, BadArgZero} = need_mem_info(Type), - case get_mem_data(ChkSup, ALCU, AA) of - notsup -> - erlang:error(notsup, [Type]); - Mem -> - Value = get_memval(Type, Mem), - case {BadArgZero, Value} of - {true, 0} -> erlang:error(badarg, [Type]); - _ -> Value - end + try + case aa_mem_data(au_mem_data(?ALL_NEEDED_ALLOCS)) of + notsup -> erlang:error(notsup); + Mem -> get_memval(Type, Mem) + end + catch + error:badarg -> erlang:error(badarg) end; memory(Types) when erlang:is_list(Types) -> - {AA, ALCU, ChkSup, BadArgZeroList} = need_mem_info_list(Types), - case get_mem_data(ChkSup, ALCU, AA) of - notsup -> - erlang:error(notsup, [Types]); - Mem -> - case memory_result_list(Types, BadArgZeroList, Mem) of - badarg -> erlang:error(badarg, [Types]); - Result -> Result - end - end. - -memory_result_list([], [], _Mem) -> - []; -memory_result_list([T|Ts], [BAZ|BAZs], Mem) -> - case memory_result_list(Ts, BAZs, Mem) of - badarg -> badarg; - TVs -> - V = get_memval(T, Mem), - case {BAZ, V} of - {true, 0} -> badarg; - _ -> [{T, V}| TVs] - end - end. - -get_mem_data(true, AlcUAllocs, NeedAllocatedAreas) -> - case memory_is_supported() of - false -> notsup; - true -> get_mem_data(false, AlcUAllocs, NeedAllocatedAreas) - end; -get_mem_data(false, AlcUAllocs, NeedAllocatedAreas) -> - AlcUMem = case AlcUAllocs of - [] -> #memory{}; - _ -> - au_mem_data(AlcUAllocs) - end, - case NeedAllocatedAreas of - true -> aa_mem_data(AlcUMem); - false -> AlcUMem + try + case aa_mem_data(au_mem_data(?ALL_NEEDED_ALLOCS)) of + notsup -> erlang:error(notsup); + Mem -> memory_1(Types, Mem) + end + catch + error:badarg -> erlang:error(badarg) end. -need_mem_info_list([]) -> - {false, [], false, []}; -need_mem_info_list([T|Ts]) -> - {MAA, MALCU, MChkSup, MBadArgZero} = need_mem_info_list(Ts), - {AA, ALCU, ChkSup, BadArgZero} = need_mem_info(T), - {case AA of - true -> true; - _ -> MAA - end, - ALCU ++ (MALCU -- ALCU), - case ChkSup of - true -> true; - _ -> MChkSup - end, - [BadArgZero|MBadArgZero]}. - -need_mem_info(Type) when Type == total; - Type == system -> - {true, ?ALL_NEEDED_ALLOCS, false, false}; -need_mem_info(Type) when Type == processes; - Type == processes_used -> - {true, [eheap_alloc, fix_alloc], true, false}; -need_mem_info(Type) when Type == atom; - Type == atom_used; - Type == code -> - {true, [], true, false}; -need_mem_info(binary) -> - {false, [binary_alloc], true, false}; -need_mem_info(ets) -> - {true, [ets_alloc], true, false}; -need_mem_info(_) -> - {false, [], false, true}. +memory_1([Type | Types], Mem) -> + [{Type, get_memval(Type, Mem)} | memory_1(Types, Mem)]; +memory_1([], _Mem) -> + []. get_memval(total, #memory{total = V}) -> V; get_memval(processes, #memory{processes = V}) -> V; @@ -3747,16 +3686,7 @@ get_memval(atom_used, #memory{atom_used = V}) -> V; get_memval(binary, #memory{binary = V}) -> V; get_memval(code, #memory{code = V}) -> V; get_memval(ets, #memory{ets = V}) -> V; -get_memval(_, #memory{}) -> 0. - -memory_is_supported() -> - {_, _, FeatureList, _} = erlang:system_info(allocator), - case ((erlang:system_info(alloc_util_allocators) - -- ?CARRIER_ALLOCS) - -- FeatureList) of - [] -> true; - _ -> false - end. +get_memval(_, #memory{}) -> erlang:error(badarg). get_blocks_size([{blocks_size, Sz, _, _} | Rest], Acc) -> get_blocks_size(Rest, Acc+Sz); @@ -3767,16 +3697,6 @@ get_blocks_size([_ | Rest], Acc) -> get_blocks_size([], Acc) -> Acc. - -blocks_size([{Carriers, SizeList} | Rest], Acc) when Carriers == mbcs; - Carriers == mbcs_pool; - Carriers == sbcs -> - blocks_size(Rest, get_blocks_size(SizeList, Acc)); -blocks_size([_ | Rest], Acc) -> - blocks_size(Rest, Acc); -blocks_size([], Acc) -> - Acc. - get_fix_proc([{ProcType, A1, U1}| Rest], {A0, U0}) when ProcType == proc; ProcType == monitor; ProcType == link; @@ -3801,64 +3721,78 @@ fix_proc([_ | Rest], Acc) -> fix_proc([], Acc) -> Acc. +au_mem_fix(#memory{ processes = Proc, + processes_used = ProcU, + system = Sys } = Mem, Data) -> + case fix_proc(Data, {0, 0}) of + {A, U} -> + Mem#memory{ processes = Proc+A, + processes_used = ProcU+U, + system = Sys-A }; + {Mask, A, U} -> + Mem#memory{ processes = Mask band (Proc+A), + processes_used = Mask band (ProcU+U), + system = Mask band (Sys-A) } + end. + +au_mem_acc(#memory{ total = Tot, + processes = Proc, + processes_used = ProcU } = Mem, + eheap_alloc, Data) -> + Sz = get_blocks_size(Data, 0), + Mem#memory{ total = Tot+Sz, + processes = Proc+Sz, + processes_used = ProcU+Sz}; +au_mem_acc(#memory{ total = Tot, + system = Sys, + ets = Ets } = Mem, ets_alloc, Data) -> + Sz = get_blocks_size(Data, 0), + Mem#memory{ total = Tot+Sz, + system = Sys+Sz, + ets = Ets+Sz }; +au_mem_acc(#memory{total = Tot, + system = Sys, + binary = Bin } = Mem, + binary_alloc, Data) -> + Sz = get_blocks_size(Data, 0), + Mem#memory{ total = Tot+Sz, + system = Sys+Sz, + binary = Bin+Sz}; +au_mem_acc(#memory{ total = Tot, + system = Sys } = Mem, + _Type, Data) -> + Sz = get_blocks_size(Data, 0), + Mem#memory{ total = Tot+Sz, + system = Sys+Sz }. + +au_mem_foreign(Mem, [{Type, SizeList} | Rest]) -> + au_mem_foreign(au_mem_acc(Mem, Type, SizeList), Rest); +au_mem_foreign(Mem, []) -> + Mem. + +au_mem_current(Mem0, Type, [{mbcs_pool, MBCS} | Rest]) -> + [Foreign] = [Foreign || {foreign_blocks, Foreign} <- MBCS], + SizeList = MBCS -- [Foreign], + Mem = au_mem_foreign(Mem0, Foreign), + au_mem_current(au_mem_acc(Mem, Type, SizeList), Type, Rest); +au_mem_current(Mem, Type, [{mbcs, SizeList} | Rest]) -> + au_mem_current(au_mem_acc(Mem, Type, SizeList), Type, Rest); +au_mem_current(Mem, Type, [{sbcs, SizeList} | Rest]) -> + au_mem_current(au_mem_acc(Mem, Type, SizeList), Type, Rest); +au_mem_current(Mem, Type, [_ | Rest]) -> + au_mem_current(Mem, Type, Rest); +au_mem_current(Mem, _Type, []) -> + Mem. + au_mem_data(notsup, _) -> notsup; au_mem_data(_, [{_, false} | _]) -> notsup; -au_mem_data(#memory{total = Tot, - processes = Proc, - processes_used = ProcU} = Mem, - [{eheap_alloc, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - au_mem_data(Mem#memory{total = Tot+Sz, - processes = Proc+Sz, - processes_used = ProcU+Sz}, - Rest); -au_mem_data(#memory{total = Tot, - system = Sys, - ets = Ets} = Mem, - [{ets_alloc, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - au_mem_data(Mem#memory{total = Tot+Sz, - system = Sys+Sz, - ets = Ets+Sz}, - Rest); -au_mem_data(#memory{total = Tot, - system = Sys, - binary = Bin} = Mem, - [{binary_alloc, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - au_mem_data(Mem#memory{total = Tot+Sz, - system = Sys+Sz, - binary = Bin+Sz}, - Rest); -au_mem_data(#memory{total = Tot, - processes = Proc, - processes_used = ProcU, - system = Sys} = Mem, - [{fix_alloc, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - case fix_proc(Data, {0, 0}) of - {A, U} -> - au_mem_data(Mem#memory{total = Tot+Sz, - processes = Proc+A, - processes_used = ProcU+U, - system = Sys+Sz-A}, - Rest); - {Mask, A, U} -> - au_mem_data(Mem#memory{total = Tot+Sz, - processes = Mask band (Proc+A), - processes_used = Mask band (ProcU+U), - system = Mask band (Sys+Sz-A)}, - Rest) - end; -au_mem_data(#memory{total = Tot, - system = Sys} = Mem, - [{_, _, Data} | Rest]) -> - Sz = blocks_size(Data, 0), - au_mem_data(Mem#memory{total = Tot+Sz, - system = Sys+Sz}, - Rest); +au_mem_data(#memory{} = Mem0, [{fix_alloc, _, Data} | Rest]) -> + Mem = au_mem_fix(Mem0, Data), + au_mem_data(au_mem_current(Mem, fix_alloc, Data), Rest); +au_mem_data(#memory{} = Mem, [{Type, _, Data} | Rest]) -> + au_mem_data(au_mem_current(Mem, Type, Data), Rest); au_mem_data(EMD, []) -> EMD. diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index 41ff38359c..517ca74301 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -580,15 +580,20 @@ read_link_info(Name, Opts) -> read_info_1(Name, 0, proplist_get_value(time, Opts, local)). read_info_1(Name, FollowLinks, TimeType) -> - try read_info_nif(encode_path(Name), FollowLinks) of - {error, Reason} -> {error, Reason}; - FileInfo -> - CTime = from_posix_seconds(FileInfo#file_info.ctime, TimeType), - MTime = from_posix_seconds(FileInfo#file_info.mtime, TimeType), - ATime = from_posix_seconds(FileInfo#file_info.atime, TimeType), - {ok, FileInfo#file_info{ ctime = CTime, mtime = MTime, atime = ATime }} + try + case read_info_nif(encode_path(Name), FollowLinks) of + {error, Reason} -> + {error, Reason}; + FileInfo -> + CTime = from_posix_seconds(FileInfo#file_info.ctime, TimeType), + MTime = from_posix_seconds(FileInfo#file_info.mtime, TimeType), + ATime = from_posix_seconds(FileInfo#file_info.atime, TimeType), + {ok, FileInfo#file_info{ ctime = CTime, + mtime = MTime, + atime = ATime }} + end catch - error:badarg -> {error, badarg} + error:_ -> {error, badarg} end. write_file_info(Filename, Info) -> diff --git a/erts/preloaded/src/prim_inet.erl b/erts/preloaded/src/prim_inet.erl index 2a3605260d..bcb54cca42 100644 --- a/erts/preloaded/src/prim_inet.erl +++ b/erts/preloaded/src/prim_inet.erl @@ -1557,7 +1557,7 @@ type_opt_1(O) when is_atom(O) -> undefined. %% Get. No supplied value. type_value(get, undefined) -> false; % Undefined type -%% These two clauses can not happen since they are only used +%% These two clauses cannot happen since they are only used %% in record fields - from record fields they must have a %% value though it might be 'undefined', so record fields %% calls type_value/3, not type_value/2. @@ -1723,7 +1723,7 @@ type_value_2(_, _) -> false. %% Get. No supplied value. %% -%% These two clauses can not happen since they are only used +%% These two clauses cannot happen since they are only used %% in record fields - from record fields they must have a %% value though it might be 'undefined', so record fields %% calls enc_value/3, not enc_value/2. diff --git a/erts/test/upgrade_SUITE.erl b/erts/test/upgrade_SUITE.erl index c32dbabe8d..f92c25bdb4 100644 --- a/erts/test/upgrade_SUITE.erl +++ b/erts/test/upgrade_SUITE.erl @@ -25,7 +25,7 @@ -define(upgr_sname,otp_upgrade). -%% Applications that are excluded from this test because they can not +%% Applications that are excluded from this test because they cannot %% just be started in a new node with out specific configuration. -define(start_exclude, [cosEvent,cosEventDomain,cosFileTransfer,cosNotification, diff --git a/erts/vsn.mk b/erts/vsn.mk index 519556119d..c0444fa483 100644 --- a/erts/vsn.mk +++ b/erts/vsn.mk @@ -18,7 +18,7 @@ # %CopyrightEnd% # -VSN = 10.0 +VSN = 10.0.5 # Port number 4365 in 4.2 # Port number 4366 in 4.3 |