diff options
Diffstat (limited to 'erts')
114 files changed, 2973 insertions, 2012 deletions
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index f6d8f20e4e..017fdbd589 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -2075,63 +2075,159 @@ esac case "$GCC-$host_cpu" in yes-i86pc | yes-i*86 | yes-x86_64 | yes-amd64) + + if test $ac_cv_sizeof_void_p = 4; then + dw_cmpxchg="cmpxchg8b" + else + dw_cmpxchg="cmpxchg16b" + fi + gcc_dw_cmpxchg_asm=no - AC_MSG_CHECKING([for gcc double word cmpxchg asm support]) - AC_TRY_COMPILE([], + gcc_pic_dw_cmpxchg_asm=no + gcc_cflags_pic=no + gcc_cmpxchg8b_pic_no_clobber_ebx=no + gcc_cmpxchg8b_pic_no_clobber_ebx_register_shortage=no + + save_CFLAGS="$CFLAGS" + + # Check if it works out of the box using passed CFLAGS + # and with -fPIC added to CFLAGS if the passed CFLAGS + # doesn't trigger position independent code + pic_cmpxchg=unknown + while true; do + + case $pic_cmpxchg in + yes) pic_text="pic ";; + *) pic_text="";; + esac + + AC_MSG_CHECKING([for gcc $pic_text$dw_cmpxchg plain asm support]) + + plain_cmpxchg=no + AC_TRY_COMPILE([], [ char xchgd; long new[2], xchg[2], *p; __asm__ __volatile__( -#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ - "pushl %%ebx\n\t" - "movl %8, %%ebx\n\t" -#endif #if ETHR_SIZEOF_PTR == 4 "lock; cmpxchg8b %0\n\t" #else "lock; cmpxchg16b %0\n\t" #endif "setz %3\n\t" -#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ - "popl %%ebx\n\t" -#endif - : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) - : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new[1]), -#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ - "r"(new[0]) -#else - "b"(new[0]) -#endif + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=q"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "c"(new[1]), "b"(new[0]) : "cc", "memory"); + ], + [plain_cmpxchg=yes]) + AC_MSG_RESULT([$plain_cmpxchg]) + + if test $pic_cmpxchg = yes; then + gcc_pic_dw_cmpxchg_asm=$plain_cmpxchg + break + fi + + gcc_dw_cmpxchg_asm=$plain_cmpxchg + + # If not already compiling to position independent + # code add -fPIC to CFLAGS and do it again. This + # since we want also want to know how to compile + # to position independent code since this might + # cause problems with the use of the EBX register + # as input to the asm on 32-bit x86 and old gcc + # compilers (gcc vsn < 5). + + AC_TRY_COMPILE([], + [ +#if !defined(__PIC__) || !__PIC__ +# error no pic +#endif ], - [gcc_dw_cmpxchg_asm=yes]) - if test $gcc_dw_cmpxchg_asm = no && test $ac_cv_sizeof_void_p = 4; then + [pic_cmpxchg=yes + gcc_cflags_pic=yes], + [pic_cmpxchg=no]) + + if test $pic_cmpxchg = yes; then + gcc_pic_dw_cmpxchg_asm=$gcc_dw_cmpxchg_asm + break + fi + + CFLAGS="$save_CFLAGS -fPIC" + pic_cmpxchg=yes + + done + + if test $gcc_pic_dw_cmpxchg_asm = no && test $ac_cv_sizeof_void_p = 4; then + + AC_MSG_CHECKING([for gcc pic cmpxchg8b asm support with EBX workaround]) + + # Check if we can work around it by managing the ebx + # register explicitly in the asm... + AC_TRY_COMPILE([], + [ + char xchgd; + long new[2], xchg[2], *p; + __asm__ __volatile__( + "pushl %%ebx\n\t" + "movl %8, %%ebx\n\t" + "lock; cmpxchg8b %0\n\t" + "setz %3\n\t" + "popl %%ebx\n\t" + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=q"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "c"(new[1]), "r"(new[0]) + : "cc", "memory"); + ], + [gcc_pic_dw_cmpxchg_asm=yes + gcc_cmpxchg8b_pic_no_clobber_ebx=yes]) + + AC_MSG_RESULT([$gcc_pic_dw_cmpxchg_asm]) + + if test $gcc_pic_dw_cmpxchg_asm = no; then + + AC_MSG_CHECKING([for gcc pic cmpxchg8b asm support with EBX and register shortage workarounds]) + # If no optimization is enabled we sometimes get a + # register shortage. Check if we can work around + # this... + + AC_TRY_COMPILE([], [ char xchgd; long new[2], xchg[2], *p; -#if !defined(__PIC__) || !__PIC__ -# error nope -#endif __asm__ __volatile__( - "pushl %%ebx\n\t" - "movl (%7), %%ebx\n\t" - "movl 4(%7), %%ecx\n\t" - "lock; cmpxchg8b %0\n\t" - "setz %3\n\t" - "popl %%ebx\n\t" - : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) - : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new) + "pushl %%ebx\n\t" + "movl (%7), %%ebx\n\t" + "movl 4(%7), %%ecx\n\t" + "lock; cmpxchg8b %0\n\t" + "setz %3\n\t" + "popl %%ebx\n\t" + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "r"(new) : "cc", "memory"); ], - [gcc_dw_cmpxchg_asm=yes]) - if test "$gcc_dw_cmpxchg_asm" = "yes"; then - AC_DEFINE(ETHR_CMPXCHG8B_REGISTER_SHORTAGE, 1, [Define if you get a register shortage with cmpxchg8b and position independent code]) + [gcc_pic_dw_cmpxchg_asm=yes + gcc_cmpxchg8b_pic_no_clobber_ebx=yes + gcc_cmpxchg8b_pic_no_clobber_ebx_register_shortage=yes]) + + AC_MSG_RESULT([$gcc_pic_dw_cmpxchg_asm]) fi + + if test $gcc_cflags_pic = yes; then + gcc_dw_cmpxchg_asm=$gcc_pic_dw_cmpxchg_asm + fi + + fi + + CFLAGS="$save_CFLAGS" + + if test "$gcc_cmpxchg8b_pic_no_clobber_ebx" = "yes"; then + AC_DEFINE(ETHR_CMPXCHG8B_PIC_NO_CLOBBER_EBX, 1, [Define if gcc wont let you clobber ebx with cmpxchg8b and position independent code]) + fi + if test "$gcc_cmpxchg8b_pic_no_clobber_ebx_register_shortage" = "yes"; then + AC_DEFINE(ETHR_CMPXCHG8B_REGISTER_SHORTAGE, 1, [Define if you get a register shortage with cmpxchg8b and position independent code]) fi - AC_MSG_RESULT([$gcc_dw_cmpxchg_asm]) if test "$gcc_dw_cmpxchg_asm" = "yes"; then AC_DEFINE(ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT, 1, [Define if you use a gcc that supports the double word cmpxchg instruction]) fi;; diff --git a/erts/configure.in b/erts/configure.in index d84651f4f9..021780ecc2 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -474,7 +474,7 @@ case $host_os in win32) # The ethread library requires _WIN32_WINNT of at least 0x0403. # -D_WIN32_WINNT=* from CPPFLAGS is saved in ETHR_DEFS. - CPPFLAGS="$CPPFLAGS -D_WIN32_WINNT=0x0501 -DWINVER=0x0501" + CPPFLAGS="$CPPFLAGS -D_WIN32_WINNT=0x0600 -DWINVER=0x0600" ;; darwin*) CPPFLAGS="$CPPFLAGS -D_XOPEN_SOURCE" @@ -584,6 +584,7 @@ fi if test "x$GCC" = xyes; then # Treat certain GCC warnings as errors LM_TRY_ENABLE_CFLAG([-Werror=return-type], [WERRORFLAGS]) + LM_TRY_ENABLE_CFLAG([-Werror=implicit], [WERRORFLAGS]) # until the emulator can handle this, I suggest we turn it off! #WFLAGS="-Wall -Wshadow -Wcast-qual -Wmissing-declarations" @@ -4071,7 +4072,7 @@ AC_ARG_WITH(ssl-rpath, AS_HELP_STRING([--with-ssl-rpath=yes|no|PATHS], [runtime library path for OpenSSL. Default is "yes", which equates to a number of standard locations. If "no", then no runtime - library paths wil be used. Anything else should be a + library paths will be used. Anything else should be a comma separated list of paths.]), [ case X$with_ssl in diff --git a/erts/doc/src/absform.xml b/erts/doc/src/absform.xml index ccdecf44ec..13756ddfdc 100644 --- a/erts/doc/src/absform.xml +++ b/erts/doc/src/absform.xml @@ -182,7 +182,7 @@ <p>Individual patterns are represented as follows:</p> <list type="bulleted"> <item>If P is an atomic literal <c>L</c>, then Rep(P) = Rep(L).</item> - <item>If P is a binary pattern + <item>If P is a bit string pattern <c><<P_1:Size_1/TSL_1, ..., P_k:Size_k/TSL_k>></c>, where each <c>Size_i</c> is an expression that can be evaluated to an integer and each <c>TSL_i</c> is a type specificer list, then @@ -241,12 +241,13 @@ <p>An expression E is one of the following alternatives:</p> <list type="bulleted"> <item>If E is an atomic literal <c>L</c>, then Rep(E) = Rep(L).</item> - <item>If E is a binary comprehension + <item>If E is a bit string comprehension <c><<E_0 || Q_1, ..., Q_k>></c>, where each <c>Q_i</c> is a qualifier, then Rep(E) = <c>{bc,LINE,Rep(E_0),[Rep(Q_1), ..., Rep(Q_k)]}</c>. For Rep(Q), see below.</item> - <item>If E is a binary constructor <c><<E_1:Size_1/TSL_1, ..., E_k:Size_k/TSL_k>></c>, + <item>If E is a bit string constructor + <c><<E_1:Size_1/TSL_1, ..., E_k:Size_k/TSL_k>></c>, where each <c>Size_i</c> is an expression and each <c>TSL_i</c> is a type specificer list, then Rep(E) = <c>{bin,LINE,[{bin_element,LINE,Rep(E_1),Rep(Size_1),Rep(TSL_1)}, ..., {bin_element,LINE,Rep(E_k),Rep(Size_k),Rep(TSL_k)}]}</c>. @@ -386,16 +387,17 @@ <item>If Q is a generator <c>P <- E</c>, where <c>P</c> is a pattern and <c>E</c> is an expression, then Rep(Q) = <c>{generate,LINE,Rep(P),Rep(E)}</c>.</item> - <item>If Q is a generator <c>P <= E</c>, where <c>P</c> is + <item>If Q is a bit string generator + <c>P <= E</c>, where <c>P</c> is a pattern and <c>E</c> is an expression, then Rep(Q) = <c>{b_generate,LINE,Rep(P),Rep(E)}</c>.</item> </list> </section> <section> - <title>Binary Element Type Specifiers</title> - <p>A type specifier list TSL for a binary element is a sequence of type - specifiers <c>TS_1 - ... - TS_k</c>, and + <title>Bit String Element Type Specifiers</title> + <p>A type specifier list TSL for a bit string element is a sequence + of type specifiers <c>TS_1 - ... - TS_k</c>, and Rep(TSL) = <c>[Rep(TS_1), ..., Rep(TS_k)]</c>.</p> <list type="bulleted"> <item>If TS is a type specifier <c>A</c>, where <c>A</c> is an atom, @@ -473,7 +475,7 @@ <p>A guard test <c>Gt</c> is one of the following alternatives:</p> <list type="bulleted"> <item>If Gt is an atomic literal <c>L</c>, then Rep(Gt) = Rep(L).</item> - <item>If Gt is a binary constructor + <item>If Gt is a bit string constructor <c><<Gt_1:Size_1/TSL_1, ..., Gt_k:Size_k/TSL_k>></c>, where each <c>Size_i</c> is a guard test and each <c>TSL_i</c> is a type specificer list, then @@ -540,7 +542,7 @@ <c>{ann_type,LINE,[Rep(A),Rep(T_0)]}</c>.</item> <item>If T is an atom or integer literal L, then Rep(T) = Rep(L). </item> - <item>If T is a bitstring type <c><<_:M,_:_*N>></c>, + <item>If T is a bit string type <c><<_:M,_:_*N>></c>, where <c>M</c> and <c>N</c> are singleton integer types, then Rep(T) = <c>{type,LINE,binary,[Rep(M),Rep(N)]}</c>.</item> <item>If T is the empty list type <c>[]</c>, then Rep(T) = diff --git a/erts/doc/src/epmd.xml b/erts/doc/src/epmd.xml index 28fcc8f7af..7f61804bea 100644 --- a/erts/doc/src/epmd.xml +++ b/erts/doc/src/epmd.xml @@ -37,7 +37,7 @@ <comsummary> <p>Erlang Port Mapper Daemon</p> <taglist> - <tag><c><![CDATA[epmd [-d|-debug] [DbgExtra...] [-port No] [-daemon] [-relaxed_command_check]]]></c></tag> + <tag><c><![CDATA[epmd [-d|-debug] [DbgExtra...] [-address Addresses] [-port No] [-daemon] [-relaxed_command_check]]]></c></tag> <item> <p>Starts the port mapper daemon</p> </item> diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml index 1c4a0056a7..096af096dc 100644 --- a/erts/doc/src/erl.xml +++ b/erts/doc/src/erl.xml @@ -387,6 +387,28 @@ <p>Replaces the path specified in the boot script. See <seealso marker="sasl:script">script(4)</seealso>.</p> </item> + <tag><c><![CDATA[-proto_dist Proto]]></c></tag> + <item> + <p>Specify a protocol for Erlang distribution.</p> + <taglist> + <tag><c>inet_tcp</c></tag> + <item> + <p>TCP over IPv4 (the default)</p> + </item> + <tag><c>inet_tls</c></tag> + <item> + <p>distribution over TLS/SSL</p> + </item> + <tag><c>inet6_tcp</c></tag> + <item> + <p>TCP over IPv6</p> + </item> + </taglist> + <p>For example, to start up IPv6 distributed nodes:</p> +<pre> +% <input>erl -name [email protected] -proto_dist inet6_tcp</input> +</pre> + </item> <tag><c><![CDATA[-remsh Node]]></c></tag> <item> <p>Starts Erlang with a remote shell connected to <c><![CDATA[Node]]></c>.</p> diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 803da382ed..350a8506f5 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -1811,7 +1811,8 @@ os_prompt% </pre> </item> </taglist> <note><p>On many platforms, the OS supports only status - codes 0-255.</p></note> + codes 0-255. A too large status code will be truncated by clearing + the high bits.</p></note> <p>For integer <c><anno>Status</anno></c>, the Erlang runtime system closes all ports and allows async threads to finish their operations before exiting. To exit without such flushing, use @@ -6468,32 +6469,44 @@ ok <p><marker id="system_flag_multi_scheduling"></marker> If multi-scheduling is enabled, more than one scheduler thread is used by the emulator. Multi-scheduling can be - blocked. When multi-scheduling is blocked, only - one scheduler thread schedules Erlang processes.</p> + blocked in two different ways. Either all schedulers but + one is blocked, or all <em>normal</em> schedulers but + one is blocked. When only normal schedulers are blocked + dirty schedulers are free to continue to schedule + processes.</p> <p>If <c><anno>BlockState</anno> =:= block</c>, multi-scheduling is - blocked. If <c><anno>BlockState</anno> =:= unblock</c> and no one + blocked. That is, one and only one scheduler thread will + execute. If <c><anno>BlockState</anno> =:= unblock</c> and no one else blocks multi-scheduling, and this process has blocked only once, multi-scheduling is unblocked.</p> - <p>One process can block multi-scheduling multiple times. - If a process has blocked multiple times, it must - unblock exactly as many times as it has blocked before it - has released its multi-scheduling block. If a process that - has blocked multi-scheduling exits, it releases its - blocking of multi-scheduling.</p> + <p>If <c><anno>BlockState</anno> =:= block_normal</c>, normal + multi-scheduling is blocked. That is, only one normal scheduler + thread will execute, but multiple dirty schedulers may execute. + If <c><anno>BlockState</anno> =:= unblock_normal</c> and no one + else blocks normal multi-scheduling, and this process has + blocked only once, normal multi-scheduling is unblocked.</p> + <p>One process can block multi-scheduling as well as normal + multi-scheduling multiple times. If a process has blocked + multiple times, it must unblock exactly as many times as it + has blocked before it has released its multi-scheduling + block. If a process that has blocked multi-scheduling or normal + multi scheduling exits, it automatically releases its blocking + of multi-scheduling and normal multi-scheduling.</p> <p>The return values are <c>disabled</c>, <c>blocked</c>, - or <c>enabled</c>. The returned value describes the - state just after the call to + <c>blocked_normal</c>, or <c>enabled</c>. The returned value + describes the state just after the call to <c>erlang:system_flag(multi_scheduling, <anno>BlockState</anno>)</c> has been made. For information about the return values, see <seealso marker="#system_info_multi_scheduling">erlang:system_info(multi_scheduling)</seealso>.</p> - <note><p>Blocking of multi-scheduling is normally not needed. - If you feel that you need to block multi-scheduling, - consider it a few more times again. Blocking multi-scheduling - is only to be used as a last resort, as it is most likely - a <em>very inefficient</em> way to solve the problem.</p> + <note><p>Blocking of multi-scheduling and normal multi-scheduling + is normally not needed. If you feel that you need to use these + features, consider it a few more times again. Blocking + multi-scheduling is only to be used as a last resort, as it is + most likely a <em>very inefficient</em> way to solve the problem.</p> </note> <p>See also <seealso marker="#system_info_multi_scheduling">erlang:system_info(multi_scheduling)</seealso>, + <seealso marker="#system_info_normal_multi_scheduling_blockers">erlang:system_info(normal_multi_scheduling_blockers)</seealso>, <seealso marker="#system_info_multi_scheduling_blockers">erlang:system_info(multi_scheduling_blockers)</seealso>, and <seealso marker="#system_info_schedulers">erlang:system_info(schedulers)</seealso>.</p> </desc> @@ -7328,6 +7341,16 @@ ok where <c><anno>MinHeapSize</anno></c> is the current system-wide minimum heap size for spawned processes.</p> </item> + <tag><marker id="system_info_message_queue_data"><c>message_queue_data</c></marker></tag> + <item> + <p>Returns the default value of the <c>message_queue_data</c> + process flag which is either <c>off_heap</c>, <c>on_heap</c>, or <c>mixed</c>. + This default is set by the <c>erl</c> command line argument + <seealso marker="erl#+xmqd"><c>+xmqd</c></seealso>. For more information on the + <c>message_queue_data</c> process flag, see documentation of + <seealso marker="#process_flag_message_queue_data"><c>process_flag(message_queue_data, + MQD)</c></seealso>.</p> + </item> <tag><c>min_bin_vheap_size</c></tag> <item> <p>Returns <c>{min_bin_vheap_size, @@ -7347,7 +7370,8 @@ ok <tag><c>multi_scheduling</c></tag> <item> <marker id="system_info_multi_scheduling"></marker> - <p>Returns <c>disabled</c>, <c>blocked</c>, or <c>enabled</c>:</p> + <p>Returns <c>disabled</c>, <c>blocked</c>, <c>blocked_normal</c>, + or <c>enabled</c>:</p> <taglist> <tag><c>disabled</c></tag> <item> @@ -7358,14 +7382,22 @@ ok <tag><c>blocked</c></tag> <item> <p>The emulator has more than one scheduler thread, - but all scheduler threads except one are blocked, - that is, only one scheduler thread schedules + but all scheduler threads except one are blocked. + That is, only one scheduler thread schedules Erlang processes and executes Erlang code.</p> </item> + <tag><c>blocked_normal</c></tag> + <item> + <p>The emulator has more than one scheduler thread, + but all normal scheduler threads except one are + blocked. Note that dirty schedulers are not + blocked, and may schedule Erlang processes and + execute native code.</p> + </item> <tag><c>enabled</c></tag> <item> <p>The emulator has more than one scheduler thread, - and no scheduler threads are blocked, that is, + and no scheduler threads are blocked. That is, all available scheduler threads schedule Erlang processes and execute Erlang code.</p> </item> @@ -7373,6 +7405,7 @@ ok <p>See also <seealso marker="#system_flag_multi_scheduling">erlang:system_flag(multi_scheduling, BlockState)</seealso>, <seealso marker="#system_info_multi_scheduling_blockers">erlang:system_info(multi_scheduling_blockers)</seealso>, + <seealso marker="#system_info_normal_multi_scheduling_blockers">erlang:system_info(normal_multi_scheduling_blockers)</seealso>, and <seealso marker="#system_info_schedulers">erlang:system_info(schedulers)</seealso>.</p> </item> @@ -7389,6 +7422,8 @@ ok <p>See also <seealso marker="#system_flag_multi_scheduling">erlang:system_flag(multi_scheduling, BlockState)</seealso>, <seealso marker="#system_info_multi_scheduling">erlang:system_info(multi_scheduling)</seealso>, + <seealso marker="#system_info_normal_multi_scheduling_blockers">erlang:system_info(normal_multi_scheduling_blockers)</seealso>, + and <seealso marker="#system_info_schedulers">erlang:system_info(schedulers)</seealso>.</p> </item> @@ -7398,15 +7433,23 @@ ok used by the runtime system. It is on the form "<major ver>.<minor ver>".</p> </item> - <tag><marker id="system_info_message_queue_data"><c>message_queue_data</c></marker></tag> + <tag><c>normal_multi_scheduling_blockers</c></tag> <item> - <p>Returns the default value of the <c>message_queue_data</c> - process flag which is either <c>off_heap</c>, <c>on_heap</c>, or <c>mixed</c>. - This default is set by the <c>erl</c> command line argument - <seealso marker="erl#+xmqd"><c>+xmqd</c></seealso>. For more information on the - <c>message_queue_data</c> process flag, see documentation of - <seealso marker="#process_flag_message_queue_data"><c>process_flag(message_queue_data, - MQD)</c></seealso>.</p> + <marker id="system_info_normal_multi_scheduling_blockers"></marker> + <p>Returns a list of <c><anno>Pid</anno></c>s when + normal multi-scheduling is blocked (i.e. all normal schedulers + but one is blocked), otherwise the empty list is returned. + The <c><anno>Pid</anno></c>s in the list represent all the + processes currently blocking normal multi-scheduling. + A <c><anno>Pid</anno></c> occurs only once in the list, even if + the corresponding process has blocked multiple times.</p> + <p>See also + <seealso marker="#system_flag_multi_scheduling">erlang:system_flag(multi_scheduling, BlockState)</seealso>, + <seealso marker="#system_info_multi_scheduling">erlang:system_info(multi_scheduling)</seealso>, + <seealso marker="#system_info_multi_scheduling_blockers">erlang:system_info(multi_scheduling_blockers)</seealso>, + + and + <seealso marker="#system_info_schedulers">erlang:system_info(schedulers)</seealso>.</p> </item> <tag><marker id="system_info_otp_release"><c>otp_release</c></marker></tag> <item> @@ -7649,6 +7692,7 @@ ok <seealso marker="#system_info_scheduler_id">erlang:system_info(scheduler_id)</seealso>, <seealso marker="#system_flag_multi_scheduling">erlang:system_flag(multi_scheduling, BlockState)</seealso>, <seealso marker="#system_info_multi_scheduling">erlang:system_info(multi_scheduling)</seealso>, + <seealso marker="#system_info_normal_multi_scheduling_blockers">erlang:system_info(normal_multi_scheduling_blockers)</seealso> and <seealso marker="#system_info_multi_scheduling_blockers">erlang:system_info(multi_scheduling_blockers)</seealso>.</p> </item> diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml index 15b78ffa10..0965f9b49c 100644 --- a/erts/doc/src/erts_alloc.xml +++ b/erts/doc/src/erts_alloc.xml @@ -52,6 +52,8 @@ <item>Allocator used for ETS data.</item> <tag><c>driver_alloc</c></tag> <item>Allocator used for driver data.</item> + <tag><c>literal_alloc</c></tag> + <item>Allocator used for constant terms in Erlang code.</item> <tag><c>sl_alloc</c></tag> <item>Allocator used for memory blocks that are expected to be short-lived.</item> @@ -77,7 +79,7 @@ instead of creating new segments. This in order to reduce the number of system calls made.</item> </taglist> - <p><c>sys_alloc</c> is always enabled and + <p><c>sys_alloc</c> and <c>literal_alloc</c> are always enabled and cannot be disabled. <c>mseg_alloc</c> is always enabled if it is available and an allocator that uses it is enabled. All other allocators can be <seealso marker="#M_e">enabled or disabled</seealso>. @@ -246,6 +248,7 @@ the currently present allocators:</p> <list type="bulleted"> <item><c>B: binary_alloc</c></item> + <item><c>I: literal_alloc</c></item> <item><c>D: std_alloc</c></item> <item><c>E: ets_alloc</c></item> <item><c>F: fix_alloc</c></item> @@ -563,6 +566,16 @@ set to <c>false</c>, <c>sys_alloc</c> carriers will never be created by allocators using the <c>alloc_util</c> framework.</item> </taglist> + <p>The following flag is special for <c>literal_alloc</c>:</p> + <taglist> + <tag><marker id="MIscs"/><c><![CDATA[+MIscs <size in MB>]]></c></tag> + <item> + <c>literal_alloc</c> super carrier size (in MB). The amount of + <em>virtual</em> address space reserved for literal terms in + Erlang code on 64-bit architectures. The default is 1024 (1GB) + and is usually sufficient. The flag is ignored on 32-bit + architectures.</item> + </taglist> <p>Instrumentation flags:</p> <taglist> <tag><marker id="Mim"/><c>+Mim true|false</c></tag> diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 6fb08ee896..169b071cd7 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -125,7 +125,9 @@ atom binary_longest_suffix_trap atom binary_to_list_continue atom binary_to_term_trap atom block +atom block_normal atom blocked +atom blocked_normal atom bm atom bnot atom bor @@ -190,7 +192,9 @@ atom dexit atom depth atom dgroup_leader atom dictionary +atom dirty_cpu atom dirty_cpu_schedulers_online +atom dirty_io atom disable_trace atom disabled atom display_items @@ -211,6 +215,7 @@ atom dsend atom dsend_continue_trap atom dunlink atom duplicate_bag +atom duplicated atom dupnames atom einval atom elib_malloc @@ -273,6 +278,7 @@ atom gather_gc_info_result atom gather_io_bytes atom gather_microstate_accounting_result atom gather_sched_wall_time_result +atom gather_system_check_result atom getting_linked atom getting_unlinked atom global @@ -303,6 +309,7 @@ atom index atom infinity atom info atom info_msg +atom init atom initial_call atom input atom internal @@ -610,6 +617,7 @@ atom use_stdio atom used atom utf8 atom unblock +atom unblock_normal atom uniq atom unless_suspending atom unloaded diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 1b4c022370..0b47fc3586 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -86,7 +86,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) ASSERT(modp->curr.num_breakpoints == 0); } - erts_start_staging_code_ix(); + erts_start_staging_code_ix(1); res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); @@ -139,6 +139,25 @@ prepare_loading_2(BIF_ALIST_2) BIF_RET(res); } +BIF_RETTYPE +has_prepared_code_on_load_1(BIF_ALIST_1) +{ + Eterm res; + ProcBin* pb; + + if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1)) { + error: + BIF_ERROR(BIF_P, BADARG); + } + + pb = (ProcBin*) binary_val(BIF_ARG_1); + res = erts_has_code_on_load(pb->val); + if (res == NIL) { + goto error; + } + BIF_RET(res); +} + struct m { Binary* code; Eterm module; @@ -163,14 +182,13 @@ exception_list(Process* p, Eterm tag, struct m* mp, Sint exceptions) Eterm* hp = HAlloc(p, 3 + 2*exceptions); Eterm res = NIL; - mp += exceptions - 1; while (exceptions > 0) { if (mp->exception) { res = CONS(hp, mp->module, res); hp += 2; exceptions--; } - mp--; + mp++; } return TUPLE2(hp, tag, res); } @@ -202,8 +220,12 @@ finish_loading_1(BIF_ALIST_1) n = erts_list_length(BIF_ARG_1); if (n < 0) { - ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); - goto done; + badarg: + if (p) { + erts_free(ERTS_ALC_T_LOADER_TMP, p); + } + erts_release_code_write_permission(); + BIF_ERROR(BIF_P, BADARG); } p = erts_alloc(ERTS_ALC_T_LOADER_TMP, n*sizeof(struct m)); @@ -218,29 +240,32 @@ finish_loading_1(BIF_ALIST_1) ProcBin* pb; if (!ERTS_TERM_IS_MAGIC_BINARY(term)) { - ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); - goto done; + goto badarg; } pb = (ProcBin*) binary_val(term); p[i].code = pb->val; p[i].module = erts_module_for_prepared_code(p[i].code); if (p[i].module == NIL) { - ERTS_BIF_PREP_ERROR(res, BIF_P, BADARG); - goto done; + goto badarg; } BIF_ARG_1 = CDR(cons); } /* * Since we cannot handle atomic loading of a group of modules - * if one or more of them uses on_load, we will only allow one - * element in the list. This limitation is intended to be - * lifted in the future. + * if one or more of them uses on_load, we will only allow + * more than one element in the list if none of the modules + * have an on_load function. */ if (n > 1) { - ERTS_BIF_PREP_ERROR(res, BIF_P, SYSTEM_LIMIT); - goto done; + for (i = 0; i < n; i++) { + if (erts_has_code_on_load(p[i].code) == am_true) { + erts_free(ERTS_ALC_T_LOADER_TMP, p); + erts_release_code_write_permission(); + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + } + } } /* @@ -252,11 +277,27 @@ finish_loading_1(BIF_ALIST_1) */ res = am_ok; - erts_start_staging_code_ix(); + erts_start_staging_code_ix(n); for (i = 0; i < n; i++) { p[i].modp = erts_put_module(p[i].module); + p[i].modp->seen = 0; } + + exceptions = 0; + for (i = 0; i < n; i++) { + p[i].exception = 0; + if (p[i].modp->seen) { + p[i].exception = 1; + exceptions++; + } + p[i].modp->seen = 1; + } + if (exceptions) { + res = exception_list(BIF_P, am_duplicated, p, exceptions); + goto done; + } + for (i = 0; i < n; i++) { if (p[i].modp->curr.num_breakpoints > 0 || p[i].modp->curr.num_traced_exports > 0 || @@ -492,7 +533,7 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } { - erts_start_staging_code_ix(); + erts_start_staging_code_ix(0); code_ix = erts_staging_code_ix(); modp = erts_get_module(BIF_ARG_1, code_ix); if (!modp) { diff --git a/erts/emulator/beam/beam_catches.c b/erts/emulator/beam/beam_catches.c index c1fd17c65d..7a1f4901aa 100644 --- a/erts/emulator/beam/beam_catches.c +++ b/erts/emulator/beam/beam_catches.c @@ -143,7 +143,7 @@ BeamInstr *beam_catches_car(unsigned i) struct bc_pool* p = &bccix[erts_active_code_ix()]; if (i >= p->tabsize ) { - erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i); } return p->beam_catches[i].cp; } @@ -157,10 +157,10 @@ void beam_catches_delmod(unsigned head, BeamInstr *code, unsigned code_bytes, ASSERT((code_ix == erts_active_code_ix()) != bccix[erts_staging_code_ix()].is_staging); for(i = head; i != (unsigned)-1;) { if (i >= p->tabsize) { - erl_exit(1, "beam_catches_delmod: index %#x is out of range\r\n", i); + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: index %#x is out of range\r\n", i); } if( (char*)p->beam_catches[i].cp - (char*)code >= code_bytes ) { - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "beam_catches_delmod: item %#x has cp %p which is not " "in module's range [%p,%p[\r\n", i, p->beam_catches[i].cp, code, ((char*)code + code_bytes)); diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 10f132abfc..a390422040 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -4730,7 +4730,7 @@ do { \ #ifdef NO_FPE_SIGNALS OpCase(fclearerror): OpCase(i_fcheckerror): - erl_exit(1, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); + erts_exit(ERTS_ERROR_EXIT, "fclearerror/i_fcheckerror without fpe signals (beam_emu)"); # define ERTS_NO_FPE_CHECK_INIT ERTS_FP_CHECK_INIT # define ERTS_NO_FPE_ERROR ERTS_FP_ERROR #else @@ -4885,7 +4885,7 @@ do { \ I = handle_error(c_p, I, reg, NULL); goto post_error_handling; default: - erl_exit(1, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); + erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %u\n", c_p->def_arg_reg[3]); } } OpCase(hipe_call_count): { @@ -4997,7 +4997,7 @@ do { \ OpCase(label_L): OpCase(on_load): OpCase(line_I): - erl_exit(1, "meta op\n"); + erts_exit(ERTS_ERROR_EXIT, "meta op\n"); /* * One-time initialization of Beam emulator. @@ -5051,7 +5051,7 @@ do { \ } #ifdef NO_JUMP_TABLE default: - erl_exit(1, "unexpected op code %d\n",Go); + erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go); } #endif return; /* Never executed */ @@ -5096,7 +5096,7 @@ translate_gc_bif(void* gcf) } else if (gcf == erts_gc_binary_part_3) { return binary_part_3; } else { - erl_exit(1, "bad gc bif"); + erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); } } @@ -5161,7 +5161,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) Eterm* hp; Eterm Value = c_p->fvalue; Eterm Args = am_true; - c_p->i = pc; /* In case we call erl_exit(). */ + c_p->i = pc; /* In case we call erts_exit(). */ ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */ @@ -5225,7 +5225,7 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) c_p->cp = 0; /* To avoid keeping stale references. */ return new_pc; } - if (c_p->catches > 0) erl_exit(1, "Catch not found"); + if (c_p->catches > 0) erts_exit(ERTS_ERROR_EXIT, "Catch not found"); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); terminate_proc(c_p, Value); diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 1511a4f935..f115df935f 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -888,6 +888,23 @@ erts_module_for_prepared_code(Binary* magic) } } +/* + * Return a non-zero value if the module has an on_load function, + * or 0 if it does not. + */ + +Eterm +erts_has_code_on_load(Binary* magic) +{ + LoaderState* stp; + + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { + return NIL; + } + stp = ERTS_MAGIC_BIN_DATA(magic); + return stp->on_load ? am_true : am_false; +} + static void free_loader_state(Binary* magic) { diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 5a2b66727a..54c337ee72 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -53,6 +53,7 @@ struct ranges { }; static struct ranges r[ERTS_NUM_CODE_IX]; static erts_smp_atomic_t mem_used; +static Range* write_ptr; #ifdef HARD_DEBUG static void check_consistency(struct ranges* p) @@ -72,6 +73,17 @@ static void check_consistency(struct ranges* p) # define CHECK(r) #endif /* HARD_DEBUG */ +static int +rangecompare(Range* a, Range* b) +{ + if (a->start < b->start) { + return -1; + } else if (a->start == b->start) { + return 0; + } else { + return 1; + } +} void erts_init_ranges(void) @@ -88,45 +100,70 @@ erts_init_ranges(void) } void -erts_start_staging_ranges(void) +erts_start_staging_ranges(int num_new) { + ErtsCodeIndex src = erts_active_code_ix(); ErtsCodeIndex dst = erts_staging_code_ix(); + Sint need; if (r[dst].modules) { erts_smp_atomic_add_nob(&mem_used, -r[dst].allocated); erts_free(ERTS_ALC_T_MODULE_REFS, r[dst].modules); - r[dst].modules = NULL; } + + need = r[dst].allocated = r[src].n + num_new; + erts_smp_atomic_add_nob(&mem_used, need); + write_ptr = erts_alloc(ERTS_ALC_T_MODULE_REFS, + need * sizeof(Range)); + r[dst].modules = write_ptr; } void erts_end_staging_ranges(int commit) { - ErtsCodeIndex dst = erts_staging_code_ix(); - - if (commit && r[dst].modules == NULL) { + if (commit) { Sint i; - Sint n; - - /* No modules added, just clone src and remove purged code. */ ErtsCodeIndex src = erts_active_code_ix(); + ErtsCodeIndex dst = erts_staging_code_ix(); + Range* mp; + Sint num_inserted; - erts_smp_atomic_add_nob(&mem_used, r[src].n); - r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS, - r[src].n * sizeof(Range)); - r[dst].allocated = r[src].n; - n = 0; + mp = r[dst].modules; + num_inserted = write_ptr - mp; for (i = 0; i < r[src].n; i++) { Range* rp = r[src].modules+i; if (rp->start < RANGE_END(rp)) { /* Only insert a module that has not been purged. */ - r[dst].modules[n] = *rp; - n++; + write_ptr->start = rp->start; + erts_smp_atomic_init_nob(&write_ptr->end, + (erts_aint_t)(RANGE_END(rp))); + write_ptr++; + } + } + + /* + * There are num_inserted new range entries (unsorted) at the + * beginning of the modules array, followed by the old entries + * (sorted). We must now sort the entire array. + */ + + r[dst].n = write_ptr - mp; + if (num_inserted > 1) { + qsort(mp, r[dst].n, sizeof(Range), + (int (*)(const void *, const void *)) rangecompare); + } else if (num_inserted == 1) { + /* Sift the new range into place. This is faster than qsort(). */ + Range t = mp[0]; + for (i = 0; i < r[dst].n-1 && t.start > mp[i+1].start; i++) { + mp[i] = mp[i+1]; } + mp[i] = t; } - r[dst].n = n; + r[dst].modules = mp; + CHECK(&r[dst]); erts_smp_atomic_set_nob(&r[dst].mid, - (erts_aint_t) (r[dst].modules + n / 2)); + (erts_aint_t) (r[dst].modules + + r[dst].n / 2)); } } @@ -135,82 +172,29 @@ erts_update_ranges(BeamInstr* code, Uint size) { ErtsCodeIndex dst = erts_staging_code_ix(); ErtsCodeIndex src = erts_active_code_ix(); - Sint i; - Sint n; - Sint need; if (src == dst) { ASSERT(!erts_initialized); /* - * During start-up of system, the indices are the same. - * Handle this by faking a source area. + * During start-up of system, the indices are the same + * and erts_start_staging_ranges() has not been called. */ - src = (src+1) % ERTS_NUM_CODE_IX; - if (r[src].modules) { - erts_smp_atomic_add_nob(&mem_used, -r[src].allocated); - erts_free(ERTS_ALC_T_MODULE_REFS, r[src].modules); + if (r[dst].modules == NULL) { + Sint need = 128; + erts_smp_atomic_add_nob(&mem_used, need); + r[dst].modules = erts_alloc(ERTS_ALC_T_MODULE_REFS, + need * sizeof(Range)); + r[dst].allocated = need; + write_ptr = r[dst].modules; } - r[src] = r[dst]; - r[dst].modules = 0; } - CHECK(&r[src]); - - ASSERT(r[dst].modules == NULL); - need = r[dst].allocated = r[src].n + 1; - erts_smp_atomic_add_nob(&mem_used, need); - r[dst].modules = (Range *) erts_alloc(ERTS_ALC_T_MODULE_REFS, - need * sizeof(Range)); - n = 0; - for (i = 0; i < r[src].n; i++) { - Range* rp = r[src].modules+i; - if (code < rp->start) { - r[dst].modules[n].start = code; - erts_smp_atomic_init_nob(&r[dst].modules[n].end, - (erts_aint_t)(((byte *)code) + size)); - ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code); - n++; - break; - } - if (rp->start < RANGE_END(rp)) { - /* Only insert a module that has not been purged. */ - r[dst].modules[n].start = rp->start; - erts_smp_atomic_init_nob(&r[dst].modules[n].end, - (erts_aint_t)(RANGE_END(rp))); - ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start); - n++; - } - } - - while (i < r[src].n) { - Range* rp = r[src].modules+i; - if (rp->start < RANGE_END(rp)) { - /* Only insert a module that has not been purged. */ - r[dst].modules[n].start = rp->start; - erts_smp_atomic_init_nob(&r[dst].modules[n].end, - (erts_aint_t)(RANGE_END(rp))); - ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < rp->start); - n++; - } - i++; - } - - if (n == 0 || code > r[dst].modules[n-1].start) { - r[dst].modules[n].start = code; - erts_smp_atomic_init_nob(&r[dst].modules[n].end, - (erts_aint_t)(((byte *)code) + size)); - ASSERT(!n || RANGE_END(&r[dst].modules[n-1]) < code); - n++; - } - - ASSERT(n <= r[src].n+1); - r[dst].n = n; - erts_smp_atomic_set_nob(&r[dst].mid, - (erts_aint_t) (r[dst].modules + n / 2)); - - CHECK(&r[dst]); - CHECK(&r[src]); + ASSERT(r[dst].modules); + write_ptr->start = code; + erts_smp_atomic_init_nob(&(write_ptr->end), + (erts_aint_t)(((byte *)code) + size)); + write_ptr++; } void diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index 2654785ffc..37bb28c6f8 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -2242,7 +2242,7 @@ BIF_RETTYPE send_3(BIF_ALIST_3) erts_dsend_export_trap_context(p, ctx)); break; default: - erl_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result); + erts_exit(ERTS_ABORT_EXIT, "send_3 invalid result %d\n", (int)result); break; } @@ -2286,7 +2286,7 @@ static BIF_RETTYPE dsend_continue_trap_1(BIF_ALIST_1) BIF_TRAP1(&dsend_continue_trap_export, BIF_P, BIF_ARG_1); } default: - erl_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result); + erts_exit(ERTS_ABORT_EXIT, "dsend_continue_trap invalid result %d\n", (int)result); break; } ASSERT(! "Can not arrive here"); @@ -2360,7 +2360,7 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg) erts_dsend_export_trap_context(p, ctx)); break; default: - erl_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result); + erts_exit(ERTS_ABORT_EXIT, "invalid send result %d\n", (int)result); break; } @@ -3738,7 +3738,7 @@ BIF_RETTYPE erts_debug_display_1(BIF_ALIST_1) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%.*T\n", INT_MAX, BIF_ARG_1); if (pres < 0) - erl_exit(1, "Failed to convert term to string: %d (%s)\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to string: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(BIF_P, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); @@ -3760,7 +3760,7 @@ BIF_RETTYPE display_string_1(BIF_ALIST_1) } str = (char *) erts_alloc(ERTS_ALC_T_TMP, sizeof(char)*(len + 1)); if (intlist_to_buf(string, str, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); str[len] = '\0'; erts_fprintf(stderr, "%s", str); erts_free(ERTS_ALC_T_TMP, (void *) str); @@ -3780,7 +3780,7 @@ BIF_RETTYPE display_nl_0(BIF_ALIST_0) BIF_RETTYPE halt_0(BIF_ALIST_0) { VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt()\n")); - erl_halt(0); + erts_halt(0); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } @@ -3793,17 +3793,18 @@ static char halt_msg[HALT_MSG_SIZE]; /* ARGSUSED */ BIF_RETTYPE halt_1(BIF_ALIST_1) { - Sint code; + Uint code; - if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { + if (term_to_Uint_mask(BIF_ARG_1, &code)) { + int pos_int_code = (int) (code & INT_MAX); VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); - erl_halt((int)(- code)); + erts_halt(pos_int_code); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_ABORT_EXIT, ""); + erts_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { Sint i; @@ -3814,11 +3815,11 @@ BIF_RETTYPE halt_1(BIF_ALIST_1) halt_msg[i] = '\0'; VERBOSE(DEBUG_SYSTEM,("System halted by BIF halt(%T)\n", BIF_ARG_1)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); } else goto error; - return NIL; /* Pedantic (lint does not know about erl_exit) */ + return NIL; /* Pedantic (lint does not know about erts_exit) */ error: BIF_ERROR(BIF_P, BADARG); } @@ -3829,7 +3830,7 @@ BIF_RETTYPE halt_1(BIF_ALIST_1) /* ARGSUSED */ BIF_RETTYPE halt_2(BIF_ALIST_2) { - Sint code; + Uint code; Eterm optlist = BIF_ARG_2; int flush = 1; @@ -3856,23 +3857,24 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) if (is_not_nil(optlist)) goto error; - if (is_small(BIF_ARG_1) && (code = signed_val(BIF_ARG_1)) >= 0) { + if (term_to_Uint_mask(BIF_ARG_1, &code)) { + int pos_int_code = (int) (code & INT_MAX); VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); if (flush) { - erl_halt((int)(- code)); + erts_halt(pos_int_code); ERTS_BIF_YIELD1(bif_export[BIF_halt_1], BIF_P, am_undefined); } else { erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit((int)(- code), ""); + erts_exit(pos_int_code, ""); } } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_ABORT_EXIT, ""); + erts_exit(ERTS_ABORT_EXIT, ""); } else if (is_string(BIF_ARG_1) || BIF_ARG_1 == NIL) { Sint i; @@ -3884,11 +3886,11 @@ BIF_RETTYPE halt_2(BIF_ALIST_2) VERBOSE(DEBUG_SYSTEM, ("System halted by BIF halt(%T, %T)\n", BIF_ARG_1, BIF_ARG_2)); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erl_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); + erts_exit(ERTS_DUMP_EXIT, "%s\n", halt_msg); } else goto error; - return NIL; /* Pedantic (lint does not know about erl_exit) */ + return NIL; /* Pedantic (lint does not know about erts_exit) */ error: BIF_ERROR(BIF_P, BADARG); } @@ -3944,7 +3946,7 @@ term2list_dsprintf(Process *p, Eterm term) erts_dsprintf_buf_t *dsbufp = erts_create_tmp_dsbuf(64); pres = erts_dsprintf(dsbufp, "%T", term); if (pres < 0) - erl_exit(1, "Failed to convert term to list: %d (%s)\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to convert term to list: %d (%s)\n", -pres, erl_errno_id(-pres)); hp = HAlloc(p, 2*dsbufp->str_len); /* we need length * 2 heap words */ res = buf_to_intlist(&hp, dsbufp->str, dsbufp->str_len, NIL); @@ -4195,22 +4197,33 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) Sint n; if (BIF_ARG_1 == am_multi_scheduling) { - if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock) { + if (BIF_ARG_2 == am_block || BIF_ARG_2 == am_unblock + || BIF_ARG_2 == am_block_normal || BIF_ARG_2 == am_unblock_normal) { #ifndef ERTS_SMP BIF_RET(am_disabled); #else + int block = (BIF_ARG_2 == am_block + || BIF_ARG_2 == am_block_normal); + int normal = (BIF_ARG_2 == am_block_normal + || BIF_ARG_2 == am_unblock_normal); if (erts_no_schedulers == 1) BIF_RET(am_disabled); else { switch (erts_block_multi_scheduling(BIF_P, ERTS_PROC_LOCK_MAIN, - BIF_ARG_2 == am_block, + block, + normal, 0)) { case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: BIF_RET(am_blocked); + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: + BIF_RET(am_blocked_normal); case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked, am_multi_scheduling); + case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: + ERTS_BIF_YIELD_RETURN_X(BIF_P, am_blocked_normal, + am_multi_scheduling); case ERTS_SCHDLR_SSPND_DONE: BIF_RET(am_enabled); case ERTS_SCHDLR_SSPND_YIELD_RESTART: @@ -4243,11 +4256,7 @@ BIF_RETTYPE system_flag_2(BIF_ALIST_2) switch (erts_set_schedulers_online(BIF_P, ERTS_PROC_LOCK_MAIN, signed_val(BIF_ARG_2), - &old_no -#ifdef ERTS_DIRTY_SCHEDULERS - , 0 -#endif - )) { + &old_no, 0)) { case ERTS_SCHDLR_SSPND_DONE: BIF_RET(make_small(old_no)); case ERTS_SCHDLR_SSPND_YIELD_RESTART: @@ -4617,29 +4626,27 @@ BIF_RETTYPE erts_internal_cmp_term_2(BIF_ALIST_2) { /* * Processes doing yield on return in a bif ends up in bif_return_trap(). */ -static BIF_RETTYPE bif_return_trap( -#ifdef DEBUG - BIF_ALIST_2 -#else - BIF_ALIST_1 -#endif - ) +static BIF_RETTYPE bif_return_trap(BIF_ALIST_2) { -#ifdef DEBUG + Eterm res = BIF_ARG_1; + switch (BIF_ARG_2) { - case am_multi_scheduling: #ifdef ERTS_SMP - erts_dbg_multi_scheduling_return_trap(BIF_P, BIF_ARG_1); -#endif - break; - case am_schedulers_online: + case am_multi_scheduling: { + int msb = erts_is_multi_scheduling_blocked(); + if (msb > 0) + res = am_blocked; + else if (msb < 0) + res = am_blocked_normal; + else + ERTS_INTERNAL_ERROR("Unexpected multi scheduling block state"); break; + } +#endif default: break; } -#endif - - BIF_RET(BIF_ARG_1); + BIF_RET(res); } /* @@ -4744,17 +4751,12 @@ void erts_init_bif(void) erts_smp_atomic_init_nob(&erts_dead_ports_ptr, (erts_aint_t) NULL); /* - * bif_return_trap/1 is a hidden BIF that bifs that need to - * yield the calling process traps to. The only thing it does: - * return the value passed as argument. + * bif_return_trap/2 is a hidden BIF that bifs that need to + * yield the calling process traps to. */ - erts_init_trap_export(&bif_return_trap_export, am_erlang, am_bif_return_trap, -#ifdef DEBUG - 2 -#else - 1 -#endif - , &bif_return_trap); + erts_init_trap_export(&bif_return_trap_export, + am_erlang, am_bif_return_trap, 2, + &bif_return_trap); erts_await_result = erts_export_put(am_erts_internal, am_await_result, diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index a62eddf36b..46af552b39 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -338,37 +338,20 @@ do { \ } while(0) extern Export bif_return_trap_export; -#ifdef DEBUG -#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL) \ +#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, OP) \ do { \ ERTS_VBUMP_ALL_REDS(P); \ - ERTS_BIF_PREP_TRAP2(RET, &bif_return_trap_export, (P), (VAL), \ - (DEBUG_VAL)); \ + ERTS_BIF_PREP_TRAP2(RET, &bif_return_trap_export, (P), (VAL), (OP));\ } while (0) -#else -#define ERTS_BIF_PREP_YIELD_RETURN_X(RET, P, VAL, DEBUG_VAL) \ -do { \ - ERTS_VBUMP_ALL_REDS(P); \ - ERTS_BIF_PREP_TRAP1(RET, &bif_return_trap_export, (P), (VAL)); \ -} while (0) -#endif #define ERTS_BIF_PREP_YIELD_RETURN(RET, P, VAL) \ ERTS_BIF_PREP_YIELD_RETURN_X(RET, (P), (VAL), am_undefined) -#ifdef DEBUG -#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL) \ +#define ERTS_BIF_YIELD_RETURN_X(P, VAL, OP) \ do { \ ERTS_VBUMP_ALL_REDS(P); \ - BIF_TRAP2(&bif_return_trap_export, (P), (VAL), (DEBUG_VAL)); \ + BIF_TRAP2(&bif_return_trap_export, (P), (VAL), (OP)); \ } while (0) -#else -#define ERTS_BIF_YIELD_RETURN_X(P, VAL, DEBUG_VAL) \ -do { \ - ERTS_VBUMP_ALL_REDS(P); \ - BIF_TRAP1(&bif_return_trap_export, (P), (VAL)); \ -} while (0) -#endif #define ERTS_BIF_RETURN_YIELD(P) ERTS_VBUMP_ALL_REDS((P)) diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 4efc055aaf..a6670c10e8 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -174,6 +174,8 @@ bif erts_internal:perf_counter_unit/0 bif erts_internal:is_system_process/1 +bif erts_internal:system_check/1 + # inet_db support bif erlang:port_set_data/2 bif erlang:port_get_data/1 @@ -649,6 +651,7 @@ bif binary:split/2 bif binary:split/3 bif erts_debug:size_shared/1 bif erts_debug:copy_shared/1 +bif erlang:has_prepared_code_on_load/1 # # Obsolete diff --git a/erts/emulator/beam/big.c b/erts/emulator/beam/big.c index 11838e24ef..c738a9ecf5 100644 --- a/erts/emulator/beam/big.c +++ b/erts/emulator/beam/big.c @@ -2035,6 +2035,32 @@ term_to_Uint(Eterm term, Uint *up) } } +/* same as term_to_Uint() + but also accept larger bignums by masking + */ +int +term_to_Uint_mask(Eterm term, Uint *up) +{ + if (is_small(term)) { + Sint i = signed_val(term); + if (i < 0) { + *up = BADARG; + return 0; + } + *up = (Uint) i; + return 1; + } else if (is_big(term) && !big_sign(term)) { + ErtsDigit* xr = big_v(term); + + ERTS_CT_ASSERT(sizeof(ErtsDigit) == sizeof(Uint)); + *up = (Uint)*xr; /* just pick first word */ + return 1; + } else { + *up = BADARG; + return 0; + } +} + int term_to_UWord(Eterm term, UWord *up) { diff --git a/erts/emulator/beam/big.h b/erts/emulator/beam/big.h index 9c92de6b55..ee51e5d313 100644 --- a/erts/emulator/beam/big.h +++ b/erts/emulator/beam/big.h @@ -154,6 +154,7 @@ Eterm bytes_to_big(byte*, dsize_t, int, Eterm*); byte* big_to_bytes(Eterm, byte*); int term_to_Uint(Eterm, Uint*); +int term_to_Uint_mask(Eterm, Uint*); int term_to_UWord(Eterm, UWord*); int term_to_Sint(Eterm, Sint*); #if HAVE_INT64 diff --git a/erts/emulator/beam/binary.c b/erts/emulator/beam/binary.c index d3e481c7f9..3fd284b589 100644 --- a/erts/emulator/beam/binary.c +++ b/erts/emulator/beam/binary.c @@ -50,7 +50,7 @@ erts_init_binary(void) if ((((UWord) &((Binary *) 0)->orig_bytes[0]) % ((UWord) 8)) != 0) { /* I assume that any compiler should be able to optimize this away. If not, this test is not very expensive... */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error: Address of orig_bytes[0] of a Binary" " is *not* 8-byte aligned\n"); } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 2c8ecf04be..8c039ef132 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -104,7 +104,7 @@ process_killer(void) erts_printf("(k)ill (n)ext (r)eturn:\n"); while(1) { if ((j = sys_get_key(0)) <= 0) - erl_exit(0, ""); + erts_exit(0, ""); switch(j) { case 'k': { ErtsProcLocks rp_locks = ERTS_PROC_LOCKS_XSIG_SEND; @@ -493,7 +493,7 @@ do_break(void) halt immediately if break is called */ mode = erts_read_env("ERL_CONSOLE_MODE"); if (mode && strcmp(mode, "window") != 0) - erl_exit(0, ""); + erts_exit(0, ""); erts_free_read_env(mode); #endif /* __WIN32__ */ @@ -503,7 +503,7 @@ do_break(void) while (1) { if ((i = sys_get_key(0)) <= 0) - erl_exit(0, ""); + erts_exit(0, ""); switch (i) { case 'q': case 'a': @@ -513,9 +513,9 @@ do_break(void) * The usual reason for a read error is Ctrl-C. Treat this as * 'a' to avoid infinite loop. */ - erl_exit(0, ""); + erts_exit(0, ""); case 'A': /* Halt generating crash dump */ - erl_exit(1, "Crash dump requested by user"); + erts_exit(ERTS_ERROR_EXIT, "Crash dump requested by user"); case 'c': return; case 'p': @@ -785,7 +785,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) erts_fdprintf(fd, "Atoms: %d\n", atom_table_size()); #ifdef USE_THREADS - /* We want to note which thread it was that called erl_exit */ + /* We want to note which thread it was that called erts_exit */ if (erts_get_scheduler_data()) { erts_fdprintf(fd, "Calling Thread: scheduler:%d\n", erts_get_scheduler_data()->no); diff --git a/erts/emulator/beam/code_ix.c b/erts/emulator/beam/code_ix.c index 209d008d18..1b0968c55c 100644 --- a/erts/emulator/beam/code_ix.c +++ b/erts/emulator/beam/code_ix.c @@ -65,12 +65,12 @@ void erts_code_ix_init(void) CIX_TRACE("init"); } -void erts_start_staging_code_ix(void) +void erts_start_staging_code_ix(int num_new) { beam_catches_start_staging(); export_start_staging(); module_start_staging(); - erts_start_staging_ranges(); + erts_start_staging_ranges(num_new); CIX_TRACE("start"); } diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 5f00b409ef..7a66211a5b 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -100,7 +100,7 @@ void erts_release_code_write_permission(void); * Must be followed by calls to either "end" and "commit" or "abort" before * code write permission can be released. */ -void erts_start_staging_code_ix(void); +void erts_start_staging_code_ix(int num_new); /* End the staging. * Preceded by "start" and must be followed by "commit". diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index 67a96f6442..b38ebe066b 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -58,7 +58,7 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { res = copy_struct(obj, size, &hp, &to->off_heap); #ifdef DEBUG if (eq(obj, res) == 0) { - erl_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); + erts_exit(ERTS_ABORT_EXIT, "copy not equal to source\n"); } #endif return res; @@ -174,7 +174,7 @@ Uint size_object(Eterm obj) } break; default: - erl_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + erts_exit(ERTS_ABORT_EXIT, "size_object: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } break; case SUB_BINARY_SUBTAG: @@ -205,7 +205,7 @@ Uint size_object(Eterm obj) } break; case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "size_object: matchstate term not allowed"); default: sum += thing_arityval(hdr) + 1; @@ -223,7 +223,7 @@ Uint size_object(Eterm obj) obj = ESTACK_POP(s); break; default: - erl_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj); + erts_exit(ERTS_ABORT_EXIT, "size_object: bad tag for %#x\n", obj); } } } @@ -438,10 +438,10 @@ Uint size_shared(Eterm obj) goto pop_next; } default: - erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); default: sum += thing_arityval(hdr) + 1; @@ -457,7 +457,7 @@ Uint size_shared(Eterm obj) obj = EQUEUE_GET(s); break; default: - erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); } } @@ -569,7 +569,7 @@ cleanup: goto cleanup_next; } default: - erl_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } default: goto cleanup_next; @@ -584,7 +584,7 @@ cleanup: obj = EQUEUE_GET(s); break; default: - erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); } } @@ -643,7 +643,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint goto L_copy_list; case TAG_PRIMARY_BOXED: argp = &res; goto L_copy_boxed; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } @@ -691,7 +691,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; case TAG_PRIMARY_BOXED: argp = tailp; goto L_copy_boxed; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } @@ -869,11 +869,11 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *argp = make_hashmap(tp); break; default: - erl_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + erts_exit(ERTS_ABORT_EXIT, "copy_struct: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } break; case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "copy_struct: matchstate term not allowed"); default: i = thing_arityval(hdr)+1; @@ -901,13 +901,13 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint } else { #ifdef DEBUG if (htop != hbot) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct() when copying %T:" " htop=%p != hbot=%p (sz=%beu)\n", org_obj, htop, hbot, org_sz); #else if (htop > hbot) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct(): htop, hbot overrun\n"); } #endif @@ -1246,10 +1246,10 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) goto pop_next; } default: - erl_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + erts_exit(ERTS_ABORT_EXIT, "copy_shared_calculate: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } case BIN_MATCHSTATE_SUBTAG: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "size_shared: matchstate term not allowed"); default: sum += thing_arityval(hdr) + 1; @@ -1278,7 +1278,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) obj = EQUEUE_GET(s); break; default: - erl_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj); + erts_exit(ERTS_ABORT_EXIT, "[pid=%T] size_shared: bad tag for %#x\n", obj); } } } @@ -1530,7 +1530,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, goto cleanup_next; } default: - erl_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); + erts_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(hdr)); } case REFC_BINARY_SUBTAG: { ProcBin* pb = (ProcBin *) ptr; @@ -1682,7 +1682,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, hscan += MAP_HEADER_ARITY(*hscan) + 1; break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "copy_shared_perform: bad hashmap type %d\n", MAP_HEADER_TYPE(*hscan)); } @@ -1711,7 +1711,7 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ASSERT(resp < hp); break; default: - erl_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); + erts_exit(ERTS_ABORT_EXIT, "size_shared: bad tag for %#x\n", obj); } } @@ -1744,7 +1744,7 @@ all_clean: if (eq(saved_obj, result) == 0) { erts_fprintf(stderr, "original = %T\n", saved_obj); erts_fprintf(stderr, "copy = %T\n", result); - erl_exit(ERTS_ABORT_EXIT, "copy (shared) not equal to source\n"); + erts_exit(ERTS_ABORT_EXIT, "copy (shared) not equal to source\n"); } #endif diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index e31ef29562..fa385f105d 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1942,7 +1942,7 @@ erts_dsig_send(ErtsDSigData *dsdp, struct erts_dsig_send_context* ctx) goto done; } default: - erl_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase); + erts_exit(ERTS_ABORT_EXIT, "dsig_send invalid phase (%d)\n", (int)ctx->phase); } } @@ -1963,7 +1963,7 @@ dist_port_command(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -2003,7 +2003,7 @@ dist_port_commandv(Port *prt, ErtsDistOutputBuf *obuf) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); if (size > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large distribution output data buffer " "(%beu bytes) passed.\n", size); @@ -3370,7 +3370,7 @@ send_nodes_mon_msgs(Process *c_p, Eterm what, Eterm node, Eterm type, Eterm reas continue; break; default: - erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); + erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); } } @@ -3679,7 +3679,7 @@ erts_processes_monitoring_nodes(Process *c_p) case ERTS_NODES_MON_OPT_TYPES: type = am_all; break; case ERTS_NODES_MON_OPT_TYPE_VISIBLE: type = am_visible; break; case ERTS_NODES_MON_OPT_TYPE_HIDDEN: type = am_hidden; break; - default: erl_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); + default: erts_exit(ERTS_ABORT_EXIT, "Bad node type found\n"); } olist = erts_bld_cons(hpp, szp, erts_bld_tuple(hpp, szp, 2, diff --git a/erts/emulator/beam/erl_afit_alloc.c b/erts/emulator/beam/erl_afit_alloc.c index 47dafa53c0..4b0541c10e 100644 --- a/erts/emulator/beam/erl_afit_alloc.c +++ b/erts/emulator/beam/erl_afit_alloc.c @@ -241,7 +241,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 806f569c38..490e0c0915 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -165,6 +165,8 @@ enum allctr_type { struct au_init { int enable; int thr_spec; + int disable_allowed; + int thr_spec_allowed; int carrier_migration_allowed; enum allctr_type atype; struct { @@ -217,7 +219,7 @@ typedef struct { struct au_init test_alloc; } erts_alc_hndl_args_init_t; -#define ERTS_AU_INIT__ {0, 0, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} +#define ERTS_AU_INIT__ {0, 0, 1, 1, 1, GOODFIT, DEFAULT_ALLCTR_INIT, {1,1,1,1}} #define SET_DEFAULT_ALLOC_OPTS(IP) \ do { \ @@ -294,6 +296,9 @@ set_default_literal_alloc_opts(struct au_init *ip) SET_DEFAULT_ALLOC_OPTS(ip); ip->enable = 1; ip->thr_spec = 0; + ip->disable_allowed = 0; + ip->thr_spec_allowed = 0; + ip->carrier_migration_allowed = 0; ip->atype = BESTFIT; ip->init.bf.ao = 1; ip->init.util.ramv = 0; @@ -665,11 +670,11 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) if (mlockall(MCL_CURRENT|MCL_FUTURE) != 0) { int err = errno; char *errstr = err ? strerror(err) : "unknown"; - erl_exit(-1, "Failed to lock physical memory: %s (%d)\n", + erts_exit(1, "Failed to lock physical memory: %s (%d)\n", errstr, err); } #else - erl_exit(-1, "Failed to lock physical memory: Not supported\n"); + erts_exit(1, "Failed to lock physical memory: Not supported\n"); #endif } @@ -769,9 +774,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) #if HAVE_ERTS_MSEG init.mseg.nos = erts_no_schedulers; erts_mseg_init(&init.mseg); -# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) - erts_mmap_init(&erts_literal_mmapper, &init.mseg.literal_mmap); -# endif #endif erts_alcu_init(&init.alloc_util); @@ -810,13 +812,13 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop) for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) { if (!erts_allctrs[i].alloc) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing alloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].realloc) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing realloc function for %s\n", ERTS_ALC_A2AD(i)); if (!erts_allctrs[i].free) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Missing free function for %s\n", ERTS_ALC_A2AD(i)); } @@ -887,7 +889,7 @@ erts_alloc_late_init(void) static void * erts_realloc_fixed_size(ErtsAlcType_t type, void *extra, void *p, Uint size) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Attempt to reallocate a block of the fixed size type %s\n", ERTS_ALC_T2TD(type)); } @@ -969,7 +971,7 @@ set_au_allocator(ErtsAlcType_t alctr_n, struct au_init *init, int ncpu) #endif { #ifdef ERTS_SMP - erl_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n", + erts_exit(ERTS_ABORT_EXIT, "%salloc is not thread safe\n", init->init.util.name_prefix); #else af->alloc = erts_alcu_alloc; @@ -1014,7 +1016,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, * tspec->size) + ERTS_CACHE_LINE_SIZE - 1)); if (!states) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to allocate allocator states for %salloc\n", init->init.util.name_prefix); tspec->allctr = (Allctr_t **) states; @@ -1042,7 +1044,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, (tot_fix_list_size + ERTS_CACHE_LINE_SIZE - 1)); if (!fix_lists) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to allocate fix lists for %salloc\n", init->init.util.name_prefix); @@ -1116,7 +1118,7 @@ start_au_allocator(ErtsAlcType_t alctr_n, } if (!as) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to start %salloc\n", init->init.util.name_prefix); ASSERT(as == (void *) as0); @@ -1352,9 +1354,17 @@ handle_au_arg(struct au_init *auip, else goto bad_switch; break; - case 'e': - auip->enable = get_bool_value(sub_param+1, argv, ip); + case 'e': { + int e = get_bool_value(sub_param + 1, argv, ip); + if (!auip->disable_allowed && !e) { + if (!u_switch) + bad_value(param, sub_param + 1, "false"); + else + ASSERT(auip->enable); /* ignore */ + } + else auip->enable = e; break; + } case 'l': if (has_prefix("lmbcs", sub_param)) { auip->default_.lmbcs = 0; @@ -1423,7 +1433,14 @@ handle_au_arg(struct au_init *auip, case 't': { int res = get_bool_value(sub_param+1, argv, ip); if (res > 0) { - auip->thr_spec = 1; + if (!auip->thr_spec_allowed) { + if (!u_switch) + bad_value(param, sub_param + 1, "true"); + else + ASSERT(!auip->thr_spec); /* ignore */ + } + else + auip->thr_spec = 1; break; } else if (res == 0) { @@ -1472,6 +1489,16 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) case 'B': handle_au_arg(&init->binary_alloc, &argv[i][3], argv, &i, 0); break; + case 'I': + if (has_prefix("scs", argv[i]+3)) { +#if HAVE_ERTS_MSEG + init->mseg.literal_mmap.scs = +#endif + get_mb_value(argv[i]+6, argv, &i); + } + else + handle_au_arg(&init->literal_alloc, &argv[i][3], argv, &i, 0); + break; case 'D': handle_au_arg(&init->std_alloc, &argv[i][3], argv, &i, 0); break; @@ -1908,7 +1935,7 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) case ERTS_ALC_O_FREE: op_str = "free"; break; default: op_str = "UNKNOWN"; break; } - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s: %s operation not supported (memory type: \"%s\")\n", allctr_str, op_str, t_str); break; @@ -1922,18 +1949,18 @@ erts_alc_fatal_error(int error, int func, ErtsAlcType_t n, ...) va_start(argp, n); size = va_arg(argp, Uint); va_end(argp); - erl_exit(-1, + erts_exit(1, "%s: Cannot %s %lu bytes of memory (of type \"%s\").\n", allctr_str, op, size, t_str); break; } case ERTS_ALC_E_NOALLCTR: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown allocator type: %d\n", ERTS_ALC_T2A(ERTS_ALC_N2T(n))); break; default: - erl_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error); + erts_exit(ERTS_ABORT_EXIT, "erts_alloc: Unknown error: %d\n", error); break; } } @@ -3120,7 +3147,7 @@ reply_alloc_info(void *vair) make_small(0), ainfo); } else { - erl_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: internal error\n", __FILE__, __LINE__); } } @@ -3354,7 +3381,7 @@ void *safe_realloc(void *ptr, Uint sz) * Keep alloc_SUITE_data/allocator_test.h updated if changes are made * * to erts_alc_test() * \* */ -#define ERTS_ALC_TEST_ABORT erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") +#define ERTS_ALC_TEST_ABORT erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n") UWord erts_alc_test(UWord op, UWord a1, UWord a2, UWord a3) { @@ -3819,7 +3846,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) found_type = GET_TYPE_OF_PATTERN(pre_pattern); if (pre_pattern != MK_PATTERN(n)) { if ((FIXED_FENCE_PATTERN_MASK & pre_pattern) != FIXED_FENCE_PATTERN) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence at beginning of memory block (p=0x%u) " "clobbered.\n", (UWord) ptr); @@ -3836,12 +3863,12 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) char *op_str; if ((FIXED_FENCE_PATTERN_MASK & post_pattern) != FIXED_FENCE_PATTERN) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence at end of memory block (p=0x%u, sz=%u) " "clobbered.\n", (UWord) ptr, (UWord) sz); if (found_type != GET_TYPE_OF_PATTERN(post_pattern)) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Fence around memory block (p=0x%u, sz=%u) " "clobbered.\n", (UWord) ptr, (UWord) sz); @@ -3864,7 +3891,7 @@ check_memory_fence(void *ptr, Uint *size, ErtsAlcType_t n, int func) default: op_str = "???"; break; } - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "ERROR: Memory block (p=0x%u, sz=%u) allocated as type \"%s\"," " but %s as type \"%s\".\n", (UWord) ptr, (UWord) sz, ftype, op_str, otype); diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 5f153ac0ab..2932adca84 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -362,6 +362,7 @@ type SCHED_WTIME_REQ SHORT_LIVED SYSTEM sched_wall_time_request type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap type MSACC DRIVER SYSTEM microstate_accounting +type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request # # Types used by system specific code diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 3230b6ef34..5e7dd7cce8 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -4718,7 +4718,7 @@ make_name_atoms(Allctr_t *allctr) size_t prefix_len = strlen(allctr->name_prefix); if (prefix_len > MAX_ATOM_CHARACTERS + sizeof(realloc) - 1) - erl_exit(1,"Too long allocator name: %salloc\n",allctr->name_prefix); + erts_exit(ERTS_ERROR_EXIT,"Too long allocator name: %salloc\n",allctr->name_prefix); memcpy((void *) buf, (void *) allctr->name_prefix, prefix_len); @@ -5912,7 +5912,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) /* erts_alcu_start assumes that allctr has been zeroed */ if (((UWord)allctr & ERTS_CRR_ALCTR_FLG_MASK) != 0) { - erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_alcu_start: Alignment error\n", __FILE__, __LINE__); } @@ -6128,7 +6128,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) if (allctr->thread_safe) erts_mtx_destroy(&allctr->mutex); #endif - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to create main carrier for %salloc\n", init->name_prefix); } @@ -6273,7 +6273,7 @@ erts_alcu_test(UWord op, UWord a1, UWord a2) case 0x019: return (UWord) PREV_BLK((Block_t *) a1); case 0x01a: return (UWord) IS_MBC_FIRST_BLK((Allctr_t*)a1, (Block_t *) a2); case 0x01b: return (UWord) sizeof(Unit_t); - case 0x01c: return (unsigned long) BLK_TO_MBC((Block_t*) a1); + case 0x01c: return (UWord) BLK_TO_MBC((Block_t*) a1); case 0x01d: ((Allctr_t*) a1)->add_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; case 0x01e: ((Allctr_t*) a1)->remove_mbc((Allctr_t*)a1, (Carrier_t*)a2); break; #ifdef ERTS_SMP @@ -6344,7 +6344,7 @@ erts_alcu_verify_unused(Allctr_t *allctr) if (no) { UWord sz = allctr->sbcs.blocks.curr.size; sz += allctr->mbcs.blocks.curr.size; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%salloc() used when expected to be unused!\n" "Total amount of blocks allocated: %bpu\n" "Total amount of bytes allocated: %bpu\n", diff --git a/erts/emulator/beam/erl_ao_firstfit_alloc.c b/erts/emulator/beam/erl_ao_firstfit_alloc.c index 19420af8ab..5cd067ff54 100644 --- a/erts/emulator/beam/erl_ao_firstfit_alloc.c +++ b/erts/emulator/beam/erl_ao_firstfit_alloc.c @@ -1035,7 +1035,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; diff --git a/erts/emulator/beam/erl_bestfit_alloc.c b/erts/emulator/beam/erl_bestfit_alloc.c index fb853b65ab..f39a18ac88 100644 --- a/erts/emulator/beam/erl_bestfit_alloc.c +++ b/erts/emulator/beam/erl_bestfit_alloc.c @@ -940,7 +940,7 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", __FILE__, __LINE__);; res = NIL; diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index 2b1d875bfe..f006856b6a 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1309,7 +1309,7 @@ static Eterm notify_when_loaded(Process *p, Eterm name_term, char *name, ErtsPro case ERL_DE_FORCE_RELOAD: break; default: - erl_exit(1,"Internal error, unknown state %u in dynamic driver.", drv->handle->status); + erts_exit(ERTS_ERROR_EXIT,"Internal error, unknown state %u in dynamic driver.", drv->handle->status); } p->flags |= F_USING_DDLL; r = add_monitor(p, drv->handle, ERL_DE_PROC_AWAIT_LOAD); diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index bc5c83e542..8c748c9bf7 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -66,6 +66,7 @@ static Export* gather_io_bytes_trap = NULL; static Export *gather_sched_wall_time_res_trap; static Export *gather_msacc_res_trap; static Export *gather_gc_info_res_trap; +static Export *gather_system_check_res_trap; #define DECL_AM(S) Eterm AM_ ## S = am_atom_put(#S, sizeof(#S) - 1) @@ -319,13 +320,11 @@ erts_print_system_version(int to, void *arg, Process *c_p) char *ov = otp_version; #ifdef ERTS_SMP Uint total, online, active; -#ifdef ERTS_DIRTY_SCHEDULERS Uint dirty_cpu, dirty_cpu_onln, dirty_io; - (void) erts_schedulers_state(&total, &online, &active, &dirty_cpu, &dirty_cpu_onln, &dirty_io, 0); -#else - (void) erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 0); -#endif + erts_schedulers_state(&total, &online, &active, + &dirty_cpu, &dirty_cpu_onln, NULL, + &dirty_io, NULL); #endif for (i = 0; i < sizeof(otp_version)-4; i++) { if (ov[i] == '-' && ov[i+1] == 'r' && ov[i+2] == 'c') @@ -931,7 +930,7 @@ BIF_RETTYPE process_info_1(BIF_ALIST_1) case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); } } @@ -971,7 +970,7 @@ BIF_RETTYPE process_info_2(BIF_ALIST_2) case ERTS_PI_FAIL_TYPE_AWAIT_EXIT: ERTS_BIF_AWAIT_X_DATA_TRAP(BIF_P, BIF_ARG_1, am_undefined); default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error", __FILE__, __LINE__); } } @@ -1726,7 +1725,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); if (intlist_to_buf(*tp, buf, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); buf[len] = '\0'; res = erts_instr_dump_memory_map(buf) ? am_true : am_false; erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -1745,7 +1744,7 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */ return res; buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1); if (intlist_to_buf(tp[1], buf, len) != len) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); buf[len] = '\0'; res = erts_instr_dump_stat(buf, 1) ? am_true : am_false; erts_free(ERTS_ALC_T_TMP, (void *) buf); @@ -2043,12 +2042,18 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) #ifndef ERTS_SMP BIF_RET(am_disabled); #else +#ifndef ERTS_DIRTY_SCHEDULERS if (erts_no_schedulers == 1) BIF_RET(am_disabled); - else { - BIF_RET(erts_is_multi_scheduling_blocked() - ? am_blocked - : am_enabled); + else +#endif + { + int msb = erts_is_multi_scheduling_blocked(); + BIF_RET(!msb + ? am_enabled + : (msb > 0 + ? am_blocked + : am_blocked_normal)); } #endif } else if (BIF_ARG_1 == am_build_type) { @@ -2517,77 +2522,120 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) res = TUPLE3(hp, make_small(1), make_small(1), make_small(1)); BIF_RET(res); #else + Eterm *hp; Uint total, online, active; - switch (erts_schedulers_state(&total, - &online, - &active, - NULL, - NULL, - NULL, - 1)) { - case ERTS_SCHDLR_SSPND_DONE: { - Eterm *hp = HAlloc(BIF_P, 4); - res = TUPLE3(hp, - make_small(total), - make_small(online), - make_small(active)); - BIF_RET(res); + erts_schedulers_state(&total, &online, &active, + NULL, NULL, NULL, NULL, NULL); + hp = HAlloc(BIF_P, 4); + res = TUPLE3(hp, + make_small(total), + make_small(online), + make_small(active)); + BIF_RET(res); +#endif + } else if (ERTS_IS_ATOM_STR("schedulers_state", BIF_ARG_1)) { +#ifndef ERTS_SMP + Eterm *hp = HAlloc(BIF_P, 4); + res = TUPLE3(hp, make_small(1), make_small(1), make_small(1)); + BIF_RET(res); +#else + Eterm *hp; + Uint total, online, active; + erts_schedulers_state(&total, &online, &active, + NULL, NULL, NULL, NULL, NULL); + hp = HAlloc(BIF_P, 4); + res = TUPLE3(hp, + make_small(total), + make_small(online), + make_small(active)); + BIF_RET(res); +#endif + } else if (ERTS_IS_ATOM_STR("all_schedulers_state", BIF_ARG_1)) { +#ifndef ERTS_SMP + Eterm *hp = HAlloc(BIF_P, 2+5); + res = CONS(hp+5, + TUPLE4(hp, + am_normal, + make_small(1), + make_small(1), + make_small(1)), + NIL); + BIF_RET(res); +#else + Eterm *hp, tpl; + Uint sz, total, online, active, + dirty_cpu_total, dirty_cpu_online, dirty_cpu_active, + dirty_io_total, dirty_io_active; + erts_schedulers_state(&total, &online, &active, + &dirty_cpu_total, &dirty_cpu_online, &dirty_cpu_active, + &dirty_io_total, &dirty_io_active); + + sz = 2+5; + if (dirty_cpu_total) + sz += 2+5; + if (dirty_io_total) + sz += 2+5; + + hp = HAlloc(BIF_P, sz); + + res = NIL; + if (dirty_io_total) { + tpl = TUPLE4(hp, + am_dirty_io, + make_small(dirty_io_total), + make_small(dirty_io_total), + make_small(dirty_io_active)); + hp += 5; + res = CONS(hp, tpl, res); + hp += 2; } - case ERTS_SCHDLR_SSPND_YIELD_RESTART: - ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP1(bif_export[BIF_system_info_1], - BIF_P, BIF_ARG_1); - default: - ASSERT(0); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); + if (dirty_cpu_total) { + tpl = TUPLE4(hp, + am_dirty_cpu, + make_small(dirty_cpu_total), + make_small(dirty_cpu_online), + make_small(dirty_cpu_active)); + hp += 5; + res = CONS(hp, tpl, res); + hp += 2; } + tpl = TUPLE4(hp, + am_normal, + make_small(total), + make_small(online), + make_small(active)); + hp += 5; + res = CONS(hp, tpl, res); + BIF_RET(res); #endif } else if (ERTS_IS_ATOM_STR("schedulers_online", BIF_ARG_1)) { #ifndef ERTS_SMP BIF_RET(make_small(1)); #else - Uint total, online, active; - switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) { - case ERTS_SCHDLR_SSPND_DONE: - BIF_RET(make_small(online)); - case ERTS_SCHDLR_SSPND_YIELD_RESTART: - ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP1(bif_export[BIF_system_info_1], - BIF_P, BIF_ARG_1); - default: - ASSERT(0); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); - } + Uint online; + erts_schedulers_state(NULL, &online, NULL, NULL, NULL, NULL, NULL, NULL); + BIF_RET(make_small(online)); #endif } else if (ERTS_IS_ATOM_STR("schedulers_active", BIF_ARG_1)) { #ifndef ERTS_SMP BIF_RET(make_small(1)); #else - Uint total, online, active; - switch (erts_schedulers_state(&total, &online, &active, NULL, NULL, NULL, 1)) { - case ERTS_SCHDLR_SSPND_DONE: - BIF_RET(make_small(active)); - case ERTS_SCHDLR_SSPND_YIELD_RESTART: - ERTS_VBUMP_ALL_REDS(BIF_P); - BIF_TRAP1(bif_export[BIF_system_info_1], - BIF_P, BIF_ARG_1); - default: - ASSERT(0); - BIF_ERROR(BIF_P, EXC_INTERNAL_ERROR); - } + Uint active; + erts_schedulers_state(NULL, NULL, &active, NULL, NULL, NULL, NULL, NULL); + BIF_RET(make_small(active)); #endif #if defined(ERTS_SMP) && defined(ERTS_DIRTY_SCHEDULERS) } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers", BIF_ARG_1)) { Uint dirty_cpu; - erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, 1); + erts_schedulers_state(NULL, NULL, NULL, &dirty_cpu, NULL, NULL, NULL, NULL); BIF_RET(make_small(dirty_cpu)); } else if (ERTS_IS_ATOM_STR("dirty_cpu_schedulers_online", BIF_ARG_1)) { Uint dirty_cpu_onln; - erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, 1); + erts_schedulers_state(NULL, NULL, NULL, NULL, &dirty_cpu_onln, NULL, NULL, NULL); BIF_RET(make_small(dirty_cpu_onln)); } else if (ERTS_IS_ATOM_STR("dirty_io_schedulers", BIF_ARG_1)) { Uint dirty_io; - erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, &dirty_io, 1); + erts_schedulers_state(NULL, NULL, NULL, NULL, NULL, NULL, &dirty_io, NULL); BIF_RET(make_small(dirty_io)); #endif } else if (ERTS_IS_ATOM_STR("run_queues", BIF_ARG_1)) { @@ -2641,7 +2689,16 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) if (erts_no_schedulers == 1) BIF_RET(NIL); else - BIF_RET(erts_multi_scheduling_blockers(BIF_P)); + BIF_RET(erts_multi_scheduling_blockers(BIF_P, 0)); +#endif + } else if (ERTS_IS_ATOM_STR("normal_multi_scheduling_blockers", BIF_ARG_1)) { +#ifndef ERTS_SMP + BIF_RET(NIL); +#else + if (erts_no_schedulers == 1) + BIF_RET(NIL); + else + BIF_RET(erts_multi_scheduling_blockers(BIF_P, 1)); #endif } else if (ERTS_IS_ATOM_STR("modified_timing_level", BIF_ARG_1)) { BIF_RET(ERTS_USE_MODIFIED_TIMING() @@ -3786,6 +3843,18 @@ BIF_RETTYPE erts_internal_is_system_process_1(BIF_ALIST_1) BIF_ERROR(BIF_P, BADARG); } +BIF_RETTYPE erts_internal_system_check_1(BIF_ALIST_1) +{ + Eterm res; + if (ERTS_IS_ATOM_STR("schedulers", BIF_ARG_1)) { + res = erts_system_check_request(BIF_P); + if (is_non_value(res)) + BIF_RET(am_undefined); + BIF_TRAP1(gather_system_check_res_trap, BIF_P, res); + } + + BIF_ERROR(BIF_P, BADARG); +} static erts_smp_atomic_t hipe_test_reschedule_flag; @@ -3800,7 +3869,7 @@ static void broken_halt_test(Eterm bif_arg_2) #if defined(ERTS_HAVE_TRY_CATCH) erts_get_scheduler_data()->run_queue = NULL; #endif - erl_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2); + erts_exit(ERTS_DUMP_EXIT, "%T", bif_arg_2); } @@ -4045,7 +4114,7 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) BIF_RET(am_true); } else if (ERTS_IS_ATOM_STR("abort", BIF_ARG_1)) { - erl_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2); + erts_exit(ERTS_ABORT_EXIT, "%T\n", BIF_ARG_2); } else if (ERTS_IS_ATOM_STR("kill_dist_connection", BIF_ARG_1)) { DistEntry *dep = erts_sysname_to_connected_dist_entry(BIF_ARG_2); @@ -4407,7 +4476,8 @@ erts_bif_info_init(void) = erts_export_put(am_erts_internal, am_gather_io_bytes, 2); gather_msacc_res_trap = erts_export_put(am_erts_internal, am_gather_microstate_accounting_result, 2); - + gather_system_check_res_trap + = erts_export_put(am_erts_internal, am_gather_system_check_result, 1); process_info_init(); os_info_init(); } diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 3acc1d7bbd..86bcf43678 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -823,7 +823,7 @@ open_port(Process* p, Eterm name, Eterm settings, int *err_typep, int *err_nump) } else { name_buf = (char *) erts_alloc(ERTS_ALC_T_TMP, i + 1); if (intlist_to_buf(name, name_buf, i) != i) - erl_exit(1, "%s:%d: Internal error\n", __FILE__, __LINE__); + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); name_buf[i] = '\0'; } driver = &vanilla_driver; @@ -1201,7 +1201,7 @@ static Eterm http_bld_uri(struct packet_callback_args* pca, return erts_bld_tuple(hpp, szp, 3, am_scheme, s1, s2); default: - erl_exit(1, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type); + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: type=%u\n", __FILE__, __LINE__, uri->type); } } diff --git a/erts/emulator/beam/erl_cpu_topology.c b/erts/emulator/beam/erl_cpu_topology.c index 8395f6ecc6..c263eebfcc 100644 --- a/erts/emulator/beam/erl_cpu_topology.c +++ b/erts/emulator/beam/erl_cpu_topology.c @@ -402,7 +402,7 @@ cpu_bind_order_sort(erts_cpu_topology_t *cpudata, break; default: cmp_func = NULL; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Bad cpu bind type: %d\n", (int) cpu_bind_order); break; @@ -1590,7 +1590,7 @@ get_cpu_topology_term(Process *c_p, int type) } break; default: - erl_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); + erts_exit(ERTS_ABORT_EXIT, "Bad cpu topology type: %d\n", type); break; } @@ -1967,7 +1967,7 @@ cpu_group_insert(erts_cpu_groups_map_t *map, ix = 0; } while (ix != start); - erl_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); + erts_exit(ERTS_ABORT_EXIT, "Reader groups map full\n"); } @@ -2290,7 +2290,7 @@ remove_cpu_groups(erts_cpu_groups_callback_t callback, void *arg) prev_cgm = cgm; } - erl_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); + erts_exit(ERTS_ABORT_EXIT, "Cpu groups not found\n"); } static int @@ -2320,7 +2320,7 @@ cpu_groups_lookup(erts_cpu_groups_map_t *map, ix = 0; } while (ix != start); - erl_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); + erts_exit(ERTS_ABORT_EXIT, "Logical cpu id %d not found\n", logical); } static void diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 55b8789e29..441dbdde4f 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1295,7 +1295,7 @@ BIF_RETTYPE ets_rename_2(BIF_ALIST_2) goto badarg; if (!remove_named_tab(tb, 1)) - erl_exit(1,"Could not find named tab %s", tb->common.id); + erts_exit(ERTS_ERROR_EXIT,"Could not find named tab %s", tb->common.id); tb->common.id = tb->common.the_name = BIF_ARG_2; @@ -1580,7 +1580,7 @@ BIF_RETTYPE ets_new_2(BIF_ALIST_2) BIF_P->common.id, make_small(slot)), 0) != DB_ERROR_NONE) { - erl_exit(1,"Could not update ets metadata."); + erts_exit(ERTS_ERROR_EXIT,"Could not update ets metadata."); } db_meta_unlock(meta_pid_to_tab, LCK_WRITE_REC); @@ -2933,7 +2933,7 @@ void init_db(ErtsDbSpinCount db_spin_count) bits = erts_fit_in_bits_int32(db_max_tabs-1); if (bits > SMALL_BITS) { - erl_exit(1,"Max limit for ets tabled too high %u (max %u).", + erts_exit(ERTS_ERROR_EXIT,"Max limit for ets tabled too high %u (max %u).", db_max_tabs, ((Uint)1)<<SMALL_BITS); } meta_main_tab_slot_mask = (((Uint)1)<<bits) - 1; @@ -2994,7 +2994,7 @@ void init_db(ErtsDbSpinCount db_spin_count) db_init_lock(meta_pid_to_tab, "meta_pid_to_tab", "meta_pid_to_tab_FIX");*/ if (db_create_hash(NULL, meta_pid_to_tab) != DB_ERROR_NONE) { - erl_exit(1,"Unable to create ets metadata tables."); + erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); } erts_smp_atomic_set_nob(&init_tb.common.memory_size, 0); @@ -3025,7 +3025,7 @@ void init_db(ErtsDbSpinCount db_spin_count) db_init_lock(meta_pid_to_fixed_tab, "meta_pid_to_fixed_tab", "meta_pid_to_fixed_tab_FIX");*/ if (db_create_hash(NULL, meta_pid_to_fixed_tab) != DB_ERROR_NONE) { - erl_exit(1,"Unable to create ets metadata tables."); + erts_exit(ERTS_ERROR_EXIT,"Unable to create ets metadata tables."); } /* Non visual BIF to trap to. */ @@ -3222,7 +3222,7 @@ retry: * yielding. */ #define ERTS_DB_INTERNAL_ERROR(LSTR) \ - erl_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_db_process_exiting(): " LSTR "\n", \ __FILE__, __LINE__) int @@ -3494,7 +3494,7 @@ static void fix_table_locked(Process* p, DbTable* tb) make_small(tb->common.slot)), 0) != DB_ERROR_NONE) { UnUseTmpHeap(3,p); - erl_exit(1,"Could not insert ets metadata in safe_fixtable."); + erts_exit(ERTS_ERROR_EXIT,"Could not insert ets metadata in safe_fixtable."); } UnUseTmpHeap(3,p); db_meta_unlock(meta_pid_to_fixed_tab, LCK_WRITE_REC); diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index cff65f244d..240b79fc60 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -3022,7 +3022,7 @@ void db_check_table_hash(DbTable *tbl) if ((list = BUCKET(tb,j)) != 0) { while (list != 0) { if (!is_tuple(make_tuple(list->dbterm.tpl))) { - erl_exit(1, "Bad term in slot %d of ets table", j); + erts_exit(ERTS_ERROR_EXIT, "Bad term in slot %d of ets table", j); } list = list->next; } diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 311b69d114..0f642f0867 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -1204,7 +1204,7 @@ static int db_select_count_continue_tree(Process *p, tptr = tuple_val(continuation); if (arityval(*tptr) != 5) - erl_exit(1,"Internal error in ets:select_count/1"); + erts_exit(ERTS_ERROR_EXIT,"Internal error in ets:select_count/1"); lastkey = tptr[2]; end_condition = tptr[3]; diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 399be6058c..973be58420 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1862,11 +1862,11 @@ restart: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } @@ -2495,7 +2495,7 @@ restart: case matchHalt: goto success; default: - erl_exit(1, "Internal error: unexpected opcode in match program."); + erts_exit(ERTS_ERROR_EXIT, "Internal error: unexpected opcode in match program."); } } fail: @@ -2512,11 +2512,11 @@ success: #ifdef DMC_DEBUG if (*heap_fence != FENCE_PATTERN) { - erl_exit(1, "Heap fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Heap fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *heap_fence); } if (*stack_fence != FENCE_PATTERN) { - erl_exit(1, "Stack fence overwritten in db_prog_match after op " + erts_exit(ERTS_ERROR_EXIT, "Stack fence overwritten in db_prog_match after op " "0x%08x, overwritten with 0x%08x.", save_op, *stack_fence); } @@ -3448,7 +3448,7 @@ static DMCRet dmc_one_term(DMCContext *context, break; } default: - erl_exit(1, "db_match_compile: " + erts_exit(ERTS_ERROR_EXIT, "db_match_compile: " "Bad object on heap: 0x%bex\n", c); } return retOk; @@ -4679,7 +4679,7 @@ static DMCRet dmc_fun(DMCContext *context, DMC_PUSH(*text, matchCall3); break; default: - erl_exit(1,"ets:match() internal error, " + erts_exit(ERTS_ERROR_EXIT,"ets:match() internal error, " "guard with more than 3 arguments."); } DMC_PUSH(*text, (UWord) b->biff); @@ -4959,7 +4959,7 @@ static Uint my_size_object(Eterm t) tmp == am_const) { sum += size_object(tuple_val(t)[2]); } else { - erl_exit(1,"Internal error, sizing unrecognized object in " + erts_exit(ERTS_ERROR_EXIT,"Internal error, sizing unrecognized object in " "(d)ets:match compilation."); } break; @@ -5004,7 +5004,7 @@ static Eterm my_copy_struct(Eterm t, Eterm **hp, ErlOffHeap* off_heap) sz = size_object(b); ret = copy_struct(b,sz,hp,off_heap); } else { - erl_exit(1, "Trying to constant-copy non constant expression " + erts_exit(ERTS_ERROR_EXIT, "Trying to constant-copy non constant expression " "0x%bex in (d)ets:match compilation.", t); } } else { diff --git a/erts/emulator/beam/erl_debug.c b/erts/emulator/beam/erl_debug.c index 7b13cf2c83..2902c98864 100644 --- a/erts/emulator/beam/erl_debug.c +++ b/erts/emulator/beam/erl_debug.c @@ -255,14 +255,14 @@ void erts_check_stack(Process *p) Eterm *stack_end = p->htop; if (p->stop > stack_start) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Stack underflow\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), internal_pid_serial(p->common.id)); if (p->stop < stack_end) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Stack overflow\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), @@ -287,7 +287,7 @@ void erts_check_stack(Process *p) if (in_mbuf) continue; - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "<%lu.%lu.%lu>: Wild stack pointer\n", internal_pid_channel_no(p->common.id), internal_pid_number(p->common.id), @@ -374,7 +374,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end) #ifdef DEBUG if (hval == DEBUG_BAD_WORD) { print_untagged_memory(start, end); - erl_exit(1, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n", + erts_exit(ERTS_ERROR_EXIT, "Uninitialized HAlloc'ed memory found @ 0x%0*lx!\n", PTR_SIZE,(unsigned long)(pos - 1)); } #endif @@ -387,7 +387,7 @@ void erts_check_memory(Process *p, Eterm *start, Eterm *end) if (verify_eterm(p,hval)) continue; - erl_exit(1, "Wild pointer found @ 0x%0*lx!\n", + erts_exit(ERTS_ERROR_EXIT, "Wild pointer found @ 0x%0*lx!\n", PTR_SIZE,(unsigned long)(pos - 1)); } } @@ -397,11 +397,11 @@ void verify_process(Process *p) #define VERIFY_AREA(name,ptr,sz) { \ int n = (sz); \ while (n--) if(!verify_eterm(p,*(ptr+n))) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } + erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); } #define VERIFY_ETERM(name,eterm) { \ if(!verify_eterm(p,eterm)) \ - erl_exit(1,"Wild pointer found in " name " of %T!\n",p->common.id); } + erts_exit(ERTS_ERROR_EXIT,"Wild pointer found in " name " of %T!\n",p->common.id); } ErtsMessage* mp = p->msg.first; diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index e0404eb5c9..184f8e8931 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -43,7 +43,7 @@ fatal_error(int err, char *func) else estr = "Unknown error"; } - erl_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err); + erts_exit(ERTS_ABORT_EXIT, "Fatal error in %s: %s [%d]\n", func, estr, err); } #define ERL_DRV_TSD_KEYS_INC 10 diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index cff476694c..ed555aa92f 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -188,7 +188,7 @@ erts_erase_fun_entry(ErlFunEntry* fe) #endif { if (fe->address != unloaded_fun) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Internal error: " "Invalid reference count found on #Fun<%T.%d.%d>: " " About to erase fun still referred by code.\n", diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 21b03ae8bd..0d125c5598 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -70,7 +70,7 @@ erts_fprintf(stderr, "stop=%p\n", (p)->stop); \ erts_fprintf(stderr, "htop=%p\n", (p)->htop); \ erts_fprintf(stderr, "heap=%p\n", (p)->heap); \ - erl_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s, line %d: %T: Overrun stack and heap\n", \ __FILE__,__LINE__,(P)->common.id); \ } @@ -289,7 +289,7 @@ erts_next_heap_size(Uint size, Uint offset) low = mid + 1; } } - erl_exit(1, "no next heap size found: %beu, offset %beu\n", size, offset); + erts_exit(ERTS_ERROR_EXIT, "no next heap size found: %beu, offset %beu\n", size, offset); } return 0; } @@ -3067,7 +3067,7 @@ within(Eterm *ptr, Process *p) #define ERTS_CHK_OFFHEAP_ASSERT(EXP) \ do { \ if (!(EXP)) \ - erl_exit(ERTS_ABORT_EXIT, \ + erts_exit(ERTS_ABORT_EXIT, \ "%s:%d: Assertion failed: %s\n", \ __FILE__, __LINE__, #EXP); \ } while (0) diff --git a/erts/emulator/beam/erl_goodfit_alloc.c b/erts/emulator/beam/erl_goodfit_alloc.c index f89f8723d9..9b4aad9d91 100644 --- a/erts/emulator/beam/erl_goodfit_alloc.c +++ b/erts/emulator/beam/erl_goodfit_alloc.c @@ -571,8 +571,8 @@ info_options(Allctr_t *allctr, if (hpp || szp) { if (!atoms_initialized) - erl_exit(1, "%s:%d: Internal error: Atoms not initialized", - __FILE__, __LINE__);; + erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error: Atoms not initialized", + __FILE__, __LINE__); res = NIL; add_2tup(hpp, szp, &res, am.as, am.gf); diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 67089df863..86c4eeb484 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -264,7 +264,7 @@ this_rel_num(void) i++; this_rel = atoi(&this_rel_str[i]); if (this_rel < 1) - erl_exit(-1, "Unexpected ERLANG_OTP_RELEASE format\n"); + erts_exit(1, "Unexpected ERLANG_OTP_RELEASE format\n"); } return this_rel; } @@ -412,7 +412,7 @@ erl_first_process_otp(char* modname, void* code, unsigned size, int argc, char** start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); if (erts_find_function(start_mod, am_start, 2, erts_active_code_ix()) == NULL) { - erl_exit(5, "No function %s:start/2\n", modname); + erts_exit(ERTS_ERROR_EXIT, "No function %s:start/2\n", modname); } /* @@ -451,7 +451,7 @@ erl_system_process_otp(Eterm parent_pid, char* modname) start_mod = erts_atom_put((byte *) modname, sys_strlen(modname), ERTS_ATOM_ENC_LATIN1, 1); if (erts_find_function(start_mod, am_start, 0, erts_active_code_ix()) == NULL) { - erl_exit(5, "No function %s:start/0\n", modname); + erts_exit(ERTS_ERROR_EXIT, "No function %s:start/0\n", modname); } parent = erts_pid2proc(NULL, 0, parent_pid, ERTS_PROC_LOCK_MAIN); @@ -532,12 +532,12 @@ load_preloaded(void) length = preload_p[i].size; module_name = erts_atom_put((byte *) name, sys_strlen(name), ERTS_ATOM_ENC_LATIN1, 1); if ((code = sys_preload_begin(&preload_p[i])) == 0) - erl_exit(1, "Failed to find preloaded code for module %s\n", + erts_exit(ERTS_ERROR_EXIT, "Failed to find preloaded code for module %s\n", name); res = erts_preload_module(NULL, 0, NIL, &module_name, code, length); sys_preload_end(&preload_p[i]); if (res != NIL) - erl_exit(1,"Failed loading preloaded module %s (%T)\n", + erts_exit(ERTS_ERROR_EXIT,"Failed loading preloaded module %s (%T)\n", name, res); i++; } @@ -670,7 +670,7 @@ void erts_usage(void) erts_fprintf(stderr, "Note that if the emulator is started with erlexec (typically\n"); erts_fprintf(stderr, "from the erl script), these flags should be specified with +.\n"); erts_fprintf(stderr, "\n\n"); - erl_exit(-1, ""); + erts_exit(1, ""); } #ifdef USE_THREADS @@ -1472,7 +1472,7 @@ erl_start(int argc, char **argv) } erts_fprintf(stderr, "(" EMULATOR ") emulator version " ERLANG_VERSION "\n"); - erl_exit(0, ""); + erts_exit(0, ""); } break; @@ -2304,23 +2304,17 @@ system_cleanup(int flush_async) } static __decl_noreturn void __noreturn -erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) +erts_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) { - unsigned int an; - system_cleanup(flush_async); save_statistics(); - if (n < 0) - an = -(unsigned int)n; - else - an = n; if (erts_mtrace_enabled) - erts_mtrace_exit((Uint32) an); + erts_mtrace_exit((Uint32) n); /* Produce an Erlang core dump if error */ - if (((n > 0 && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) + if (((n == ERTS_ERROR_EXIT && erts_no_crash_dump == 0) || n == ERTS_DUMP_EXIT) && erts_initialized) { erl_crash_dump_v((char*) NULL, 0, fmt, args1); } @@ -2333,29 +2327,29 @@ erl_exit_vv(int n, int flush_async, char *fmt, va_list args1, va_list args2) exit(0); else if (n == ERTS_DUMP_EXIT) ERTS_EXIT_AFTER_DUMP(1); - else if (n > 0 || n == ERTS_ABORT_EXIT) + else if (n == ERTS_ERROR_EXIT || n == ERTS_ABORT_EXIT) abort(); - exit(an); + exit(n); } /* Exit without flushing async threads */ -__decl_noreturn void __noreturn erl_exit(int n, char *fmt, ...) +__decl_noreturn void __noreturn erts_exit(int n, char *fmt, ...) { va_list args1, args2; va_start(args1, fmt); va_start(args2, fmt); - erl_exit_vv(n, 0, fmt, args1, args2); + erts_exit_vv(n, 0, fmt, args1, args2); va_end(args2); va_end(args1); } /* Exit after flushing async threads */ -__decl_noreturn void __noreturn erl_exit_flush_async(int n, char *fmt, ...) +__decl_noreturn void __noreturn erts_flush_async_exit(int n, char *fmt, ...) { va_list args1, args2; va_start(args1, fmt); va_start(args2, fmt); - erl_exit_vv(n, 1, fmt, args1, args2); + erts_exit_vv(n, 1, fmt, args1, args2); va_end(args2); va_end(args1); } diff --git a/erts/emulator/beam/erl_map.c b/erts/emulator/beam/erl_map.c index d0ffb11e79..4b6931f848 100644 --- a/erts/emulator/beam/erl_map.c +++ b/erts/emulator/beam/erl_map.c @@ -1270,7 +1270,7 @@ recurse: break; } default: - erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA); + erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrA); } } @@ -1297,7 +1297,7 @@ recurse: break; } default: - erl_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB); + erts_exit(ERTS_ABORT_EXIT, "bad header %ld\r\n", hdrB); } } } @@ -1387,7 +1387,7 @@ resume_from_trap: res = make_boxed(nhp); break; default: - erl_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix); + erts_exit(ERTS_ABORT_EXIT, "strange mix %d\r\n", sp->mix); } } @@ -1882,7 +1882,7 @@ void hashmap_iterator_init(ErtsWStack* s, Eterm node, int reverse) { sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: - erl_exit(ERTS_ABORT_EXIT, "bad header"); + erts_exit(ERTS_ABORT_EXIT, "bad header"); } WSTACK_PUSH3((*s), (UWord)THE_NON_VALUE, /* end marker */ @@ -1919,7 +1919,7 @@ Eterm* hashmap_iterator_next(ErtsWStack* s) { ASSERT(sz < 17); break; default: - erl_exit(ERTS_ABORT_EXIT, "bad header"); + erts_exit(ERTS_ABORT_EXIT, "bad header"); } idx++; @@ -1969,7 +1969,7 @@ Eterm* hashmap_iterator_prev(ErtsWStack* s) { ASSERT(sz < 17); break; default: - erl_exit(1, "bad header"); + erts_exit(ERTS_ERROR_EXIT, "bad header"); } if (idx > sz) @@ -2153,12 +2153,12 @@ int erts_hashmap_insert_down(Uint32 hx, Eterm key, Eterm node, Uint *sz, size += HAMT_HEAD_BITMAP_SZ(n+1); goto unroll; default: - erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); break; } break; default: - erl_exit(1, "bad primary tag %p\r\n", node); + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node); break; } } @@ -2273,12 +2273,12 @@ Eterm erts_hashmap_insert_up(Eterm *hp, Eterm key, Eterm value, res = make_hashmap(nhp); break; default: - erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); break; } break; default: - erl_exit(1, "bad primary tag %x\r\n", primary_tag(node)); + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %x\r\n", primary_tag(node)); break; } @@ -2396,12 +2396,12 @@ static Eterm hashmap_delete(Process *p, Uint32 hx, Eterm key, Eterm map) { /* not occupied */ goto not_found; default: - erl_exit(1, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + erts_exit(ERTS_ERROR_EXIT, "bad header tag %ld\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); break; } break; default: - erl_exit(1, "bad primary tag %p\r\n", node); + erts_exit(ERTS_ERROR_EXIT, "bad primary tag %p\r\n", node); break; } } @@ -2578,7 +2578,7 @@ unroll: res = make_hashmap(nhp); break; default: - erl_exit(1, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); + erts_exit(ERTS_ERROR_EXIT, "bad header tag %x\r\n", hdr & _HEADER_MAP_SUBTAG_MASK); break; } } while(!ESTACK_ISEMPTY(stack)); @@ -2728,7 +2728,7 @@ BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) { case MAP_HEADER_TAG_HAMT_NODE_BITMAP : BIF_RET(ERTS_MAKE_AM("hashmap_node")); default: - erl_exit(ERTS_ABORT_EXIT, "term_type: bad map header type %d\n", MAP_HEADER_TYPE(hdr)); + erts_exit(ERTS_ABORT_EXIT, "term_type: bad map header type %d\n", MAP_HEADER_TYPE(hdr)); } case REFC_BINARY_SUBTAG: BIF_RET(ERTS_MAKE_AM("refc_binary")); @@ -2752,7 +2752,7 @@ BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) { case FLOAT_SUBTAG: BIF_RET(ERTS_MAKE_AM("hfloat")); default: - erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", hdr); + erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", hdr); } } case TAG_PRIMARY_IMMED1: @@ -2772,13 +2772,13 @@ BIF_RETTYPE erts_internal_term_type_1(BIF_ALIST_1) { case _TAG_IMMED2_NIL: BIF_RET(ERTS_MAKE_AM("nil")); default: - erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); + erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); } default: - erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); + erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); } default: - erl_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); + erts_exit(ERTS_ABORT_EXIT, "term_type: Invalid tag (0x%X)\n", obj); } } @@ -2811,7 +2811,7 @@ BIF_RETTYPE erts_internal_map_hashmap_children_1(BIF_ALIST_1) { ptr += 2; break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); break; } ASSERT(sz < 17); @@ -2889,7 +2889,7 @@ static Eterm hashmap_info(Process *p, Eterm node) { } break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); break; } } diff --git a/erts/emulator/beam/erl_monitors.c b/erts/emulator/beam/erl_monitors.c index 7dfa01c8ac..bd899fa2f9 100644 --- a/erts/emulator/beam/erl_monitors.c +++ b/erts/emulator/beam/erl_monitors.c @@ -356,7 +356,7 @@ void erts_add_monitor(ErtsMonitor **root, Uint type, Eterm ref, Eterm pid, tstack[tpos++] = this; this = &((*this)->right); } else { /* Equal key is an error for monitors */ - erl_exit(1,"Insertion of already present monitor!"); + erts_exit(ERTS_ERROR_EXIT,"Insertion of already present monitor!"); break; } } diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 2204231748..8580bc2689 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -331,7 +331,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, #ifdef ERTS_SMP c_p = NULL; #else - erl_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); + erts_exit(ERTS_ABORT_EXIT,"enif_send: env==NULL on non-SMP VM"); #endif } diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 13f14adbab..8617f42d7b 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -1053,7 +1053,7 @@ insert_dist_entry(DistEntry *dist, int type, Eterm id, Uint creation) } if(!rdp) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing distribution table entry found!\n"); insert_dist_referrer(rdp, type, id, creation); @@ -1114,7 +1114,7 @@ insert_node(ErlNode *node, int type, Eterm id) } if (!rnp) - erl_exit(1, "Reference to non-existing node table entry found!\n"); + erts_exit(ERTS_ERROR_EXIT, "Reference to non-existing node table entry found!\n"); insert_node_referrer(rnp, type, id); } diff --git a/erts/emulator/beam/erl_port_task.c b/erts/emulator/beam/erl_port_task.c index 197b328fe2..e85395e062 100644 --- a/erts/emulator/beam/erl_port_task.c +++ b/erts/emulator/beam/erl_port_task.c @@ -1786,7 +1786,7 @@ erts_port_task_execute(ErtsRunQueue *runq, Port **curr_port_pp) reds = erts_dist_command(pp, CONTEXT_REDS - pp->reds); break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); break; @@ -2051,7 +2051,7 @@ begin_port_cleanup(Port *pp, ErtsPortTask **execqp, int *processing_busy_q_p) break; } default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid port task type: %d\n", (int) ptp->type); } diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index fd29c22810..a706ebf595 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -153,10 +153,8 @@ int ERTS_WRITE_UNLIKELY(erts_eager_check_io) = 1; int ERTS_WRITE_UNLIKELY(erts_sched_compact_load); int ERTS_WRITE_UNLIKELY(erts_sched_balance_util) = 0; Uint ERTS_WRITE_UNLIKELY(erts_no_schedulers); -#ifdef ERTS_DIRTY_SCHEDULERS -Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers); -Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers); -#endif +Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_cpu_schedulers) = 0; +Uint ERTS_WRITE_UNLIKELY(erts_no_dirty_io_schedulers) = 0; static char *erts_aux_work_flag_descr[ERTS_SSI_AUX_WORK_NO_FLAGS] = {0}; int erts_aux_work_no_flags = ERTS_SSI_AUX_WORK_NO_FLAGS; @@ -188,84 +186,176 @@ int erts_disable_proc_not_running_opt; static ErtsAuxWorkData *aux_thread_aux_work_data; -#define ERTS_SCHDLR_SSPND_CHNG_WAITER (((erts_aint32_t) 1) << 0) +#define ERTS_SCHDLR_SSPND_CHNG_NMSB (((erts_aint32_t) 1) << 0) #define ERTS_SCHDLR_SSPND_CHNG_MSB (((erts_aint32_t) 1) << 1) #define ERTS_SCHDLR_SSPND_CHNG_ONLN (((erts_aint32_t) 1) << 2) +#define ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN (((erts_aint32_t) 1) << 3) -#ifndef DEBUG +typedef enum { + ERTS_SCHED_NORMAL, + ERTS_SCHED_DIRTY_CPU, + ERTS_SCHED_DIRTY_IO +} ErtsSchedType; -#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, (VAL)) +typedef struct { + int ongoing; + ErtsProcList *blckrs; + ErtsProcList *chngq; +} ErtsMultiSchedulingBlock; -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_cpu_changing, (VAL)) -#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \ - erts_smp_atomic32_set_nob(&schdlr_sspnd.dirty_io_changing, (VAL)) -#endif +static struct { + erts_smp_mtx_t mtx; + Uint32 online; + Uint32 curr_online; + Uint32 active; + erts_smp_atomic32_t changing; + ErtsProcList *chngq; + Eterm changer; + ErtsMultiSchedulingBlock nmsb; /* Normal multi Scheduling Block */ + ErtsMultiSchedulingBlock msb; /* Multi Scheduling Block */ +} schdlr_sspnd; -#else +#define ERTS_SCHDLR_SSPND_S_BITS 10 +#define ERTS_SCHDLR_SSPND_DCS_BITS 11 +#define ERTS_SCHDLR_SSPND_DIS_BITS 11 -#define ERTS_SCHDLR_SSPND_CHNG_SET(VAL, OLD_VAL) \ -do { \ - erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.changing, \ - (VAL)); \ - ASSERT(old_val__ == (OLD_VAL)); \ -} while (0) +#define ERTS_SCHDLR_SSPND_S_MASK ((1 << ERTS_SCHDLR_SSPND_S_BITS)-1) +#define ERTS_SCHDLR_SSPND_DCS_MASK ((1 << ERTS_SCHDLR_SSPND_DCS_BITS)-1) +#define ERTS_SCHDLR_SSPND_DIS_MASK ((1 << ERTS_SCHDLR_SSPND_DIS_BITS)-1) -#ifdef ERTS_DIRTY_SCHEDULERS -#define ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(VAL, OLD_VAL) \ -do { \ - erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_cpu_changing, \ - (VAL)); \ - ASSERT(old_val__ == (OLD_VAL)); \ -} while (0) -#define ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(VAL, OLD_VAL) \ -do { \ - erts_aint32_t old_val__; \ - old_val__ = erts_smp_atomic32_xchg_nob(&schdlr_sspnd.dirty_io_changing, \ - (VAL)); \ - ASSERT(old_val__ == (OLD_VAL)); \ -} while (0) -#endif +#define ERTS_SCHDLR_SSPND_S_SHIFT 0 +#define ERTS_SCHDLR_SSPND_DCS_SHIFT (ERTS_SCHDLR_SSPND_S_SHIFT \ + + ERTS_SCHDLR_SSPND_S_BITS) +#define ERTS_SCHDLR_SSPND_DIS_SHIFT (ERTS_SCHDLR_SSPND_DCS_SHIFT \ + + ERTS_SCHDLR_SSPND_DCS_BITS) +#if (ERTS_SCHDLR_SSPND_S_BITS \ + + ERTS_SCHDLR_SSPND_DCS_BITS \ + + ERTS_SCHDLR_SSPND_DIS_BITS) > 32 +# error Wont fit in Uint32 #endif - -static struct { - erts_smp_mtx_t mtx; - erts_smp_cnd_t cnd; - int online; - int curr_online; - int wait_curr_online; -#ifdef ERTS_DIRTY_SCHEDULERS - int dirty_cpu_online; - int dirty_cpu_curr_online; - int dirty_cpu_wait_curr_online; - int dirty_io_online; - int dirty_io_curr_online; - int dirty_io_wait_curr_online; +#if (ERTS_MAX_NO_OF_SCHEDULERS-1) > ERTS_SCHDLR_SSPND_S_MASK +# error Max no schedulers wont fit in its bit-field #endif - erts_smp_atomic32_t changing; - erts_smp_atomic32_t active; -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_t dirty_cpu_changing; - erts_smp_atomic32_t dirty_cpu_active; - erts_smp_atomic32_t dirty_io_changing; - erts_smp_atomic32_t dirty_io_active; +#if ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS > ERTS_SCHDLR_SSPND_DCS_MASK +# error Max no dirty cpu schedulers wont fit in its bit-field #endif - struct { - int ongoing; - long wait_active; -#ifdef ERTS_DIRTY_SCHEDULERS - long dirty_cpu_wait_active; - long dirty_io_wait_active; +#if ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS > ERTS_SCHDLR_SSPND_DIS_MASK +# error Max no dirty io schedulers wont fit in its bit-field #endif - ErtsProcList *procs; - } msb; /* Multi Scheduling Block */ -} schdlr_sspnd; + +#define ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(S, DCS, DIS) \ + ((((Uint32) (((S) & ERTS_SCHDLR_SSPND_S_MASK))-1) \ + << ERTS_SCHDLR_SSPND_S_SHIFT) \ + | ((((Uint32) ((DCS) & ERTS_SCHDLR_SSPND_DCS_MASK)) \ + << ERTS_SCHDLR_SSPND_DCS_SHIFT)) \ + | ((((Uint32) ((DIS) & ERTS_SCHDLR_SSPND_DIS_MASK)) \ + << ERTS_SCHDLR_SSPND_DIS_SHIFT))) + +static void init_scheduler_suspend(void); + +static ERTS_INLINE Uint32 +schdlr_sspnd_get_nscheds(Uint32 *valp, ErtsSchedType type) +{ + Uint32 res = (Uint32) (*valp); + switch (type) { + case ERTS_SCHED_NORMAL: + res >>= ERTS_SCHDLR_SSPND_S_SHIFT; + res &= (Uint32) ERTS_SCHDLR_SSPND_S_MASK; + res++; + break; + case ERTS_SCHED_DIRTY_CPU: + res >>= ERTS_SCHDLR_SSPND_DCS_SHIFT; + res &= (Uint32) ERTS_SCHDLR_SSPND_DCS_MASK; + break; + case ERTS_SCHED_DIRTY_IO: + res >>= ERTS_SCHDLR_SSPND_DIS_SHIFT; + res &= (Uint32) ERTS_SCHDLR_SSPND_DIS_MASK; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + return 0; + } + + return res; +} + +static ERTS_INLINE void +schdlr_sspnd_dec_nscheds(Uint32 *valp, ErtsSchedType type) +{ + ASSERT(schdlr_sspnd_get_nscheds(valp, type) > 0); + + switch (type) { + case ERTS_SCHED_NORMAL: + *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + break; + case ERTS_SCHED_DIRTY_CPU: + *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + break; + case ERTS_SCHED_DIRTY_IO: + *valp -= ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + } +} + +static ERTS_INLINE void +schdlr_sspnd_inc_nscheds(Uint32 *valp, ErtsSchedType type) +{ + switch (type) { + case ERTS_SCHED_NORMAL: + ASSERT(schdlr_sspnd_get_nscheds(valp, type) + < ERTS_MAX_NO_OF_SCHEDULERS-1); + *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_S_SHIFT; + break; + case ERTS_SCHED_DIRTY_CPU: + ASSERT(schdlr_sspnd_get_nscheds(valp, type) + < ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS); + *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DCS_SHIFT; + break; + case ERTS_SCHED_DIRTY_IO: + ASSERT(schdlr_sspnd_get_nscheds(valp, type) + < ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS); + *valp += ((Uint32) 1) << ERTS_SCHDLR_SSPND_DIS_SHIFT; + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + } +} + +static ERTS_INLINE void +schdlr_sspnd_set_nscheds(Uint32 *valp, ErtsSchedType type, Uint32 no) +{ + Uint32 val = *valp; + + switch (type) { + case ERTS_SCHED_NORMAL: + ASSERT(no > 0); + val &= ~(((Uint32) ERTS_SCHDLR_SSPND_S_MASK) + << ERTS_SCHDLR_SSPND_S_SHIFT); + val |= (((no-1) & ((Uint32) ERTS_SCHDLR_SSPND_S_MASK)) + << ERTS_SCHDLR_SSPND_S_SHIFT); + break; + case ERTS_SCHED_DIRTY_CPU: + val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK) + << ERTS_SCHDLR_SSPND_DCS_SHIFT); + val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DCS_MASK)) + << ERTS_SCHDLR_SSPND_DCS_SHIFT); + break; + case ERTS_SCHED_DIRTY_IO: + val &= ~(((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK) + << ERTS_SCHDLR_SSPND_DIS_SHIFT); + val |= ((no & ((Uint32) ERTS_SCHDLR_SSPND_DIS_MASK)) + << ERTS_SCHDLR_SSPND_DIS_SHIFT); + break; + default: + ERTS_INTERNAL_ERROR("Invalid scheduler type"); + } + + *valp = val; +} static struct { erts_smp_mtx_t update_mtx; @@ -426,8 +516,10 @@ do { \ do { \ ErtsRunQueue *RQVAR; \ int ix__; \ + int online__ = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, \ + ERTS_SCHED_NORMAL); \ ERTS_SMP_LC_ASSERT(erts_smp_lc_mtx_is_locked(&schdlr_sspnd.mtx)); \ - for (ix__ = 0; ix__ < schdlr_sspnd.online; ix__++) { \ + for (ix__ = 0; ix__ < online__; ix__++) { \ RQVAR = ERTS_RUNQ_IX(ix__); \ erts_smp_runq_lock(RQVAR); \ { DO; } \ @@ -505,7 +597,7 @@ dbg_chk_aux_work_val(erts_aint32_t value) valid |= ERTS_SSI_AUX_WORK_DEBUG_WAIT_COMPLETED; if (~valid & value) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid aux_work value found: 0x%x\n", ~valid & value); } @@ -959,10 +1051,24 @@ typedef struct { erts_smp_atomic32_t refc; } ErtsSchedWallTimeReq; +typedef struct { + Process *proc; + Eterm ref; + Eterm ref_heap[REF_THING_SIZE]; + Uint req_sched; + erts_smp_atomic32_t refc; +} ErtsSystemCheckReq; + + ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(swtreq, - ErtsSchedWallTimeReq, - 5, - ERTS_ALC_T_SCHED_WTIME_REQ) + ErtsSchedWallTimeReq, + 5, + ERTS_ALC_T_SCHED_WTIME_REQ) + +ERTS_SCHED_PREF_QUICK_ALLOC_IMPL(screq, + ErtsSystemCheckReq, + 5, + ERTS_ALC_T_SYS_CHECK_REQ) static void reply_sched_wall_time(void *vswtrp) @@ -1095,6 +1201,75 @@ erts_sched_wall_time_request(Process *c_p, int set, int enable) return ref; } +static void +reply_system_check(void *vscrp) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ErtsSystemCheckReq *scrp = (ErtsSystemCheckReq *) vscrp; + ErtsProcLocks rp_locks = (scrp->req_sched == esdp->no ? ERTS_PROC_LOCK_MAIN : 0); + Process *rp = scrp->proc; + Eterm msg; + Eterm *hp = NULL; + Eterm **hpp; + Uint sz; + ErlOffHeap *ohp = NULL; + ErtsMessage *mp = NULL; + + ASSERT(esdp); +#ifdef ERTS_DIRTY_SCHEDULERS + ASSERT(!ERTS_SCHEDULER_IS_DIRTY(esdp)); +#endif + + sz = REF_THING_SIZE; + mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); + hpp = &hp; + msg = STORE_NC(hpp, ohp, scrp->ref); + + erts_queue_message(rp, &rp_locks, mp, msg, NIL); + + if (scrp->req_sched == esdp->no) + rp_locks &= ~ERTS_PROC_LOCK_MAIN; + + if (rp_locks) + erts_smp_proc_unlock(rp, rp_locks); + + erts_proc_dec_refc(rp); + + if (erts_smp_atomic32_dec_read_nob(&scrp->refc) == 0) + screq_free(vscrp); +} + + +Eterm erts_system_check_request(Process *c_p) { + ErtsSchedulerData *esdp = ERTS_PROC_GET_SCHDATA(c_p); + Eterm ref; + ErtsSystemCheckReq *scrp; + Eterm *hp; + + scrp = screq_alloc(); + ref = erts_make_ref(c_p); + hp = &scrp->ref_heap[0]; + + scrp->proc = c_p; + scrp->ref = STORE_NC(&hp, NULL, ref); + scrp->req_sched = esdp->no; + erts_smp_atomic32_init_nob(&scrp->refc, (erts_aint32_t) erts_no_schedulers); + + erts_proc_add_refc(c_p, (Sint) erts_no_schedulers); + +#ifdef ERTS_SMP + if (erts_no_schedulers > 1) + erts_schedule_multi_misc_aux_work(1, + erts_no_schedulers, + reply_system_check, + (void *) scrp); +#endif + + reply_system_check((void *) scrp); + + return ref; +} + static ERTS_INLINE ErtsProcList * proclist_create(Process *p) { @@ -1105,6 +1280,15 @@ proclist_create(Process *p) return plp; } +static ERTS_INLINE ErtsProcList * +proclist_copy(ErtsProcList *plp0) +{ + ErtsProcList *plp1 = proclist_alloc(); + plp1->pid = plp0->pid; + plp1->started_interval = plp0->started_interval; + return plp1; +} + static ERTS_INLINE void proclist_destroy(ErtsProcList *plp) { @@ -1112,6 +1296,12 @@ proclist_destroy(ErtsProcList *plp) } ErtsProcList * +erts_proclist_copy(ErtsProcList *plp) +{ + return proclist_copy(plp); +} + +ErtsProcList * erts_proclist_create(Process *p) { return proclist_create(p); @@ -1189,7 +1379,7 @@ erts_sched_finish_poke(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flags) case 0: break; default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__); break; } @@ -2147,7 +2337,7 @@ handle_reap_ports(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting) erts_port_release(prt); } if (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0) { - erl_exit_flush_async(erts_halt_code, ""); + erts_flush_async_exit(erts_halt_code, ""); } } return aux_work & ~ERTS_SSI_AUX_WORK_REAP_PORTS; @@ -2549,13 +2739,6 @@ sched_active(Uint no, ErtsRunQueue *rq) profile_scheduler(make_small(no), am_active); } -static int ERTS_INLINE -ongoing_multi_scheduling_block(void) -{ - ERTS_SMP_LC_ASSERT(erts_lc_mtx_is_locked(&schdlr_sspnd.mtx)); - return schdlr_sspnd.msb.ongoing; -} - static ERTS_INLINE void empty_runq_aux(ErtsRunQueue *rq, Uint32 old_flags) { @@ -2942,8 +3125,10 @@ scheduler_wait(int *fcalls, ErtsSchedulerData *esdp, ErtsRunQueue *rq) timeout_time = erts_check_next_timeout_time(esdp); current_time = erts_get_monotonic_time(esdp); do_timeout = (current_time >= timeout_time); - } else + } else { + current_time = 0; timeout_time = ERTS_MONOTONIC_TIME_MAX; + } if (do_timeout) { if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); @@ -3665,7 +3850,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) prio = ERTS_PORT_PRIO_LEVEL; break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Invalid immigrate queue mask", __FILE__, __LINE__, __func__); prio = 0; @@ -3689,7 +3874,7 @@ immigrate(ErtsRunQueue *c_rq, ErtsMigrationPath *mp) rq = erts_port_runq(prt); if (rq) { if (rq != c_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error", __FILE__, __LINE__, __func__); erts_enqueue_port(c_rq, prt); @@ -3754,24 +3939,33 @@ static ERTS_INLINE void resume_run_queue(ErtsRunQueue *rq) { int pix; + Uint32 oflgs; erts_smp_runq_lock(rq); - (void) ERTS_RUNQ_FLGS_READ_BSET(rq, - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK - | ERTS_RUNQ_FLG_SUSPENDED), - (ERTS_RUNQ_FLG_OUT_OF_WORK - | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + oflgs = ERTS_RUNQ_FLGS_READ_BSET(rq, + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK + | ERTS_RUNQ_FLG_SUSPENDED), + (ERTS_RUNQ_FLG_OUT_OF_WORK + | ERTS_RUNQ_FLG_HALFTIME_OUT_OF_WORK)); + + if (oflgs & ERTS_RUNQ_FLG_SUSPENDED) { + erts_aint32_t len; + + rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; + for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { + len = erts_smp_atomic32_read_dirty(&rq->procs.prio_info[pix].len); + rq->procs.prio_info[pix].max_len = len; + rq->procs.prio_info[pix].reds = 0; + } + len = erts_smp_atomic32_read_dirty(&rq->ports.info.len); + rq->ports.info.max_len = len; + rq->ports.info.reds = 0; + len = erts_smp_atomic32_read_dirty(&rq->len); + rq->max_len = len; - rq->check_balance_reds = ERTS_RUNQ_CALL_CHECK_BALANCE_REDS; - for (pix = 0; pix < ERTS_NO_PROC_PRIO_LEVELS; pix++) { - rq->procs.prio_info[pix].max_len = 0; - rq->procs.prio_info[pix].reds = 0; } - rq->ports.info.max_len = 0; - rq->ports.info.reds = 0; - rq->max_len = 0; erts_smp_runq_unlock(rq); @@ -3880,7 +4074,7 @@ evacuate_run_queue(ErtsRunQueue *rq, prt_rq = erts_port_runq(prt); if (prt_rq) { if (prt_rq != to_rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); erts_enqueue_port(to_rq, prt); @@ -4125,7 +4319,7 @@ no_procs: return 0; else { if (prt_rq != rq) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s() internal error\n", __FILE__, __LINE__, __func__); *rq_lockedp = 1; @@ -5550,6 +5744,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online char *daww_ptr; size_t daww_sz; size_t size_runqs; +#ifdef ERTS_SMP + erts_aint32_t set_schdlr_sspnd_change_flags; +#endif init_misc_op_list_alloc(); init_proc_sys_task_queues_alloc(); @@ -5776,6 +5973,7 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online init_misc_aux_work(); init_swtreq_alloc(); + init_screq_alloc(); erts_atomic32_init_nob(&debug_wait_completed_count, 0); /* debug only */ debug_wait_completed_flags = 0; @@ -5786,25 +5984,6 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online erts_alloc_permanent_cache_aligned(ERTS_ALC_T_SCHDLR_DATA, sizeof(ErtsAuxWorkData)); - erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); - erts_smp_cnd_init(&schdlr_sspnd.cnd); - - erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); - schdlr_sspnd.online = no_schedulers_online; - schdlr_sspnd.curr_online = no_schedulers; - schdlr_sspnd.msb.ongoing = 0; - erts_smp_atomic32_init_nob(&schdlr_sspnd.active, no_schedulers); -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_changing, 0); - schdlr_sspnd.dirty_cpu_online = no_dirty_cpu_schedulers_online; - schdlr_sspnd.dirty_cpu_curr_online = no_dirty_cpu_schedulers; - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_cpu_active, no_dirty_cpu_schedulers); - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_changing, 0); - schdlr_sspnd.dirty_io_online = no_dirty_io_schedulers; - schdlr_sspnd.dirty_io_curr_online = no_dirty_io_schedulers; - erts_smp_atomic32_init_nob(&schdlr_sspnd.dirty_io_active, no_dirty_io_schedulers); -#endif - schdlr_sspnd.msb.procs = NULL; init_no_runqs(no_schedulers_online, no_schedulers_online); balance_info.last_active_runqs = no_schedulers; erts_smp_mtx_init(&balance_info.update_mtx, "migration_info_update"); @@ -5819,32 +5998,66 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online init_migration_paths(); - if (no_schedulers_online < no_schedulers) { + init_scheduler_suspend(); + + set_schdlr_sspnd_change_flags = 0; + + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL, + no_schedulers_online); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_NORMAL, + no_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL, + no_schedulers); + + if (no_schedulers_online != no_schedulers) { + ASSERT(no_schedulers_online < no_schedulers); + set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN; + schdlr_sspnd.changer = am_init; change_no_used_runqs(no_schedulers_online); for (ix = no_schedulers_online; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); } - schdlr_sspnd.wait_curr_online = no_schedulers_online; - schdlr_sspnd.curr_online *= 2; /* Boot strapping... */ - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); #ifdef ERTS_DIRTY_SCHEDULERS - schdlr_sspnd.dirty_cpu_wait_curr_online = no_dirty_cpu_schedulers_online; - schdlr_sspnd.dirty_cpu_curr_online *= 2; - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) { - ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); - erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU, + no_dirty_cpu_schedulers_online); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_DIRTY_CPU, + no_dirty_cpu_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_CPU, + no_dirty_cpu_schedulers); + + if (no_dirty_cpu_schedulers_online != no_dirty_cpu_schedulers) { + ASSERT(no_dirty_cpu_schedulers_online < no_dirty_cpu_schedulers); + set_schdlr_sspnd_change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + for (ix = no_dirty_cpu_schedulers_online; ix < no_dirty_cpu_schedulers; ix++) { + ErtsSchedulerData* esdp = ERTS_DIRTY_CPU_SCHEDULER_IX(ix); + erts_smp_atomic32_read_bor_nob(&esdp->ssi->flags, ERTS_SSI_FLG_SUSPENDED); + } } - schdlr_sspnd.dirty_io_wait_curr_online = no_dirty_io_schedulers; - schdlr_sspnd.dirty_io_curr_online *= 2; - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_IO, + no_dirty_io_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_DIRTY_IO, + no_dirty_io_schedulers); + schdlr_sspnd_set_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_IO, + no_dirty_io_schedulers); + #endif + if (set_schdlr_sspnd_change_flags) + erts_smp_atomic32_set_nob(&schdlr_sspnd.changing, + set_schdlr_sspnd_change_flags); + erts_smp_atomic32_init_nob(&doing_sys_schedule, 0); init_misc_aux_work(); @@ -5859,11 +6072,9 @@ erts_init_scheduling(int no_schedulers, int no_schedulers_online #endif } erts_no_schedulers = 1; -#ifdef ERTS_DIRTY_SCHEDULERS erts_no_dirty_cpu_schedulers = 0; erts_no_dirty_io_schedulers = 0; #endif -#endif erts_smp_atomic32_init_nob(&function_calls, 0); @@ -6578,9 +6789,6 @@ suspend_process(Process *c_p, Process *p) if (suspended) { - ASSERT(!(ERTS_PSFLG_RUNNING & state) - || p == erts_get_current_process()); - if (suspended > 0 && erts_system_profile_flags.runnable_procs) { /* 'state' is before our change... */ @@ -6625,19 +6833,6 @@ resume_process(Process *p, ErtsProcLocks locks) add2runq(enqueue, enq_prio, p, state, NULL); } -int -erts_get_max_no_executing_schedulers(void) -{ -#ifdef ERTS_SMP - if (erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - return (int) erts_no_schedulers; - ERTS_THR_MEMORY_BARRIER; - return (int) erts_smp_atomic32_read_nob(&schdlr_sspnd.active); -#else - return 1; -#endif -} - #ifdef ERTS_SMP static void @@ -6740,42 +6935,82 @@ sched_set_suspended_sleeptype(ErtsSchedulerSleepInfo *ssi) } } -#ifdef ERTS_DIRTY_SCHEDULERS +static void +init_scheduler_suspend(void) +{ + erts_smp_mtx_init(&schdlr_sspnd.mtx, "schdlr_sspnd"); + schdlr_sspnd.online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + schdlr_sspnd.curr_online = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + schdlr_sspnd.active = ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0); + erts_smp_atomic32_init_nob(&schdlr_sspnd.changing, 0); + schdlr_sspnd.chngq = NULL; + schdlr_sspnd.changer = am_false; + schdlr_sspnd.nmsb.ongoing = 0; + schdlr_sspnd.nmsb.blckrs = NULL; + schdlr_sspnd.nmsb.chngq = NULL; + schdlr_sspnd.msb.ongoing = 0; + schdlr_sspnd.msb.blckrs = NULL; + schdlr_sspnd.msb.chngq = NULL; +} + +typedef struct { + struct { + Eterm chngr; + Eterm nxt; + } onln; + struct { + ErtsProcList *chngrs; + } msb; +} ErtsSchdlrSspndResume; + +static void +schdlr_sspnd_resume_proc(Eterm pid) +{ + Process *p = erts_pid2proc(NULL, 0, pid, ERTS_PROC_LOCK_STATUS); + if (p) { + resume_process(p, ERTS_PROC_LOCK_STATUS); + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } +} + +static ERTS_INLINE void +schdlr_sspnd_resume_procs(ErtsSchedType sched_type, + ErtsSchdlrSspndResume *resume) +{ + if (is_internal_pid(resume->onln.chngr)) { + schdlr_sspnd_resume_proc(resume->onln.chngr); + resume->onln.chngr = NIL; + } + if (is_internal_pid(resume->onln.nxt)) { + schdlr_sspnd_resume_proc(resume->onln.nxt); + resume->onln.nxt = NIL; + } + while (resume->msb.chngrs) { + ErtsProcList *plp = resume->msb.chngrs; + resume->msb.chngrs = plp->next; + schdlr_sspnd_resume_proc(plp->pid); + proclist_destroy(plp); + } +} static void suspend_scheduler(ErtsSchedulerData *esdp) { erts_aint32_t flgs; erts_aint32_t changing; -#ifdef ERTS_DIRTY_SCHEDULERS - long no = (long) (ERTS_SCHEDULER_IS_DIRTY(esdp) - ? ERTS_DIRTY_SCHEDULER_NO(esdp) - : esdp->no); -#else - long no = (long) esdp->no; -#endif + long no; ErtsSchedulerSleepInfo *ssi = esdp->ssi; - long active_schedulers; int curr_online = 1; - int wake = 0; + ErtsSchdlrSspndResume resume = {{NIL, NIL}, {NULL}}; erts_aint32_t aux_work; int thr_prgr_active = 1; ErtsStuckBoundProcesses sbp = {NULL, NULL}; - int* ss_onlinep; - int* ss_curr_onlinep; - int* ss_wait_curr_onlinep; - long* ss_wait_activep; - long ss_wait_active_target; - erts_smp_atomic32_t* ss_changingp; - erts_smp_atomic32_t* ss_activep; + ErtsSchedType sched_type; + erts_aint32_t online_flag; /* * Schedulers may be suspended in two different ways: * - A scheduler may be suspended since it is not online. - * All schedulers with scheduler ids greater than - * schdlr_sspnd.online are suspended; same for dirty - * schedulers and schdlr_sspnd.dirty_cpu_online and - * schdlr_sspnd.dirty_io_online. * - Multi scheduling is blocked. All schedulers except the * scheduler with scheduler id 1 are suspended, and all * dirty CPU and dirty I/O schedulers are suspended. @@ -6783,27 +7018,43 @@ suspend_scheduler(ErtsSchedulerData *esdp) * Regardless of why a scheduler is suspended, it ends up here. */ - ASSERT(ERTS_SCHEDULER_IS_DIRTY(esdp) || no != 1); - #ifdef ERTS_DIRTY_SCHEDULERS if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { + no = ERTS_DIRTY_SCHEDULER_NO(esdp); + if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { + online_flag = ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + sched_type = ERTS_SCHED_DIRTY_CPU; + } + else { + online_flag = 0; + sched_type = ERTS_SCHED_DIRTY_IO; + } + } + else +#endif + { + online_flag = ERTS_SCHDLR_SSPND_CHNG_ONLN; + no = esdp->no; + sched_type = ERTS_SCHED_NORMAL; + } + + ASSERT(sched_type != ERTS_SCHED_NORMAL || no != 1); + + if (sched_type != ERTS_SCHED_NORMAL) { if (erts_smp_mtx_trylock(&schdlr_sspnd.mtx) == EBUSY) { erts_smp_runq_unlock(esdp->run_queue); erts_smp_mtx_lock(&schdlr_sspnd.mtx); erts_smp_runq_lock(esdp->run_queue); } - if (ongoing_multi_scheduling_block()) + if (schdlr_sspnd.msb.ongoing) evacuate_run_queue(esdp->run_queue, &sbp); - } else -#endif + erts_smp_runq_unlock(esdp->run_queue); + } + else { evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); + erts_smp_runq_unlock(esdp->run_queue); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { erts_sched_check_cpu_bind_prep_suspend(esdp); if (erts_system_profile_flags.scheduler) @@ -6817,318 +7068,115 @@ suspend_scheduler(ErtsSchedulerData *esdp) flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); if (flgs & ERTS_SSI_FLG_SUSPENDED) { -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (ERTS_RUNQ_IS_DIRTY_CPU_RUNQ(esdp->run_queue)) { - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_cpu_active); - ASSERT(active_schedulers >= 0); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing); - ss_onlinep = &schdlr_sspnd.dirty_cpu_online; - ss_curr_onlinep = &schdlr_sspnd.dirty_cpu_curr_online; - ss_wait_curr_onlinep = &schdlr_sspnd.dirty_cpu_wait_curr_online; - ss_changingp = &schdlr_sspnd.dirty_cpu_changing; - ss_wait_activep = &schdlr_sspnd.msb.dirty_cpu_wait_active; - ss_activep = &schdlr_sspnd.dirty_cpu_active; - } else { - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.dirty_io_active); - ASSERT(active_schedulers >= 0); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing); - ss_onlinep = &schdlr_sspnd.dirty_io_online; - ss_curr_onlinep = &schdlr_sspnd.dirty_io_curr_online; - ss_wait_curr_onlinep = &schdlr_sspnd.dirty_io_wait_curr_online; - ss_changingp = &schdlr_sspnd.dirty_io_changing; - ss_wait_activep = &schdlr_sspnd.msb.dirty_io_wait_active; - ss_activep = &schdlr_sspnd.dirty_io_active; - } - ss_wait_active_target = 0; - } - else -#endif - { - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - ss_onlinep = &schdlr_sspnd.online; - ss_curr_onlinep = &schdlr_sspnd.curr_online; - ss_wait_curr_onlinep = &schdlr_sspnd.wait_curr_online; - ss_changingp = &schdlr_sspnd.changing; - ss_wait_activep = &schdlr_sspnd.msb.wait_active; - ss_activep = &schdlr_sspnd.active; - ss_wait_active_target = 1; - } - if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { - if (active_schedulers == *ss_wait_activep) - wake = 1; - if (active_schedulers == ss_wait_active_target) { - changing = erts_smp_atomic32_read_band_nob(ss_changingp, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; - } - } - - while (1) { - if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { - int changed = 0; - if (no > *ss_onlinep && curr_online) { - (*ss_curr_onlinep)--; - curr_online = 0; - changed = 1; - } - else if (no <= *ss_onlinep && !curr_online) { - (*ss_curr_onlinep)++; - curr_online = 1; - changed = 1; - } - if (changed - && *ss_curr_onlinep == *ss_wait_curr_onlinep) - wake = 1; - if (*ss_onlinep == *ss_curr_onlinep) { - changing = erts_smp_atomic32_read_band_nob(ss_changingp, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; - } - } + schdlr_sspnd_dec_nscheds(&schdlr_sspnd.active, sched_type); - if (wake) { - erts_smp_cnd_signal(&schdlr_sspnd.cnd); - wake = 0; - } - - if (curr_online && !ongoing_multi_scheduling_block()) { - flgs = erts_smp_atomic32_read_acqb(&ssi->flags); - if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) - break; - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) >= 1); - while (1) { - erts_aint32_t qmask; - erts_aint32_t flgs; + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) - & ERTS_RUNQ_FLGS_QMASK); - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work|qmask) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - if (aux_work) - aux_work = handle_aux_work(&esdp->aux_work_data, - aux_work, - 1); + while (1) { - if (aux_work && erts_thr_progress_update(esdp)) - erts_thr_progress_leader_update(esdp); + if (changing & (ERTS_SCHDLR_SSPND_CHNG_NMSB + | ERTS_SCHDLR_SSPND_CHNG_MSB)) { + int i = 0; + ErtsMultiSchedulingBlock *msb[3] = {0}; + if (changing & ERTS_SCHDLR_SSPND_CHNG_NMSB) + msb[i++] = &schdlr_sspnd.nmsb; + if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) + msb[i++] = &schdlr_sspnd.msb; + + for (i = 0; msb[i]; i++) { + erts_aint32_t clr_flg = 0; + + if (msb[i] == &schdlr_sspnd.nmsb + && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1) { + clr_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB; } - if (qmask) { -#ifdef ERTS_DIRTY_SCHEDULERS - if (ERTS_SCHEDULER_IS_DIRTY(esdp)) { - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - erts_smp_runq_lock(esdp->run_queue); - if (ongoing_multi_scheduling_block()) - evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - } else -#endif - { - erts_smp_runq_lock(esdp->run_queue); - evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); - } + else if (schdlr_sspnd.active + == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)) { + clr_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; } - } - if (!aux_work) { -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { - if (thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 0); - sched_wall_time_change(esdp, 0); + if (clr_flg) { + ErtsProcList *plp, *end_plp; + changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~clr_flg); + changing &= ~clr_flg; + (void) erts_proclist_fetch(&msb[i]->chngq, &end_plp); + /* resume processes that initiated the multi scheduling block... */ + plp = msb[i]->chngq; + while (plp) { + erts_proclist_store_last(&msb[i]->blckrs, + proclist_copy(plp)); + plp = plp->next; } - erts_thr_progress_prepare_wait(esdp); + if (end_plp) + end_plp->next = resume.msb.chngrs; + resume.msb.chngrs = msb[i]->chngq; + msb[i]->chngq = NULL; } - flgs = sched_spin_suspended(ssi, - ERTS_SCHED_SUSPEND_SLEEP_SPINCOUNT); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - flgs = sched_set_suspended_sleeptype(ssi); - if (flgs == (ERTS_SSI_FLG_SLEEPING - | ERTS_SSI_FLG_TSE_SLEEPING - | ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)) { - int res; - - do { - res = erts_tse_twait(ssi->event, -1); - } while (res == EINTR); - } - } -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - erts_thr_progress_finalize_wait(esdp); } - - flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING - | ERTS_SSI_FLG_SUSPENDED)); - if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) - break; - changing = erts_smp_atomic32_read_nob(ss_changingp); - if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) - break; - } - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(ss_changingp); - } - - active_schedulers = erts_smp_atomic32_inc_read_nob(ss_activep); - changing = erts_smp_atomic32_read_nob(ss_changingp); - if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) - && *ss_onlinep == active_schedulers) { - erts_smp_atomic32_read_band_nob(ss_changingp, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - } - -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - ASSERT(no <= *ss_onlinep); - ASSERT(!ongoing_multi_scheduling_block()); - - } - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - ASSERT(curr_online); - -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_active); - - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - } - - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) - (void) erts_get_monotonic_time(esdp); - erts_smp_runq_lock(esdp->run_queue); - non_empty_runq(esdp->run_queue); - -#ifdef ERTS_DIRTY_SCHEDULERS - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) -#endif - { - schedule_bound_processes(esdp->run_queue, &sbp); - - erts_sched_check_cpu_bind_post_suspend(esdp); - } -} - -#else /* !ERTS_DIRTY_SCHEDULERS */ - -static void -suspend_scheduler(ErtsSchedulerData *esdp) -{ - erts_aint32_t flgs; - erts_aint32_t changing; - long no = (long) esdp->no; - ErtsSchedulerSleepInfo *ssi = esdp->ssi; - long active_schedulers; - int curr_online = 1; - int wake = 0; - erts_aint32_t aux_work; - int thr_prgr_active = 1; - ErtsStuckBoundProcesses sbp = {NULL, NULL}; - - /* - * Schedulers may be suspended in two different ways: - * - A scheduler may be suspended since it is not online. - * All schedulers with scheduler ids greater than - * schdlr_sspnd.online are suspended. - * - Multi scheduling is blocked. All schedulers except the - * scheduler with scheduler id 1 are suspended. - * - * Regardless of why a scheduler is suspended, it ends up here. - */ - - ASSERT(no != 1); - - evacuate_run_queue(esdp->run_queue, &sbp); - - erts_smp_runq_unlock(esdp->run_queue); - - erts_sched_check_cpu_bind_prep_suspend(esdp); - - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_inactive); - - sched_wall_time_change(esdp, 0); - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - flgs = sched_prep_spin_suspended(ssi, ERTS_SSI_FLG_SUSPENDED); - if (flgs & ERTS_SSI_FLG_SUSPENDED) { - - active_schedulers = erts_smp_atomic32_dec_read_nob(&schdlr_sspnd.active); - ASSERT(active_schedulers >= 1); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if (changing & ERTS_SCHDLR_SSPND_CHNG_MSB) { - if (active_schedulers == schdlr_sspnd.msb.wait_active) - wake = 1; - if (active_schedulers == 1) { - changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_MSB); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_MSB; } - } - while (1) { - if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + if (changing & online_flag) { int changed = 0; - if (no > schdlr_sspnd.online && curr_online) { - schdlr_sspnd.curr_online--; + Uint32 st_online; + + st_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + sched_type); + if (no > st_online && curr_online) { + schdlr_sspnd_dec_nscheds(&schdlr_sspnd.curr_online, + sched_type); curr_online = 0; changed = 1; } - else if (no <= schdlr_sspnd.online && !curr_online) { - schdlr_sspnd.curr_online++; + else if (no <= st_online && !curr_online) { + schdlr_sspnd_inc_nscheds(&schdlr_sspnd.curr_online, + sched_type); curr_online = 1; changed = 1; } if (changed - && schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) - wake = 1; - if (schdlr_sspnd.online == schdlr_sspnd.curr_online) { + && (schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + sched_type) + == schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, + sched_type))) { + ErtsProcList *plp; changing = erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - changing &= ~ERTS_SCHDLR_SSPND_CHNG_ONLN; + ~online_flag); + changing &= ~online_flag; + if (sched_type == ERTS_SCHED_NORMAL) { + ASSERT(is_internal_pid(schdlr_sspnd.changer) + || schdlr_sspnd.changer == am_init); + /* resume process that initiated this change... */ + resume.onln.chngr = schdlr_sspnd.changer; + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (!plp) + schdlr_sspnd.changer = am_false; + else { + schdlr_sspnd.changer = am_true; /* change right in transit */ + /* resume process that is queued for next change... */ + resume.onln.nxt = plp->pid; + ASSERT(is_internal_pid(resume.onln.nxt)); + } + } } } - if (wake) { - erts_smp_cnd_signal(&schdlr_sspnd.cnd); - wake = 0; - } - - if (curr_online && !ongoing_multi_scheduling_block()) { + if (curr_online + && (sched_type == ERTS_SCHED_NORMAL + ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) + : !schdlr_sspnd.msb.ongoing)) { flgs = erts_smp_atomic32_read_acqb(&ssi->flags); if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + schdlr_sspnd_resume_procs(sched_type, &resume); + while (1) { ErtsMonotonicTime current_time; erts_aint32_t qmask; @@ -7136,9 +7184,23 @@ suspend_scheduler(ErtsSchedulerData *esdp) qmask = (ERTS_RUNQ_FLGS_GET(esdp->run_queue) & ERTS_RUNQ_FLGS_QMASK); - aux_work = erts_atomic32_read_acqb(&ssi->aux_work); - if (aux_work|qmask) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + + if (sched_type != ERTS_SCHED_NORMAL) { + if (qmask) { + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + erts_smp_runq_lock(esdp->run_queue); + if (schdlr_sspnd.msb.ongoing) + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + } + aux_work = 0; + } + else { + + aux_work = erts_atomic32_read_acqb(&ssi->aux_work); + + if (aux_work|qmask) { if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); @@ -7147,45 +7209,53 @@ suspend_scheduler(ErtsSchedulerData *esdp) aux_work = handle_aux_work(&esdp->aux_work_data, aux_work, 1); + if (aux_work && erts_thr_progress_update(esdp)) erts_thr_progress_leader_update(esdp); + if (qmask) { + erts_smp_runq_lock(esdp->run_queue); + evacuate_run_queue(esdp->run_queue, &sbp); + erts_smp_runq_unlock(esdp->run_queue); + } } - if (qmask) { - erts_smp_runq_lock(esdp->run_queue); - evacuate_run_queue(esdp->run_queue, &sbp); - erts_smp_runq_unlock(esdp->run_queue); - } + } if (aux_work) { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { - current_time = erts_get_monotonic_time(esdp); - if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); - } - erts_bump_timers(esdp->timer_wheel, current_time); + ASSERT(sched_type == ERTS_SCHED_NORMAL); + current_time = erts_get_monotonic_time(esdp); + if (current_time >= erts_next_timeout_time(esdp->next_tmo_ref)) { + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); } + erts_bump_timers(esdp->timer_wheel, current_time); } } else { ErtsMonotonicTime timeout_time; - int do_timeout = 0; - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + int do_timeout; + + if (sched_type == ERTS_SCHED_NORMAL) { timeout_time = erts_check_next_timeout_time(esdp); current_time = erts_get_monotonic_time(esdp); do_timeout = (current_time >= timeout_time); - } else + } + else { timeout_time = ERTS_MONOTONIC_TIME_MAX; + current_time = 0; + do_timeout = 0; + } + if (do_timeout) { + ASSERT(sched_type == ERTS_SCHED_NORMAL); if (!thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 1); sched_wall_time_change(esdp, 1); } } else { - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (sched_type == ERTS_SCHED_NORMAL) { if (thr_prgr_active) { erts_thr_progress_active(esdp, thr_prgr_active = 0); sched_wall_time_change(esdp, 0); @@ -7204,30 +7274,39 @@ suspend_scheduler(ErtsSchedulerData *esdp) | ERTS_SSI_FLG_SUSPENDED)) { int res; - current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : - erts_get_monotonic_time(esdp); + if (sched_type == ERTS_SCHED_NORMAL) + current_time = erts_get_monotonic_time(esdp); + else + current_time = 0; + do { Sint64 timeout; if (current_time >= timeout_time) break; - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (sched_type != ERTS_SCHED_NORMAL) + timeout = -1; + else timeout = ERTS_MONOTONIC_TO_NSEC(timeout_time - current_time - 1) + 1; - } else - timeout = -1; res = erts_tse_twait(ssi->event, timeout); - current_time = ERTS_SCHEDULER_IS_DIRTY(esdp) ? 0 : - erts_get_monotonic_time(esdp); + + if (sched_type == ERTS_SCHED_NORMAL) + current_time = erts_get_monotonic_time(esdp); + else + current_time = 0; + } while (res == EINTR); } } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) + if (sched_type == ERTS_SCHED_NORMAL) erts_thr_progress_finalize_wait(esdp); } - if (!ERTS_SCHEDULER_IS_DIRTY(esdp) && current_time >= timeout_time) + if (current_time >= timeout_time) { + ASSERT(sched_type == ERTS_SCHED_NORMAL); erts_bump_timers(esdp->timer_wheel, current_time); + } } flgs = sched_prep_spin_suspended(ssi, (ERTS_SSI_FLG_WAITING @@ -7235,7 +7314,7 @@ suspend_scheduler(ErtsSchedulerData *esdp) if (!(flgs & ERTS_SSI_FLG_SUSPENDED)) break; changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER) + if (changing) break; } @@ -7243,612 +7322,494 @@ suspend_scheduler(ErtsSchedulerData *esdp) changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); } - active_schedulers = erts_smp_atomic32_inc_read_nob(&schdlr_sspnd.active); + schdlr_sspnd_inc_nscheds(&schdlr_sspnd.active, sched_type); changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); if ((changing & ERTS_SCHDLR_SSPND_CHNG_MSB) - && schdlr_sspnd.online == active_schedulers) { + && schdlr_sspnd.online == schdlr_sspnd.active) { erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, ~ERTS_SCHDLR_SSPND_CHNG_MSB); } - ASSERT(no <= schdlr_sspnd.online); - ASSERT(!ongoing_multi_scheduling_block()); - + ASSERT(no <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, sched_type)); + ASSERT((sched_type == ERTS_SCHED_NORMAL + ? !(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) + : !schdlr_sspnd.msb.ongoing)); } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + ASSERT(!resume.msb.chngrs); + schdlr_sspnd_resume_procs(sched_type, &resume); + ASSERT(curr_online); - if (erts_system_profile_flags.scheduler) - profile_scheduler(make_small(esdp->no), am_active); + if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { + if (erts_system_profile_flags.scheduler) + profile_scheduler(make_small(esdp->no), am_active); - if (!thr_prgr_active) { - erts_thr_progress_active(esdp, thr_prgr_active = 1); - sched_wall_time_change(esdp, 1); + if (!thr_prgr_active) { + erts_thr_progress_active(esdp, thr_prgr_active = 1); + sched_wall_time_change(esdp, 1); + } } + if (sched_type == ERTS_SCHED_NORMAL) + (void) erts_get_monotonic_time(esdp); erts_smp_runq_lock(esdp->run_queue); non_empty_runq(esdp->run_queue); - schedule_bound_processes(esdp->run_queue, &sbp); + if (sched_type == ERTS_SCHED_NORMAL) { + schedule_bound_processes(esdp->run_queue, &sbp); - erts_sched_check_cpu_bind_post_suspend(esdp); + erts_sched_check_cpu_bind_post_suspend(esdp); + } } -#endif - -ErtsSchedSuspendResult +void erts_schedulers_state(Uint *total, Uint *online, Uint *active, Uint *dirty_cpu, Uint *dirty_cpu_online, + Uint *dirty_cpu_active, Uint *dirty_io, - int yield_allowed) + Uint *dirty_io_active) { - int res = ERTS_SCHDLR_SSPND_EINVAL; - erts_aint32_t changing; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); -#ifdef ERTS_DIRTY_SCHEDULERS - changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) - | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)); -#endif - if (yield_allowed && (changing & ~ERTS_SCHDLR_SSPND_CHNG_WAITER)) - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; - else { + if (active || online || dirty_cpu_online + || dirty_cpu_active || dirty_io_active) { + erts_smp_mtx_lock(&schdlr_sspnd.mtx); if (active) - *active = schdlr_sspnd.online; + *active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL); if (online) - *online = schdlr_sspnd.online; - if (ongoing_multi_scheduling_block() && active) - *active = 1; -#ifdef ERTS_DIRTY_SCHEDULERS + *online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_NORMAL); + if (dirty_cpu_active) + *dirty_cpu_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_CPU); if (dirty_cpu_online) - *dirty_cpu_online = schdlr_sspnd.dirty_cpu_online; -#endif - res = ERTS_SCHDLR_SSPND_DONE; + *dirty_cpu_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.curr_online, + ERTS_SCHED_DIRTY_CPU); + if (dirty_io_active) + *dirty_io_active = schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_DIRTY_IO); + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (total) *total = erts_no_schedulers; -#ifdef ERTS_DIRTY_SCHEDULERS if (dirty_cpu) *dirty_cpu = erts_no_dirty_cpu_schedulers; if (dirty_io) *dirty_io = erts_no_dirty_io_schedulers; -#endif - return res; } -#ifdef ERTS_DIRTY_SCHEDULERS +static void +abort_sched_onln_chng_waitq(Process *p) +{ + Eterm resume = NIL; + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + +#ifdef DEBUG + { + int found_it = 0; + ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + while (plp) { + if (erts_proclist_same(plp, p)) + found_it++; + plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp); + } + ASSERT(found_it == !!(p->flags & F_SCHDLR_ONLN_WAITQ)); + } +#endif + + if (p->flags & F_SCHDLR_ONLN_WAITQ) { + ErtsProcList *plp = NULL; + + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (plp) { + if (erts_proclist_same(plp, p) + && schdlr_sspnd.changer == am_true) { + p->flags &= ~F_SCHDLR_ONLN_WAITQ; + /* + * Change right was in transit to us; + * transfer it to the next process by + * resuming it... + */ + erts_proclist_remove(&schdlr_sspnd.chngq, plp); + proclist_destroy(plp); + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (plp) + resume = plp->pid; + else + schdlr_sspnd.changer = am_false; + } + else { + do { + if (erts_proclist_same(plp, p)) { + p->flags &= ~F_SCHDLR_ONLN_WAITQ; + erts_proclist_remove(&schdlr_sspnd.chngq, plp); + proclist_destroy(plp); + break; + } + plp = erts_proclist_peek_next(schdlr_sspnd.chngq, plp); + } while (plp); + } + } + } + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + + if (is_internal_pid(resume)) + schdlr_sspnd_resume_proc(resume); +} ErtsSchedSuspendResult erts_set_schedulers_online(Process *p, ErtsProcLocks plocks, Sint new_no, - Sint *old_no -#ifdef ERTS_DIRTY_SCHEDULERS - , int dirty_only -#endif - ) + Sint *old_no, + int dirty_only) { - ErtsSchedulerData *esdp; - int ix, res = -1, no, have_unlocked_plocks, end_wait; - erts_aint32_t changing = 0; + int resume_proc, ix, res = -1, no, have_unlocked_plocks; + erts_aint32_t changing = 0, change_flags; + int online, increase; + ErtsProcList *plp; #ifdef ERTS_DIRTY_SCHEDULERS - ErtsSchedulerSleepInfo* ssi; - int dirty_no, change_dirty; + int dirty_no, change_dirty, dirty_online; +#else + ASSERT(!dirty_only); #endif if (new_no < 1) return ERTS_SCHDLR_SSPND_EINVAL; -#ifdef ERTS_DIRTY_SCHEDULERS else if (dirty_only && erts_no_dirty_cpu_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; -#endif else if (erts_no_schedulers < new_no) return ERTS_SCHDLR_SSPND_EINVAL; - esdp = ERTS_PROC_GET_SCHDATA(p); - end_wait = 0; +#ifdef ERTS_DIRTY_SCHEDULERS + if (dirty_only) + resume_proc = 0; + else +#endif + { + resume_proc = 1; + /* + * If we suspend current process we need to suspend before + * requesting the change; otherwise, we got a resume/suspend + * race... + */ + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + suspend_process(p, p); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } erts_smp_mtx_lock(&schdlr_sspnd.mtx); + change_flags = 0; have_unlocked_plocks = 0; no = (int) new_no; #ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(schdlr_sspnd.dirty_cpu_online <= erts_no_dirty_cpu_schedulers); + if (!dirty_only) +#endif + { + changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); + if (changing & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + enqueue_wait: + p->flags |= F_SCHDLR_ONLN_WAITQ; + plp = proclist_create(p); + erts_proclist_store_last(&schdlr_sspnd.chngq, plp); + resume_proc = 0; + res = ERTS_SCHDLR_SSPND_YIELD_RESTART; + goto done; + } + plp = erts_proclist_peek_first(schdlr_sspnd.chngq); + if (!plp) { + ASSERT(schdlr_sspnd.changer == am_false); + } + else { + ASSERT(schdlr_sspnd.changer == am_true); + if (!erts_proclist_same(plp, p)) + goto enqueue_wait; + p->flags &= ~F_SCHDLR_ONLN_WAITQ; + erts_proclist_remove(&schdlr_sspnd.chngq, plp); + proclist_destroy(plp); + } + } + + *old_no = online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); +#ifndef ERTS_DIRTY_SCHEDULERS + if (no == online) { + res = ERTS_SCHDLR_SSPND_DONE; + goto done; + } +#else /* ERTS_DIRTY_SCHEDULERS */ + dirty_online = schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU); + if (dirty_only) + *old_no = dirty_online; + + ASSERT(dirty_online <= erts_no_dirty_cpu_schedulers); + if (dirty_only) { - if (no > schdlr_sspnd.online) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - return ERTS_SCHDLR_SSPND_EINVAL; + if (no > online) { + res = ERTS_SCHDLR_SSPND_EINVAL; + goto done; } dirty_no = no; + if (dirty_no == dirty_online) { + res = ERTS_SCHDLR_SSPND_DONE; + goto done; + } + change_dirty = 1; } else { /* * Adjust the number of dirty CPU schedulers online relative to the * adjustment made to the number of normal schedulers online. */ int total_pct = erts_no_dirty_cpu_schedulers*100/erts_no_schedulers; - int onln_pct = no*total_pct/schdlr_sspnd.online; - dirty_no = schdlr_sspnd.dirty_cpu_online*onln_pct/100; + int onln_pct = no*total_pct/online; + dirty_no = dirty_online*onln_pct/100; if (dirty_no == 0) dirty_no = 1; ASSERT(dirty_no <= erts_no_dirty_cpu_schedulers); - } -#endif - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); -#ifdef ERTS_DIRTY_SCHEDULERS - changing |= erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing); -#endif - if (changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; - } - else { - int online = *old_no = schdlr_sspnd.online; -#ifdef ERTS_DIRTY_SCHEDULERS - int dirty_online = schdlr_sspnd.dirty_cpu_online; - if (dirty_only) { - *old_no = schdlr_sspnd.dirty_cpu_online; - if (dirty_no == schdlr_sspnd.dirty_cpu_online) { + if (no != online) + change_dirty = (dirty_no != dirty_online); + else { + dirty_only = 1; + if (dirty_no == dirty_online) { res = ERTS_SCHDLR_SSPND_DONE; + goto done; } change_dirty = 1; - } else { -#endif - if (no == schdlr_sspnd.online) { -#ifdef ERTS_DIRTY_SCHEDULERS - dirty_only = 1; - if (dirty_no == schdlr_sspnd.dirty_cpu_online) -#endif - res = ERTS_SCHDLR_SSPND_DONE; -#ifdef ERTS_DIRTY_SCHEDULERS - else - change_dirty = 1; -#endif - } -#ifdef ERTS_DIRTY_SCHEDULERS - else - change_dirty = (dirty_no != schdlr_sspnd.dirty_cpu_online); } -#endif - if (res == -1) - { - int increase = (no > online); -#ifdef ERTS_DIRTY_SCHEDULERS - if (!dirty_only) { -#endif - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - schdlr_sspnd.online = no; -#ifdef ERTS_DIRTY_SCHEDULERS - } else - increase = (dirty_no > dirty_online); - if (change_dirty) { - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - schdlr_sspnd.dirty_cpu_online = dirty_no; - } -#endif - if (increase) { - int ix; -#ifdef ERTS_DIRTY_SCHEDULERS - if (!dirty_only) { -#endif - schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) { - for (ix = online; ix < no; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); - } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - change_no_used_runqs(no); + } + if (change_dirty) { + change_flags |= ERTS_SCHDLR_SSPND_CHNG_DCPU_ONLN; + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU, + dirty_no); + } + + if (dirty_only) + increase = (dirty_no > dirty_online); + else +#endif /* ERTS_DIRTY_SCHEDULERS */ + { + change_flags |= ERTS_SCHDLR_SSPND_CHNG_ONLN; + schdlr_sspnd_set_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL, + no); + increase = (no > online); + } - for (ix = online; ix < no; ix++) - resume_run_queue(ERTS_RUNQ_IX(ix)); + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, change_flags); - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - } + res = ERTS_SCHDLR_SSPND_DONE; + if (increase) { + int ix; #ifdef ERTS_DIRTY_SCHEDULERS + if (change_dirty) { + ErtsSchedulerSleepInfo* ssi; + if (schdlr_sspnd.msb.ongoing) { + for (ix = dirty_online; ix < dirty_no; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_sched_poke(ssi); } - if (change_dirty) { - schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no; - ASSERT(schdlr_sspnd.dirty_cpu_curr_online != - schdlr_sspnd.dirty_cpu_wait_curr_online); - if (ongoing_multi_scheduling_block()) { - for (ix = dirty_online; ix < dirty_no; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_sched_poke(ssi); - } - } else { - for (ix = dirty_online; ix < dirty_no; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - erts_smp_atomic32_read_band_nob(&ssi->flags, - ~ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - } + } else { + for (ix = dirty_online; ix < dirty_no; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); } -#endif - res = ERTS_SCHDLR_SSPND_DONE; } - else /* if (no < online) */ { -#ifdef ERTS_DIRTY_SCHEDULERS - if (change_dirty) { - schdlr_sspnd.dirty_cpu_wait_curr_online = dirty_no; - ASSERT(schdlr_sspnd.dirty_cpu_curr_online != - schdlr_sspnd.dirty_cpu_wait_curr_online); - if (ongoing_multi_scheduling_block()) { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_sched_poke(ssi); - } - } else { - for (ix = dirty_no; ix < dirty_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - } - } - if (dirty_only) { - res = ERTS_SCHDLR_SSPND_DONE; - } - else + } + if (!dirty_only) #endif - { - if (p->scheduler_data->no <= no) { - res = ERTS_SCHDLR_SSPND_DONE; - schdlr_sspnd.wait_curr_online = no; - } - else { - /* - * Yield! Current process needs to migrate - * before bif returns. - */ - res = ERTS_SCHDLR_SSPND_YIELD_DONE; - schdlr_sspnd.wait_curr_online = no+1; - } - - if (ongoing_multi_scheduling_block()) { - for (ix = no; ix < online; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); - } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - - change_no_used_runqs(no); - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - - for (ix = no; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq); - } - } - } + { + if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { + for (ix = online; ix < no; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - -#ifdef ERTS_DIRTY_SCHEDULERS - if (change_dirty) { - while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - if (!dirty_only) -#endif - { - if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (plocks && !have_unlocked_plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - end_wait = 1; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + else { + if (plocks) { + have_unlocked_plocks = 1; + erts_smp_proc_unlock(p, plocks); } + change_no_used_runqs(no); - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); + for (ix = online; ix < no; ix++) + resume_run_queue(ERTS_RUNQ_IX(ix)); - ASSERT(res != ERTS_SCHDLR_SSPND_DONE - ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); } } } - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + else /* if decrease */ { #ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(schdlr_sspnd.dirty_cpu_online <= schdlr_sspnd.online); - if (!dirty_only) -#endif - { - if (end_wait) { - erts_thr_progress_finalize_wait(esdp); - erts_thr_progress_active(esdp, 1); - } - if (have_unlocked_plocks) - erts_smp_proc_lock(p, plocks); - } - - return res; -} - -#else /* !ERTS_DIRTY_SCHEDULERS */ - -ErtsSchedSuspendResult -erts_set_schedulers_online(Process *p, - ErtsProcLocks plocks, - Sint new_no, - Sint *old_no) -{ - ErtsSchedulerData *esdp; - int ix, res, no, have_unlocked_plocks, end_wait; - erts_aint32_t changing; - - if (new_no < 1 || erts_no_schedulers < new_no) - return ERTS_SCHDLR_SSPND_EINVAL; - - esdp = ERTS_PROC_GET_SCHDATA(p); - end_wait = 0; - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - have_unlocked_plocks = 0; - no = (int) new_no; - - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); - if (changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; - } - else { - int online = *old_no = schdlr_sspnd.online; - if (no == schdlr_sspnd.online) { - res = ERTS_SCHDLR_SSPND_DONE; - } - else { - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_ONLN - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - schdlr_sspnd.online = no; - if (no > online) { - int ix; - schdlr_sspnd.wait_curr_online = no; - if (ongoing_multi_scheduling_block()) { - for (ix = online; ix < no; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); + if (change_dirty) { + ErtsSchedulerSleepInfo* ssi; + if (schdlr_sspnd.msb.ongoing) { + for (ix = dirty_no; ix < dirty_online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_sched_poke(ssi); } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - change_no_used_runqs(no); - - for (ix = online; ix < no; ix++) - resume_run_queue(ERTS_RUNQ_IX(ix)); - - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); + } else { + for (ix = dirty_no; ix < dirty_online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); } - res = ERTS_SCHDLR_SSPND_DONE; + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); } - else /* if (no < online) */ { - if (p->scheduler_data->no <= no) { - res = ERTS_SCHDLR_SSPND_DONE; - schdlr_sspnd.wait_curr_online = no; - } - else { - /* - * Yield! Current process needs to migrate - * before bif returns. - */ - res = ERTS_SCHDLR_SSPND_YIELD_DONE; - schdlr_sspnd.wait_curr_online = no+1; - } - - if (ongoing_multi_scheduling_block()) { - for (ix = no; ix < online; ix++) - erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); - } - else { - if (plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); - } - - change_no_used_runqs(no); - for (ix = no; ix < erts_no_run_queues; ix++) - suspend_run_queue(ERTS_RUNQ_IX(ix)); - - for (ix = no; ix < online; ix++) { - ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); - wake_scheduler(rq); - } - } + } + if (!dirty_only) +#endif + { + if (schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing) { + for (ix = no; ix < online; ix++) + erts_sched_poke(ERTS_SCHED_SLEEP_INFO_IX(ix)); } - - if (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) { - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (plocks && !have_unlocked_plocks) { + else { + if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - end_wait = 1; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - } - - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ASSERT(res != ERTS_SCHDLR_SSPND_DONE - ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + change_no_used_runqs(no); + for (ix = no; ix < erts_no_run_queues; ix++) + suspend_run_queue(ERTS_RUNQ_IX(ix)); + for (ix = no; ix < online; ix++) { + ErtsRunQueue *rq = ERTS_RUNQ_IX(ix); + wake_scheduler(rq); + } + } } } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - if (end_wait) { - erts_thr_progress_finalize_wait(esdp); - erts_thr_progress_active(esdp, 1); + if (change_flags & ERTS_SCHDLR_SSPND_CHNG_ONLN) { + /* Suspend and wait for requested change to complete... */ + schdlr_sspnd.changer = p->common.id; + resume_proc = 0; + res = ERTS_SCHDLR_SSPND_YIELD_DONE; } + +done: + + ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU) + <= schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL)); + + erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (have_unlocked_plocks) erts_smp_proc_lock(p, plocks); + if (resume_proc) { + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + resume_process(p, plocks|ERTS_PROC_LOCK_STATUS); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } + return res; } -#endif - ErtsSchedSuspendResult -erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) +erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int normal, int all) { - int ix, res, have_unlocked_plocks = 0, online; - erts_aint32_t changing; + int resume_proc, ix, res, have_unlocked_plocks = 0; ErtsProcList *plp; #ifdef ERTS_DIRTY_SCHEDULERS ErtsSchedulerSleepInfo* ssi; #endif + ErtsMultiSchedulingBlock *msbp; + erts_aint32_t chng_flg; + int have_blckd_flg; - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - changing = erts_smp_atomic32_read_nob(&schdlr_sspnd.changing); -#ifdef ERTS_DIRTY_SCHEDULERS - changing |= (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) - | erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing)); -#endif - if (changing) { - res = ERTS_SCHDLR_SSPND_YIELD_RESTART; /* Yield */ + if (normal) { + chng_flg = ERTS_SCHDLR_SSPND_CHNG_NMSB; + have_blckd_flg = F_HAVE_BLCKD_NMSCHED; + msbp = &schdlr_sspnd.nmsb; + } + else { + chng_flg = ERTS_SCHDLR_SSPND_CHNG_MSB; + have_blckd_flg = F_HAVE_BLCKD_MSCHED; + msbp = &schdlr_sspnd.msb; } - else if (on) { /* ------ BLOCK ------ */ - if (schdlr_sspnd.msb.procs) { + + /* + * If we suspend current process we need to suspend before + * requesting the change; otherwise, we got a resume/suspend + * race... + */ + if (!on) { + /* We never suspend current process when unblocking... */ + resume_proc = 0; + } + else { + resume_proc = 1; + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + suspend_process(p, p); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); + } + + erts_smp_mtx_lock(&schdlr_sspnd.mtx); + if (on) { /* ------ BLOCK ------ */ + if (msbp->chngq) { + ASSERT(msbp->ongoing); + p->flags |= have_blckd_flg; + goto wait_until_msb; + } + else if (msbp->blckrs) { + ASSERT(msbp->ongoing); plp = proclist_create(p); - erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); - p->flags |= F_HAVE_BLCKD_MSCHED; - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 0); - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) == 0); -#endif + erts_proclist_store_last(&msbp->blckrs, plp); + p->flags |= have_blckd_flg; + ASSERT(schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0)); ASSERT(p->scheduler_data->no == 1); - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - } else { - int online = schdlr_sspnd.online; - p->flags |= F_HAVE_BLCKD_MSCHED; + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; + } + else { + int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); + ASSERT(!msbp->ongoing); + p->flags |= have_blckd_flg; if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); } - ASSERT(!ongoing_multi_scheduling_block()); - schdlr_sspnd.msb.ongoing = 1; - if (online == 1) { - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); -#ifdef ERTS_DIRTY_SCHEDULERS - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) == 1); - ASSERT(!(erts_smp_atomic32_read_nob(&ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0)->flags) - & ERTS_SSI_FLG_SUSPENDED)); - schdlr_sspnd.msb.dirty_cpu_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(0); - erts_smp_atomic32_read_bor_nob(&ssi->flags, ERTS_SSI_FLG_SUSPENDED); - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) - != schdlr_sspnd.msb.dirty_cpu_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - - schdlr_sspnd.msb.dirty_io_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) - != schdlr_sspnd.msb.dirty_io_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); -#endif + ASSERT(!msbp->ongoing); + msbp->ongoing = 1; + if (schdlr_sspnd.active == ERTS_SCHDLR_SSPND_MAKE_NSCHEDS_VAL(1, 0, 0) + || (normal && schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1)) { ASSERT(p->scheduler_data->no == 1); + plp = proclist_create(p); + erts_proclist_store_last(&msbp->blckrs, plp); + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; } else { - ERTS_SCHDLR_SSPND_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - if (p->scheduler_data->no == 1) { - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - schdlr_sspnd.msb.wait_active = 1; - } - else { - /* - * Yield! Current process needs to migrate - * before bif returns. - */ - res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; - schdlr_sspnd.msb.wait_active = 2; - } - -#ifdef ERTS_DIRTY_SCHEDULERS - schdlr_sspnd.msb.dirty_cpu_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_active) - != schdlr_sspnd.msb.dirty_cpu_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - ASSERT(schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_online); - - schdlr_sspnd.msb.dirty_io_wait_active = 0; - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET((ERTS_SCHDLR_SSPND_CHNG_MSB - | ERTS_SCHDLR_SSPND_CHNG_WAITER), 0); - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - erts_smp_atomic32_read_bor_nob(&ssi->flags, - ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_active) - != schdlr_sspnd.msb.dirty_io_wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - ASSERT(schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_online); -#endif + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + chng_flg); change_no_used_runqs(1); for (ix = 1; ix < erts_no_run_queues; ix++) suspend_run_queue(ERTS_RUNQ_IX(ix)); @@ -7858,84 +7819,84 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) wake_scheduler(rq); } - if (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) - != schdlr_sspnd.msb.wait_active) { - ErtsSchedulerData *esdp; - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - if (plocks && !have_unlocked_plocks) { - have_unlocked_plocks = 1; - erts_smp_proc_unlock(p, plocks); +#ifdef ERTS_DIRTY_SCHEDULERS + if (!normal) { + for (ix = 0; ix < erts_no_dirty_cpu_schedulers; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); } + wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - esdp = ERTS_PROC_GET_SCHDATA(p); - - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - while (erts_smp_atomic32_read_nob(&schdlr_sspnd.active) - != schdlr_sspnd.msb.wait_active) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, - &schdlr_sspnd.mtx); - - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - erts_thr_progress_active(esdp, 1); - erts_thr_progress_finalize_wait(esdp); + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + erts_smp_atomic32_read_bor_nob(&ssi->flags, + ERTS_SSI_FLG_SUSPENDED); + } + wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); + } +#endif - erts_smp_mtx_lock(&schdlr_sspnd.mtx); + wait_until_msb: - } + ASSERT(chng_flg & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)); - ASSERT(res != ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED - ? (ERTS_SCHDLR_SSPND_CHNG_WAITER - & erts_smp_atomic32_read_nob(&schdlr_sspnd.changing)) - : (ERTS_SCHDLR_SSPND_CHNG_WAITER - == erts_smp_atomic32_read_nob(&schdlr_sspnd.changing))); - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_WAITER); + plp = proclist_create(p); + erts_proclist_store_last(&msbp->chngq, plp); + resume_proc = 0; + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED; } - plp = proclist_create(p); - erts_proclist_store_last(&schdlr_sspnd.msb.procs, plp); ASSERT(p->scheduler_data); } } - else if (!ongoing_multi_scheduling_block()) { - /* unblock not ongoing */ - ASSERT(!schdlr_sspnd.msb.procs); - res = ERTS_SCHDLR_SSPND_DONE; + else if (!msbp->ongoing) { + ASSERT(!msbp->blckrs); + goto unblock_res; } else { /* ------ UNBLOCK ------ */ - if (p->flags & F_HAVE_BLCKD_MSCHED) { - ErtsProcList *plp = erts_proclist_peek_first(schdlr_sspnd.msb.procs); - - while (plp) { - ErtsProcList *tmp_plp = plp; - plp = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp); - if (erts_proclist_same(tmp_plp, p)) { - erts_proclist_remove(&schdlr_sspnd.msb.procs, tmp_plp); - proclist_destroy(tmp_plp); - if (!all) - break; + if (p->flags & have_blckd_flg) { + ErtsProcList *plps[2]; + ErtsProcList *plp; + int limit = 0; + + plps[limit++] = erts_proclist_peek_first(msbp->blckrs); + if (all) + plps[limit++] = erts_proclist_peek_first(msbp->chngq); + + for (ix = 0; ix < limit; ix++) { + plp = plps[ix]; + while (plp) { + ErtsProcList *tmp_plp = plp; + plp = erts_proclist_peek_next(msbp->blckrs, plp); + if (erts_proclist_same(tmp_plp, p)) { + erts_proclist_remove(&msbp->blckrs, tmp_plp); + proclist_destroy(tmp_plp); + if (!all) + break; + } } } } - if (schdlr_sspnd.msb.procs) - res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; - else { - ERTS_SCHDLR_SSPND_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); - p->flags &= ~F_HAVE_BLCKD_MSCHED; - schdlr_sspnd.msb.ongoing = 0; - if (schdlr_sspnd.online == 1) { + if (!msbp->blckrs && !msbp->chngq) { + int online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_NORMAL); + erts_smp_atomic32_read_bor_nob(&schdlr_sspnd.changing, + chng_flg); + p->flags &= ~have_blckd_flg; + msbp->ongoing = 0; + if (online == 1) { /* No normal schedulers to resume */ - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.active) == 1); - ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_MSB); + ASSERT(schdlr_sspnd_get_nscheds(&schdlr_sspnd.active, + ERTS_SCHED_NORMAL) == 1); +#ifndef ERTS_DIRTY_SCHEDULERS + erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, + ~chng_flg); +#endif } - else { - online = schdlr_sspnd.online; + else if (!(schdlr_sspnd.msb.ongoing|schdlr_sspnd.nmsb.ongoing)) { if (plocks) { have_unlocked_plocks = 1; erts_smp_proc_unlock(p, plocks); @@ -7951,83 +7912,91 @@ erts_block_multi_scheduling(Process *p, ErtsProcLocks plocks, int on, int all) suspend_run_queue(ERTS_RUNQ_IX(ix)); } #ifdef ERTS_DIRTY_SCHEDULERS - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); - schdlr_sspnd.msb.dirty_cpu_wait_active = schdlr_sspnd.dirty_cpu_online; - for (ix = 0; ix < schdlr_sspnd.dirty_cpu_online; ix++) { - ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - erts_smp_atomic32_read_band_nob(&ssi->flags, - ~ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_CPU_RUNQ, 0); - - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(ERTS_SCHDLR_SSPND_CHNG_MSB, 0); - schdlr_sspnd.msb.dirty_io_wait_active = erts_no_dirty_io_schedulers; - for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { - ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); - scheduler_ssi_resume_wake(ssi); - erts_smp_atomic32_read_band_nob(&ssi->flags, - ~ERTS_SSI_FLG_SUSPENDED); - } - wake_dirty_schedulers(ERTS_DIRTY_IO_RUNQ, 0); + if (!normal) { + ASSERT(!schdlr_sspnd.msb.ongoing); + online = (int) schdlr_sspnd_get_nscheds(&schdlr_sspnd.online, + ERTS_SCHED_DIRTY_CPU); + for (ix = 0; ix < online; ix++) { + ssi = ERTS_DIRTY_CPU_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); + } + + for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { + ssi = ERTS_DIRTY_IO_SCHED_SLEEP_INFO_IX(ix); + scheduler_ssi_resume_wake(ssi); + } + } #endif - res = ERTS_SCHDLR_SSPND_DONE; } + + unblock_res: + if (schdlr_sspnd.msb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED; + else if (schdlr_sspnd.nmsb.ongoing) + res = ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED; + else + res = ERTS_SCHDLR_SSPND_DONE; } erts_smp_mtx_unlock(&schdlr_sspnd.mtx); + if (have_unlocked_plocks) erts_smp_proc_lock(p, plocks); - return res; -} -#ifdef DEBUG -void -erts_dbg_multi_scheduling_return_trap(Process *p, Eterm return_value) -{ - if (return_value == am_blocked) { - erts_aint32_t active = erts_smp_atomic32_read_nob(&schdlr_sspnd.active); - ASSERT(1 <= active && active <= 2); - ASSERT(ERTS_PROC_GET_SCHDATA(p)->no == 1); + if (resume_proc) { + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_lock(p, ERTS_PROC_LOCK_STATUS); + resume_process(p, plocks|ERTS_PROC_LOCK_STATUS); + if (!(plocks & ERTS_PROC_LOCK_STATUS)) + erts_smp_proc_unlock(p, ERTS_PROC_LOCK_STATUS); } + + return res; } -#endif int erts_is_multi_scheduling_blocked(void) { int res; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - res = schdlr_sspnd.msb.procs != NULL; + if (schdlr_sspnd.msb.blckrs) + res = 1; + else if (schdlr_sspnd.nmsb.blckrs) + res = -1; + else + res = 0; erts_smp_mtx_unlock(&schdlr_sspnd.mtx); return res; } Eterm -erts_multi_scheduling_blockers(Process *p) +erts_multi_scheduling_blockers(Process *p, int normal) { Eterm res = NIL; + ErtsMultiSchedulingBlock *msbp; + + msbp = normal ? &schdlr_sspnd.nmsb : &schdlr_sspnd.msb; erts_smp_mtx_lock(&schdlr_sspnd.mtx); - if (!erts_proclist_is_empty(schdlr_sspnd.msb.procs)) { + if (!erts_proclist_is_empty(msbp->blckrs)) { Eterm *hp, *hp_end; ErtsProcList *plp1, *plp2; Uint max_size = 0; - for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + for (plp1 = erts_proclist_peek_first(msbp->blckrs); plp1; - plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { + plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) { max_size += 2; } ASSERT(max_size); hp = HAlloc(p, max_size); hp_end = hp + max_size; - for (plp1 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + for (plp1 = erts_proclist_peek_first(msbp->blckrs); plp1; - plp1 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp1)) { - for (plp2 = erts_proclist_peek_first(schdlr_sspnd.msb.procs); + plp1 = erts_proclist_peek_next(msbp->blckrs, plp1)) { + for (plp2 = erts_proclist_peek_first(msbp->blckrs); plp2->pid != plp1->pid; - plp2 = erts_proclist_peek_next(schdlr_sspnd.msb.procs, plp2)); + plp2 = erts_proclist_peek_next(msbp->blckrs, plp2)); if (plp2 == plp1) { res = CONS(hp, plp1->pid, res); hp += 2; @@ -8096,39 +8065,6 @@ sched_thread_func(void *vesdp) #endif erts_thread_init_float(); - if (no == 1) { - erts_thr_progress_active(esdp, 0); - erts_thr_progress_prepare_wait(esdp); - } - - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.changing) - & ERTS_SCHDLR_SSPND_CHNG_ONLN); - - if (--schdlr_sspnd.curr_online == schdlr_sspnd.wait_curr_online) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (no != 1) -#ifdef ERTS_DIRTY_SCHEDULERS - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); -#else - erts_smp_cnd_signal(&schdlr_sspnd.cnd); -#endif - } - - if (no == 1) { - while (schdlr_sspnd.curr_online != schdlr_sspnd.wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - - if (no == 1) { - erts_thr_progress_finalize_wait(esdp); - erts_thr_progress_active(esdp, 1); - } - #ifdef ERTS_DO_VERIFY_UNUSED_TEMP_ALLOC esdp->verify_unused_temp_alloc = erts_alloc_get_verify_unused_temp_alloc( @@ -8138,7 +8074,7 @@ sched_thread_func(void *vesdp) process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", no); return NULL; @@ -8183,27 +8119,9 @@ sched_dirty_cpu_thread_func(void *vesdp) #endif erts_thread_init_float(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_cpu_changing) - & ERTS_SCHDLR_SSPND_CHNG_ONLN); - - if (--schdlr_sspnd.dirty_cpu_curr_online == schdlr_sspnd.dirty_cpu_wait_curr_online) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_cpu_changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (no != 1) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - } - - if (no == 1) { - while (schdlr_sspnd.dirty_cpu_curr_online != schdlr_sspnd.dirty_cpu_wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_CPU_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Dirty CPU scheduler thread number %beu terminated\n", no); return NULL; @@ -8246,27 +8164,9 @@ sched_dirty_io_thread_func(void *vesdp) #endif erts_thread_init_float(); - erts_smp_mtx_lock(&schdlr_sspnd.mtx); - ASSERT(erts_smp_atomic32_read_nob(&schdlr_sspnd.dirty_io_changing) - & ERTS_SCHDLR_SSPND_CHNG_ONLN); - - if (--schdlr_sspnd.dirty_io_curr_online == schdlr_sspnd.dirty_io_wait_curr_online) { - erts_smp_atomic32_read_band_nob(&schdlr_sspnd.dirty_io_changing, - ~ERTS_SCHDLR_SSPND_CHNG_ONLN); - if (no != 1) - erts_smp_cnd_broadcast(&schdlr_sspnd.cnd); - } - - if (no == 1) { - while (schdlr_sspnd.dirty_io_curr_online != schdlr_sspnd.dirty_io_wait_curr_online) - erts_smp_cnd_wait(&schdlr_sspnd.cnd, &schdlr_sspnd.mtx); - ERTS_SCHDLR_SSPND_DIRTY_IO_CHNG_SET(0, ERTS_SCHDLR_SSPND_CHNG_WAITER); - } - erts_smp_mtx_unlock(&schdlr_sspnd.mtx); - process_main(); /* No schedulers should *ever* terminate */ - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Dirty I/O scheduler thread number %beu terminated\n", no); return NULL; @@ -8296,12 +8196,12 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "runq_supervisor"); erts_atomic_init_nob(&runq_supervisor_sleeping, 0); if (0 != ethr_event_init(&runq_supervision_event)) - erl_exit(1, "Failed to create run-queue supervision event\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision event\n"); if (0 != ethr_thr_create(&runq_supervisor_tid, runq_supervisor, NULL, &opts)) - erl_exit(1, "Failed to create run-queue supervision thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create run-queue supervision thread\n"); } #endif @@ -8339,14 +8239,14 @@ erts_start_schedulers(void) erts_snprintf(opts.name, 16, "%d_dirty_cpu_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid,sched_dirty_cpu_thread_func,(void*)esdp,&opts); if (res != 0) - erl_exit(1, "Failed to create dirty cpu scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty cpu scheduler thread %d\n", ix); } for (ix = 0; ix < erts_no_dirty_io_schedulers; ix++) { ErtsSchedulerData *esdp = ERTS_DIRTY_IO_SCHEDULER_IX(ix); erts_snprintf(opts.name, 16, "%d_dirty_io_scheduler", ix + 1); res = ethr_thr_create(&esdp->tid,sched_dirty_io_thread_func,(void*)esdp,&opts); if (res != 0) - erl_exit(1, "Failed to create dirty io scheduler thread %d\n", ix); + erts_exit(ERTS_ERROR_EXIT, "Failed to create dirty io scheduler thread %d\n", ix); } } #endif @@ -8358,10 +8258,10 @@ erts_start_schedulers(void) res = ethr_thr_create(&aux_tid, aux_thread, NULL, &opts); if (res != 0) - erl_exit(1, "Failed to create aux thread\n"); + erts_exit(ERTS_ERROR_EXIT, "Failed to create aux thread\n"); if (actual < 1) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Failed to create any scheduler-threads: %s (%d)\n", erl_errno_id(res), res); @@ -8515,9 +8415,8 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, resume_process(rp, rp_locks); } else { - rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, - pid, pid_locks|ERTS_PROC_LOCK_STATUS); + pid, ERTS_PROC_LOCK_STATUS); if (!rp) { c_p->flags &= ~F_P2PNR_RESCHED; @@ -8526,40 +8425,84 @@ pid2proc_not_running(Process *c_p, ErtsProcLocks c_p_locks, ASSERT(!(c_p->flags & F_P2PNR_RESCHED)); - if (suspend) { - if (suspend_process(c_p, rp)) - goto done; - } - else { - if (!((ERTS_PSFLG_RUNNING|ERTS_PSFLG_RUNNING_SYS) - & erts_smp_atomic32_read_acqb(&rp->state))) - goto done; + /* + * Suspend the other process in order to prevent + * it from being selected for normal execution. + * This will however not prevent it from being + * selected for execution of a system task. If + * it is selected for execution of a system task + * we might be blocked for quite a while if the + * try-lock below fails. That is, there is room + * for improvement here... + */ - } + if (!suspend_process(c_p, rp)) { + /* Other process running */ - /* Other process running */ + ASSERT(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state)); - /* - * If we got pending suspenders and suspend ourselves waiting - * to suspend another process we might deadlock. - * In this case we have to yield, be suspended by - * someone else and then do it all over again. - */ - if (!c_p->pending_suspenders) { - /* Mark rp pending for suspend by c_p */ - add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); - ASSERT(is_nil(c_p->suspendee)); + running: - /* Suspend c_p; when rp is suspended c_p will be resumed. */ - suspend_process(c_p, c_p); - c_p->flags |= F_P2PNR_RESCHED; + /* + * If we got pending suspenders and suspend ourselves waiting + * to suspend another process we might deadlock. + * In this case we have to yield, be suspended by + * someone else and then do it all over again. + */ + if (!c_p->pending_suspenders) { + /* Mark rp pending for suspend by c_p */ + add_pend_suspend(rp, c_p->common.id, handle_pend_sync_suspend); + ASSERT(is_nil(c_p->suspendee)); + + /* Suspend c_p; when rp is suspended c_p will be resumed. */ + suspend_process(c_p, c_p); + c_p->flags |= F_P2PNR_RESCHED; + } + /* Yield (caller is assumed to yield immediately in bif). */ + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + rp = ERTS_PROC_LOCK_BUSY; + } + else { + ErtsProcLocks need_locks = pid_locks & ~ERTS_PROC_LOCK_STATUS; + if (need_locks && erts_smp_proc_trylock(rp, need_locks) == EBUSY) { + if (ERTS_PSFLG_RUNNING_SYS + & erts_smp_atomic32_read_nob(&rp->state)) { + /* Executing system task... */ + resume_process(rp, ERTS_PROC_LOCK_STATUS); + goto running; + } + erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); + /* + * If we are unlucky, the process just got selected for + * execution of a system task. In this case we may be + * blocked here for quite a while... Execution of system + * tasks are fortunately quite rare events. We try to + * avoid this by checking if it is in a state executing + * system tasks (above), but it will not prevent all + * scenarios for a long block here... + */ + rp = erts_pid2proc(c_p, c_p_locks|ERTS_PROC_LOCK_STATUS, + pid, pid_locks|ERTS_PROC_LOCK_STATUS); + if (!rp) + goto done; + } + + /* + * The previous suspend has prevented the process + * from being selected for normal execution regardless + * of locks held or not held on it... + */ + ASSERT(!(ERTS_PSFLG_RUNNING + & erts_smp_atomic32_read_nob(&rp->state))); + + if (!suspend) + resume_process(rp, pid_locks|ERTS_PROC_LOCK_STATUS); } - /* Yield (caller is assumed to yield immediately in bif). */ - erts_smp_proc_unlock(rp, pid_locks|ERTS_PROC_LOCK_STATUS); - rp = ERTS_PROC_LOCK_BUSY; } done: + if (rp && rp != ERTS_PROC_LOCK_BUSY && !(pid_locks & ERTS_PROC_LOCK_STATUS)) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_STATUS); if (unlock_c_p_status) @@ -9493,11 +9436,11 @@ Process *schedule(Process *p, int calls) if (!ERTS_SCHEDULER_IS_DIRTY(esdp)) { ErtsProcList *pnd_xtrs = rq->procs.pending_exiters; if (erts_proclist_fetch(&pnd_xtrs, NULL)) { - rq->procs.pending_exiters = NULL; - erts_smp_runq_unlock(rq); - handle_pending_exiters(pnd_xtrs); - erts_smp_runq_lock(rq); - } + rq->procs.pending_exiters = NULL; + erts_smp_runq_unlock(rq); + handle_pending_exiters(pnd_xtrs); + erts_smp_runq_lock(rq); + } if (rq->check_balance_reds <= 0) check_balance(rq); @@ -12269,7 +12212,7 @@ static void doit_exit_link(ErtsLink *lnk, void *vpcontext) break; default: - erl_exit(1, "bad type in link list\n"); + erts_exit(ERTS_ERROR_EXIT, "bad type in link list\n"); break; } erts_destroy_link(lnk); @@ -12310,7 +12253,7 @@ erts_do_exit_process(Process* p, Eterm reason) #endif if (p->static_flags & ERTS_STC_FLG_SYSTEM_PROC) - erl_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n", + erts_exit(ERTS_DUMP_EXIT, "System process %T terminated: %T\n", p->common.id, reason); #ifdef ERTS_SMP @@ -12414,21 +12357,46 @@ erts_continue_exit_process(Process *p) #endif #ifdef ERTS_SMP + if (p->flags & F_SCHDLR_ONLN_WAITQ) + abort_sched_onln_chng_waitq(p); + if (p->flags & F_HAVE_BLCKD_MSCHED) { ErtsSchedSuspendResult ssr; - ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1); + ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 0, 1); + switch (ssr) { + case ERTS_SCHDLR_SSPND_YIELD_RESTART: + goto yield; + case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE: + case ERTS_SCHDLR_SSPND_YIELD_DONE: + p->flags &= ~F_HAVE_BLCKD_MSCHED; + break; + case ERTS_SCHDLR_SSPND_EINVAL: + default: + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + __FILE__, __LINE__, (int) ssr); + } + } + if (p->flags & F_HAVE_BLCKD_NMSCHED) { + ErtsSchedSuspendResult ssr; + ssr = erts_block_multi_scheduling(p, ERTS_PROC_LOCK_MAIN, 0, 1, 1); switch (ssr) { case ERTS_SCHDLR_SSPND_YIELD_RESTART: goto yield; case ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED: case ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED: + case ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED: case ERTS_SCHDLR_SSPND_DONE: case ERTS_SCHDLR_SSPND_YIELD_DONE: p->flags &= ~F_HAVE_BLCKD_MSCHED; break; case ERTS_SCHDLR_SSPND_EINVAL: default: - erl_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: %d\n", __FILE__, __LINE__, (int) ssr); } } @@ -12925,10 +12893,10 @@ erts_print_scheduler_info(int to, void *to_arg, ErtsSchedulerData *esdp) { * The same global atomic is used as refcount. * * A BIF that calls this should make sure to schedule out to never come back: - * erl_halt((int)(- code)); + * erts_halt(code); * ERTS_BIF_YIELD1(bif_export[BIF_erlang_halt_1], BIF_P, NIL); */ -void erl_halt(int code) +void erts_halt(int code) { if (-1 == erts_smp_atomic32_cmpxchg_acqb(&erts_halt_progress, erts_no_schedulers, diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index ef4aab7234..c88bd7056c 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -79,10 +79,8 @@ struct ErtsNodesMonitor_; #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT 0 #define ERTS_MAX_NO_OF_SCHEDULERS 1024 -#ifdef ERTS_DIRTY_SCHEDULERS #define ERTS_MAX_NO_OF_DIRTY_CPU_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS #define ERTS_MAX_NO_OF_DIRTY_IO_SCHEDULERS ERTS_MAX_NO_OF_SCHEDULERS -#endif #define ERTS_DEFAULT_MAX_PROCESSES (1 << 18) @@ -246,7 +244,9 @@ extern int erts_sched_thread_suggested_stack_size; typedef enum { ERTS_SCHDLR_SSPND_DONE_MSCHED_BLOCKED, + ERTS_SCHDLR_SSPND_DONE_NMSCHED_BLOCKED, ERTS_SCHDLR_SSPND_YIELD_DONE_MSCHED_BLOCKED, + ERTS_SCHDLR_SSPND_YIELD_DONE_NMSCHED_BLOCKED, ERTS_SCHDLR_SSPND_DONE, ERTS_SCHDLR_SSPND_YIELD_RESTART, ERTS_SCHDLR_SSPND_YIELD_DONE, @@ -1339,6 +1339,8 @@ extern int erts_system_profile_ts_type; #define F_OFF_HEAP_MSGQ_CHNG (1 << 14) /* Off heap msg queue changing */ #define F_ABANDONED_HEAP_USE (1 << 15) /* Have usage of abandoned heap */ #define F_DELAY_GC (1 << 16) /* Similar to disable GC (see below) */ +#define F_SCHDLR_ONLN_WAITQ (1 << 17) /* Process enqueued waiting to change schedulers online */ +#define F_HAVE_BLCKD_NMSCHED (1 << 18) /* Process has blocked normal multi-scheduling */ /* * F_DISABLE_GC and F_DELAY_GC are similar. Both will prevent @@ -1523,6 +1525,7 @@ void erts_init_scheduling(int, int int erts_set_gc_state(Process *c_p, int enable); Eterm erts_sched_wall_time_request(Process *c_p, int set, int enable); +Eterm erts_system_check_request(Process *c_p); Eterm erts_gc_info_request(Process *c_p); Uint64 erts_get_proc_interval(void); Uint64 erts_ensure_later_proc_interval(Uint64); @@ -1532,6 +1535,7 @@ int erts_setup_nif_gc(Process* proc, Eterm** objv, int* nobj); /* see erl_nif.c void erts_destroy_nif_export(void *); /* see erl_nif.c */ ErtsProcList *erts_proclist_create(Process *); +ErtsProcList *erts_proclist_copy(ErtsProcList *); void erts_proclist_destroy(ErtsProcList *); ERTS_GLB_INLINE int erts_proclist_same(ErtsProcList *, Process *); @@ -1717,28 +1721,21 @@ void erts_schedule_complete_off_heap_message_queue_change(Eterm pid); #if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) int erts_dbg_check_halloc_lock(Process *p); #endif -#ifdef DEBUG -void erts_dbg_multi_scheduling_return_trap(Process *, Eterm); -#endif -int erts_get_max_no_executing_schedulers(void); #if defined(ERTS_SMP) || defined(ERTS_DIRTY_SCHEDULERS) -ErtsSchedSuspendResult -erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, int); +void +erts_schedulers_state(Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *, Uint *); #endif #ifdef ERTS_SMP ErtsSchedSuspendResult erts_set_schedulers_online(Process *p, ErtsProcLocks plocks, Sint new_no, - Sint *old_no -#ifdef ERTS_DIRTY_SCHEDULERS - , int dirty_only -#endif - ); + Sint *old_no, + int dirty_only); ErtsSchedSuspendResult -erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int); +erts_block_multi_scheduling(Process *, ErtsProcLocks, int, int, int); int erts_is_multi_scheduling_blocked(void); -Eterm erts_multi_scheduling_blockers(Process *); +Eterm erts_multi_scheduling_blockers(Process *, int); void erts_start_schedulers(void); void erts_alloc_notify_delayed_dealloc(int); void erts_alloc_ensure_handle_delayed_dealloc_call(int); @@ -2461,6 +2458,6 @@ erts_sched_poke(ErtsSchedulerSleepInfo *ssi) #endif -void erl_halt(int code); +void erts_halt(int code); extern erts_smp_atomic32_t erts_halt_progress; extern int erts_halt_code; diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 36d16f7f42..487b944fc0 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -392,7 +392,7 @@ static void pd_hash_erase(Process *p, Eterm id, Eterm *ret) "display term found in line %d:\n" "%T\n", p->common.id, __LINE__, old); #endif - erl_exit(1, "Damaged process dictionary found during erase/1."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during erase/1."); } if ((range = HASH_RANGE(p->dictionary)) > INITIAL_SIZE && range / 2 > (p->dictionary->numElements)) { @@ -452,7 +452,7 @@ Eterm pd_hash_get_with_hval(Process *p, Eterm bucket, Eterm id) "display term found in line %d:\n" "%T\n", p->common.id, __LINE__, bucket); #endif - erl_exit(1, "Damaged process dictionary found during get/1."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during get/1."); } return am_undefined; } @@ -707,7 +707,7 @@ static Eterm pd_hash_put(Process *p, Eterm id, Eterm value) "%T\n", p->common.id, __LINE__, old); #endif - erl_exit(1, "Damaged process dictionary found during put/2."); + erts_exit(ERTS_ERROR_EXIT, "Damaged process dictionary found during put/2."); } if (HASH_RANGE(p->dictionary) <= p->dictionary->numElements) { grow(p); @@ -1061,7 +1061,7 @@ static void pd_check(ProcDict *pd) } continue; } else { - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "Found tag 0x%08x in process dictionary at position %d", (unsigned long) t, (int) i); } diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index a64c993e8f..9c59301086 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -523,6 +523,10 @@ erts_smp_proc_lock__(Process *p, ERTS_LC_ASSERT((locks & ~ERTS_PROC_LOCKS_ALL) == 0); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_proc_lc_lock(p, locks, file, line); +#endif + old_lflgs = erts_smp_proc_raw_trylock__(p, locks); if (old_lflgs != 0) { @@ -544,9 +548,6 @@ erts_smp_proc_lock__(Process *p, #ifdef ERTS_ENABLE_LOCK_COUNT erts_lcnt_proc_lock_post_x(&(p->lock), locks, file, line); #endif -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_proc_lc_lock(p, locks, file, line); -#endif #ifdef ERTS_PROC_LOCK_DEBUG erts_proc_lock_op_debug(p, locks, 1); diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index f7997df051..9ed175fe01 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -1255,7 +1255,7 @@ ptab_list_bif_engine(Process *c_p, Eterm *res_accp, Binary *mbp) return 1; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:ptab_list_bif_engine(): Invalid state: %d\n", __FILE__, __LINE__, (int) ptlbdp->state); } diff --git a/erts/emulator/beam/erl_term.c b/erts/emulator/beam/erl_term.c index e31d3a951d..d2343912b4 100644 --- a/erts/emulator/beam/erl_term.c +++ b/erts/emulator/beam/erl_term.c @@ -72,7 +72,7 @@ et_abort(const char *expr, const char *file, unsigned line) * Prevent infinite loop. */ have_been_called = 1; - erl_exit(1, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); + erts_exit(ERTS_ERROR_EXIT, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); } #else erts_fprintf(stderr, "TYPE ASSERTION FAILED, file %s, line %u: %s\n", file, line, expr); diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 7148b756e7..7b06fd840f 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -502,7 +502,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) if (tpd) { if (!tpd->is_temporary) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Double register of thread\n", __FILE__, __LINE__, __func__); is_blocking = tpd->is_blocking; @@ -524,7 +524,7 @@ erts_thr_progress_register_unmanaged_thread(ErtsThrPrgrCallbacks *callbacks) #endif ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->unmanaged.no) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Too many unmanaged registered threads\n", __FILE__, __LINE__, __func__); @@ -547,7 +547,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, if (tpd) { if (!tpd->is_temporary) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Double register of thread\n", __FILE__, __LINE__, __func__); is_blocking = tpd->is_blocking; @@ -568,7 +568,7 @@ erts_thr_progress_register_managed_thread(ErtsSchedulerData *esdp, tpd->id = erts_atomic32_inc_read_nob(&intrnl->misc.data.managed_id); ASSERT(tpd->id >= 0); if (tpd->id >= intrnl->managed.no) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Too many managed registered threads\n", __FILE__, __LINE__, __func__); @@ -1033,7 +1033,7 @@ has_reached_wakeup(ErtsThrPrgrVal wakeup) limit += 1; if (!erts_thr_progress_has_passed__(limit, wakeup)) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Invalid wakeup request value found:" " current=%b64u, wakeup=%b64u, limit=%b64u", current, wakeup, limit); @@ -1102,7 +1102,7 @@ request_wakeup_managed(ErtsThrPrgrData *tpd, ErtsThrPrgrVal value) ix = erts_atomic32_inc_read_nob(&mwd->len) - 1; #if ERTS_THR_PRGR_DBG_CHK_WAKEUP_REQUEST_VALUE if (ix >= intrnl->managed.no) - erl_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n"); + erts_exit(ERTS_ABORT_EXIT, "Internal error: Too many wakeup requests\n"); #endif mwd->id[ix] = tpd->id; diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index 3a91ca9dbe..7ff456b915 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -224,7 +224,7 @@ ErtsThrQCleanState_t erts_thr_q_destroy(ErtsThrQ_t *q) { if (!q->q.blk) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Trying to destroy not created thread queue\n"); return erts_thr_q_finalize(q); } @@ -589,7 +589,7 @@ enqueue(ErtsThrQ_t *q, void *data, ErtsThrQElement_t *this) #if ERTS_THR_Q_DBG_CHK_DATA if (!data) - erl_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); + erts_exit(ERTS_ABORT_EXIT, "Missing data in enqueue\n"); #endif ASSERT(!q->q.finalizing); @@ -771,7 +771,7 @@ erts_thr_q_dequeue(ErtsThrQ_t *q) #if ERTS_THR_Q_DBG_CHK_DATA head->data.ptr = NULL; if (!res) - erl_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); + erts_exit(ERTS_ABORT_EXIT, "Missing data in dequeue\n"); #endif clean(q, (q->head.deq_fini.automatic diff --git a/erts/emulator/beam/erl_time_sup.c b/erts/emulator/beam/erl_time_sup.c index 98159fdf72..f3f528eaf6 100644 --- a/erts/emulator/beam/erl_time_sup.c +++ b/erts/emulator/beam/erl_time_sup.c @@ -289,7 +289,7 @@ read_corrected_time(int os_drift_corrected) ci = time_sup.inf.c.parmon.cdata.insts.curr; else { if (os_mtime < time_sup.inf.c.parmon.cdata.insts.prev.os_mtime) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "OS monotonic time stepped backwards\n"); ci = time_sup.inf.c.parmon.cdata.insts.prev; } @@ -381,7 +381,7 @@ check_time_correction(void *vesdp) erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); if (os_mtime < ci.os_mtime) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "OS monotonic time stepped backwards\n"); erl_mtime = calc_corrected_erl_mtime(os_mtime, &ci, &mdiff, @@ -798,7 +798,7 @@ finalize_corrected_time_offset(ErtsSystemTime *stimep) erts_smp_rwmtx_runlock(&time_sup.inf.c.parmon.rwmtx); if (os_mtime < ci.os_mtime) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "OS monotonic time stepped backwards\n"); return calc_corrected_erl_mtime(os_mtime, &ci, NULL, diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 25393369a7..41876b8c62 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -661,7 +661,7 @@ WRITE_SYS_MSG_TO_PORT(Eterm unused_to, erts_encode_ext(message, &ptr); if (!(ptr <= buffer+size)) { - erl_exit(1, "Internal error in do_send_to_port: %d\n", ptr-buffer); + erts_exit(ERTS_ERROR_EXIT, "Internal error in do_send_to_port: %d\n", ptr-buffer); } #ifndef ERTS_SMP @@ -1286,7 +1286,7 @@ seq_trace_output_generic(Eterm token, Eterm msg, Uint type, case SEQ_TRACE_PRINT: type_atom = am_print; break; case SEQ_TRACE_RECEIVE: type_atom = am_receive; break; default: - erl_exit(1, "invalid type in seq_trace_output_generic: %d:\n", type); + erts_exit(ERTS_ERROR_EXIT, "invalid type in seq_trace_output_generic: %d:\n", type); return; /* To avoid warning */ } diff --git a/erts/emulator/beam/erl_unicode.c b/erts/emulator/beam/erl_unicode.c index 551717139d..36d85b0a22 100644 --- a/erts/emulator/beam/erl_unicode.c +++ b/erts/emulator/beam/erl_unicode.c @@ -2507,7 +2507,7 @@ void erts_copy_utf8_to_utf16_little(byte *target, byte *bytes, int num_chars) ((Uint) (bytes[3] & ((byte) 0x3F))); bytes += 4; } else { - erl_exit(1,"Internal unicode error in prim_file:internal_name2native/1"); + erts_exit(ERTS_ERROR_EXIT,"Internal unicode error in prim_file:internal_name2native/1"); } *target++ = (byte) (unipoint & 0xFF); *target++ = (byte) ((unipoint >> 8) & 0xFF); diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index 58010dcb72..aac1490f0c 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -570,7 +570,7 @@ void erts_encode_ext(Eterm term, byte **ext) *ep++ = VERSION_MAGIC; ep = enc_term(NULL, term, ep, TERM_TO_BINARY_DFLAGS, NULL); if (!ep) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d:erts_encode_ext(): Internal data structure error\n", __FILE__, __LINE__); *ext = ep; @@ -1559,7 +1559,7 @@ static BIF_RETTYPE binary_to_term_int(Process* p, Uint32 flags, Eterm bin, Binar b2t_destroy_context(ctx); if (ctx->u.dc.factory.hp > ctx->u.dc.factory.hp_end) { - erl_exit(1, ":%s, line %d: heap overrun by %d words(s)\n", + erts_exit(ERTS_ERROR_EXIT, ":%s, line %d: heap overrun by %d words(s)\n", __FILE__, __LINE__, ctx->u.dc.factory.hp - ctx->u.dc.factory.hp_end); } erts_factory_close(&ctx->u.dc.factory); @@ -1708,12 +1708,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl if ((endp = enc_term(NULL, Term, bytes, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, real_size - size); } @@ -1753,12 +1753,12 @@ erts_term_to_binary_simple(Process* p, Eterm Term, Uint size, int level, Uint fl bytes[0] = VERSION_MAGIC; if ((endp = enc_term(NULL, Term, bytes+1, flags, NULL)) == NULL) { - erl_exit(1, "%s, line %d: bad term: %x\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: bad term: %x\n", __FILE__, __LINE__, Term); } real_size = endp - bytes; if (real_size > size) { - erl_exit(1, "%s, line %d: buffer overflow: %d word(s)\n", + erts_exit(ERTS_ERROR_EXIT, "%s, line %d: buffer overflow: %d word(s)\n", __FILE__, __LINE__, endp - (bytes + size)); } return erts_realloc_binary(bin, real_size); @@ -2631,7 +2631,7 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ASSERT(node_sz < 17); break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); } ptr++; @@ -4086,7 +4086,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, ASSERT(node_sz < 17); break; default: - erl_exit(1, "bad header\r\n"); + erts_exit(ERTS_ERROR_EXIT, "bad header\r\n"); } ptr++; @@ -4174,7 +4174,7 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, break; default: - erl_exit(1,"Internal data structure error (in encode_size_struct2)%x\n", + erts_exit(ERTS_ERROR_EXIT,"Internal data structure error (in encode_size_struct2)%x\n", obj); } diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 2a8bdb6ee3..d7edf451be 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -425,7 +425,7 @@ void erl_grow_estack(ErtsEStack*, Uint need); #define ESTACK_CHANGE_ALLOCATOR(s,t) \ do { \ if ((s).start != ESTK_DEF_STACK(s)) { \ - erl_exit(1, "Internal error - trying to change allocator " \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ "type of active estack\n"); \ } \ (s).alloc_type = (t); \ @@ -586,7 +586,7 @@ do { \ #define WSTACK_CHANGE_ALLOCATOR(s,t) \ do { \ if (s.wstart != WSTK_DEF_STACK(s)) { \ - erl_exit(1, "Internal error - trying to change allocator " \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ "type of active wstack\n"); \ } \ s.alloc_type = (t); \ @@ -778,7 +778,7 @@ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ #define PSTACK_CHANGE_ALLOCATOR(s,t) \ do { \ if (s.pstart != (byte*)PSTK_DEF_STACK(s)) { \ - erl_exit(1, "Internal error - trying to change allocator " \ + erts_exit(ERTS_ERROR_EXIT, "Internal error - trying to change allocator " \ "type of active pstack\n"); \ } \ s.alloc_type = (t); \ @@ -1008,6 +1008,7 @@ typedef struct { Binary* erts_alloc_loader_state(void); Eterm erts_module_for_prepared_code(Binary* magic); +Eterm erts_has_code_on_load(Binary* magic); Eterm erts_prepare_loading(Binary* loader_state, Process *c_p, Eterm group_leader, Eterm* modp, byte* code, Uint size); @@ -1026,7 +1027,7 @@ Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); /* beam_ranges.c */ void erts_init_ranges(void); -void erts_start_staging_ranges(void); +void erts_start_staging_ranges(int num_new); void erts_end_staging_ranges(int commit); void erts_update_ranges(BeamInstr* code, Uint size); void erts_remove_from_ranges(BeamInstr* code); @@ -1047,8 +1048,8 @@ double erts_get_positive_zero_float(void); /* config.c */ -__decl_noreturn void __noreturn erl_exit(int n, char*, ...); -__decl_noreturn void __noreturn erl_exit_flush_async(int n, char*, ...); +__decl_noreturn void __noreturn erts_exit(int n, char*, ...); +__decl_noreturn void __noreturn erts_flush_async_exit(int n, char*, ...); void erl_error(char*, va_list); /* This controls whether sharing-preserving copy is used by Erlang */ diff --git a/erts/emulator/beam/index.c b/erts/emulator/beam/index.c index 06d0b5123d..5f6ea14732 100644 --- a/erts/emulator/beam/index.c +++ b/erts/emulator/beam/index.c @@ -84,7 +84,7 @@ index_put_entry(IndexTable* t, void* tmpl) Uint sz; if (ix >= t->limit) { /* A core dump is unnecessary */ - erl_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n", + erts_exit(ERTS_DUMP_EXIT, "no more index entries in %s (max=%d)\n", t->htable.name, t->limit); } sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); @@ -123,7 +123,7 @@ void erts_index_merge(Hash* src, IndexTable* dst) ix = dst->entries++; if (ix >= dst->size) { if (ix >= dst->limit) { - erl_exit(1, "no more index entries in %s (max=%d)\n", + erts_exit(ERTS_ERROR_EXIT, "no more index entries in %s (max=%d)\n", dst->htable.name, dst->limit); } sz = INDEX_PAGE_SIZE*sizeof(IndexSlot*); diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 41b0fcdd1b..797e4cdadc 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -3587,7 +3587,7 @@ terminate_port(Port *prt) if ((state & ERTS_PORT_SFLG_HALT) && (erts_smp_atomic32_dec_read_nob(&erts_halt_progress) == 0)) { erts_port_release(prt); /* We will exit and never return */ - erl_exit_flush_async(erts_halt_code, ""); + erts_flush_async_exit(erts_halt_code, ""); } if (is_internal_port(send_closed_port_id)) deliver_result(send_closed_port_id, connected_id, am_closed); @@ -5018,7 +5018,7 @@ void erts_raw_port_command(Port* p, byte* buf, Uint len) ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(p)); if (len > (Uint) INT_MAX) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Absurdly large data buffer (%beu bytes) passed to" "output callback of %s driver.\n", len, @@ -7396,7 +7396,7 @@ driver_system_info(ErlDrvSysInfo *sip, size_t si_size) * of ErlDrvSysInfo (introduced in driver version 1.0). */ if (!sip || si_size < ERL_DRV_SYS_INFO_SIZE(smp_support)) - erl_exit(1, + erts_exit(ERTS_ERROR_EXIT, "driver_system_info(%p, %ld) called with invalid arguments\n", sip, si_size); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index e66d628ca9..b7468b0926 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -37,6 +37,7 @@ struct erl_module_instance { typedef struct erl_module { IndexSlot slot; /* Must be located at top of struct! */ int module; /* Atom index for module (not tagged). */ + int seen; /* Used by finish_loading() */ struct erl_module_instance curr; struct erl_module_instance old; /* protected by "old_code" rwlock */ diff --git a/erts/emulator/beam/register.c b/erts/emulator/beam/register.c index fdb6cbc813..ebc35b0c4d 100644 --- a/erts/emulator/beam/register.c +++ b/erts/emulator/beam/register.c @@ -125,7 +125,7 @@ static RegProc* reg_alloc(RegProc *tmpl) { RegProc* obj = (RegProc*) erts_alloc(ERTS_ALC_T_REG_PROC, sizeof(RegProc)); if (!obj) { - erl_exit(1, "Can't allocate %d bytes of memory\n", sizeof(RegProc)); + erts_exit(ERTS_ERROR_EXIT, "Can't allocate %d bytes of memory\n", sizeof(RegProc)); } obj->name = tmpl->name; obj->p = tmpl->p; diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 068d636a40..03b9088adc 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -580,15 +580,16 @@ static unsigned long zero_value = 0, one_value = 1; # endif /* !__WIN32__ */ #endif /* WANT_NONBLOCKING */ -__decl_noreturn void __noreturn erl_exit(int n, char*, ...); +__decl_noreturn void __noreturn erts_exit(int n, char*, ...); -/* Some special erl_exit() codes: */ -#define ERTS_INTR_EXIT INT_MIN /* called from signal handler */ -#define ERTS_ABORT_EXIT (INT_MIN + 1) /* no crash dump; only abort() */ -#define ERTS_DUMP_EXIT (INT_MIN + 2) /* crash dump; then exit() */ +/* Some special erts_exit() codes: */ +#define ERTS_INTR_EXIT -1 /* called from signal handler */ +#define ERTS_ABORT_EXIT -2 /* no crash dump; only abort() */ +#define ERTS_DUMP_EXIT -3 /* crash dump; then exit() */ +#define ERTS_ERROR_EXIT -4 /* crash dump; then abort() */ #define ERTS_INTERNAL_ERROR(What) \ - erl_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ + erts_exit(ERTS_ABORT_EXIT, "%s:%d:%s(): Internal error: %s\n", \ __FILE__, __LINE__, __func__, What) Eterm erts_check_io_info(void *p); @@ -903,7 +904,7 @@ erts_refc_inc(erts_refc_t *refcp, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_inc(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else @@ -917,7 +918,7 @@ erts_refc_inctest(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_inc_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_inctest(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -930,7 +931,7 @@ erts_refc_dec(erts_refc_t *refcp, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_dec(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #else @@ -944,7 +945,7 @@ erts_refc_dectest(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_dec_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_dectest(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif @@ -957,7 +958,7 @@ erts_refc_add(erts_refc_t *refcp, erts_aint_t diff, erts_aint_t min_val) #ifdef ERTS_REFC_DEBUG erts_aint_t val = erts_smp_atomic_add_read_nob((erts_smp_atomic_t *) refcp, diff); if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_add(%ld): Bad refc found (refc=%ld < %ld)!\n", diff, val, min_val); #else @@ -971,7 +972,7 @@ erts_refc_read(erts_refc_t *refcp, erts_aint_t min_val) erts_aint_t val = erts_smp_atomic_read_nob((erts_smp_atomic_t *) refcp); #ifdef ERTS_REFC_DEBUG if (val < min_val) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "erts_refc_read(): Bad refc found (refc=%ld < %ld)!\n", val, min_val); #endif diff --git a/erts/emulator/beam/time.c b/erts/emulator/beam/time.c index 48cb39333a..0ce4f443f3 100644 --- a/erts/emulator/beam/time.c +++ b/erts/emulator/beam/time.c @@ -567,7 +567,7 @@ erts_init_time(int time_correction, ErtsTimeWarpMode time_warp_mode) itime = erts_init_time_sup(time_correction, time_warp_mode); #ifdef TIW_ITIME_IS_CONSTANT if (itime != TIW_ITIME) { - erl_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); + erts_exit(ERTS_ABORT_EXIT, "timer resolution mismatch %d != %d", itime, TIW_ITIME); } #else tiw_itime = itime; diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 3ba02023a1..66fcfcf6ce 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -1069,7 +1069,7 @@ tail_recur: } default: - erl_exit(1, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash(0x%X,0x%X)\n", term, op); return 0; } if (WSTACK_ISEMPTY(stack)) break; @@ -1342,7 +1342,7 @@ make_hash2(Eterm term) i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: - erl_exit(1, "bad header"); + erts_exit(ERTS_ERROR_EXIT, "bad header"); } while (i) { if (is_list(*ptr)) { @@ -1505,7 +1505,7 @@ make_hash2(Eterm term) break; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } } break; @@ -1536,7 +1536,7 @@ make_hash2(Eterm term) UINT32_HASH(NIL_DEF, HCONST_2); goto hash2_common; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } case _TAG_IMMED1_SMALL: { @@ -1552,7 +1552,7 @@ make_hash2(Eterm term) } break; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); hash2_common: /* Uint32 hash always has the hash value of the previous term, @@ -1747,7 +1747,7 @@ make_internal_hash(Eterm term) i = hashmap_bitcount(MAP_HEADER_VAL(hdr)); break; default: - erl_exit(1, "bad header"); + erts_exit(ERTS_ERROR_EXIT, "bad header"); } while (i) { if (is_list(*ptr)) { @@ -1923,7 +1923,7 @@ make_internal_hash(Eterm term) goto pop_next; } default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); } } break; @@ -1936,7 +1936,7 @@ make_internal_hash(Eterm term) goto pop_next; default: - erl_exit(1, "Invalid tag in make_hash2(0x%X)\n", term); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_hash2(0x%X)\n", term); pop_next: if (ESTACK_ISEMPTY(s)) { @@ -2219,7 +2219,7 @@ tail_recur: } default: - erl_exit(1, "Invalid tag in make_broken_hash\n"); + erts_exit(ERTS_ERROR_EXIT, "Invalid tag in make_broken_hash\n"); return 0; } if (WSTACK_ISEMPTY(stack)) break; @@ -2893,7 +2893,7 @@ tailrecur_ne: ASSERT(sz > 0 && sz < 17); break; default: - erl_exit(1, "Unknown hashmap subsubtag\n"); + erts_exit(ERTS_ERROR_EXIT, "Unknown hashmap subsubtag\n"); } goto term_array; } diff --git a/erts/emulator/drivers/common/efile_drv.c b/erts/emulator/drivers/common/efile_drv.c index c5c780dce5..3088dfd572 100644 --- a/erts/emulator/drivers/common/efile_drv.c +++ b/erts/emulator/drivers/common/efile_drv.c @@ -119,8 +119,6 @@ #include "dtrace-wrapper.h" -void erl_exit(int n, char *fmt, ...); - static ErlDrvSysInfo sys_info; /* For explanation of this var, see comment for same var in erl_async.c */ @@ -509,21 +507,10 @@ struct t_data static void *ef_safe_alloc(Uint s) { void *p = EF_ALLOC(s); - if (!p) erl_exit(1, "efile drv: Can't allocate %lu bytes of memory\n", (unsigned long)s); - return p; -} - -#if 0 /* Currently not used */ - -static void *ef_safe_realloc(void *op, Uint s) -{ - void *p = EF_REALLOC(op, s); - if (!p) erl_exit(1, "efile drv: Can't reallocate %lu bytes of memory\n", (unsigned long)s); + if (!p) erts_exit(ERTS_ERROR_EXIT, "efile drv: Can't allocate %lu bytes of memory\n", (unsigned long)s); return p; } -#endif - /********************************************************************* * ErlIOVec manipulation functions. */ diff --git a/erts/emulator/drivers/common/inet_drv.c b/erts/emulator/drivers/common/inet_drv.c index f910b5955a..b219669ce5 100644 --- a/erts/emulator/drivers/common/inet_drv.c +++ b/erts/emulator/drivers/common/inet_drv.c @@ -1345,7 +1345,7 @@ static const struct in6_addr in6addr_loopback = #endif /* HAVE_IN6 */ /* XXX: is this a driver interface function ??? */ -void erl_exit(int n, char*, ...); +void erts_exit(int n, char*, ...); /* * Malloc wrapper, @@ -1358,7 +1358,7 @@ void erl_exit(int n, char*, ...); static void *alloc_wrapper(ErlDrvSizeT size){ void *ret = driver_alloc(size); if(ret == NULL) - erl_exit(1,"Out of virtual memory in malloc (%s)", __FILE__); + erts_exit(ERTS_ERROR_EXIT,"Out of virtual memory in malloc (%s)", __FILE__); return ret; } #define ALLOC(X) alloc_wrapper(X) @@ -1366,7 +1366,7 @@ static void *alloc_wrapper(ErlDrvSizeT size){ static void *realloc_wrapper(void *current, ErlDrvSizeT size){ void *ret = driver_realloc(current,size); if(ret == NULL) - erl_exit(1,"Out of virtual memory in realloc (%s)", __FILE__); + erts_exit(ERTS_ERROR_EXIT,"Out of virtual memory in realloc (%s)", __FILE__); return ret; } #define REALLOC(X,Y) realloc_wrapper(X,Y) @@ -1627,7 +1627,7 @@ check_double_release(InetDrvBufStk *bs, ErlDrvBinary* buf) int i; for (i = 0; i < bs->buf.pos; ++i) { if (bs->buf.stk[i] == buf) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Multiple buffer release in inet_drv, this " "is a bug, save the core and send it to " "[email protected]!"); @@ -6803,7 +6803,7 @@ static ErlDrvSSizeT inet_fill_opts(inet_descriptor* desc, do { \ ErlDrvSizeT new_need = ((Ptr) - (*dest)) + (Size); \ if (new_need > dest_used) { \ - erl_exit(1,"Internal error in inet_drv, " \ + erts_exit(ERTS_ERROR_EXIT,"Internal error in inet_drv, " \ "miscalculated buffer size"); \ } \ dest_used = new_need; \ @@ -7180,7 +7180,7 @@ static ErlDrvSSizeT sctp_fill_opts(inet_descriptor* desc, do { \ int need; \ if ((Index) > spec_allocated) { \ - erl_exit(1,"Internal error in inet_drv, " \ + erts_exit(ERTS_ERROR_EXIT,"Internal error in inet_drv, " \ "miscalculated buffer size"); \ } \ need = (Index) + (N); \ diff --git a/erts/emulator/drivers/win32/win_con.c b/erts/emulator/drivers/win32/win_con.c index 0d63d46698..7fe708dc7b 100644 --- a/erts/emulator/drivers/win32/win_con.c +++ b/erts/emulator/drivers/win32/win_con.c @@ -279,7 +279,7 @@ ConInit(void) } /* - ConNormalExit() is called from erl_exit() when the emulator + ConNormalExit() is called from erts_exit() when the emulator is stopping. If the exit has not been initiated by this console thread (WM_DESTROY or ID_BREAK), the function must invoke the console thread to save the user preferences. @@ -529,7 +529,7 @@ ConThreadInit(LPVOID param) /* PostQuitMessage() results in WM_QUIT which makes GetMessage() return 0 (which stops the main loop). Before we return from - the console thread, the ctrl_handler is called to do erl_exit. + the console thread, the ctrl_handler is called to do erts_exit. */ (*ctrl_handler)(CTRL_CLOSE_EVENT); return msg.wParam; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index 6f495b8825..70354aa9af 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -406,7 +406,7 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) nrcallees = arityval(tuple_val(BIF_ARG_2)[0]); else nrcallees = 0; - erl_exit(1, "%s: failed to allocate %lu bytes and %lu trampolines\r\n", + erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n", __func__, (unsigned long)nrbytes, (unsigned long)nrcallees); } memcpy(address, bytes, nrbytes); @@ -1322,7 +1322,7 @@ static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote) export_entry = erts_export_get_or_make_stub(m, f, arity); StubAddress = hipe_make_native_stub(export_entry, arity); if (!StubAddress) - erl_exit(1, "hipe_make_stub: code allocation failed\r\n"); + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); return StubAddress; } diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index 976180cc30..804c458b6c 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -108,7 +108,7 @@ static const char *code_str(unsigned code) static void __noreturn hipe_abort(const char *expr, const char *file, unsigned line) { - erl_exit(1, "ASSERTION FAILED, file %s, line %u: %s\r\n", file, line, expr); + erts_exit(ERTS_ERROR_EXIT, "ASSERTION FAILED, file %s, line %u: %s\r\n", file, line, expr); } #define HIPE_ASSERT3(expr, file, line) \ @@ -316,7 +316,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) break; } default: - erl_exit(1, "hipe_mode_switch: cmd %#x\r\n", cmd); + erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: cmd %#x\r\n", cmd); } do_return_from_native: DPRINTF("result == %#x (%s)", result, code_str(result)); @@ -560,7 +560,7 @@ Process *hipe_mode_switch(Process *p, unsigned cmd, Eterm reg[]) goto do_throw_to_native; } default: - erl_exit(1, "hipe_mode_switch: result %#x\r\n", result); + erts_exit(ERTS_ERROR_EXIT, "hipe_mode_switch: result %#x\r\n", result); } HIPE_CHECK_PCB(p); p->def_arg_reg[3] = result; diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 16cc70234d..0ec417999b 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -180,7 +180,7 @@ void hipe_fclearerror_error(Process *p) #if !defined(NO_FPE_SIGNALS) erts_fp_check_init_error(&p->fp_exception); #else - erl_exit(ERTS_ABORT_EXIT, "Emulated FPE not cleared by HiPE"); + erts_exit(ERTS_ABORT_EXIT, "Emulated FPE not cleared by HiPE"); #endif } diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index 03ca080c14..889d6f3868 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -1341,7 +1341,7 @@ os_unreserve_physical(char *ptr, UWord size) void *res = mmap((void *) ptr, (size_t) size, ERTS_MMAP_UNRESERVE_PROT, ERTS_MMAP_UNRESERVE_FLAGS, ERTS_MMAP_FD, 0); if (res == (void *) MAP_FAILED) - erl_exit(ERTS_ABORT_EXIT, "Failed to unreserve memory"); + erts_exit(ERTS_ABORT_EXIT, "Failed to unreserve memory"); } static void * @@ -2136,7 +2136,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) #endif erts_page_inv_mask = pagesize - 1; if (pagesize & erts_page_inv_mask) - erl_exit(-1, "erts_mmap: Invalid pagesize: %bpu\n", + erts_exit(1, "erts_mmap: Invalid pagesize: %bpu\n", pagesize); ERTS_MMAP_OP_RINGBUF_INIT(); @@ -2148,7 +2148,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) #if HAVE_MMAP && !defined(MAP_ANON) mm->mmap_fd = open("/dev/zero", O_RDWR); if (mm->mmap_fd < 0) - erl_exit(-1, "erts_mmap: Failed to open /dev/zero\n"); + erts_exit(1, "erts_mmap: Failed to open /dev/zero\n"); #endif erts_smp_mtx_init(&mm->mtx, "erts_mmap"); @@ -2165,7 +2165,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) sz = end - ptr; start = os_mmap_virtual(ptr, sz); if (!start || start > ptr || start >= end) - erl_exit(-1, + erts_exit(1, "erts_mmap: Failed to create virtual range for super carrier\n"); sz = start - ptr; if (sz) @@ -2203,7 +2203,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) start = os_mmap(NULL, sz, 1); } if (!start) - erl_exit(-1, + erts_exit(1, "erts_mmap: Failed to create super carrier of size %bpu MB\n", init->scs/1024/1024); end = start + sz; @@ -2250,7 +2250,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) if ((desc_size + ERTS_SUPERALIGNED_SIZE + ERTS_PAGEALIGNED_SIZE) > end - start) - erl_exit(-1, "erts_mmap: No space for segments in super carrier\n"); + erts_exit(1, "erts_mmap: No space for segments in super carrier\n"); mm->sa.bot = start; mm->sa.bot += desc_size; @@ -2288,7 +2288,7 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init) */ #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION if (virtual_map && !os_reserve_physical(start, mm->sa.bot - start)) - erl_exit(-1, "erts_mmap: Failed to reserve physical memory for descriptors\n"); + erts_exit(1, "erts_mmap: Failed to reserve physical memory for descriptors\n"); #endif mm->desc.unused_start = start; mm->desc.unused_end = mm->sa.bot; diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 61d912fd28..67e131b53e 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -50,8 +50,10 @@ typedef struct { #define ERTS_MMAP_INIT_DEFAULT_INITER \ {{NULL, NULL}, {NULL, NULL}, 0, 1, (1 << 16), 1} +#define ERTS_LITERAL_VIRTUAL_AREA_SIZE (UWORD_CONSTANT(1)*1024*1024*1024) + #define ERTS_MMAP_INIT_LITERAL_INITER \ - {{NULL, NULL}, {NULL, NULL}, 1024*1024*1024, 1, (1 << 16), 0} + {{NULL, NULL}, {NULL, NULL}, ERTS_LITERAL_VIRTUAL_AREA_SIZE, 1, (1 << 16), 0} typedef struct ErtsMemMapper_ ErtsMemMapper; diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 20695899eb..286130bece 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1403,9 +1403,12 @@ erts_mseg_init(ErtsMsegInit_t *init) erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); erts_mmap_init(&erts_dflt_mmapper, &init->dflt_mmap); +#if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + erts_mmap_init(&erts_literal_mmapper, &init->literal_mmap); +#endif if (!IS_2POW(GET_PAGE_SIZE)) - erl_exit(ERTS_ABORT_EXIT, "erts_mseg: Unexpected page_size %beu\n", GET_PAGE_SIZE); + erts_exit(ERTS_ABORT_EXIT, "erts_mseg: Unexpected page_size %beu\n", GET_PAGE_SIZE); ASSERT((MSEG_ALIGNED_SIZE % GET_PAGE_SIZE) == 0); diff --git a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c index b79485241f..367b22d989 100644 --- a/erts/emulator/sys/common/erl_os_monotonic_time_extender.c +++ b/erts/emulator/sys/common/erl_os_monotonic_time_extender.c @@ -44,7 +44,7 @@ static void *os_monotonic_time_extender(void *vstatep) erts_milli_sleep(sleep_time); } - erl_exit(ERTS_ABORT_EXIT, "os_monotonic_time_extender thread terminating"); + erts_exit(ERTS_ABORT_EXIT, "os_monotonic_time_extender thread terminating"); return NULL; } diff --git a/erts/emulator/sys/unix/erl_child_setup.c b/erts/emulator/sys/unix/erl_child_setup.c index 4e61530cf1..8dd14bbac6 100644 --- a/erts/emulator/sys/unix/erl_child_setup.c +++ b/erts/emulator/sys/unix/erl_child_setup.c @@ -61,6 +61,7 @@ #include "erl_driver.h" #include "sys_uds.h" #include "hash.h" +#include "erl_term.h" #include "erl_child_setup.h" #define SET_CLOEXEC(fd) fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC) diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 2ad5f3b4d5..0a18e54389 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -631,7 +631,7 @@ break_requested(void) fprintf(stderr,"break!\n"); #endif if (ERTS_BREAK_REQUESTED) - erl_exit(ERTS_INTR_EXIT, ""); + erts_exit(ERTS_INTR_EXIT, ""); ERTS_SET_BREAK_REQUESTED; ERTS_CHK_IO_AS_INTR(); /* Make sure we don't sleep in poll */ @@ -670,7 +670,7 @@ sigusr1_exit(void) } prepare_crash_dump(secs); - erl_exit(1, "Received SIGUSR1\n"); + erts_exit(ERTS_ERROR_EXIT, "Received SIGUSR1\n"); } #ifdef ETHR_UNUSABLE_SIGUSRX @@ -726,7 +726,7 @@ static RETSIGTYPE suspend_signal(int signum) static void quit_requested(void) { - erl_exit(ERTS_INTR_EXIT, ""); + erts_exit(ERTS_INTR_EXIT, ""); } #if (defined(SIG_SIGSET) || defined(SIG_SIGNAL)) @@ -1252,7 +1252,7 @@ signal_dispatcher_thread_func(void *unused) if (res < 0) { if (errno == EINTR) continue; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "signal-dispatcher thread got unexpected error: %s (%d)\n", erl_errno_id(errno), errno); @@ -1289,7 +1289,7 @@ signal_dispatcher_thread_func(void *unused) sigusr1_exit(); break; default: - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "signal-dispatcher thread received unknown " "signal notification: '%c'\n", buf[i]); @@ -1308,7 +1308,7 @@ init_smp_sig_notify(void) thr_opts.name = "sys_sig_dispatcher"; if (pipe(sig_notify_fds) < 0) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to create signal-dispatcher pipe: %s (%d)\n", erl_errno_id(errno), errno); @@ -1324,7 +1324,7 @@ init_smp_sig_notify(void) static void init_smp_sig_suspend(void) { if (pipe(sig_suspend_fds) < 0) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to create sig_suspend pipe: %s (%d)\n", erl_errno_id(errno), errno); @@ -1340,7 +1340,7 @@ static void initialize_darwin_main_thread_pipes(void) { if (pipe(erts_darwin_main_thread_pipe) < 0 || pipe(erts_darwin_main_thread_result_pipe) < 0) { - erl_exit(1,"Fatal error initializing Darwin main thread stealing"); + erts_exit(ERTS_ERROR_EXIT,"Fatal error initializing Darwin main thread stealing"); } } diff --git a/erts/emulator/sys/unix/sys_drivers.c b/erts/emulator/sys/unix/sys_drivers.c index 2a7cd91265..82fea1b330 100644 --- a/erts/emulator/sys/unix/sys_drivers.c +++ b/erts/emulator/sys/unix/sys_drivers.c @@ -1665,14 +1665,14 @@ static ErlDrvData forker_start(ErlDrvPort port_num, char* name, res = erts_sys_getenv_raw("BINDIR", bindir, &bindirsz); if (res != 0) { if (res < 0) - erl_exit(-1, + erts_exit(1, "Environment variable BINDIR is not set\n"); if (res > 0) - erl_exit(-1, + erts_exit(1, "Value of environment variable BINDIR is too large\n"); } if (bindir[0] != DIR_SEPARATOR_CHAR) - erl_exit(-1, + erts_exit(1, "Environment variable BINDIR does not contain an" " absolute path\n"); csp_path_sz = (strlen(bindir) @@ -1686,7 +1686,7 @@ static ErlDrvData forker_start(ErlDrvPort port_num, char* name, DIR_SEPARATOR_CHAR, CHILD_SETUP_PROG_NAME); if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) < 0) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Could not open unix domain socket in spawn_init: %d\n", errno); } @@ -1754,11 +1754,11 @@ static void forker_ready_input(ErlDrvData e, ErlDrvEvent fd) if ((res = read(fd, proto, sizeof(*proto))) < 0) { if (errno == ERRNO_BLOCK) return; - erl_exit(ERTS_DUMP_EXIT, "Failed to read from erl_child_setup: %d\n", errno); + erts_exit(ERTS_DUMP_EXIT, "Failed to read from erl_child_setup: %d\n", errno); } if (res == 0) - erl_exit(ERTS_DUMP_EXIT, "erl_child_setup closed\n"); + erts_exit(ERTS_DUMP_EXIT, "erl_child_setup closed\n"); ASSERT(res == sizeof(*proto)); @@ -1814,7 +1814,7 @@ static void forker_ready_output(ErlDrvData e, ErlDrvEvent fd) proto->u.start.fds, 3, 0) < 0) { if (errno == ERRNO_BLOCK) return; - erl_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno); + erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno); } #ifndef FORKER_PROTO_START_ACK close(proto->u.start.fds[0]); @@ -1846,7 +1846,7 @@ static ErlDrvSSizeT forker_control(ErlDrvData e, unsigned int cmd, char *buf, driver_select(port_num, forker_fd, ERL_DRV_WRITE|ERL_DRV_USE, 1); return 0; } - erl_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno); + erts_exit(ERTS_DUMP_EXIT, "Failed to write to erl_child_setup: %d\n", errno); } #ifndef FORKER_PROTO_START_ACK diff --git a/erts/emulator/sys/unix/sys_float.c b/erts/emulator/sys/unix/sys_float.c index 1ef9e5eef7..8fe7e599e5 100644 --- a/erts/emulator/sys/unix/sys_float.c +++ b/erts/emulator/sys/unix/sys_float.c @@ -90,7 +90,7 @@ void erts_fp_check_init_error(volatile unsigned long *fpexnp) snprintf(buf, sizeof buf, "ERTS_FP_CHECK_INIT at %p: detected unhandled FPE at %p\r\n", __builtin_return_address(0), (void*)*fpexnp); if (write(2, buf, strlen(buf)) <= 0) - erl_exit(ERTS_ABORT_EXIT, "%s", buf); + erts_exit(ERTS_ABORT_EXIT, "%s", buf); *fpexnp = 0; #if defined(__i386__) || defined(__x86_64__) erts_restore_fpu(); diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c index 03d39c7ce6..8ce02506ab 100644 --- a/erts/emulator/sys/unix/sys_time.c +++ b/erts/emulator/sys/unix/sys_time.c @@ -344,7 +344,7 @@ sys_init_time(ErtsSysInitTimeResult *init_resp) * times() (CLK_TCK), the resolution is always one millisecond.. */ if ((erts_sys_time_data__.r.o.ticks_per_sec = TICKS_PER_SEC()) < 0) - erl_exit(ERTS_ABORT_EXIT, "Can't get clock ticks/sec\n"); + erts_exit(ERTS_ABORT_EXIT, "Can't get clock ticks/sec\n"); #if defined(OS_MONOTONIC_TIME_USING_TIMES) #if ERTS_COMPILE_TIME_MONOTONIC_TIME_UNIT @@ -454,7 +454,7 @@ posix_clock_gettime(clockid_t id, char *name) if (clock_gettime(id, &ts) != 0) { int err = errno; char *errstr = err ? strerror(err) : "unknown"; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "clock_gettime(%s, _) failed: %s (%d)\n", name, errstr, err); } @@ -499,13 +499,13 @@ posix_clock_gettime_times(clockid_t mid, char *mname, if (mres != 0) { char *errstr = merr ? strerror(merr) : "unknown"; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "clock_gettime(%s, _) failed: %s (%d)\n", mname, errstr, merr); } if (sres != 0) { char *errstr = serr ? strerror(serr) : "unknown"; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "clock_gettime(%s, _) failed: %s (%d)\n", sname, errstr, serr); } @@ -678,7 +678,7 @@ mach_clocks_init(void) clck_srv_p = &internal_state.r.o.mach.clock.monotonic.srv; kret = host_get_clock_service(host, id, clck_srv_p); if (kret != KERN_SUCCESS) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "host_get_clock_service(_, %s, _) failed\n", name); } @@ -690,7 +690,7 @@ mach_clocks_init(void) clck_srv_p = &internal_state.r.o.mach.clock.wall.srv; kret = host_get_clock_service(host, id, clck_srv_p); if (kret != KERN_SUCCESS) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "host_get_clock_service(_, %s, _) failed\n", name); } @@ -699,7 +699,7 @@ mach_clocks_init(void) if (atexit(mach_clocks_fini) != 0) { int err = errno; char *errstr = err ? strerror(err) : "unknown"; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "Failed to register mach_clocks_fini() " "for call at exit: %s (%d)\n", errstr, err); @@ -721,7 +721,7 @@ mach_clock_getres(ErtsMachClock *clk) (clock_attr_t) attr, &cnt); if (kret != KERN_SUCCESS || cnt != 1) { - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "clock_get_attributes(%s, _) failed\n", clk->name); } @@ -739,7 +739,7 @@ mach_clock_get_time(ErtsMachClock *clk) kret = clock_get_time(clk->srv, &time_spec); if (kret != KERN_SUCCESS) - erl_exit(ERTS_ABORT_EXIT, "clock_get_time(%s, _) failed\n", clk->name); + erts_exit(ERTS_ABORT_EXIT, "clock_get_time(%s, _) failed\n", clk->name); return ERTS_TimeSpec2Sint64(&time_spec); } @@ -785,11 +785,11 @@ erts_os_times(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep) &sys_time_spec); if (mkret != KERN_SUCCESS) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "clock_get_time(%s, _) failed\n", internal_state.r.o.mach.clock.monotonic.name); if (skret != KERN_SUCCESS) - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "clock_get_time(%s, _) failed\n", internal_state.r.o.mach.clock.wall.name); @@ -854,7 +854,7 @@ erts_os_system_time(void) if (gettimeofday(&tv, NULL) != 0) { int err = errno; char *errstr = err ? strerror(err) : "unknown"; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "gettimeofday(_, NULL) failed: %s (%d)\n", errstr, err); } diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c index 660ded297a..9c1bef5f4f 100644 --- a/erts/emulator/sys/win32/erl_poll.c +++ b/erts/emulator/sys/win32/erl_poll.c @@ -437,7 +437,7 @@ wakeup_cause(ErtsPollSet ps) break; default: res = 0; - erl_exit(ERTS_ABORT_EXIT, + erts_exit(ERTS_ABORT_EXIT, "%s:%d: Internal error: Invalid wakeup_state=%d\n", __FILE__, __LINE__, (int) wakeup_state); } @@ -577,7 +577,7 @@ static void signal_standby(ErtsPollSet ps) --(ps->standby_wait_counter); if (ps->standby_wait_counter < 0) { LeaveCriticalSection(&(ps->standby_crit)); - erl_exit(1,"Standby signalled by more threads than expected"); + erts_exit(ERTS_ERROR_EXIT,"Standby signalled by more threads than expected"); } if (!(ps->standby_wait_counter)) { SetEvent(ps->standby_wait_event); @@ -739,7 +739,7 @@ static void *break_waiter(void *param) erts_mtx_unlock(&break_waiter_lock); break; default: - erl_exit(1,"Unexpected event in break_waiter"); + erts_exit(ERTS_ERROR_EXIT,"Unexpected event in break_waiter"); } } } @@ -1158,7 +1158,7 @@ int erts_poll_wait(ErtsPollSet ps, HARDDEBUGF(("Oups!")); /* Oups, got signalled before we took the lock, can't reset */ if(!is_io_ready(ps)) { - erl_exit(1,"Internal error: " + erts_exit(ERTS_ERROR_EXIT,"Internal error: " "Inconsistent io structures in erl_poll.\n"); } START_WAITER(ps,w); @@ -1219,7 +1219,7 @@ int erts_poll_wait(ErtsPollSet ps, ERTS_SET_BREAK_REQUESTED; break; case BREAK_WAITER_GOT_HALT: - erl_exit(0,""); + erts_exit(0,""); break; default: break; diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 76ce25916a..ea7523ddc2 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -38,7 +38,7 @@ void erts_sys_init_float(void); void erl_start(int, char**); -void erl_exit(int n, char*, ...); +void erts_exit(int n, char*, ...); void erl_error(char*, va_list); void erl_crash_dump(char*, int, char*, ...); @@ -200,7 +200,7 @@ erts_sys_misc_mem_sz(void) */ void sys_tty_reset(int exit_code) { - if (exit_code > 0) + if (exit_code == ERTS_ERROR_EXIT) ConWaitForExit(); else ConNormalExit(); @@ -3108,13 +3108,13 @@ check_supported_os_version(void) || int_os_version.dwMajorVersion < major || (int_os_version.dwMajorVersion == major && int_os_version.dwMinorVersion < minor)) - erl_exit(-1, + erts_exit(1, "Windows version not supported " "(min required: winnt %d.%d)\n", major, minor); } #else - erl_exit(-1, + erts_exit(1, "Windows version not supported " "(min required: win %d.%d)\n", nt_major, nt_minor); diff --git a/erts/emulator/sys/win32/sys_interrupt.c b/erts/emulator/sys/win32/sys_interrupt.c index d6178de03c..a89211fa8e 100644 --- a/erts/emulator/sys/win32/sys_interrupt.c +++ b/erts/emulator/sys/win32/sys_interrupt.c @@ -81,7 +81,7 @@ BOOL WINAPI ctrl_handler_ignore_break(DWORD dwCtrlType) return TRUE; /* else pour through... */ case CTRL_CLOSE_EVENT: - erl_exit(0, ""); + erts_exit(0, ""); break; } return TRUE; @@ -106,7 +106,7 @@ BOOL WINAPI ctrl_handler_replace_intr(DWORD dwCtrlType) /* else pour through... */ case CTRL_CLOSE_EVENT: case CTRL_SHUTDOWN_EVENT: - erl_exit(0, ""); + erts_exit(0, ""); break; } return TRUE; @@ -133,7 +133,7 @@ BOOL WINAPI ctrl_handler(DWORD dwCtrlType) return TRUE; /* else pour through... */ case CTRL_CLOSE_EVENT: - erl_exit(0, ""); + erts_exit(0, ""); break; } return TRUE; diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c index 3b4fd26d63..9a9d90610d 100644 --- a/erts/emulator/sys/win32/sys_time.c +++ b/erts/emulator/sys/win32/sys_time.c @@ -147,7 +147,7 @@ os_monotonic_time_qpc(void) LARGE_INTEGER pc; if (!(*internal_state.r.o.pQueryPerformanceCounter)(&pc)) - erl_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n"); + erts_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n"); return (ErtsMonotonicTime) pc.QuadPart; } @@ -164,7 +164,7 @@ os_times_qpc(ErtsMonotonicTime *mtimep, ErtsSystemTime *stimep) GetSystemTime(&st); if (!qpcr) - erl_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n"); + erts_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n"); *mtimep = (ErtsMonotonicTime) pc.QuadPart; @@ -251,7 +251,7 @@ sys_hrtime_qpc(void) LARGE_INTEGER pc; if (!(*internal_state.r.o.pQueryPerformanceCounter)(&pc)) - erl_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n"); + erts_exit(ERTS_ABORT_EXIT, "QueryPerformanceCounter() failed\n"); ASSERT(pc.QuadPart > 0); diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index 987a655c3d..318db4b45e 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -74,6 +74,7 @@ MODULES= \ match_spec_SUITE \ module_info_SUITE \ monitor_SUITE \ + multi_load_SUITE \ nested_SUITE \ nif_SUITE \ node_container_SUITE \ diff --git a/erts/emulator/test/distribution_SUITE.erl b/erts/emulator/test/distribution_SUITE.erl index 7a33dbbb36..d562f79bb7 100644 --- a/erts/emulator/test/distribution_SUITE.erl +++ b/erts/emulator/test/distribution_SUITE.erl @@ -164,7 +164,7 @@ bulk_send(Terms, BinSize) -> ?line stop_node(Node), ?line test_server:timetrap_cancel(Dog), - {comment, integer_to_list(trunc(Size/1024/Elapsed+0.5)) ++ " K/s"}. + {comment, integer_to_list(trunc(Size/1024/max(1,Elapsed)+0.5)) ++ " K/s"}. bulk_sendsend(Terms, BinSize) -> {Rate1, MonitorCount1} = bulk_sendsend2(Terms, BinSize, 5), diff --git a/erts/emulator/test/estone_SUITE.erl b/erts/emulator/test/estone_SUITE.erl index 7be55eca8d..7710424894 100644 --- a/erts/emulator/test/estone_SUITE.erl +++ b/erts/emulator/test/estone_SUITE.erl @@ -382,11 +382,11 @@ apply_micro(M) -> {weight_percentage, M#micro.weight}, {loops, M#micro.loops}, {microsecs,MicroSecs}, - {estones, (M#micro.weight * M#micro.weight * ?STONEFACTOR) div MicroSecs}, + {estones, (M#micro.weight * M#micro.weight * ?STONEFACTOR) div max(1,MicroSecs)}, {gcs, GC1 - GC0}, {kilo_word_reclaimed, (Words1 - Words0) div 1000}, {kilo_reductions, Reds div 1000}, - {gc_intensity, gci(Elapsed, GC1 - GC0, Words1 - Words0)}]. + {gc_intensity, gci(max(1,Elapsed), GC1 - GC0, Words1 - Words0)}]. monotonic_time() -> try erlang:monotonic_time() catch error:undef -> erlang:now() end. diff --git a/erts/emulator/test/multi_load_SUITE.erl b/erts/emulator/test/multi_load_SUITE.erl new file mode 100644 index 0000000000..784b239116 --- /dev/null +++ b/erts/emulator/test/multi_load_SUITE.erl @@ -0,0 +1,196 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 1999-2012. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% + +-module(multi_load_SUITE). +-export([all/0, suite/0,groups/0,init_per_suite/1, end_per_suite/1, + init_per_group/2,end_per_group/2, + many/1,on_load/1,errors/1]). + +-include_lib("common_test/include/ct.hrl"). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [many,on_load,errors]. + +groups() -> + []. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + ok. + +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, Config) -> + Config. + +many(_Config) -> + Ms = make_modules(100, fun many_module/1), + + io:put_chars("Light load\n" + "=========="), + many_measure(Ms), + + _ = [spawn_link(fun many_worker/0) || _ <- lists:seq(1, 8)], + erlang:yield(), + io:put_chars("Heavy load\n" + "=========="), + many_measure(Ms), + + ok. + +many_module(M) -> + ["-module("++M++").", + "-compile(export_all).", + "f1() -> ok.", + "f2() -> ok.", + "f3() -> ok.", + "f4() -> ok."]. + +many_measure(Ms) -> + many_purge(Ms), + MsPrep1 = prepare_modules(Ms), + Us1 = ms(fun() -> many_load_seq(MsPrep1) end), + many_try_call(Ms), + many_purge(Ms), + MsPrep2 = prepare_modules(Ms), + Us2 = ms(fun() -> many_load_par(MsPrep2) end), + many_try_call(Ms), + io:format("# modules: ~9w\n" + "Sequential: ~9w µs\n" + "Parallel: ~9w µs\n" + "Ratio: ~9w\n", + [length(Ms),Us1,Us2,round(Us1/Us2)]), + ok. + +many_load_seq(Ms) -> + [erlang:finish_loading([M]) || M <- Ms], + ok. + +many_load_par(Ms) -> + erlang:finish_loading(Ms). + +many_purge(Ms) -> + _ = [catch erlang:purge_module(M) || {M,_} <- Ms], + ok. + +many_try_call(Ms) -> + _ = [begin + ok = M:f1(), + ok = M:f2(), + ok = M:f3(), + ok = M:f4() + end || {M,_} <- Ms], + ok. + +many_worker() -> + many_worker(lists:seq(1, 100)). + +many_worker(L) -> + N0 = length(L), + N1 = N0 * N0 * N0, + N2 = N1 div (N0 * N0), + N3 = N2 + 10, + _ = N3 - 10, + many_worker(L). + + +on_load(_Config) -> + On = make_modules(2, fun on_load_module/1), + OnPrep = prepare_modules(On), + {'EXIT',{system_limit,_}} = (catch erlang:finish_loading(OnPrep)), + + Normal = make_modules(1, fun on_load_normal/1), + Mixed = Normal ++ tl(On), + MixedPrep = prepare_modules(Mixed), + {'EXIT',{system_limit,_}} = (catch erlang:finish_loading(MixedPrep)), + + [false,true] = [erlang:has_prepared_code_on_load(Code) || + Code <- MixedPrep], + {'EXIT',{badarg,_}} = (catch erlang:has_prepared_code_on_load(<<1,2,3>>)), + Magic = ets:match_spec_compile([{'_',[true],['$_']}]), + {'EXIT',{badarg,_}} = (catch erlang:has_prepared_code_on_load(Magic)), + + SingleOnPrep = tl(OnPrep), + {on_load,[OnLoadMod]} = erlang:finish_loading(SingleOnPrep), + ok = erlang:call_on_load_function(OnLoadMod), + + ok. + +on_load_module(M) -> + ["-module("++M++").", + "-on_load(f/0).", + "f() -> ok."]. + +on_load_normal(M) -> + ["-module("++M++")."]. + + +errors(_Config) -> + finish_loading_badarg(x), + finish_loading_badarg([x|y]), + finish_loading_badarg([x]), + finish_loading_badarg([<<>>]), + + Mods = make_modules(2, fun errors_module/1), + Ms = lists:sort([M || {M,_} <- Mods]), + Prep = prepare_modules(Mods), + {duplicated,Dups} = erlang:finish_loading(Prep ++ Prep), + Ms = lists:sort(Dups), + ok. + +finish_loading_badarg(Arg) -> + {'EXIT',{badarg,[{erlang,finish_loading,[Arg],_}|_]}} = + (catch erlang:finish_loading(Arg)). + +errors_module(M) -> + ["-module("++M++").", + "-export([f/0]).", + "f() -> ok."]. + +%%% +%%% Common utilities +%%% + +ms(Fun) -> + {Ms,ok} = timer:tc(Fun), + Ms. + +make_modules(0, _) -> + []; +make_modules(N, Fun) -> + U = erlang:unique_integer([positive]), + M0 = "m__" ++ integer_to_list(N) ++ "_" ++ integer_to_list(U), + Contents = Fun(M0), + Forms = [make_form(S) || S <- Contents], + {ok,M,Code} = compile:forms(Forms), + [{M,Code}|make_modules(N-1, Fun)]. + +make_form(S) -> + {ok,Toks,_} = erl_scan:string(S), + {ok,Form} = erl_parse:parse_form(Toks), + Form. + +prepare_modules(Ms) -> + [erlang:prepare_loading(M, Code) || {M,Code} <- Ms]. diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 7ce37b04b3..c6c617f97f 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -1711,7 +1711,9 @@ tmpmem() -> false -> undefined; MemInfo -> MSBCS = lists:foldl( - fun ({instance, _, L}, Acc) -> + fun ({instance, 0, _}, Acc) -> + Acc; % Ignore instance 0 + ({instance, _, L}, Acc) -> {value,{_,MBCS}} = lists:keysearch(mbcs, 1, L), {value,{_,SBCS}} = lists:keysearch(sbcs, 1, L), [MBCS,SBCS | Acc] diff --git a/erts/emulator/test/scheduler_SUITE.erl b/erts/emulator/test/scheduler_SUITE.erl index 1f284228db..f998e7d4d2 100644 --- a/erts/emulator/test/scheduler_SUITE.erl +++ b/erts/emulator/test/scheduler_SUITE.erl @@ -54,6 +54,7 @@ sct_cmd/1, sbt_cmd/1, scheduler_threads/1, + scheduler_suspend_basic/1, scheduler_suspend/1, dirty_scheduler_threads/1, dirty_scheduler_exit/1, @@ -69,8 +70,10 @@ all() -> [equal, few_low, many_low, equal_with_part_time_high, equal_with_part_time_max, equal_and_high_with_part_time_max, equal_with_high, - equal_with_high_max, bound_process, - {group, scheduler_bind}, scheduler_threads, scheduler_suspend, + equal_with_high_max, + bound_process, + {group, scheduler_bind}, scheduler_threads, + scheduler_suspend_basic, scheduler_suspend, dirty_scheduler_threads, dirty_scheduler_exit, reader_groups]. @@ -1131,6 +1134,7 @@ dirty_schedulers_online_test(true) -> dirty_schedulers_online_smp_test(erlang:system_info(schedulers_online)). dirty_schedulers_online_smp_test(SchedOnln) when SchedOnln < 4 -> ok; dirty_schedulers_online_smp_test(SchedOnln) -> + receive after 500 -> ok end, DirtyCPUSchedOnln = erlang:system_info(dirty_cpu_schedulers_online), SchedOnln = DirtyCPUSchedOnln, HalfSchedOnln = SchedOnln div 2, @@ -1139,9 +1143,11 @@ dirty_schedulers_online_smp_test(SchedOnln) -> HalfDirtyCPUSchedOnln = erlang:system_flag(schedulers_online, SchedOnln), DirtyCPUSchedOnln = erlang:system_flag(dirty_cpu_schedulers_online, HalfDirtyCPUSchedOnln), + receive after 500 -> ok end, HalfDirtyCPUSchedOnln = erlang:system_info(dirty_cpu_schedulers_online), QrtrDirtyCPUSchedOnln = HalfDirtyCPUSchedOnln div 2, SchedOnln = erlang:system_flag(schedulers_online, HalfSchedOnln), + receive after 500 -> ok end, QrtrDirtyCPUSchedOnln = erlang:system_info(dirty_cpu_schedulers_online), ok. @@ -1214,6 +1220,113 @@ wait_dse([Pid|Pids]) -> dirty_sleeper() -> erlang:nif_error({error,?MODULE}). +scheduler_suspend_basic(Config) when is_list(Config) -> + case erlang:system_info(multi_scheduling) of + disabled -> + {skip, "Nothing to test"}; + _ -> + Onln = erlang:system_info(schedulers_online), + try + scheduler_suspend_basic_test() + after + erlang:system_flag(schedulers_online, Onln) + end + end. + +scheduler_suspend_basic_test() -> + %% The receives after setting scheduler states are there + %% since the operation is not fully synchronous. For example, + %% we do not wait for dirty cpu schedulers online to complete + %% before returning from erlang:system_flag(schedulers_online, _). + + erlang:system_flag(schedulers_online, + erlang:system_info(schedulers)), + try + erlang:system_flag(dirty_cpu_schedulers_online, + erlang:system_info(dirty_cpu_schedulers)), + receive after 500 -> ok end + catch + _ : _ -> + ok + end, + + S0 = sched_state(), + io:format("~p~n", [S0]), + {{normal,NTot0,NOnln0,NAct0}, + {dirty_cpu,DCTot0,DCOnln0,DCAct0}, + {dirty_io,DITot0,DIOnln0,DIAct0}} = S0, + enabled = erlang:system_info(multi_scheduling), + + DCOne = case DCTot0 of + 0 -> 0; + _ -> 1 + end, + + blocked_normal = erlang:system_flag(multi_scheduling, block_normal), + blocked_normal = erlang:system_info(multi_scheduling), + {{normal,NTot0,NOnln0,1}, + {dirty_cpu,DCTot0,DCOnln0,DCAct0}, + {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(), + + NOnln0 = erlang:system_flag(schedulers_online, 1), + receive after 500 -> ok end, + {{normal,NTot0,1,1}, + {dirty_cpu,DCTot0,DCOne,DCOne}, + {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(), + + 1 = erlang:system_flag(schedulers_online, NOnln0), + receive after 500 -> ok end, + {{normal,NTot0,NOnln0,1}, + {dirty_cpu,DCTot0,DCOnln0,DCAct0}, + {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(), + + blocked = erlang:system_flag(multi_scheduling, block), + blocked = erlang:system_info(multi_scheduling), + receive after 500 -> ok end, + {{normal,NTot0,NOnln0,1}, + {dirty_cpu,DCTot0,DCOnln0,0}, + {dirty_io,DITot0,DIOnln0,0}} = sched_state(), + + NOnln0 = erlang:system_flag(schedulers_online, 1), + receive after 500 -> ok end, + {{normal,NTot0,1,1}, + {dirty_cpu,DCTot0,DCOne,0}, + {dirty_io,DITot0,DIOnln0,0}} = sched_state(), + + 1 = erlang:system_flag(schedulers_online, NOnln0), + receive after 500 -> ok end, + {{normal,NTot0,NOnln0,1}, + {dirty_cpu,DCTot0,DCOnln0,0}, + {dirty_io,DITot0,DIOnln0,0}} = sched_state(), + + blocked = erlang:system_flag(multi_scheduling, unblock_normal), + blocked = erlang:system_info(multi_scheduling), + {{normal,NTot0,NOnln0,1}, + {dirty_cpu,DCTot0,DCOnln0,0}, + {dirty_io,DITot0,DIOnln0,0}} = sched_state(), + + enabled = erlang:system_flag(multi_scheduling, unblock), + enabled = erlang:system_info(multi_scheduling), + receive after 500 -> ok end, + {{normal,NTot0,NOnln0,NAct0}, + {dirty_cpu,DCTot0,DCOnln0,DCAct0}, + {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(), + + NOnln0 = erlang:system_flag(schedulers_online, 1), + receive after 500 -> ok end, + {{normal,NTot0,1,1}, + {dirty_cpu,DCTot0,DCOne,DCOne}, + {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(), + + 1 = erlang:system_flag(schedulers_online, NOnln0), + receive after 500 -> ok end, + {{normal,NTot0,NOnln0,NAct0}, + {dirty_cpu,DCTot0,DCOnln0,DCAct0}, + {dirty_io,DITot0,DIOnln0,DIAct0}} = sched_state(), + + ok. + + scheduler_suspend(Config) when is_list(Config) -> ?line Dog = ?t:timetrap(?t:minutes(5)), ?line lists:foreach(fun (S) -> scheduler_suspend_test(Config, S) end, @@ -1238,12 +1351,17 @@ scheduler_suspend_test(Config, Schedulers) -> ?line [ok] = mcall(Node, [fun () -> sst0_loop(300) end]), ?line [ok] = mcall(Node, [fun () -> sst1_loop(300) end]), ?line [ok] = mcall(Node, [fun () -> sst2_loop(300) end]), - ?line [ok, ok, ok, ok, ok] = mcall(Node, - [fun () -> sst0_loop(200) end, - fun () -> sst1_loop(200) end, - fun () -> sst2_loop(200) end, - fun () -> sst2_loop(200) end, - fun () -> sst3_loop(Sched, 200) end]), + ?line [ok] = mcall(Node, [fun () -> sst4_loop(300) end]), + ?line [ok] = mcall(Node, [fun () -> sst5_loop(300) end]), + ?line [ok, ok, ok, ok, + ok, ok, ok] = mcall(Node, + [fun () -> sst0_loop(200) end, + fun () -> sst1_loop(200) end, + fun () -> sst2_loop(200) end, + fun () -> sst2_loop(200) end, + fun () -> sst3_loop(Sched, 200) end, + fun () -> sst4_loop(200) end, + fun () -> sst5_loop(200) end]), ?line [SState] = mcall(Node, [fun () -> case Sched == SchedOnln of false -> @@ -1320,6 +1438,20 @@ sst3_loop_with_dirty_schedulers(S, DS, N) -> erlang:system_flag(dirty_cpu_schedulers_online, DS), sst3_loop_with_dirty_schedulers(S, DS, N-1). +sst4_loop(0) -> + ok; +sst4_loop(N) -> + erlang:system_flag(multi_scheduling, block_normal), + erlang:system_flag(multi_scheduling, unblock_normal), + sst4_loop(N-1). + +sst5_loop(0) -> + ok; +sst5_loop(N) -> + erlang:system_flag(multi_scheduling, block_normal), + erlang:system_flag(multi_scheduling, unblock_normal), + sst5_loop(N-1). + reader_groups(Config) when is_list(Config) -> %% White box testing. These results are correct, but other results %% could be too... @@ -1598,6 +1730,34 @@ reader_groups_map(CPUT, Groups) -> %% Utils %% +sched_state() -> + sched_state(erlang:system_info(all_schedulers_state), + undefined, + {dirty_cpu,0,0,0}, + {dirty_io,0,0,0}). + + +sched_state([], N, DC, DI) -> + try + chk_basic(N), + chk_basic(DC), + chk_basic(DI), + {N, DC, DI} + catch + _ : _ -> + ?t:fail({inconsisten_scheduler_state, {N, DC, DI}}) + end; +sched_state([{normal, _, _, _} = S | Rest], _S, DC, DI) -> + sched_state(Rest, S, DC, DI); +sched_state([{dirty_cpu, _, _, _} = DC | Rest], S, _DC, DI) -> + sched_state(Rest, S, DC, DI); +sched_state([{dirty_io, _, _, _} = DI | Rest], S, DC, _DI) -> + sched_state(Rest, S, DC, DI). + +chk_basic({_Type, Tot, Onln, Act}) -> + true = Tot >= Onln, + true = Onln >= Act. + l(Id) -> {logical, Id}. diff --git a/erts/emulator/valgrind/suppress.patched.3.6.0 b/erts/emulator/valgrind/suppress.patched.3.6.0 index 16cecf2dba..fcde4a0123 100644 --- a/erts/emulator/valgrind/suppress.patched.3.6.0 +++ b/erts/emulator/valgrind/suppress.patched.3.6.0 @@ -368,7 +368,7 @@ Memcheck:Addr4 ... fun:erts_print_scheduler_info ... -fun:erl_exit +fun:erts_exit fun:broken_halt_test fun:erts_debug_set_internal_state_2 fun:process_main diff --git a/erts/emulator/valgrind/suppress.standard b/erts/emulator/valgrind/suppress.standard index a1f3f82364..bb07c92fc1 100644 --- a/erts/emulator/valgrind/suppress.standard +++ b/erts/emulator/valgrind/suppress.standard @@ -336,7 +336,7 @@ Memcheck:Addr4 ... fun:erts_print_scheduler_info ... -fun:erl_exit +fun:erts_exit fun:broken_halt_test fun:erts_debug_set_internal_state_2 fun:process_main diff --git a/erts/epmd/src/epmd.c b/erts/epmd/src/epmd.c index 21ce4e52b0..324ae6ac40 100644 --- a/erts/epmd/src/epmd.c +++ b/erts/epmd/src/epmd.c @@ -343,7 +343,7 @@ static void run_daemon(EpmdVars *g) for (fd = 0; fd < g->max_conn ; fd++) /* close all files ... */ close(fd); /* Syslog on linux will try to write to whatever if we dont - inform it of that the log is closed. */ + inform it that the log is closed. */ closelog(); /* These shouldn't be needed but for safety... */ diff --git a/erts/epmd/src/epmd_cli.c b/erts/epmd/src/epmd_cli.c index a8fe865d9a..6fc05e153e 100644 --- a/erts/epmd/src/epmd_cli.c +++ b/erts/epmd/src/epmd_cli.c @@ -136,19 +136,33 @@ void epmd_call(EpmdVars *g,int what) static int conn_to_epmd(EpmdVars *g) { struct EPMD_SOCKADDR_IN address; + size_t salen = 0; int connect_sock; - - connect_sock = socket(FAMILY, SOCK_STREAM, 0); - if (connect_sock<0) - goto error; + unsigned short sport = g->port; + +#if defined(EPMD6) + SET_ADDR6(address, in6addr_loopback, sport); + salen = sizeof(struct sockaddr_in6); + + connect_sock = socket(AF_INET6, SOCK_STREAM, 0); + if (connect_sock>=0) { + + if (connect(connect_sock, (struct sockaddr*)&address, salen) == 0) + return connect_sock; - { /* store port number in unsigned short */ - unsigned short sport = g->port; - SET_ADDR(address, EPMD_ADDR_LOOPBACK, sport); + close(connect_sock); } +#endif + SET_ADDR(address, htonl(INADDR_LOOPBACK), sport); + salen = sizeof(struct sockaddr_in); - if (connect(connect_sock, (struct sockaddr*)&address, sizeof address) < 0) + connect_sock = socket(AF_INET, SOCK_STREAM, 0); + if (connect_sock<0) goto error; + + if (connect(connect_sock, (struct sockaddr*)&address, salen) < 0) + goto error; + return connect_sock; error: diff --git a/erts/epmd/src/epmd_int.h b/erts/epmd/src/epmd_int.h index e222abb4b7..e127eff097 100644 --- a/erts/epmd/src/epmd_int.h +++ b/erts/epmd/src/epmd_int.h @@ -48,6 +48,7 @@ # ifndef WINDOWS_H_INCLUDES_WINSOCK2_H # include <winsock2.h> # endif +# include <ws2tcpip.h> # include <windows.h> # include <process.h> #endif @@ -114,6 +115,10 @@ # include <systemd/sd-daemon.h> #endif /* HAVE_SYSTEMD_DAEMON */ +#if defined(HAVE_IN6) && defined(AF_INET6) && defined(HAVE_INET_PTON) +# define EPMD6 +#endif + /* ************************************************************************ */ /* Replace some functions by others by making the function name a macro */ @@ -167,33 +172,53 @@ /* ************************************************************************ */ /* Macros that let us use IPv6 */ -#if defined(HAVE_IN6) && defined(AF_INET6) && defined(EPMD6) +#if HAVE_IN6 +# if ! defined(HAVE_IN6ADDR_ANY) || ! HAVE_IN6ADDR_ANY +# if HAVE_DECL_IN6ADDR_ANY_INIT +static const struct in6_addr in6addr_any = { { IN6ADDR_ANY_INIT } }; +# else +static const struct in6_addr in6addr_any = + { { { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 } } }; +# endif /* HAVE_IN6ADDR_ANY_INIT */ +# endif /* ! HAVE_DECL_IN6ADDR_ANY */ + +# if ! defined(HAVE_IN6ADDR_LOOPBACK) || ! HAVE_IN6ADDR_LOOPBACK +# if HAVE_DECL_IN6ADDR_LOOPBACK_INIT +static const struct in6_addr in6addr_loopback = + { { IN6ADDR_LOOPBACK_INIT } }; +# else +static const struct in6_addr in6addr_loopback = + { { { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1 } } }; +# endif /* HAVE_IN6ADDR_LOOPBACK_INIT */ +# endif /* ! HAVE_DECL_IN6ADDR_LOOPBACK */ +#endif /* HAVE_IN6 */ + +#define IS_ADDR_LOOPBACK(addr) ((addr).s_addr == htonl(INADDR_LOOPBACK)) + +#if defined(EPMD6) -#define EPMD_SOCKADDR_IN sockaddr_in6 -#define EPMD_IN_ADDR in6_addr -#define EPMD_S_ADDR s6_addr -#define EPMD_ADDR_LOOPBACK in6addr_loopback.s6_addr -#define EPMD_ADDR_ANY in6addr_any.s6_addr +#define EPMD_SOCKADDR_IN sockaddr_storage #define FAMILY AF_INET6 -#define SET_ADDR(dst, addr, port) do { \ - memset((char*)&(dst), 0, sizeof(dst)); \ - memcpy((char*)&(dst).sin6_addr.s6_addr, (char*)&(addr), 16); \ - (dst).sin6_family = AF_INET6; \ - (dst).sin6_flowinfo = 0; \ - (dst).sin6_port = htons(port); \ +#define SET_ADDR6(dst, addr, port) do { \ + struct sockaddr_in6 *sa = (struct sockaddr_in6 *)&(dst); \ + memset(sa, 0, sizeof(dst)); \ + sa->sin6_family = AF_INET6; \ + sa->sin6_addr = (addr); \ + sa->sin6_port = htons(port); \ } while(0) -#define IS_ADDR_LOOPBACK(addr) \ - (memcmp((addr).s6_addr, in6addr_loopback.s6_addr, 16) == 0) +#define SET_ADDR(dst, addr, port) do { \ + struct sockaddr_in *sa = (struct sockaddr_in *)&(dst); \ + memset(sa, 0, sizeof(dst)); \ + sa->sin_family = AF_INET; \ + sa->sin_addr.s_addr = (addr); \ + sa->sin_port = htons(port); \ + } while(0) #else /* Not IP v6 */ #define EPMD_SOCKADDR_IN sockaddr_in -#define EPMD_IN_ADDR in_addr -#define EPMD_S_ADDR s_addr -#define EPMD_ADDR_LOOPBACK htonl(INADDR_LOOPBACK) -#define EPMD_ADDR_ANY htonl(INADDR_ANY) #define FAMILY AF_INET #define SET_ADDR(dst, addr, port) do { \ @@ -203,8 +228,6 @@ (dst).sin_port = htons(port); \ } while(0) -#define IS_ADDR_LOOPBACK(addr) ((addr).s_addr == htonl(INADDR_LOOPBACK)) - #endif /* Not IP v6 */ /* ************************************************************************ */ diff --git a/erts/epmd/src/epmd_srv.c b/erts/epmd/src/epmd_srv.c index fb8cae05df..75a33f28cb 100644 --- a/erts/epmd/src/epmd_srv.c +++ b/erts/epmd/src/epmd_srv.c @@ -71,6 +71,7 @@ static time_t current_time(EpmdVars*); static Connection *conn_init(EpmdVars*); static int conn_open(EpmdVars*,int); +static int conn_local_peer_check(EpmdVars*, int); static int conn_close_fd(EpmdVars*,int); static void node_init(EpmdVars*); @@ -201,10 +202,11 @@ void run(EpmdVars *g) { struct EPMD_SOCKADDR_IN iserv_addr[MAX_LISTEN_SOCKETS]; int listensock[MAX_LISTEN_SOCKETS]; - int num_sockets; + int num_sockets = 0; int i; int opt; unsigned short sport = g->port; + int bound = 0; node_init(g); g->conn = conn_init(g); @@ -247,64 +249,82 @@ void run(EpmdVars *g) if (g->addresses != NULL && /* String contains non-separator characters if: */ g->addresses[strspn(g->addresses," ,")] != '\000') { - char *tmp; - char *token; - int loopback_ok = 0; + char *tmp = NULL; + char *token = NULL; + + /* Always listen on the loopback. */ + SET_ADDR(iserv_addr[num_sockets],htonl(INADDR_LOOPBACK),sport); + num_sockets++; +#if defined(EPMD6) + SET_ADDR6(iserv_addr[num_sockets],in6addr_loopback,sport); + num_sockets++; +#endif - if ((tmp = (char *)malloc(strlen(g->addresses) + 1)) == NULL) + if ((tmp = strdup(g->addresses)) == NULL) { dbg_perror(g,"cannot allocate memory"); epmd_cleanup_exit(g,1); } - strcpy(tmp,g->addresses); - for(token = strtok(tmp,", "), num_sockets = 0; + for(token = strtok(tmp,", "); token != NULL; - token = strtok(NULL,", "), num_sockets++) + token = strtok(NULL,", ")) { - struct EPMD_IN_ADDR addr; -#ifdef HAVE_INET_PTON - int ret; + struct in_addr addr; +#if defined(EPMD6) + struct in6_addr addr6; + struct sockaddr_storage *sa = &iserv_addr[num_sockets]; - if ((ret = inet_pton(FAMILY,token,&addr)) == -1) + if (inet_pton(AF_INET6,token,&addr6) == 1) { - dbg_perror(g,"cannot convert IP address to network format"); - epmd_cleanup_exit(g,1); + SET_ADDR6(iserv_addr[num_sockets],addr6,sport); + } + else if (inet_pton(AF_INET,token,&addr) == 1) + { + SET_ADDR(iserv_addr[num_sockets],addr.s_addr,sport); + } + else +#else + if ((addr.s_addr = inet_addr(token)) != INADDR_NONE) + { + SET_ADDR(iserv_addr[num_sockets],addr.s_addr,sport); } - else if (ret == 0) -#elif !defined(EPMD6) - if ((addr.EPMD_S_ADDR = inet_addr(token)) == INADDR_NONE) + else #endif { dbg_tty_printf(g,0,"cannot parse IP address \"%s\"",token); epmd_cleanup_exit(g,1); } +#if defined(EPMD6) + if (sa->ss_family == AF_INET6 && IN6_IS_ADDR_LOOPBACK(&addr6)) + continue; + + if (sa->ss_family == AF_INET) +#endif if (IS_ADDR_LOOPBACK(addr)) - loopback_ok = 1; + continue; - if (num_sockets - loopback_ok == MAX_LISTEN_SOCKETS - 1) + num_sockets++; + + if (num_sockets >= MAX_LISTEN_SOCKETS) { dbg_tty_printf(g,0,"cannot listen on more than %d IP addresses", MAX_LISTEN_SOCKETS); epmd_cleanup_exit(g,1); } - - SET_ADDR(iserv_addr[num_sockets],addr.EPMD_S_ADDR,sport); } free(tmp); - - if (!loopback_ok) - { - SET_ADDR(iserv_addr[num_sockets],EPMD_ADDR_LOOPBACK,sport); - num_sockets++; - } } else { - SET_ADDR(iserv_addr[0],EPMD_ADDR_ANY,sport); - num_sockets = 1; + SET_ADDR(iserv_addr[num_sockets],htonl(INADDR_ANY),sport); + num_sockets++; +#if defined(EPMD6) + SET_ADDR6(iserv_addr[num_sockets],in6addr_any,sport); + num_sockets++; +#endif } #ifdef HAVE_SYSTEMD_DAEMON } @@ -335,13 +355,39 @@ void run(EpmdVars *g) #endif /* HAVE_SYSTEMD_DAEMON */ for (i = 0; i < num_sockets; i++) { - if ((listensock[i] = socket(FAMILY,SOCK_STREAM,0)) < 0) + struct sockaddr *sa = (struct sockaddr *)&iserv_addr[i]; +#if defined(EPMD6) + size_t salen = (sa->sa_family == AF_INET6 ? + sizeof(struct sockaddr_in6) : + sizeof(struct sockaddr_in)); +#else + size_t salen = sizeof(struct sockaddr_in); +#endif + + if ((listensock[i] = socket(sa->sa_family,SOCK_STREAM,0)) < 0) { - dbg_perror(g,"error opening stream socket"); + switch (errno) { + case EAFNOSUPPORT: + case EPROTONOSUPPORT: + continue; + default: + dbg_perror(g,"error opening stream socket"); + epmd_cleanup_exit(g,1); + } + } + g->listenfd[bound++] = listensock[i]; + +#if HAVE_DECL_IPV6_V6ONLY + opt = 1; + if (sa->sa_family == AF_INET6 && + setsockopt(listensock[i],IPPROTO_IPV6,IPV6_V6ONLY,&opt, + sizeof(opt)) <0) + { + dbg_perror(g,"can't set IPv6 only socket option"); epmd_cleanup_exit(g,1); } - g->listenfd[i] = listensock[i]; - +#endif + /* * Note that we must not enable the SO_REUSEADDR on Windows, * because addresses will be reused even if they are still in use. @@ -373,8 +419,7 @@ void run(EpmdVars *g) dbg_perror(g,"failed to set non-blocking mode of listening socket %d", listensock[i]); - if (bind(listensock[i], (struct sockaddr*) &iserv_addr[i], - sizeof(iserv_addr[i])) < 0) + if (bind(listensock[i], (struct sockaddr*) &iserv_addr[i], salen) < 0) { if (errno == EADDRINUSE) { @@ -395,6 +440,11 @@ void run(EpmdVars *g) } select_fd_set(g, listensock[i]); } + if (bound == 0) { + dbg_perror(g,"unable to bind any address"); + epmd_cleanup_exit(g,1); + } + num_sockets = bound; #ifdef HAVE_SYSTEMD_DAEMON } sd_notifyf(0, "READY=1\n" @@ -439,8 +489,8 @@ void run(EpmdVars *g) } for (i = 0; i < num_sockets; i++) - if (FD_ISSET(listensock[i],&read_mask)) { - if (do_accept(g, listensock[i]) && g->active_conn < g->max_conn) { + if (FD_ISSET(g->listenfd[i],&read_mask)) { + if (do_accept(g, g->listenfd[i]) && g->active_conn < g->max_conn) { /* * The accept() succeeded, and we have at least one file * descriptor still free, which means that another accept() @@ -1000,15 +1050,6 @@ static int conn_open(EpmdVars *g,int fd) for (i = 0; i < g->max_conn; i++) { if (g->conn[i].open == EPMD_FALSE) { - struct sockaddr_in si; - struct sockaddr_in di; -#ifdef HAVE_SOCKLEN_T - socklen_t st; -#else - int st; -#endif - st = sizeof(si); - g->active_conn++; s = &g->conn[i]; @@ -1019,20 +1060,7 @@ static int conn_open(EpmdVars *g,int fd) s->open = EPMD_TRUE; s->keep = EPMD_FALSE; - /* Determine if connection is from localhost */ - if (getpeername(s->fd,(struct sockaddr*) &si,&st) || - st < sizeof(si)) { - /* Failure to get peername is regarded as non local host */ - s->local_peer = EPMD_FALSE; - } else { - /* Only 127.x.x.x and connections from the host's IP address - allowed, no false positives */ - s->local_peer = - (((((unsigned) ntohl(si.sin_addr.s_addr)) & 0xFF000000U) == - 0x7F000000U) || - (getsockname(s->fd,(struct sockaddr*) &di,&st) ? - EPMD_FALSE : si.sin_addr.s_addr == di.sin_addr.s_addr)); - } + s->local_peer = conn_local_peer_check(g, s->fd); dbg_tty_printf(g,2,(s->local_peer) ? "Local peer connected" : "Non-local peer connected"); @@ -1040,7 +1068,7 @@ static int conn_open(EpmdVars *g,int fd) s->got = 0; s->mod_time = current_time(g); /* Note activity */ - s->buf = (char *)malloc(INBUF_SIZE); + s->buf = malloc(INBUF_SIZE); if (s->buf == NULL) { dbg_printf(g,0,"epmd: Insufficient memory"); @@ -1058,6 +1086,60 @@ static int conn_open(EpmdVars *g,int fd) return EPMD_FALSE; } +static int conn_local_peer_check(EpmdVars *g, int fd) +{ + struct EPMD_SOCKADDR_IN si; + struct EPMD_SOCKADDR_IN di; + + struct sockaddr_in *si4 = (struct sockaddr_in *)&si; + struct sockaddr_in *di4 = (struct sockaddr_in *)&di; + +#if defined(EPMD6) + struct sockaddr_in6 *si6 = (struct sockaddr_in6 *)&si; + struct sockaddr_in6 *di6 = (struct sockaddr_in6 *)&di; +#endif + +#ifdef HAVE_SOCKLEN_T + socklen_t st; +#else + int st; +#endif + + st = sizeof(si); + + /* Determine if connection is from localhost */ + if (getpeername(fd,(struct sockaddr*) &si,&st) || + st > sizeof(si)) { + /* Failure to get peername is regarded as non local host */ + return EPMD_FALSE; + } + + /* Only 127.x.x.x and connections from the host's IP address + allowed, no false positives */ +#if defined(EPMD6) + if (si.ss_family == AF_INET6 && IN6_IS_ADDR_LOOPBACK(&(si6->sin6_addr))) + return EPMD_TRUE; + + if (si.ss_family == AF_INET) +#endif + if ((((unsigned) ntohl(si4->sin_addr.s_addr)) & 0xFF000000U) == + 0x7F000000U) + return EPMD_TRUE; + + if (getsockname(fd,(struct sockaddr*) &di,&st)) + return EPMD_FALSE; + +#if defined(EPMD6) + if (si.ss_family == AF_INET6) + return IN6_ARE_ADDR_EQUAL( &(si6->sin6_addr), &(di6->sin6_addr)); + if (si.ss_family == AF_INET) +#endif + return si4->sin_addr.s_addr == di4->sin_addr.s_addr; +#if defined(EPMD6) + return EPMD_FALSE; +#endif +} + static int conn_close_fd(EpmdVars *g,int fd) { int i; diff --git a/erts/epmd/test/epmd_SUITE.erl b/erts/epmd/test/epmd_SUITE.erl index 0c8f01ef6b..44d3c5a880 100644 --- a/erts/epmd/test/epmd_SUITE.erl +++ b/erts/epmd/test/epmd_SUITE.erl @@ -43,6 +43,7 @@ -export( [ register_name/1, + register_name_ipv6/1, register_names_1/1, register_names_2/1, register_duplicate_name/1, @@ -113,7 +114,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [register_name, register_names_1, register_names_2, + [register_name, register_name_ipv6, + register_names_1, register_names_2, register_duplicate_name, unicode_name, long_unicode_name, get_port_nr, slow_get_port_nr, unregister_others_name_1, unregister_others_name_2, @@ -172,6 +174,24 @@ register_name(Config) when is_list(Config) -> ?line ok = close(Sock), % Unregister ok. +register_name_ipv6(doc) -> + ["Register a name over IPv6"]; +register_name_ipv6(suite) -> + []; +register_name_ipv6(Config) when is_list(Config) -> + % Test if the host has an IPv6 loopback address + Res = gen_tcp:listen(0, [inet6, {ip, {0,0,0,0,0,0,0,1}}]), + case Res of + {ok,LSock} -> + gen_tcp:close(LSock), + ?line ok = epmdrun(), + ?line {ok,Sock} = register_node6("foobar6"), + ?line ok = close(Sock), % Unregister + ok; + _Error -> + {skip, "Host does not have an IPv6 loopback address"} + end. + register_names_1(doc) -> ["Register and unregister two nodes"]; register_names_1(suite) -> @@ -245,9 +265,14 @@ register_node(Name) -> register_node(Name,Port) -> register_node_v2(Port,$M,0,5,5,Name,""). +register_node6(Name) -> + register_node_v2({0,0,0,0,0,0,0,1},?DUMMY_PORT,$M,0,5,5,Name,""). + register_node_v2(Port, NodeType, Prot, HVsn, LVsn, Name, Extra) -> + register_node_v2("localhost", Port, NodeType, Prot, HVsn, LVsn, Name, Extra). +register_node_v2(Addr, Port, NodeType, Prot, HVsn, LVsn, Name, Extra) -> Req = alive2_req(Port, NodeType, Prot, HVsn, LVsn, Name, Extra), - case send_req(Req) of + case send_req(Req, Addr) of {ok,Sock} -> case recv(Sock,4) of {ok, [?EPMD_ALIVE2_RESP,_Res=0,_C0,_C1]} -> @@ -1186,7 +1211,9 @@ send_direct(Sock, Bytes) -> end. send_req(Req) -> - case connect() of + send_req(Req, "localhost"). +send_req(Req, Addr) -> + case connect(Addr) of {ok,Sock} -> case send(Sock, [size16(Req), Req]) of ok -> diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c index f21671e837..54da59e50d 100644 --- a/erts/etc/common/erlexec.c +++ b/erts/etc/common/erlexec.c @@ -65,6 +65,7 @@ static const char plusM_au_allocs[]= { 'u', /* all alloc_util allocators */ 'B', /* binary_alloc */ + 'I', /* literal_alloc */ 'D', /* std_alloc */ 'E', /* ets_alloc */ 'F', /* fix_alloc */ @@ -121,6 +122,7 @@ static char *plusM_other_switches[] = { "Ym", "Ytp", "Ytt", + "Iscs", NULL }; diff --git a/erts/etc/common/heart.c b/erts/etc/common/heart.c index 9571b83ffd..1a826221fb 100644 --- a/erts/etc/common/heart.c +++ b/erts/etc/common/heart.c @@ -472,10 +472,6 @@ message_loop(erlin_fd, erlout_fd) switch (mp->op) { case HEART_BEAT: timestamp(&last_received); -#ifdef USE_WATCHDOG - /* reset the hardware watchdog timer */ - wd_reset(); -#endif break; case SHUT_DOWN: return R_SHUT_DOWN; diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index 9cabd0591a..f4b08cfced 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -166,6 +166,10 @@ /* Define if you use a gcc that supports the double word cmpxchg instruction */ #undef ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT +/* Define if gcc wont let you clobber ebx with cmpxchg8b and position + independent code */ +#undef ETHR_CMPXCHG8B_PIC_NO_CLOBBER_EBX + /* Define if you get a register shortage with cmpxchg8b and position independent code */ #undef ETHR_CMPXCHG8B_REGISTER_SHORTAGE diff --git a/erts/include/internal/i386/ethr_dw_atomic.h b/erts/include/internal/i386/ethr_dw_atomic.h index e8c4119ef0..5444a6345c 100644 --- a/erts/include/internal/i386/ethr_dw_atomic.h +++ b/erts/include/internal/i386/ethr_dw_atomic.h @@ -115,13 +115,19 @@ ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var) return (ethr_sint_t *) ETHR_DW_NATMC_MEM__(var); } -#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__ +#if defined(ETHR_CMPXCHG8B_PIC_NO_CLOBBER_EBX) && defined(__PIC__) && __PIC__ +#if ETHR_SIZEOF_PTR != 4 +# error unexpected pic issue +#endif /* * When position independent code is used in 32-bit mode, the EBX register - * is used for storage of global offset table address, and we may not - * use it as input or output in an asm. We need to save and restore the - * EBX register explicitly (for some reason gcc doesn't provide this - * service to us). + * is used for storage of global offset table address. When compiling with + * an old gcc (< vsn 5) we may not use it as input or output in an inline + * asm. We then need to save and restore the EBX register explicitly (for + * some reason old gcc compilers didn't provide this service to us). + * ETHR_CMPXCHG8B_PIC_NO_CLOBBER_EBX will be defined if we need to + * explicitly manage EBX ourselves. + * */ # define ETHR_NO_CLOBBER_EBX__ 1 #else @@ -151,36 +157,53 @@ ethr_native_dw_atomic_cmpxchg_mb(ethr_native_dw_atomic_t *var, ETHR_DW_DBG_ALIGNED__(p); +#if ETHR_NO_CLOBBER_EBX__ && ETHR_CMPXCHG8B_REGISTER_SHORTAGE + /* + * gcc wont let us use ebx as input and we + * get a register shortage + */ + __asm__ __volatile__( -#if ETHR_NO_CLOBBER_EBX__ "pushl %%ebx\n\t" -# if ETHR_CMPXCHG8B_REGISTER_SHORTAGE "movl (%7), %%ebx\n\t" "movl 4(%7), %%ecx\n\t" -# else - "movl %8, %%ebx\n\t" -# endif -#endif - "lock; cmpxchg" ETHR_DW_CMPXCHG_SFX__ " %0\n\t" + "lock; cmpxchg8b %0\n\t" "setz %3\n\t" -#if ETHR_NO_CLOBBER_EBX__ "popl %%ebx\n\t" -#endif : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd) - : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), -#if ETHR_NO_CLOBBER_EBX__ -# if ETHR_CMPXCHG8B_REGISTER_SHORTAGE - "3"(new) -# else - "3"(new[1]), - "r"(new[0]) -# endif + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "r"(new) + : "cc", "memory"); + +#elif ETHR_NO_CLOBBER_EBX__ + /* + * gcc wont let us use ebx as input + */ + + __asm__ __volatile__( + "pushl %%ebx\n\t" + "movl %8, %%ebx\n\t" + "lock; cmpxchg8b %0\n\t" + "setz %3\n\t" + "popl %%ebx\n\t" + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=q"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "c"(new[1]), "r"(new[0]) + : "cc", "memory"); + #else - "3"(new[1]), - "b"(new[0]) -#endif + /* + * gcc lets us place values in the registers where + * we want them + */ + + __asm__ __volatile__( + "lock; cmpxchg" ETHR_DW_CMPXCHG_SFX__ " %0\n\t" + "setz %3\n\t" + : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=q"(xchgd) + : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "c"(new[1]), "b"(new[0]) : "cc", "memory"); +#endif + return (int) xchgd; } diff --git a/erts/preloaded/ebin/erl_prim_loader.beam b/erts/preloaded/ebin/erl_prim_loader.beam Binary files differindex ca9e6bd20f..6d777fa811 100644 --- a/erts/preloaded/ebin/erl_prim_loader.beam +++ b/erts/preloaded/ebin/erl_prim_loader.beam diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex b353129a34..2ea2de4c70 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex 5590f5a911..88da34b192 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/prim_file.beam b/erts/preloaded/ebin/prim_file.beam Binary files differindex a386613eef..426e764127 100644 --- a/erts/preloaded/ebin/prim_file.beam +++ b/erts/preloaded/ebin/prim_file.beam diff --git a/erts/preloaded/src/erl_prim_loader.erl b/erts/preloaded/src/erl_prim_loader.erl index 824ed3435d..cbcced5512 100644 --- a/erts/preloaded/src/erl_prim_loader.erl +++ b/erts/preloaded/src/erl_prim_loader.erl @@ -55,6 +55,9 @@ %% Used by test suites -export([purge_archive_cache/0]). +%% Used by init and the code server. +-export([get_modules/3]). + -include_lib("kernel/include/file.hrl"). -type host() :: atom(). @@ -236,6 +239,16 @@ set_primary_archive(File, ArchiveBin, FileInfo, ParserFun) purge_archive_cache() -> request(purge_archive_cache). + +-spec get_modules([module()], + fun((atom(), string(), binary()) -> + {'ok',any()} | {'error',any()}), + [string()]) -> + {'ok',{[any()],[any()]}}. + +get_modules(Modules, Fun, Path) -> + request({get_modules,{Modules,Fun,Path}}). + request(Req) -> Loader = whereis(erl_prim_loader), Loader ! {self(),Req}, @@ -325,6 +338,8 @@ handle_request(Req, Paths, St0) -> {{ok,Paths},St0}; {get_file,File} -> handle_get_file(St0, Paths, File); + {get_modules,{Modules,Fun,ModPaths}} -> + handle_get_modules(St0, Modules, Fun, ModPaths); {list_dir,Dir} -> handle_list_dir(St0, Dir); {read_file_info,File} -> @@ -465,6 +480,124 @@ efile_timeout_handler(State, _Parent) -> State. %%% -------------------------------------------------------- +%%% Read and process severals modules in parallel. +%%% -------------------------------------------------------- + +handle_get_modules(#state{loader=efile}=St, Ms, Process, Paths) -> + Primary = (St#state.prim_state)#prim_state.primary_archive, + Res = case efile_any_archives(Paths, Primary) of + false -> + efile_get_mods_par(Ms, Process, Paths); + true -> + Get = fun efile_get_file_from_port/3, + gm_get_mods(St, Get, Ms, Process, Paths) + end, + {Res,St}; +handle_get_modules(#state{loader=inet}=St, Ms, Process, Paths) -> + Get = fun inet_get_file_from_port/3, + {gm_get_mods(St, Get, Ms, Process, Paths),St}. + +efile_get_mods_par(Ms, Process, Paths) -> + Self = self(), + Ref = make_ref(), + GmSpawn = fun() -> + efile_gm_spawn({Self,Ref}, Ms, Process, Paths) + end, + _ = spawn_link(GmSpawn), + N = length(Ms), + efile_gm_recv(N, Ref, [], []). + +efile_any_archives([H|T], Primary) -> + case name_split(Primary, H) of + {file,_} -> efile_any_archives(T, Primary); + {archive,_,_} -> true + end; +efile_any_archives([], _) -> + false. + +efile_gm_recv(0, _Ref, Succ, Fail) -> + {ok,{Succ,Fail}}; +efile_gm_recv(N, Ref, Succ, Fail) -> + receive + {Ref,Mod,{ok,Res}} -> + efile_gm_recv(N-1, Ref, [{Mod,Res}|Succ], Fail); + {Ref,Mod,{error,Res}} -> + efile_gm_recv(N-1, Ref, Succ, [{Mod,Res}|Fail]) + end. + +efile_gm_spawn(ParentRef, Ms, Process, Paths) -> + efile_gm_spawn_1(0, Ms, ParentRef, Process, Paths). + +efile_gm_spawn_1(N, Ms, ParentRef, Process, Paths) when N >= 32 -> + receive + {'DOWN',_,process,_,_} -> + efile_gm_spawn_1(N-1, Ms, ParentRef, Process, Paths) + end; +efile_gm_spawn_1(N, [M|Ms], ParentRef, Process, Paths) -> + Get = fun() -> efile_gm_get(Paths, M, ParentRef, Process) end, + _ = spawn_monitor(Get), + efile_gm_spawn_1(N+1, Ms, ParentRef, Process, Paths); +efile_gm_spawn_1(_, [], _, _, _) -> + ok. + +efile_gm_get(Paths, Mod, ParentRef, Process) -> + File = atom_to_list(Mod) ++ init:objfile_extension(), + efile_gm_get_1(Paths, File, Mod, ParentRef, Process). + +efile_gm_get_1([P|Ps], File0, Mod, {Parent,Ref}=PR, Process) -> + File = join(P, File0), + Res = try prim_file:read_file(File) of + {ok,Bin} -> + gm_process(Mod, File, Bin, Process); + {error,enoent} -> + efile_gm_get_1(Ps, File0, Mod, PR, Process); + Error -> + check_file_result(get_modules, File, Error), + Error + catch + _:Reason -> + {error,{crash,Reason}} + end, + Parent ! {Ref,Mod,Res}; +efile_gm_get_1([], _, Mod, {Parent,Ref}, _Process) -> + Parent ! {Ref,Mod,{error,enoent}}. + +gm_get_mods(St, Get, Ms, Process, Paths) -> + gm_get_mods(St, Get, Ms, Process, Paths, [], []). + +gm_get_mods(St, Get, [M|Ms], Process, Paths, Succ, Fail) -> + File = atom_to_list(M) ++ init:objfile_extension(), + case gm_arch_get(St, Get, M, File, Paths, Process) of + {ok,Res} -> + gm_get_mods(St, Get, Ms, Process, Paths, + [{M,Res}|Succ], Fail); + {error,Res} -> + gm_get_mods(St, Get, Ms, Process, Paths, + Succ, [{M,Res}|Fail]) + end; +gm_get_mods(_St, _Get, [], _, _, Succ, Fail) -> + {ok,{Succ,Fail}}. + +gm_arch_get(St, Get, Mod, File, Paths, Process) -> + case Get(St, File, Paths) of + {{error,_}=E,_} -> + E; + {{ok,Bin,Full},_} -> + gm_process(Mod, Full, Bin, Process) + end. + +gm_process(Mod, File, Bin, Process) -> + try Process(Mod, File, Bin) of + {ok,_}=Res -> Res; + {error,_}=Res -> Res; + Other -> {error,{bad_return,Other}} + catch + _:Error -> + {error,{crash,Error}} + end. + + +%%% -------------------------------------------------------- %%% Functions which handle inet prim_loader %%% -------------------------------------------------------- diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index ab51cf385c..dfbd116d6e 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -105,7 +105,9 @@ -export([garbage_collect/0, garbage_collect/1, garbage_collect/2]). -export([garbage_collect_message_area/0, get/0, get/1, get_keys/0, get_keys/1]). -export([get_module_info/1, get_stacktrace/0, group_leader/0]). --export([group_leader/2, halt/0, halt/1, halt/2, hash/2, hibernate/3]). +-export([group_leader/2]). +-export([halt/0, halt/1, halt/2, hash/2, + has_prepared_code_on_load/1, hibernate/3]). -export([insert_element/3]). -export([integer_to_binary/1, integer_to_list/1]). -export([iolist_size/1, iolist_to_binary/1]). @@ -997,6 +999,12 @@ halt(_Status, _Options) -> hash(_Term, _Range) -> erlang:nif_error(undefined). +%% has_prepared_code_on_load/1 +-spec erlang:has_prepared_code_on_load(PreparedCode) -> boolean() when + PreparedCode :: binary(). +has_prepared_code_on_load(_PreparedCode) -> + erlang:nif_error(undefined). + %% hibernate/3 -spec erlang:hibernate(Module, Function, Args) -> no_return() when Module :: module(), @@ -2292,8 +2300,8 @@ subtract(_,_) -> MinBinVHeapSize :: non_neg_integer(), OldMinBinVHeapSize :: non_neg_integer(); (multi_scheduling, BlockState) -> OldBlockState when - BlockState :: block | unblock, - OldBlockState :: block | unblock | enabled; + BlockState :: block | unblock | block_normal | unblock_normal, + OldBlockState :: blocked | disabled | enabled; (scheduler_bind_type, How) -> OldBindType when How :: scheduler_bind_type() | default_bind, OldBindType :: scheduler_bind_type(); @@ -2438,14 +2446,15 @@ tuple_to_list(_Tuple) -> logical_processors_available | logical_processors_online) -> unknown | pos_integer(); (machine) -> string(); + (message_queue_data) -> message_queue_data(); (min_heap_size) -> {min_heap_size, MinHeapSize :: pos_integer()}; (min_bin_vheap_size) -> {min_bin_vheap_size, MinBinVHeapSize :: pos_integer()}; (modified_timing_level) -> integer() | undefined; - (multi_scheduling) -> disabled | blocked | enabled; + (multi_scheduling) -> disabled | blocked | blocked_normal | enabled; (multi_scheduling_blockers) -> [Pid :: pid()]; (nif_version) -> string(); - (message_queue_data) -> message_queue_data(); + (normal_multi_scheduling_blockers) -> [Pid :: pid()]; (otp_release) -> string(); (os_monotonic_time_source) -> [{atom(),term()}]; (os_system_time_source) -> [{atom(),term()}]; diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl index 26025d6704..330fcc4a9c 100644 --- a/erts/preloaded/src/erts_internal.erl +++ b/erts/preloaded/src/erts_internal.erl @@ -35,6 +35,9 @@ -export([open_port/2, port_command/3, port_connect/2, port_close/1, port_control/3, port_call/3, port_info/1, port_info/2]). +-export([system_check/1, + gather_system_check_result/1]). + -export([request_system_task/3]). -export([check_process_code/3]). @@ -281,6 +284,23 @@ copy_literals(_Mod, _Bool) -> purge_module(_Module) -> erlang:nif_error(undefined). +-spec system_check(Type) -> 'ok' when + Type :: 'schedulers'. + +system_check(_Type) -> + erlang:nif_error(undefined). + +gather_system_check_result(Ref) when is_reference(Ref) -> + gather_system_check_result(Ref, erlang:system_info(schedulers)). + +gather_system_check_result(_Ref, 0) -> + ok; +gather_system_check_result(Ref, N) -> + receive + Ref -> + gather_system_check_result(Ref, N - 1) + end. + %% term compare where integer() < float() = true -spec cmp_term(A,B) -> Result when diff --git a/erts/preloaded/src/prim_file.erl b/erts/preloaded/src/prim_file.erl index 2eb1b1d408..ab5359ebbc 100644 --- a/erts/preloaded/src/prim_file.erl +++ b/erts/preloaded/src/prim_file.erl @@ -1,7 +1,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2000-2013. All Rights Reserved. +%% Copyright Ericsson AB 2000-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. diff --git a/erts/test/otp_SUITE.erl b/erts/test/otp_SUITE.erl index 6777c205b0..2efb5ae200 100644 --- a/erts/test/otp_SUITE.erl +++ b/erts/test/otp_SUITE.erl @@ -298,7 +298,7 @@ call_to_now_0(Config) when is_list(Config) -> Apps = [asn1,common_test,compiler,debugger,dialyzer, gs,kernel,mnesia,observer,parsetools,reltool, runtime_tools,sasl,stdlib,syntax_tools, - test_server,tools], + tools], not_recommended_calls(Config, Apps, {erlang,now,0}). not_recommended_calls(Config, Apps0, MFA) -> @@ -459,24 +459,7 @@ runtime_dependencies(Config) -> end, {undefined, []}, SAE), - [] = lists:filter(fun ({missing_runtime_dependency, - AppFile, - common_test}) -> - %% The test_server app is contaminated by - %% common_test when run in a source tree. It - %% should however *not* be contaminated - %% when run in an installation. - case {filename:basename(AppFile), - is_run_in_src_tree()} of - {"test_server.app", true} -> - false; - _ -> - true - end; - (_) -> - true - end, - check_apps_deps([AppDep|AppDeps], IgnoreApps)), + check_apps_deps([AppDep|AppDeps], IgnoreApps), case IgnoreApps of [] -> ok; @@ -488,17 +471,6 @@ runtime_dependencies(Config) -> {comment, Comment} end. -is_run_in_src_tree() -> - %% At least currently run_erl is not present in <code-root>/bin - %% in the source tree, but present in <code-root>/bin of an - %% ordinary installation. - case file:read_file_info(filename:join([code:root_dir(), - "bin", - "run_erl"])) of - {ok, _} -> false; - {error, _} -> true - end. - have_rdep(_App, [], _Dep) -> false; have_rdep(App, [RDep | RDeps], Dep) -> |