diff options
Diffstat (limited to 'erts')
139 files changed, 4606 insertions, 4114 deletions
diff --git a/erts/configure.in b/erts/configure.in index ad9a66126f..98f3e6bcc5 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -121,7 +121,6 @@ AS_HELP_STRING([--enable-bootstrap-only], enable_hipe=no enable_sctp=no enable_dirty_schedulers=no - enable_new_purge=no fi ]) @@ -140,13 +139,6 @@ AS_HELP_STRING([--enable-dirty-schedulers], [enable dirty scheduler support]), *) enable_dirty_schedulers=yes ;; esac ], enable_dirty_schedulers=default) -AC_ARG_ENABLE(new-purge-strategy, -AS_HELP_STRING([--enable-new-purge-strategy], [enable new code purge strategy]), -[ case "$enableval" in - no) enable_new_purge=no ;; - *) enable_new_purge=yes ;; - esac ], enable_new_purge=default) - AC_ARG_ENABLE(smp-support, AS_HELP_STRING([--enable-smp-support], [enable smp support]) AS_HELP_STRING([--disable-smp-support], [disable smp support]), @@ -541,6 +533,7 @@ if test "x$GCC" = xyes; then # Treat certain GCC warnings as errors LM_TRY_ENABLE_CFLAG([-Werror=return-type], [WERRORFLAGS]) LM_TRY_ENABLE_CFLAG([-Werror=implicit], [WERRORFLAGS]) + LM_TRY_ENABLE_CFLAG([-Werror=undef], [WERRORFLAGS]) # until the emulator can handle this, I suggest we turn it off! #WFLAGS="-Wall -Wshadow -Wcast-qual -Wmissing-declarations" @@ -1046,8 +1039,7 @@ case $ERTS_BUILD_SMP_EMU-$enable_dirty_schedulers in yes-yes) DIRTY_SCHEDULER_SUPPORT=yes;; yes-default) - ## Maybe yes for OTP 19... - DIRTY_SCHEDULER_SUPPORT=no;; + DIRTY_SCHEDULER_SUPPORT=yes;; no-default) DIRTY_SCHEDULER_SUPPORT=no;; no-yes) @@ -1058,27 +1050,6 @@ esac AC_MSG_RESULT($DIRTY_SCHEDULER_SUPPORT) AC_SUBST(DIRTY_SCHEDULER_SUPPORT) -AC_MSG_CHECKING(whether the new code purge strategy should be enabled) -case $enable_new_purge-$enable_dirty_schedulers in - yes-*) - AC_MSG_RESULT(yes) - enable_new_purge=yes;; - default-yes) - AC_MSG_RESULT(yes; forced by dirty scheduler support) - enable_new_purge=yes;; - no-yes) - AC_MSG_ERROR([Dirty schedulers enabled, but new code purge strategy disabled]);; - *) - AC_MSG_RESULT(no) - enable_new_purge=no;; -esac - -if test $enable_new_purge = yes; then - AC_DEFINE(ERTS_NEW_PURGE_STRATEGY, 1, [Define if you want to use the new code purge strategy]) -fi -NEW_PURGE_STRATEGY=$enable_new_purge -AC_SUBST(NEW_PURGE_STRATEGY) - if test $ERTS_BUILD_SMP_EMU = yes; then if test $found_threads = no; then @@ -1410,6 +1381,8 @@ AC_ARG_ENABLE(builtin-zlib, Z_LIB= if test "x$enable_builtin_zlib" = "xyes"; then + AC_DEFINE(HAVE_ZLIB_INFLATEGETDICTIONARY, 1, + [Define if your zlib version defines inflateGetDictionary.]) AC_MSG_NOTICE([Using our own built-in zlib source]) else AC_MSG_CHECKING(for zlib 1.2.5 or higher) @@ -1436,6 +1409,11 @@ error AC_MSG_RESULT(no) ]) LIBS=$zlib_save_LIBS + +AC_MSG_CHECKING(for zlib inflateGetDictionary presence) +AC_SEARCH_LIBS(inflateGetDictionary, [z], + AC_DEFINE(HAVE_ZLIB_INFLATEGETDICTIONARY, 1, + [Define if your zlib version defines inflateGetDictionary.])) fi AC_SUBST(Z_LIB) @@ -2061,7 +2039,7 @@ AC_CHECK_FUNCS([getipnodebyname getipnodebyaddr gethostbyname2]) AC_CHECK_FUNCS([ieee_handler fpsetmask finite isnan isinf res_gethostbyname dlopen \ pread pwrite memmove strerror strerror_r strncasecmp \ - gethrtime localtime_r gmtime_r inet_pton \ + gethrtime localtime_r gmtime_r inet_pton mprotect \ mmap mremap memcpy mallopt sbrk _sbrk __sbrk brk _brk __brk \ flockfile fstat strlcpy strlcat setsid posix2time time2posix \ setlocale nl_langinfo poll mlockall ppoll]) @@ -2767,17 +2745,29 @@ ERL_TIME_CORRECTION AC_CHECK_PROG(M4, m4, m4) -dnl HiPE cannot run on 64-bit without MAP_FIXED and MAP_NORESERVE -if test X${enable_hipe} != Xno && test X$ac_cv_sizeof_void_p != X4; then - AC_CHECK_DECLS([MAP_FIXED, MAP_NORESERVE], [], [], [#include <sys/mman.h>]) - if test X$ac_cv_have_decl_MAP_FIXED != Xyes || test X$ac_cv_have_decl_MAP_NORESERVE != Xyes; then - if test X${enable_hipe} = Xyes; then - AC_MSG_ERROR([HiPE on 64-bit needs MAP_FIXED and MAP_NORESERVE flags for mmap()]) - else - enable_hipe=no - AC_MSG_WARN([Disable HiPE due to lack of MAP_FIXED and MAP_NORESERVE flags for mmap()]) - fi - fi +if test X${enable_hipe} != Xno; then + if test X$ac_cv_sizeof_void_p != X4 && test X$ARCH = Xamd64; then + dnl HiPE cannot run on x86_64 without MAP_FIXED and MAP_NORESERVE + AC_CHECK_DECLS([MAP_FIXED, MAP_NORESERVE], [], [], [#include <sys/mman.h>]) + if test X$ac_cv_have_decl_MAP_FIXED != Xyes || test X$ac_cv_have_decl_MAP_NORESERVE != Xyes; then + if test X${enable_hipe} = Xyes; then + AC_MSG_ERROR([HiPE on x86_64 needs MAP_FIXED and MAP_NORESERVE flags for mmap()]) + else + enable_hipe=no + AC_MSG_WARN([Disable HiPE due to lack of MAP_FIXED and MAP_NORESERVE flags for mmap()]) + fi + fi + else + dnl HiPE cannot run without mprotect() + if test X$ac_cv_func_mprotect != Xyes; then + if test X${enable_hipe} = Xyes; then + AC_MSG_ERROR([HiPE needs mprotect() on $ARCH]) + else + enable_hipe=no + AC_MSG_WARN([Disable HiPE due to lack of mprotect()]) + fi + fi + fi fi dnl check to auto-enable hipe here... @@ -3426,13 +3416,8 @@ if test X${enable_hipe} = Xyes; then AC_DEFINE(HIPE,[1],[Define to enable HiPE]) HIPE_HELPERS="xmerl syntax_tools edoc" ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS hipe" - case "$ARCH" in - amd64) - # For now exec_alloc is only used for hipe on amd64 - AC_MSG_NOTICE([Enable exec_alloc for hipe code allocation]) - ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS exec_alloc" - ;; - esac + AC_MSG_NOTICE([Enable exec_alloc for hipe code allocation]) + ENABLE_ALLOC_TYPE_VARS="$ENABLE_ALLOC_TYPE_VARS exec_alloc" fi fi AC_SUBST(HIPE_HELPERS) @@ -3968,6 +3953,7 @@ dnl use "PATH/include" and "PATH/lib". AC_SUBST(SSL_INCLUDE) AC_SUBST(SSL_INCDIR) AC_SUBST(SSL_LIBDIR) +AC_SUBST(SSL_FLAGS) AC_SUBST(SSL_CRYPTO_LIBNAME) AC_SUBST(SSL_SSL_LIBNAME) AC_SUBST(SSL_CC_RUNTIME_LIBRARY_PATH) @@ -4657,6 +4643,31 @@ no) # Use no ssl runtime library path esac +AC_ARG_ENABLE(fips, +AS_HELP_STRING([--enable-fips], [enable OpenSSL FIPS mode support]) +AS_HELP_STRING([--disable-fips], [disable OpenSSL FIPS mode support (default)]), +[ case "$enableval" in + yes) enable_fips_support=yes ;; + *) enable_fips_support=no ;; + esac ], enable_fips_support=no) + +if test "x$enable_fips_support" = "xyes" && test "$CRYPTO_APP" != ""; then + saveCFLAGS="$CFLAGS" + saveLDFLAGS="$LDFLAGS" + saveLIBS="$LIBS" + CFLAGS="$CFLAGS $SSL_INCLUDE" + LDFLAGS="$LDFLAGS $SSL_LD_RUNTIME_LIBRARY_PATH -L$SSL_LIBDIR" + LIBS="-lcrypto" + AC_CHECK_FUNC([FIPS_mode_set], + [SSL_FLAGS="-DFIPS_SUPPORT"], + [SSL_FLAGS=]) + CFLAGS="$saveCFLAGS" + LDFLAGS="$saveLDFLAGS" + LIBS="$saveLIBS" +else + SSL_FLAGS= +fi + #-------------------------------------------------------------------- # Os mon stuff. #-------------------------------------------------------------------- diff --git a/erts/doc/src/erl_nif.xml b/erts/doc/src/erl_nif.xml index b5dc9037c4..906c1be17b 100644 --- a/erts/doc/src/erl_nif.xml +++ b/erts/doc/src/erl_nif.xml @@ -80,7 +80,7 @@ <code type="none"> /* niftest.c */ -#include "erl_nif.h" +#include <erl_nif.h> static ERL_NIF_TERM hello(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { @@ -296,8 +296,8 @@ return term;</code> synchronization. This includes terms in process-independent environments that are shared between threads. Resource objects also require synchronization if you treat them as mutable.</p> - <p>The library initialization callbacks <c>load</c>, <c>reload</c>, and - <c>upgrade</c> are all thread-safe even for shared state data.</p> + <p>The library initialization callbacks <c>load</c> and + <c>upgrade</c> are thread-safe even for shared state data.</p> </item> <tag><marker id="version_management"/>Version Management</tag> <item> @@ -498,7 +498,7 @@ return term;</code> <title>Initialization</title> <taglist> <tag><marker id="ERL_NIF_INIT"/><c>ERL_NIF_INIT(MODULE, - ErlNifFunc funcs[], load, reload, upgrade, unload)</c></tag> + ErlNifFunc funcs[], load, NULL, upgrade, unload)</c></tag> <item> <p>This is the magic macro to initialize a NIF library. It is to be evaluated in global file scope.</p> @@ -507,11 +507,14 @@ return term;</code> the macro.</p> <p><c>funcs</c> is a static array of function descriptors for all the implemented NIFs in this library.</p> - <p><c>load</c>, <c>reload</c>, <c>upgrade</c> and <c>unload</c> - are pointers to functions. One of <c>load</c>, <c>reload</c>, or + <p><c>load</c>, <c>upgrade</c> and <c>unload</c> + are pointers to functions. One of <c>load</c> or <c>upgrade</c> is called to initialize the library. <c>unload</c> is called to release the library. All are described individually below.</p> + <p>The fourth argument <c>NULL</c> is ignored. It + was earlier used for the deprectated <c>reload</c> callback + which is no longer supported since OTP 20.</p> <p>If compiling a NIF for static inclusion through <c>--enable-static-nifs</c>, you must define <c>STATIC_ERLANG_NIF</c> before the <c>ERL_NIF_INIT</c> declaration.</p> @@ -539,7 +542,7 @@ return term;</code> and there is old code of this module with a loaded NIF library.</p> <p>Works as <c>load</c>, except that <c>*old_priv_data</c> already contains the value set by the last call to <c>load</c> or - <c>reload</c> for the old module code. <c>*priv_data</c> is + <c>upgrade</c> for the old module code. <c>*priv_data</c> is initialized to <c>NULL</c> when <c>upgrade</c> is called. It is allowed to write to both <c>*priv_data</c> and <c>*old_priv_data.</c></p> @@ -551,27 +554,7 @@ return term;</code> <item> <p><c>unload</c> is called when the module code that the NIF library belongs to is purged as old. New code of the same - module may or may not exist. Notice that <c>unload</c> is not - called for a replaced library as a consequence of <c>reload</c>.</p> - </item> - <tag><marker id="reload"/><c>int (*reload)(ErlNifEnv* env, void** - priv_data, ERL_NIF_TERM load_info)</c></tag> - <item> - <note> - <p><em>The reload mechanism is deprecated.</em> It was only intended - as a development feature. Do not use it as an upgrade method for - live production systems. It can be removed in future releases. - Ensure to pass <c>reload</c> as <c>NULL</c> to - <seealso marker="#ERL_NIF_INIT"><c>ERL_NIF_INIT</c></seealso> - to disable it when not used.</p> - </note> - <p><c>reload</c> is called when the NIF library is loaded and a - previously loaded library already exists for this module code.</p> - <p>Works as <c>load</c>, except that - <c>*priv_data</c> already contains the value set by the - previous call to <c>load</c> or <c>reload</c>.</p> - <p>The library fails to load if <c>reload</c> returns - anything other than <c>0</c> or if <c>reload</c> is <c>NULL</c>.</p> + module may or may not exist.</p> </item> </taglist> </section> @@ -2249,9 +2232,8 @@ enif_map_iterator_destroy(env, &iter);</code> returns <c>NULL</c> and sets <c>*tried</c> to <c>flags</c>. It is allowed to set <c>tried</c> to <c>NULL</c>.</p> <p>Notice that <c>enif_open_resource_type</c> is only allowed to be - called in the three callbacks - <seealso marker="#load"><c>load</c></seealso>, - <seealso marker="#reload"><c>reload</c></seealso>, and + called in the two callbacks + <seealso marker="#load"><c>load</c></seealso> and <seealso marker="#upgrade"><c>upgrade</c></seealso>.</p> </desc> </func> @@ -2305,10 +2287,8 @@ enif_map_iterator_destroy(env, &iter);</code> <fsummary>Get the private data of a NIF library.</fsummary> <desc> <p>Returns the pointer to the private data that was set by - <seealso marker="#load"><c>load</c></seealso>, - <seealso marker="#reload"><c>reload</c></seealso>, or + <seealso marker="#load"><c>load</c></seealso> or <seealso marker="#upgrade"><c>upgrade</c></seealso>.</p> - <p>Was previously named <c>enif_get_data</c>.</p> </desc> </func> diff --git a/erts/doc/src/erl_tracer.xml b/erts/doc/src/erl_tracer.xml index 83eef374ca..43613c31b1 100644 --- a/erts/doc/src/erl_tracer.xml +++ b/erts/doc/src/erl_tracer.xml @@ -684,7 +684,7 @@ trace(_, _, _, _, _) -> <p><c>erl_msg_tracer.c</c>:</p> <pre> -#include "erl_nif.h" +#include <erl_nif.h> /* NIF interface declarations */ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info); diff --git a/erts/doc/src/erlang.xml b/erts/doc/src/erlang.xml index 9646953518..3dad09365e 100644 --- a/erts/doc/src/erlang.xml +++ b/erts/doc/src/erlang.xml @@ -733,6 +733,19 @@ hello </func> <func> + <name name="ceil" arity="1"/> + <fsummary>Returns the smallest integer not less than the argument</fsummary> + <desc> + <p>Returns the smallest integer not less than + <c><anno>Number</anno></c>. + For example:</p> + <pre> +> <input>ceil(5.5).</input> +6</pre> + <p>Allowed in guard tests.</p> + </desc> + </func> + <func> <name name="check_old_code" arity="1"/> <fsummary>Check if a module has old code.</fsummary> <desc> @@ -1480,6 +1493,20 @@ true</pre> </func> <func> + <name name="floor" arity="1"/> + <fsummary>Returns the largest integer not greater than the argument</fsummary> + <desc> + <p>Returns the largest integer not greater than + <c><anno>Number</anno></c>. + For example:</p> + <pre> +> <input>floor(-10.5).</input> +-11</pre> + <p>Allowed in guard tests.</p> + </desc> + </func> + + <func> <name name="fun_info" arity="1"/> <fsummary>Information about a fun.</fsummary> <desc> @@ -1667,6 +1694,12 @@ true</pre> the form <c>{garbage_collect, <anno>RequestId</anno>, <anno>GCResult</anno>}</c>. </item> + + <tag><c>{type, 'major' | 'minor'}</c></tag> + <item>Triggers garbage collection of requested type. Default value is + <c>'major'</c>, which would trigger a fullsweep GC. + The option <c>'minor'</c> is considered a hint and may lead to + either minor or major GC run.</item> </taglist> <p>If <c><anno>Pid</anno></c> equals <c>self()</c>, and no <c>async</c> option has been passed, the garbage @@ -2544,13 +2577,6 @@ os_prompt%</pre> <name name="load_nif" arity="2"/> <fsummary>Load NIF library.</fsummary> <desc> - <note> - <p>Before Erlang/OTP R14B, NIFs were an - experimental feature. Versions before Erlang/OTP R14B can - have different and possibly incompatible NIF semantics and - interfaces. For example, in Erlang/OTP R13B03 the return value on - failure was <c>{error,Reason,Text}</c>.</p> - </note> <p>Loads and links a dynamic library containing native implemented functions (NIFs) for a module. <c><anno>Path</anno></c> is a file path to the shareable object/dynamic library file minus @@ -2580,14 +2606,22 @@ os_prompt%</pre> <item>The library did not fulfill the requirements as a NIF library of the calling module. </item> - <tag><c>load | reload | upgrade</c></tag> + <tag><c>load | upgrade</c></tag> <item>The corresponding library callback was unsuccessful. </item> + <tag><c>reload</c></tag> + <item>A NIF library is already loaded for this module instance. + The previously deprecated <c>reload</c> feature was removed in OTP 20. + </item> <tag><c>old_code</c></tag> <item>The call to <c>load_nif/2</c> was made from the old code of a module that has been upgraded; this is not allowed. </item> + <tag><c>notsup</c></tag> + <item>Lack of support. Such as loading NIF library for a + HiPE compiled module. + </item> </taglist> </desc> </func> @@ -7190,8 +7224,8 @@ ok </func> <func> - <name name="system_info" arity="1" clause_i="10"/> <name name="system_info" arity="1" clause_i="11"/> + <name name="system_info" arity="1" clause_i="12"/> <fsummary>Information about the CPU topology of the system.</fsummary> <type name="cpu_topology"/> <type name="level_entry"/> @@ -7291,12 +7325,12 @@ ok </func> <func> - <name name="system_info" arity="1" clause_i="27"/> <name name="system_info" arity="1" clause_i="28"/> - <name name="system_info" arity="1" clause_i="36"/> + <name name="system_info" arity="1" clause_i="29"/> <name name="system_info" arity="1" clause_i="37"/> <name name="system_info" arity="1" clause_i="38"/> <name name="system_info" arity="1" clause_i="39"/> + <name name="system_info" arity="1" clause_i="40"/> <fsummary>Information about the default process heap settings.</fsummary> <type name="message_queue_data"/> <type name="max_heap_size"/> @@ -7339,12 +7373,6 @@ ok see <seealso marker="#process_flag_max_heap_size"> <c>process_flag(max_heap_size, MaxHeapSize)</c></seealso>.</p> </item> - <tag><c>min_heap_size</c></tag> - <item> - <p>Returns <c>{min_heap_size, <anno>MinHeapSize</anno>}</c>, - where <c><anno>MinHeapSize</anno></c> is the current - system-wide minimum heap size for spawned processes.</p> - </item> <tag><marker id="system_info_message_queue_data"/> <c>message_queue_data</c></tag> <item> @@ -7357,6 +7385,12 @@ ok <seealso marker="#process_flag_message_queue_data"> <c>process_flag(message_queue_data, MQD)</c></seealso>.</p> </item> + <tag><c>min_heap_size</c></tag> + <item> + <p>Returns <c>{min_heap_size, <anno>MinHeapSize</anno>}</c>, + where <c><anno>MinHeapSize</anno></c> is the current + system-wide minimum heap size for spawned processes.</p> + </item> <tag><c>min_bin_vheap_size</c></tag> <item> <p>Returns <c>{min_bin_vheap_size, @@ -7373,7 +7407,7 @@ ok <name name="system_info" arity="1" clause_i="7"/> <name name="system_info" arity="1" clause_i="8"/> <name name="system_info" arity="1" clause_i="9"/> - <name name="system_info" arity="1" clause_i="12"/> + <name name="system_info" arity="1" clause_i="10"/> <name name="system_info" arity="1" clause_i="13"/> <name name="system_info" arity="1" clause_i="14"/> <name name="system_info" arity="1" clause_i="15"/> @@ -7388,14 +7422,14 @@ ok <name name="system_info" arity="1" clause_i="24"/> <name name="system_info" arity="1" clause_i="25"/> <name name="system_info" arity="1" clause_i="26"/> - <name name="system_info" arity="1" clause_i="29"/> + <name name="system_info" arity="1" clause_i="27"/> <name name="system_info" arity="1" clause_i="30"/> <name name="system_info" arity="1" clause_i="31"/> <name name="system_info" arity="1" clause_i="32"/> <name name="system_info" arity="1" clause_i="33"/> <name name="system_info" arity="1" clause_i="34"/> <name name="system_info" arity="1" clause_i="35"/> - <name name="system_info" arity="1" clause_i="40"/> + <name name="system_info" arity="1" clause_i="36"/> <name name="system_info" arity="1" clause_i="41"/> <name name="system_info" arity="1" clause_i="42"/> <name name="system_info" arity="1" clause_i="43"/> @@ -7425,16 +7459,21 @@ ok <name name="system_info" arity="1" clause_i="67"/> <name name="system_info" arity="1" clause_i="68"/> <name name="system_info" arity="1" clause_i="69"/> + <name name="system_info" arity="1" clause_i="70"/> <fsummary>Information about the system.</fsummary> <desc> <p>Returns various information about the current system (emulator) as specified by <c><anno>Item</anno></c>:</p> <taglist> - <tag><c>allocated_areas</c>, <c>allocator</c>, - <c>alloc_util_allocators</c>, <c>allocator_sizes</c></tag> + <tag><c>atom_limit</c></tag> <item> - <p>See <seealso marker="#system_info_allocator_tags"> - above</seealso>.</p> + <marker id="system_info_atom_limit"></marker> + <p>Returns the maximum number of atoms allowed. + This limit can be increased at startup by passing + command-line flag + <seealso marker="erts:erl#+t"><c>+t</c></seealso> to + <c>erl(1)</c>. + </p> </item> <tag><c>build_type</c></tag> <item> diff --git a/erts/doc/src/zlib.xml b/erts/doc/src/zlib.xml index 138414a880..e1924fffee 100644 --- a/erts/doc/src/zlib.xml +++ b/erts/doc/src/zlib.xml @@ -576,6 +576,18 @@ unpack(Z, Compressed, Dict) -> </func> <func> + <name name="inflateGetDictionary" arity="1"/> + <fsummary>Return the decompression dictionary.</fsummary> + <desc> + <p>Returns the decompression dictionary currently in use + by the stream. This function must be called between + <seealso marker="#inflateInit/1"><c>inflateInit/1,2</c></seealso> + and <seealso marker="#inflateEnd/1"><c>inflateEnd</c></seealso>.</p> + <p>Only supported if ERTS was compiled with zlib >= 1.2.8.</p> + </desc> + </func> + + <func> <name name="open" arity="0"/> <fsummary>Open a stream and return a stream reference.</fsummary> <desc> diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 8772befe27..ce50022683 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -52,7 +52,6 @@ OMIT_OMIT_FP=no TYPE_LIBS= DIRTY_SCHEDULER_SUPPORT=@DIRTY_SCHEDULER_SUPPORT@ -NEW_PURGE_STRATEGY=@NEW_PURGE_STRATEGY@ ifeq ($(TYPE),debug) PURIFY = @@ -566,9 +565,9 @@ $(TARGET)/erl_bif_table.c \ $(TARGET)/erl_bif_table.h \ $(TARGET)/erl_bif_wrap.c \ $(TARGET)/erl_bif_list.h \ +$(TARGET)/erl_gc_bifs.c \ $(TARGET)/erl_atom_table.c \ $(TARGET)/erl_atom_table.h \ -$(TARGET)/erl_pbifs.c \ : $(TARGET)/TABLES-GENERATED $(TARGET)/TABLES-GENERATED: $(ATOMS) $(BIFS) utils/make_tables $(gen_verbose)LANG=C $(PERL) utils/make_tables -src $(TARGET) -include $(TARGET)\ @@ -742,7 +741,6 @@ EMU_OBJS = \ $(OBJDIR)/beam_ranges.o RUN_OBJS = \ - $(OBJDIR)/erl_pbifs.o \ $(OBJDIR)/erl_alloc.o $(OBJDIR)/erl_mtrace.o \ $(OBJDIR)/erl_alloc_util.o $(OBJDIR)/erl_goodfit_alloc.o \ $(OBJDIR)/erl_bestfit_alloc.o $(OBJDIR)/erl_afit_alloc.o \ @@ -753,6 +751,7 @@ RUN_OBJS = \ $(OBJDIR)/erl_bif_os.o $(OBJDIR)/erl_bif_lists.o \ $(OBJDIR)/erl_bif_trace.o $(OBJDIR)/erl_bif_unique.o \ $(OBJDIR)/erl_bif_wrap.o \ + $(OBJDIR)/erl_gc_bifs.o \ $(OBJDIR)/erl_trace.o $(OBJDIR)/copy.o \ $(OBJDIR)/utils.o $(OBJDIR)/bif.o \ $(OBJDIR)/io.o $(OBJDIR)/erl_printf_term.o\ @@ -888,7 +887,9 @@ HIPE_OBJS= \ $(OBJDIR)/hipe_bif2.o \ $(OBJDIR)/hipe_debug.o \ $(OBJDIR)/hipe_gc.o \ + $(OBJDIR)/hipe_load.o \ $(OBJDIR)/hipe_mode_switch.o \ + $(OBJDIR)/hipe_module.o \ $(OBJDIR)/hipe_native_bif.o \ $(OBJDIR)/hipe_stack.o $(HIPE_ARCH_OBJS) ifdef HIPE_ENABLED diff --git a/erts/emulator/beam/atom.c b/erts/emulator/beam/atom.c index 2052afe52b..b47739059b 100644 --- a/erts/emulator/beam/atom.c +++ b/erts/emulator/beam/atom.c @@ -483,3 +483,9 @@ dump_atoms(fmtfn_t to, void *to_arg) } } } + +Uint +erts_get_atom_limit(void) +{ + return erts_atom_table.limit; +}
\ No newline at end of file diff --git a/erts/emulator/beam/atom.h b/erts/emulator/beam/atom.h index a82efabb9f..f3b21e1687 100644 --- a/erts/emulator/beam/atom.h +++ b/erts/emulator/beam/atom.h @@ -138,6 +138,7 @@ int atom_static_put(byte*, int); void init_atom_table(void); void atom_info(fmtfn_t, void *); void dump_atoms(fmtfn_t, void *); +Uint erts_get_atom_limit(void); int erts_atom_get(const char* name, int len, Eterm* ap, ErtsAtomEncoding enc); void erts_atom_get_text_space_sizes(Uint *reserved, Uint *used); #endif diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 2b66bf6f4e..66af05c1f2 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -366,6 +366,7 @@ atom long_schedule atom low atom Lt='<' atom machine +atom major atom match atom match_limit atom match_limit_recursion @@ -395,6 +396,7 @@ atom milli_seconds atom millisecond atom min_heap_size atom min_bin_vheap_size +atom minor atom minor_version atom Minus='-' atom module diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index 8af7703f51..e7301b1b4b 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -36,6 +36,12 @@ #include "erl_nif.h" #include "erl_bits.h" #include "erl_thr_progress.h" +#ifdef HIPE +# include "hipe_bif0.h" +# define IF_HIPE(X) (X) +#else +# define IF_HIPE(X) (0) +#endif #ifdef HIPE # include "hipe_stack.h" @@ -65,7 +71,6 @@ erts_smp_atomic_t erts_copy_literal_area__; #define ERTS_SET_COPY_LITERAL_AREA(LA) \ erts_smp_atomic_set_nob(&erts_copy_literal_area__, \ (erts_aint_t) (LA)) -#ifdef ERTS_NEW_PURGE_STRATEGY Process *erts_literal_area_collector = NULL; typedef struct ErtsLiteralAreaRef_ ErtsLiteralAreaRef; @@ -79,10 +84,9 @@ struct { ErtsLiteralAreaRef *first; ErtsLiteralAreaRef *last; } release_literal_areas; -#endif static void set_default_trace_pattern(Eterm module); -static Eterm check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls); +static Eterm check_process_code(Process* rp, Module* modp, int *redsp, int fcalls); static void delete_code(Module* modp); static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); static int any_heap_refs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size); @@ -113,11 +117,9 @@ init_purge_state(void) void erts_beam_bif_load_init(void) { -#ifdef ERTS_NEW_PURGE_STRATEGY erts_smp_mtx_init(&release_literal_areas.mtx, "release_literal_areas"); release_literal_areas.first = NULL; release_literal_areas.last = NULL; -#endif erts_smp_atomic_init_nob(&erts_copy_literal_area__, (erts_aint_t) NULL); @@ -147,8 +149,17 @@ BIF_RETTYPE code_is_module_native_1(BIF_ALIST_1) BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) { +#if !defined(HIPE) + BIF_ERROR(BIF_P, EXC_NOTSUP); +#else Module* modp; - Eterm res; + Eterm res, mod; + + if (!ERTS_TERM_IS_MAGIC_BINARY(BIF_ARG_1) || + is_not_atom(mod = erts_module_for_prepared_code + (((ProcBin*)binary_val(BIF_ARG_1))->val))) { + BIF_ERROR(BIF_P, BADARG); + } if (!erts_try_seize_code_write_permission(BIF_P)) { ERTS_BIF_YIELD3(bif_export[BIF_code_make_stub_module_3], @@ -158,7 +169,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - modp = erts_get_module(BIF_ARG_1, erts_active_code_ix()); + modp = erts_get_module(mod, erts_active_code_ix()); if (modp && modp->curr.num_breakpoints > 0) { ASSERT(modp->curr.code_hdr != NULL); @@ -170,9 +181,12 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) res = erts_make_stub_module(BIF_P, BIF_ARG_1, BIF_ARG_2, BIF_ARG_3); - if (res == BIF_ARG_1) { + if (res == mod) { erts_end_staging_code_ix(); erts_commit_staging_code_ix(); + if (!modp) + modp = erts_get_module(mod, erts_active_code_ix()); + hipe_redirect_to_module(modp); } else { erts_abort_staging_code_ix(); @@ -181,6 +195,7 @@ BIF_RETTYPE code_make_stub_module_3(BIF_ALIST_3) erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_release_code_write_permission(); return res; +#endif } BIF_RETTYPE @@ -245,7 +260,7 @@ struct m { Uint exception; }; -static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int); +static Eterm staging_epilogue(Process* c_p, int, Eterm res, int, struct m*, int, int); #ifdef ERTS_SMP static void smp_code_ix_commiter(void*); @@ -381,8 +396,9 @@ finish_loading_1(BIF_ALIST_1) for (i = 0; i < n; i++) { if (p[i].modp->curr.num_breakpoints > 0 || p[i].modp->curr.num_traced_exports > 0 || - erts_is_default_trace_enabled()) { - /* tracing involved, fallback with thread blocking */ + erts_is_default_trace_enabled() || + IF_HIPE(hipe_need_blocking(p[i].modp))) { + /* tracing or hipe need thread blocking */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -440,32 +456,36 @@ finish_loading_1(BIF_ALIST_1) } done: - return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n); + return staging_epilogue(BIF_P, do_commit, res, is_blocking, p, n, 1); } static Eterm staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, - struct m* loaded, int nloaded) + struct m* mods, int nmods, int free_mods) { #ifdef ERTS_SMP if (is_blocking || !commit) #endif { if (commit) { + int i; erts_end_staging_code_ix(); erts_commit_staging_code_ix(); - if (loaded) { - int i; - for (i=0; i < nloaded; i++) { - set_default_trace_pattern(loaded[i].module); + + for (i=0; i < nmods; i++) { + if (mods[i].modp->curr.code_hdr) { + set_default_trace_pattern(mods[i].module); } + #ifdef HIPE + hipe_redirect_to_module(mods[i].modp); + #endif } } else { erts_abort_staging_code_ix(); } - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } if (is_blocking) { erts_smp_thr_progress_unblock(); @@ -478,8 +498,8 @@ staging_epilogue(Process* c_p, int commit, Eterm res, int is_blocking, else { ASSERT(is_value(res)); - if (loaded) { - erts_free(ERTS_ALC_T_LOADER_TMP, loaded); + if (free_mods) { + erts_free(ERTS_ALC_T_LOADER_TMP, mods); } erts_end_staging_code_ix(); /* @@ -547,7 +567,7 @@ check_old_code_1(BIF_ALIST_1) } Eterm -erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls) +erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls) { Module* modp; Eterm res; @@ -562,31 +582,23 @@ erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int if (!modp) return am_false; erts_rlock_old_code(code_ix); - res = (!modp->old.code_hdr ? am_false : - check_process_code(c_p, modp, flags, redsp, fcalls)); + res = (!modp->old.code_hdr + ? am_false + : check_process_code(c_p, modp, redsp, fcalls)); erts_runlock_old_code(code_ix); return res; } -BIF_RETTYPE erts_internal_check_process_code_2(BIF_ALIST_2) +BIF_RETTYPE erts_internal_check_process_code_1(BIF_ALIST_1) { int reds = 0; - Uint flags; Eterm res; if (is_not_atom(BIF_ARG_1)) goto badarg; - if (is_not_small(BIF_ARG_2)) - goto badarg; - - flags = unsigned_val(BIF_ARG_2); - if (flags & ~ERTS_CPC_ALL) { - goto badarg; - } - - res = erts_check_process_code(BIF_P, BIF_ARG_1, flags, &reds, BIF_P->fcalls); + res = erts_check_process_code(BIF_P, BIF_ARG_1, &reds, BIF_P->fcalls); ASSERT(is_value(res)); @@ -622,7 +634,7 @@ BIF_RETTYPE erts_internal_check_dirty_process_code_2(BIF_ALIST_2) if (!rp) BIF_RET(am_false); - res = erts_check_process_code(rp, BIF_ARG_2, 0, &reds, BIF_P->fcalls); + res = erts_check_process_code(rp, BIF_ARG_2, &reds, BIF_P->fcalls); if (BIF_P != rp) erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MAIN); @@ -665,8 +677,9 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) } else { if (modp->curr.num_breakpoints > 0 || - modp->curr.num_traced_exports > 0) { - /* we have tracing, retry single threaded */ + modp->curr.num_traced_exports > 0 || + IF_HIPE(hipe_need_blocking(modp))) { + /* tracing or hipe need to go single threaded */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; @@ -680,7 +693,14 @@ BIF_RETTYPE delete_module_1(BIF_ALIST_1) success = 1; } } - return staging_epilogue(BIF_P, success, res, is_blocking, NULL, 0); + { + struct m mod; + Eterm retval; + mod.module = BIF_ARG_1; + mod.modp = modp; + retval = staging_epilogue(BIF_P, success, res, is_blocking, &mod, 1, 0); + return retval; + } } BIF_RETTYPE module_loaded_1(BIF_ALIST_1) @@ -804,23 +824,26 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) */ for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->code[0] != BIF_ARG_1) { + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[4] != 0) { - ep->addressv[code_ix] = (void *) ep->code[4]; - ep->code[4] = 0; + if (ep->beam[1] != 0) { + ep->addressv[code_ix] = (void *) ep->beam[1]; + ep->beam[1] = 0; } else { - if (ep->addressv[code_ix] == ep->code+3 && - ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->beam && + ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - ep->addressv[code_ix] = ep->code+3; - ep->code[3] = (BeamInstr) em_call_error_handler; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = (BeamInstr) em_call_error_handler; } } modp->curr.code_hdr->on_load_function_ptr = NULL; set_default_trace_pattern(BIF_ARG_1); + #ifdef HIPE + hipe_redirect_to_module(modp); + #endif } else if (BIF_ARG_2 == am_false) { int i; @@ -831,13 +854,13 @@ BIF_RETTYPE finish_after_on_load_2(BIF_ALIST_2) for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i,code_ix); - if (ep == NULL || ep->code[0] != BIF_ARG_1) { + if (ep == NULL || ep->info.mfa.module != BIF_ARG_1) { continue; } - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - ep->code[4] = 0; + ep->beam[1] = 0; } } erts_smp_thr_progress_unblock(); @@ -861,9 +884,9 @@ set_default_trace_pattern(Eterm module) &trace_pattern_flags, &meta_tracer); if (trace_pattern_is_on) { - Eterm mfa[1]; - mfa[0] = module; - (void) erts_set_trace_pattern(0, mfa, 1, + ErtsCodeMFA mfa; + mfa.module = module; + (void) erts_set_trace_pattern(0, &mfa, 1, match_spec, meta_match_spec, 1, trace_pattern_flags, @@ -871,32 +894,12 @@ set_default_trace_pattern(Eterm module) } } -#ifndef ERTS_NEW_PURGE_STRATEGY - -static ERTS_INLINE int -check_mod_funs(Process *p, ErlOffHeap *off_heap, char *area, size_t area_size) -{ - struct erl_off_heap_header* oh; - for (oh = off_heap->first; oh; oh = oh->next) { - if (thing_subtag(oh->thing_word) == FUN_SUBTAG) { - ErlFunThing* funp = (ErlFunThing*) oh; - if (ErtsInArea(funp->fe->address, area, area_size)) - return !0; - } - } - return 0; -} - -#endif - static Uint hfrag_literal_size(Eterm* start, Eterm* end, char* lit_start, Uint lit_size); static void hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, Eterm *start, Eterm *end, char *lit_start, Uint lit_size); -#ifdef ERTS_NEW_PURGE_STRATEGY - Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed) { @@ -1063,12 +1066,16 @@ literal_gc: } static Eterm -check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) +check_process_code(Process* rp, Module* modp, int *redsp, int fcalls) { BeamInstr* start; char* mod_start; Uint mod_size; Eterm* sp; +#ifdef HIPE + void *nat_start = NULL; + Uint nat_size = 0; +#endif *redsp += 1; @@ -1098,78 +1105,19 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } } - /* - * Check all continuation pointers stored in stackdump - * and clear exception stackdump if there is a pointer - * to the module. - */ - if (rp->ftrace != NIL) { - struct StackTrace *s; - ASSERT(is_list(rp->ftrace)); - s = (struct StackTrace *) big_val(CDR(list_val(rp->ftrace))); - if ((s->pc && ErtsInArea(s->pc, mod_start, mod_size)) || - (s->current && ErtsInArea(s->current, mod_start, mod_size))) { - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - } else { - int i; - for (i = 0; i < s->depth; i++) { - if (ErtsInArea(s->trace[i], mod_start, mod_size)) { - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - break; - } - } - } - } - - return am_false; -} - -#else /* !ERTS_NEW_PURGE_STRATEGY, i.e, old style purge... */ - -static Eterm -check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls) -{ - BeamInstr* start; - char* literals; - Uint lit_bsize; - char* mod_start; - Uint mod_size; - Eterm* sp; - int done_gc = 0; - int need_gc = 0; - ErtsMessage *msgp; - ErlHeapFragment *hfrag; - -#define ERTS_ORDINARY_GC__ (1 << 0) -#define ERTS_LITERAL_GC__ (1 << 1) - - /* - * Pick up limits for the module. - */ - start = (BeamInstr*) modp->old.code_hdr; - mod_start = (char *) start; - mod_size = modp->old.code_length; - - /* - * Check if current instruction or continuation pointer points into module. - */ - if (ErtsInArea(rp->i, mod_start, mod_size) - || ErtsInArea(rp->cp, mod_start, mod_size)) { - return am_true; - } - +#ifdef HIPE /* - * Check all continuation pointers stored on the stack. + * Check all continuation pointers stored on the native stack if the module + * has native code. */ - for (sp = rp->stop; sp < STACK_START(rp); sp++) { - if (is_CP(*sp) && ErtsInArea(cp_val(*sp), mod_start, mod_size)) { + if (modp->old.hipe_code) { + nat_start = modp->old.hipe_code->text_segment; + nat_size = modp->old.hipe_code->text_segment_size; + if (nat_size && nstack_any_cps_in_segment(rp, nat_start, nat_size)) { return am_true; } } +#endif /* * Check all continuation pointers stored in stackdump @@ -1187,8 +1135,16 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls rp->ftrace = NIL; } else { int i; + char *area_start = mod_start; + Uint area_size = mod_size; +#ifdef HIPE + if (rp->freason & EXF_NATIVE) { + area_start = nat_start; + area_size = nat_size; + } +#endif for (i = 0; i < s->depth; i++) { - if (ErtsInArea(s->trace[i], mod_start, mod_size)) { + if (ErtsInArea(s->trace[i], area_start, area_size)) { rp->freason = EXC_NULL; rp->fvalue = NIL; rp->ftrace = NIL; @@ -1198,183 +1154,9 @@ check_process_code(Process* rp, Module* modp, Uint flags, int *redsp, int fcalls } } - if (rp->flags & F_DISABLE_GC) { - /* - * Cannot proceed. Process has disabled gc in order to - * safely leave inconsistent data on the heap and/or - * off heap lists. Need to wait for gc to be enabled - * again. - */ - return THE_NON_VALUE; - } - - /* - * Message queue can contains funs, and may contain - * literals. If we got references to this module from the message - * queue. - * - * If a literal is in the message queue we maka an explicit copy of - * and attach it to the heap fragment. Each message needs to be - * self contained, we cannot save the literal in the old_heap or - * any other heap than the message it self. - */ - - erts_smp_proc_lock(rp, ERTS_PROC_LOCK_MSGQ); - ERTS_SMP_MSGQ_MV_INQ2PRIVQ(rp); - erts_smp_proc_unlock(rp, ERTS_PROC_LOCK_MSGQ); - - if (modp->old.code_hdr->literal_area) { - literals = (char*) modp->old.code_hdr->literal_area->start; - lit_bsize = (char*) modp->old.code_hdr->literal_area->end - literals; - } - else { - literals = NULL; - lit_bsize = 0; - } - - for (msgp = rp->msg.first; msgp; msgp = msgp->next) { - if (msgp->data.attached == ERTS_MSG_COMBINED_HFRAG) - hfrag = &msgp->hfrag; - else if (is_value(ERL_MESSAGE_TERM(msgp)) && msgp->data.heap_frag) - hfrag = msgp->data.heap_frag; - else - continue; - { - ErlHeapFragment *hf; - Uint lit_sz; - for (hf=hfrag; hf; hf = hf->next) { - if (check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)) - return am_true; - lit_sz = hfrag_literal_size(&hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); - } - if (lit_sz > 0) { - ErlHeapFragment *bp = new_message_buffer(lit_sz); - Eterm *hp = bp->mem; - - for (hf=hfrag; hf; hf = hf->next) { - hfrag_literal_copy(&hp, &bp->off_heap, - &hf->mem[0], &hf->mem[hf->used_size], - literals, lit_bsize); - hfrag=hf; - } - /* link new hfrag last */ - ASSERT(hfrag->next == NULL); - hfrag->next = bp; - bp->next = NULL; - } - } - } - - while (1) { - - /* Check heap, stack etc... */ - if (check_mod_funs(rp, &rp->off_heap, mod_start, mod_size)) - goto try_gc; - if (any_heap_ref_ptrs(&rp->fvalue, &rp->fvalue+1, literals, lit_bsize)) { - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - } - if (any_heap_ref_ptrs(rp->stop, rp->hend, literals, lit_bsize)) - goto try_literal_gc; -#ifdef HIPE - if (nstack_any_heap_ref_ptrs(rp, literals, lit_bsize)) - goto try_literal_gc; -#endif - if (any_heap_refs(rp->heap, rp->htop, literals, lit_bsize)) - goto try_literal_gc; - if (any_heap_refs(rp->old_heap, rp->old_htop, literals, lit_bsize)) - goto try_literal_gc; - - /* Check dictionary */ - if (rp->dictionary) { - Eterm* start = ERTS_PD_START(rp->dictionary); - Eterm* end = start + ERTS_PD_SIZE(rp->dictionary); - - if (any_heap_ref_ptrs(start, end, literals, lit_bsize)) - goto try_literal_gc; - } - - /* Check heap fragments */ - for (hfrag = rp->mbuf; hfrag; hfrag = hfrag->next) { - Eterm *hp, *hp_end; - /* Off heap lists should already have been moved into process */ - ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - - hp = &hfrag->mem[0]; - hp_end = &hfrag->mem[hfrag->used_size]; - if (any_heap_refs(hp, hp_end, literals, lit_bsize)) - goto try_literal_gc; - } - - /* - * Message buffer fragments (matched messages) - * - off heap lists should already have been moved into - * process off heap structure. - * - Check for literals - */ - for (msgp = rp->msg_frag; msgp; msgp = msgp->next) { - hfrag = erts_message_to_heap_frag(msgp); - for (; hfrag; hfrag = hfrag->next) { - Eterm *hp, *hp_end; - ASSERT(!check_mod_funs(rp, &hfrag->off_heap, mod_start, mod_size)); - - hp = &hfrag->mem[0]; - hp_end = &hfrag->mem[hfrag->used_size]; - - if (any_heap_refs(hp, hp_end, literals, lit_bsize)) - goto try_literal_gc; - } - } - - return am_false; - - try_literal_gc: - need_gc |= ERTS_LITERAL_GC__; - - try_gc: - need_gc |= ERTS_ORDINARY_GC__; - - if ((done_gc & need_gc) == need_gc) - return am_true; - - if (!(flags & ERTS_CPC_ALLOW_GC)) - return am_aborted; - - need_gc &= ~done_gc; - - /* - * Try to get rid of literals by by garbage collecting. - * Clear both fvalue and ftrace. - */ - - rp->freason = EXC_NULL; - rp->fvalue = NIL; - rp->ftrace = NIL; - - if (need_gc & ERTS_ORDINARY_GC__) { - FLAGS(rp) |= F_NEED_FULLSWEEP; - *redsp += erts_garbage_collect_nobump(rp, 0, rp->arg_reg, rp->arity, fcalls); - done_gc |= ERTS_ORDINARY_GC__; - } - if (need_gc & ERTS_LITERAL_GC__) { - struct erl_off_heap_header* oh; - oh = modp->old.code_hdr->literal_area->off_heap; - *redsp += lit_bsize / 64; /* Need, better value... */ - erts_garbage_collect_literals(rp, (Eterm*)literals, lit_bsize, oh); - done_gc |= ERTS_LITERAL_GC__; - } - need_gc = 0; - } - -#undef ERTS_ORDINARY_GC__ -#undef ERTS_LITERAL_GC__ - + return am_false; } -#endif /* !ERTS_NEW_PURGE_STRATEGY */ - static int any_heap_ref_ptrs(Eterm* start, Eterm* end, char* mod_start, Uint mod_size) { @@ -1503,8 +1285,6 @@ hfrag_literal_copy(Eterm **hpp, ErlOffHeap *ohp, } } -#ifdef ERTS_NEW_PURGE_STRATEGY - #ifdef ERTS_SMP ErtsThrPrgrLaterOp later_literal_area_switch; @@ -1535,13 +1315,8 @@ complete_literal_area_switch(void *literal_area) } #endif -#endif /* ERTS_NEW_PURGE_STRATEGY */ - BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) { -#ifndef ERTS_NEW_PURGE_STRATEGY - BIF_ERROR(BIF_P, EXC_NOTSUP); -#else ErtsLiteralArea *unused_la; ErtsLiteralAreaRef *la_ref; @@ -1600,7 +1375,6 @@ BIF_RETTYPE erts_internal_release_literal_area_switch_0(BIF_ALIST_0) BIF_RET(am_true); #endif -#endif /* ERTS_NEW_PURGE_STRATEGY */ } void @@ -1794,10 +1568,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) code = (BeamInstr*) modp->old.code_hdr; end = (BeamInstr *)((char *)code + modp->old.code_length); erts_fun_purge_prepare(code, end); -#if !defined(ERTS_NEW_PURGE_STRATEGY) - ASSERT(!ERTS_COPY_LITERAL_AREA()); - ERTS_SET_COPY_LITERAL_AREA(modp->old.code_hdr->literal_area); -#endif } if (BIF_ARG_2 == am_prepare_on_load) { @@ -1842,10 +1612,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) erts_fun_purge_abort_prepare(purge_state.funs, purge_state.fe_ix); -#if !defined(ERTS_NEW_PURGE_STRATEGY) - ASSERT(ERTS_COPY_LITERAL_AREA()); - ERTS_SET_COPY_LITERAL_AREA(NULL); -#endif #ifndef ERTS_SMP erts_fun_purge_abort_finalize(purge_state.funs, purge_state.fe_ix); finalize_purge_operation(BIF_P, 0); @@ -1914,15 +1680,18 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) /* * Unload any NIF library */ - if (modp->old.nif != NULL) { + if (modp->old.nif != NULL + || IF_HIPE(hipe_purge_need_blocking(modp))) { /* ToDo: Do unload nif without blocking */ erts_rwunlock_old_code(code_ix); erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); is_blocking = 1; erts_rwlock_old_code(code_ix); - erts_unload_nif(modp->old.nif); - modp->old.nif = NULL; + if (modp->old.nif) { + erts_unload_nif(modp->old.nif); + modp->old.nif = NULL; + } } /* @@ -1941,7 +1710,9 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) modp->old.code_length = 0; modp->old.catches = BEAM_CATCHES_NIL; erts_remove_from_ranges(code); - +#ifdef HIPE + hipe_purge_module(modp, is_blocking); +#endif ERTS_BIF_PREP_RET(ret, am_true); } @@ -1960,14 +1731,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) finalize_purge_operation(BIF_P, ret == am_true); -#if !defined(ERTS_NEW_PURGE_STRATEGY) - - ASSERT(ERTS_COPY_LITERAL_AREA() == literals); - ERTS_SET_COPY_LITERAL_AREA(NULL); - erts_release_literal_area(literals); - -#else /* ERTS_NEW_PURGE_STRATEGY */ - if (literals) { ErtsLiteralAreaRef *ref; ref = erts_alloc(ERTS_ALC_T_LITERAL_REF, @@ -1991,8 +1754,6 @@ BIF_RETTYPE erts_internal_purge_module_2(BIF_ALIST_2) BIF_P->common.id); } -#endif /* ERTS_NEW_PURGE_STRATEGY */ - return ret; } @@ -2015,34 +1776,34 @@ delete_code(Module* modp) for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i, code_ix); - if (ep != NULL && (ep->code[0] == module)) { - if (ep->addressv[code_ix] == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep != NULL && (ep->info.mfa.module == module)) { + if (ep->addressv[code_ix] == ep->beam) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; } - else if (ep->code[3] == + else if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(modp->curr.num_traced_exports > 0); - erts_clear_export_break(modp, ep->code+3); + DBG_TRACE_MFA_P(&ep->info.mfa, + "export trace cleared, code_ix=%d", code_ix); + erts_clear_export_break(modp, &ep->info); } - else ASSERT(ep->code[3] == (BeamInstr) em_call_error_handler + else ASSERT(ep->beam[0] == (BeamInstr) em_call_error_handler || !erts_initialized); } - ep->addressv[code_ix] = ep->code+3; - ep->code[3] = (BeamInstr) em_call_error_handler; - ep->code[4] = 0; + ep->addressv[code_ix] = ep->beam; + ep->beam[0] = (BeamInstr) em_call_error_handler; + ep->beam[1] = 0; + DBG_TRACE_MFA_P(&ep->info.mfa, + "export invalidation, code_ix=%d", code_ix); } } ASSERT(modp->curr.num_breakpoints == 0); ASSERT(modp->curr.num_traced_exports == 0); modp->old = modp->curr; - modp->curr.code_hdr = NULL; - modp->curr.code_length = 0; - modp->curr.catches = BEAM_CATCHES_NIL; - modp->curr.nif = NULL; - + erts_module_instance_init(&modp->curr); } @@ -2056,9 +1817,11 @@ beam_make_current_old(Process *c_p, ErtsProcLocks c_p_locks, Eterm module) * if not, delete old code; error if old code already exists. */ - if (modp->curr.code_hdr && modp->old.code_hdr) { - return am_not_purged; - } else if (!modp->old.code_hdr) { /* Make the current version old. */ + if (modp->curr.code_hdr) { + if (modp->old.code_hdr) { + return am_not_purged; + } + /* Make the current version old. */ delete_code(modp); } return NIL; diff --git a/erts/emulator/beam/beam_bp.c b/erts/emulator/beam/beam_bp.c index 920c8b1ed0..62464f3864 100644 --- a/erts/emulator/beam/beam_bp.c +++ b/erts/emulator/beam/beam_bp.c @@ -92,27 +92,27 @@ get_mtime(Process *c_p) /* ** Helpers */ -static ErtsTracer do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, +static ErtsTracer do_call_trace(Process* c_p, ErtsCodeInfo *info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer); static void set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer); -static void set_function_break(BeamInstr *pc, +static void set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer); static void clear_break(BpFunctions* f, Uint break_flags); -static int clear_function_break(BeamInstr *pc, Uint break_flags); +static int clear_function_break(ErtsCodeInfo *ci, Uint break_flags); -static BpDataTime* get_time_break(BeamInstr *pc); -static GenericBpData* check_break(BeamInstr *pc, Uint break_flags); +static BpDataTime* get_time_break(ErtsCodeInfo *ci); +static GenericBpData* check_break(ErtsCodeInfo *ci, Uint break_flags); -static void bp_meta_unref(BpMetaTracer* bmt); -static void bp_count_unref(BpCount* bcp); -static void bp_time_unref(BpDataTime* bdt); -static void consolidate_bp_data(Module* modp, BeamInstr* pc, int local); -static void uninstall_breakpoint(BeamInstr* pc); +static void bp_meta_unref(BpMetaTracer *bmt); +static void bp_count_unref(BpCount *bcp); +static void bp_time_unref(BpDataTime *bdt); +static void consolidate_bp_data(Module *modp, ErtsCodeInfo *ci, int local); +static void uninstall_breakpoint(ErtsCodeInfo *ci); /* bp_hash */ #define BP_TIME_ADD(pi0, pi1) \ @@ -139,7 +139,7 @@ erts_bp_init(void) { void -erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) +erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified) { ErtsCodeIndex code_ix = erts_active_code_ix(); Uint max_funcs = 0; @@ -164,41 +164,44 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) i = 0; for (current = 0; current < num_modules; current++) { BeamCodeHeader* code_hdr = module[current]->curr.code_hdr; - BeamInstr* code; + ErtsCodeInfo* ci; Uint num_functions = (Uint)(UWord) code_hdr->num_functions; Uint fi; if (specified > 0) { - if (mfa[0] != make_atom(module[current]->module)) { + if (mfa->module != make_atom(module[current]->module)) { /* Wrong module name */ continue; } } for (fi = 0; fi < num_functions; fi++) { - BeamInstr* pc; - int wi; - code = code_hdr->functions[fi]; - ASSERT(code[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - pc = code+5; - if (erts_is_native_break(pc)) { + ci = code_hdr->functions[fi]; + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(ci)) { continue; } - if (is_nil(code[3])) { /* Ignore BIF stub */ + if (is_nil(ci->mfa.module)) { /* Ignore BIF stub */ continue; } - for (wi = 0; - wi < specified && (Eterm) code[2+wi] == mfa[wi]; - wi++) { - /* Empty loop body */ - } - if (wi == specified) { - /* Store match */ - f->matching[i].pc = pc; - f->matching[i].mod = module[current]; - i++; - } + switch (specified) { + case 3: + if (ci->mfa.arity != mfa->arity) + continue; + case 2: + if (ci->mfa.function != mfa->function) + continue; + case 1: + if (ci->mfa.module != mfa->module) + continue; + case 0: + break; + } + /* Store match */ + f->matching[i].ci = ci; + f->matching[i].mod = module[current]; + i++; } } f->matched = i; @@ -206,7 +209,7 @@ erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified) } void -erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) +erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified) { ErtsCodeIndex code_ix = erts_active_code_ix(); int i; @@ -218,27 +221,36 @@ erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified) for (i = 0; i < num_exps; i++) { Export* ep = export_list(i, code_ix); BeamInstr* pc; - int j; - for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { - /* Empty loop body */ - } - if (j < specified) { - continue; - } - pc = ep->code+3; + switch (specified) { + case 3: + if (mfa->arity != ep->info.mfa.arity) + continue; + case 2: + if (mfa->function != ep->info.mfa.function) + continue; + case 1: + if (mfa->module != ep->info.mfa.module) + continue; + case 0: + break; + default: + ASSERT(0); + } + + pc = ep->beam; if (ep->addressv[code_ix] == pc) { if ((*pc == (BeamInstr) em_apply_bif || *pc == (BeamInstr) em_call_error_handler)) { continue; } ASSERT(*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - } else if (erts_is_native_break(ep->addressv[code_ix])) { + } else if (erts_is_native_break(erts_code_to_codeinfo(ep->addressv[code_ix]))) { continue; } - f->matching[ne].pc = pc; - f->matching[ne].mod = erts_get_module(ep->code[0], code_ix); + f->matching[ne].ci = &ep->info; + f->matching[ne].mod = erts_get_module(ep->info.mfa.module, code_ix); ne++; } @@ -264,7 +276,7 @@ erts_consolidate_bp_data(BpFunctions* f, int local) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < n; i++) { - consolidate_bp_data(fs[i].mod, fs[i].pc, local); + consolidate_bp_data(fs[i].mod, fs[i].ci, local); } } @@ -276,14 +288,14 @@ erts_consolidate_bif_bp_data(void) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); for (i = 0; i < BIF_SIZE; i++) { Export *ep = bif_export[i]; - consolidate_bp_data(0, ep->code+3, 0); + consolidate_bp_data(0, &ep->info, 0); } } static void -consolidate_bp_data(Module* modp, BeamInstr* pc, int local) +consolidate_bp_data(Module* modp, ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; GenericBpData* src; GenericBpData* dst; Uint flags; @@ -329,9 +341,10 @@ consolidate_bp_data(Module* modp, BeamInstr* pc, int local) } ASSERT(modp->curr.num_breakpoints >= 0); ASSERT(modp->curr.num_traced_exports >= 0); - ASSERT(*pc != (BeamInstr) BeamOp(op_i_generic_breakpoint)); + ASSERT(*erts_codeinfo_to_code(ci) != + (BeamInstr) BeamOp(op_i_generic_breakpoint)); } - pc[-4] = 0; + ci->native = 0; Free(g); return; } @@ -380,8 +393,9 @@ erts_install_breakpoints(BpFunctions* f) BeamInstr br = (BeamInstr) BeamOp(op_i_generic_breakpoint); for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - GenericBp* g = (GenericBp *) pc[-4]; + ErtsCodeInfo* ci = f->matching[i].ci; + BeamInstr *pc = erts_codeinfo_to_code(ci); + GenericBp* g = (GenericBp *) ci->native; if (*pc != br && g) { Module* modp = f->matching[i].mod; @@ -413,16 +427,16 @@ erts_uninstall_breakpoints(BpFunctions* f) Uint n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - uninstall_breakpoint(pc); + uninstall_breakpoint(f->matching[i].ci); } } static void -uninstall_breakpoint(BeamInstr* pc) +uninstall_breakpoint(ErtsCodeInfo *ci) { + BeamInstr *pc = erts_codeinfo_to_code(ci); if (*pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; if (g->data[erts_active_bp_ix()].flags == 0) { /* * The following write is not protected by any lock. We @@ -449,30 +463,30 @@ erts_set_mtrace_break(BpFunctions* f, Binary *match_spec, ErtsTracer tracer) } void -erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local) +erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; - set_function_break(pc, match_spec, flags, 0, erts_tracer_nil); + set_function_break(ci, match_spec, flags, 0, erts_tracer_nil); } void -erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, ErtsTracer tracer) +erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer) { - set_function_break(pc, match_spec, ERTS_BPF_META_TRACE, 0, tracer); + set_function_break(ci, match_spec, ERTS_BPF_META_TRACE, 0, tracer); } void -erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op count_op) +erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op count_op) { - set_function_break(pc, NULL, + set_function_break(ci, NULL, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE, count_op, erts_tracer_nil); } void -erts_clear_time_trace_bif(BeamInstr *pc) { - clear_function_break(pc, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); +erts_clear_time_trace_bif(ErtsCodeInfo *ci) { + clear_function_break(ci, ERTS_BPF_TIME_TRACE|ERTS_BPF_TIME_TRACE_ACTIVE); } void @@ -501,14 +515,14 @@ erts_clear_trace_break(BpFunctions* f) } void -erts_clear_call_trace_bif(BeamInstr *pc, int local) +erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; if (g) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; if (g->data[erts_staging_bp_ix()].flags & flags) { - clear_function_break(pc, flags); + clear_function_break(ci, flags); } } } @@ -520,9 +534,9 @@ erts_clear_mtrace_break(BpFunctions* f) } void -erts_clear_mtrace_bif(BeamInstr *pc) +erts_clear_mtrace_bif(ErtsCodeInfo *ci) { - clear_function_break(pc, ERTS_BPF_META_TRACE); + clear_function_break(ci, ERTS_BPF_META_TRACE); } void @@ -564,52 +578,48 @@ erts_clear_module_break(Module *modp) { } n = (Uint)(UWord) code_hdr->num_functions; for (i = 0; i < n; ++i) { - BeamInstr* pc; - - pc = code_hdr->functions[i] + 5; - if (erts_is_native_break(pc)) { + ErtsCodeInfo *ci = code_hdr->functions[i]; + if (erts_is_native_break(ci)) continue; - } - clear_function_break(pc, ERTS_BPF_ALL); + clear_function_break(ci, ERTS_BPF_ALL); } erts_commit_staged_bp(); for (i = 0; i < n; ++i) { - BeamInstr* pc; - - pc = code_hdr->functions[i] + 5; - if (erts_is_native_break(pc)) { + ErtsCodeInfo *ci = code_hdr->functions[i]; + if (erts_is_native_break(ci)) continue; - } - uninstall_breakpoint(pc); - consolidate_bp_data(modp, pc, 1); - ASSERT(pc[-4] == 0); + uninstall_breakpoint(ci); + consolidate_bp_data(modp, ci, 1); + ASSERT(ci->native == 0); } return n; } void -erts_clear_export_break(Module* modp, BeamInstr* pc) +erts_clear_export_break(Module* modp, ErtsCodeInfo *ci) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); - clear_function_break(pc, ERTS_BPF_ALL); + clear_function_break(ci, ERTS_BPF_ALL); erts_commit_staged_bp(); - *pc = (BeamInstr) 0; - consolidate_bp_data(modp, pc, 0); - ASSERT(pc[-4] == 0); + *erts_codeinfo_to_code(ci) = (BeamInstr) 0; + consolidate_bp_data(modp, ci, 0); + ASSERT(ci->native == 0); } BeamInstr -erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) +erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *info, Eterm* reg) { GenericBp* g; GenericBpData* bp; Uint bp_flags; ErtsBpIndex ix = erts_active_bp_ix(); - g = (GenericBp *) I[-4]; + ASSERT(info->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + + g = (GenericBp *) info->native; bp = &g->data[ix]; bp_flags = bp->flags; ASSERT((bp_flags & ~ERTS_BPF_ALL) == 0); @@ -628,9 +638,9 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) if (bp_flags & ERTS_BPF_LOCAL_TRACE) { ASSERT((bp_flags & ERTS_BPF_GLOBAL_TRACE) == 0); - (void) do_call_trace(c_p, I, reg, 1, bp->local_ms, erts_tracer_true); + (void) do_call_trace(c_p, info, reg, 1, bp->local_ms, erts_tracer_true); } else if (bp_flags & ERTS_BPF_GLOBAL_TRACE) { - (void) do_call_trace(c_p, I, reg, 0, bp->local_ms, erts_tracer_true); + (void) do_call_trace(c_p, info, reg, 0, bp->local_ms, erts_tracer_true); } if (bp_flags & ERTS_BPF_META_TRACE) { @@ -638,7 +648,8 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) old_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); - new_tracer = do_call_trace(c_p, I, reg, 1, bp->meta_ms, old_tracer); + new_tracer = do_call_trace(c_p, info, reg, 1, bp->meta_ms, old_tracer); + if (!ERTS_TRACER_COMPARE(new_tracer, old_tracer)) { if (old_tracer == erts_smp_atomic_cmpxchg_acqb( &bp->meta_tracer->tracer, @@ -657,7 +668,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE) { Eterm w; - erts_trace_time_call(c_p, I, bp->time); + erts_trace_time_call(c_p, info, bp->time); w = (BeamInstr) *c_p->cp; if (! (w == (BeamInstr) BeamOp(op_i_return_time_trace) || w == (BeamInstr) BeamOp(op_return_trace) || @@ -665,7 +676,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) Eterm* E = c_p->stop; ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - 2 < c_p->htop) { - (void) erts_garbage_collect(c_p, 2, reg, I[-1]); + (void) erts_garbage_collect(c_p, 2, reg, info->mfa.arity); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); } E = c_p->stop; @@ -673,7 +684,7 @@ erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg) ASSERT(c_p->htop <= E && E <= c_p->hend); E -= 2; - E[0] = make_cp(I); + E[0] = make_cp(erts_codeinfo_to_code(info)); E[1] = make_cp(c_p->cp); /* original return address */ c_p->cp = beam_return_time_trace; c_p->stop = E; @@ -701,9 +712,9 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) Export* ep = bif_export[bif_index]; Uint32 flags = 0, flags_meta = 0; ErtsTracer meta_tracer = erts_tracer_nil; - int applying = (I == &(ep->code[3])); /* Yup, the apply code for a bif - * is actually in the - * export entry */ + int applying = (I == ep->beam); /* Yup, the apply code for a bif + * is actually in the + * export entry */ BeamInstr *cp = p->cp; GenericBp* g; GenericBpData* bp = NULL; @@ -711,7 +722,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) ERTS_SMP_CHK_HAVE_ONLY_MAIN_PROC_LOCK(p); - g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + g = (GenericBp *) ep->info.native; if (g) { bp = &g->data[erts_active_bp_ix()]; bp_flags = bp->flags; @@ -727,7 +738,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) if (bp_flags & (ERTS_BPF_LOCAL_TRACE|ERTS_BPF_GLOBAL_TRACE) && IS_TRACED_FL(p, F_TRACE_CALLS)) { int local = !!(bp_flags & ERTS_BPF_LOCAL_TRACE); - flags = erts_call_trace(p, ep->code, bp->local_ms, args, + flags = erts_call_trace(p, &ep->info, bp->local_ms, args, local, &ERTS_TRACER(p)); } if (bp_flags & ERTS_BPF_META_TRACE) { @@ -735,7 +746,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) meta_tracer = erts_smp_atomic_read_nob(&bp->meta_tracer->tracer); old_tracer = meta_tracer; - flags_meta = erts_call_trace(p, ep->code, bp->meta_ms, args, + flags_meta = erts_call_trace(p, &ep->info, bp->meta_ms, args, 0, &meta_tracer); if (!ERTS_TRACER_COMPARE(old_tracer, meta_tracer)) { @@ -753,8 +764,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } if (bp_flags & ERTS_BPF_TIME_TRACE_ACTIVE && IS_TRACED_FL(p, F_TRACE_CALLS)) { - BeamInstr *pc = (BeamInstr *)ep->code+3; - erts_trace_time_call(p, pc, bp->time); + erts_trace_time_call(p, &ep->info, bp->time); } /* Restore original continuation pointer (if changed). */ @@ -817,11 +827,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) class = exception_tag[GET_EXC_CLASS(reason)]; if (flags_meta & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, + erts_trace_exception(p, &ep->info.mfa, class, value, &meta_tracer); } if (flags & MATCH_SET_EXCEPTION_TRACE) { - erts_trace_exception(p, ep->code, class, value, + erts_trace_exception(p, &ep->info.mfa, class, value, &ERTS_TRACER(p)); } if ((flags & MATCH_SET_RETURN_TO_TRACE) && p->catches > 0) { @@ -852,11 +862,11 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } } else { if (flags_meta & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &meta_tracer); + erts_trace_return(p, &ep->info.mfa, result, &meta_tracer); } /* MATCH_SET_RETURN_TO_TRACE cannot occur if(meta) */ if (flags & MATCH_SET_RX_TRACE) { - erts_trace_return(p, ep->code, result, &ERTS_TRACER(p)); + erts_trace_return(p, &ep->info.mfa, result, &ERTS_TRACER(p)); } if (flags & MATCH_SET_RETURN_TO_TRACE && IS_TRACED_FL(p, F_TRACE_RETURN_TO)) { @@ -875,7 +885,7 @@ erts_bif_trace(int bif_index, Process* p, Eterm* args, BeamInstr* I) } static ErtsTracer -do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, +do_call_trace(Process* c_p, ErtsCodeInfo* info, Eterm* reg, int local, Binary* ms, ErtsTracer tracer) { Eterm* cpp; @@ -916,7 +926,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, ASSERT(is_CP(*cpp)); } ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - flags = erts_call_trace(c_p, I-3, ms, reg, local, &tracer); + flags = erts_call_trace(c_p, info, ms, reg, local, &tracer); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); if (cpp) { c_p->cp = cp_save; @@ -932,7 +942,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, if (need) { ASSERT(c_p->htop <= E && E <= c_p->hend); if (E - need < c_p->htop) { - (void) erts_garbage_collect(c_p, need, reg, I[-1]); + (void) erts_garbage_collect(c_p, need, reg, info->mfa.arity); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); E = c_p->stop; } @@ -948,12 +958,12 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, E -= 3; c_p->stop = E; ASSERT(c_p->htop <= E && E <= c_p->hend); - ASSERT(is_CP((Eterm) (UWord) (I - 3))); + ASSERT(is_CP((Eterm) (UWord) (&info->mfa.module))); ASSERT(IS_TRACER_VALID(tracer)); E[2] = make_cp(c_p->cp); E[1] = copy_object(tracer, c_p); - E[0] = make_cp(I - 3); /* We ARE at the beginning of an - instruction, + E[0] = make_cp(&info->mfa.module); + /* We ARE at the beginning of an instruction, the funcinfo is above i. */ c_p->cp = (flags & MATCH_SET_EXCEPTION_TRACE) ? beam_exception_trace : beam_return_trace; @@ -966,7 +976,7 @@ do_call_trace(Process* c_p, BeamInstr* I, Eterm* reg, } void -erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) +erts_trace_time_call(Process* c_p, ErtsCodeInfo *info, BpDataTime* bdt) { ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; @@ -995,14 +1005,14 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) pbt = Alloc(sizeof(process_breakpoint_time_t)); (void) ERTS_PROC_SET_CALL_TIME(c_p, pbt); } else { - ASSERT(pbt->pc); + ASSERT(pbt->ci); /* add time to previous code */ sitem.time = time - pbt->time; sitem.pid = c_p->common.id; sitem.count = 0; /* previous breakpoint */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); /* if null then the breakpoint was removed */ if (pbdt) { @@ -1039,12 +1049,12 @@ erts_trace_time_call(Process* c_p, BeamInstr* I, BpDataTime* bdt) BP_TIME_ADD(item, &sitem); } - pbt->pc = I; + pbt->ci = info; pbt->time = time; } void -erts_trace_time_return(Process *p, BeamInstr *pc) +erts_trace_time_return(Process *p, ErtsCodeInfo *ci) { ErtsMonotonicTime time; process_breakpoint_time_t *pbt = NULL; @@ -1074,14 +1084,14 @@ erts_trace_time_return(Process *p, BeamInstr *pc) /* might have been removed due to * trace_pattern(false) */ - ASSERT(pbt->pc); + ASSERT(pbt->ci); sitem.time = time - pbt->time; sitem.pid = p->common.id; sitem.count = 0; /* previous breakpoint */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); /* beware, the trace_pattern might have been removed */ if (pbdt) { @@ -1098,16 +1108,16 @@ erts_trace_time_return(Process *p, BeamInstr *pc) } } - pbt->pc = pc; + pbt->ci = ci; pbt->time = time; } } int -erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) +erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local) { Uint flags = local ? ERTS_BPF_LOCAL_TRACE : ERTS_BPF_GLOBAL_TRACE; - GenericBpData* bp = check_break(pc, flags); + GenericBpData* bp = check_break(ci, flags); if (bp) { if (match_spec_ret) { @@ -1119,10 +1129,10 @@ erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local) } int -erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, +erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret) { - GenericBpData* bp = check_break(pc, ERTS_BPF_META_TRACE); + GenericBpData* bp = check_break(ci, ERTS_BPF_META_TRACE); if (bp) { if (match_spec_ret) { @@ -1137,20 +1147,20 @@ erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, } int -erts_is_native_break(BeamInstr *pc) { +erts_is_native_break(ErtsCodeInfo *ci) { #ifdef HIPE - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - return pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call) - || pc[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + return erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call) + || erts_codeinfo_to_code(ci)[0] == (BeamInstr) BeamOp(op_hipe_trap_call_closure); #else return 0; #endif } int -erts_is_count_break(BeamInstr *pc, Uint *count_ret) +erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret) { - GenericBpData* bp = check_break(pc, ERTS_BPF_COUNT); + GenericBpData* bp = check_break(ci, ERTS_BPF_COUNT); if (bp) { if (count_ret) { @@ -1161,13 +1171,13 @@ erts_is_count_break(BeamInstr *pc, Uint *count_ret) return 0; } -int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { +int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *retval) { Uint i, ix; bp_time_hash_t hash; Uint size; Eterm *hp, t; bp_data_time_item_t *item = NULL; - BpDataTime *bdt = get_time_break(pc); + BpDataTime *bdt = get_time_break(ci); if (bdt) { if (retval) { @@ -1221,26 +1231,25 @@ int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *retval) { } -BeamInstr * -erts_find_local_func(Eterm mfa[3]) { +ErtsCodeInfo * +erts_find_local_func(ErtsCodeMFA *mfa) { Module *modp; BeamCodeHeader* code_hdr; - BeamInstr* code_ptr; + ErtsCodeInfo* ci; Uint i,n; - if ((modp = erts_get_module(mfa[0], erts_active_code_ix())) == NULL) + if ((modp = erts_get_module(mfa->module, erts_active_code_ix())) == NULL) return NULL; if ((code_hdr = modp->curr.code_hdr) == NULL) return NULL; n = (BeamInstr) code_hdr->num_functions; for (i = 0; i < n; ++i) { - code_ptr = code_hdr->functions[i]; - ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == code_ptr[0]); - ASSERT(mfa[0] == ((Eterm) code_ptr[2]) || - is_nil((Eterm) code_ptr[2])); - if (mfa[1] == ((Eterm) code_ptr[3]) && - ((BeamInstr) mfa[2]) == code_ptr[4]) { - return code_ptr + 5; + ci = code_hdr->functions[i]; + ASSERT(((BeamInstr) BeamOp(op_i_func_info_IaaI)) == ci->op); + ASSERT(mfa->module == ci->mfa.module || is_nil(ci->mfa.module)); + if (mfa->function == ci->mfa.function && + mfa->arity == ci->mfa.arity) { + return ci; } } return NULL; @@ -1369,7 +1378,7 @@ void erts_schedule_time_break(Process *p, Uint schedule) { * the previous breakpoint. */ - pbdt = get_time_break(pbt->pc); + pbdt = get_time_break(pbt->ci); if (pbdt) { sitem.time = get_mtime(p) - pbt->time; sitem.pid = p->common.id; @@ -1417,14 +1426,14 @@ set_break(BpFunctions* f, Binary *match_spec, Uint break_flags, n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - set_function_break(pc, match_spec, break_flags, + set_function_break(f->matching[i].ci, + match_spec, break_flags, count_op, tracer); } } static void -set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, +set_function_break(ErtsCodeInfo *ci, Binary *match_spec, Uint break_flags, enum erts_break_op count_op, ErtsTracer tracer) { GenericBp* g; @@ -1433,7 +1442,7 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, ErtsBpIndex ix = erts_staging_bp_ix(); ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - g = (GenericBp *) pc[-4]; + g = (GenericBp *) ci->native; if (g == 0) { int i; if (count_op == ERTS_BREAK_RESTART || count_op == ERTS_BREAK_PAUSE) { @@ -1441,11 +1450,11 @@ set_function_break(BeamInstr *pc, Binary *match_spec, Uint break_flags, return; } g = Alloc(sizeof(GenericBp)); - g->orig_instr = *pc; + g->orig_instr = *erts_codeinfo_to_code(ci); for (i = 0; i < ERTS_NUM_BP_IX; i++) { g->data[i].flags = 0; } - pc[-4] = (BeamInstr) g; + ci->native = (BeamInstr) g; } bp = &g->data[ix]; @@ -1537,13 +1546,12 @@ clear_break(BpFunctions* f, Uint break_flags) n = f->matched; for (i = 0; i < n; i++) { - BeamInstr* pc = f->matching[i].pc; - clear_function_break(pc, break_flags); + clear_function_break(f->matching[i].ci, break_flags); } } static int -clear_function_break(BeamInstr *pc, Uint break_flags) +clear_function_break(ErtsCodeInfo *ci, Uint break_flags) { GenericBp* g; GenericBpData* bp; @@ -1552,7 +1560,7 @@ clear_function_break(BeamInstr *pc, Uint break_flags) ERTS_SMP_LC_ASSERT(erts_has_code_write_permission()); - if ((g = (GenericBp *) pc[-4]) == 0) { + if ((g = (GenericBp *) ci->native) == 0) { return 1; } @@ -1638,19 +1646,19 @@ bp_time_unref(BpDataTime* bdt) } static BpDataTime* -get_time_break(BeamInstr *pc) +get_time_break(ErtsCodeInfo *ci) { - GenericBpData* bp = check_break(pc, ERTS_BPF_TIME_TRACE); + GenericBpData* bp = check_break(ci, ERTS_BPF_TIME_TRACE); return bp ? bp->time : 0; } static GenericBpData* -check_break(BeamInstr *pc, Uint break_flags) +check_break(ErtsCodeInfo *ci, Uint break_flags) { - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ci->native; - ASSERT(pc[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (erts_is_native_break(pc)) { + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (erts_is_native_break(ci)) { return 0; } if (g) { diff --git a/erts/emulator/beam/beam_bp.h b/erts/emulator/beam/beam_bp.h index 541af77211..d32a6bcba5 100644 --- a/erts/emulator/beam/beam_bp.h +++ b/erts/emulator/beam/beam_bp.h @@ -46,7 +46,7 @@ typedef struct bp_data_time { /* Call time */ typedef struct { ErtsMonotonicTime time; - BeamInstr *pc; + ErtsCodeInfo *ci; } process_breakpoint_time_t; /* used within psd */ typedef struct { @@ -95,7 +95,7 @@ enum erts_break_op{ typedef Uint32 ErtsBpIndex; typedef struct { - BeamInstr* pc; + ErtsCodeInfo *ci; Module* mod; } BpFunction; @@ -116,8 +116,8 @@ void erts_commit_staged_bp(void); ERTS_GLB_INLINE ErtsBpIndex erts_active_bp_ix(void); ERTS_GLB_INLINE ErtsBpIndex erts_staging_bp_ix(void); -void erts_bp_match_functions(BpFunctions* f, Eterm mfa[3], int specified); -void erts_bp_match_export(BpFunctions* f, Eterm mfa[3], int specified); +void erts_bp_match_functions(BpFunctions* f, ErtsCodeMFA *mfa, int specified); +void erts_bp_match_export(BpFunctions* f, ErtsCodeMFA *mfa, int specified); void erts_bp_free_matched_functions(BpFunctions* f); void erts_install_breakpoints(BpFunctions* f); @@ -128,15 +128,15 @@ void erts_consolidate_bif_bp_data(void); void erts_set_trace_break(BpFunctions *f, Binary *match_spec); void erts_clear_trace_break(BpFunctions *f); -void erts_set_call_trace_bif(BeamInstr *pc, Binary *match_spec, int local); -void erts_clear_call_trace_bif(BeamInstr *pc, int local); +void erts_set_call_trace_bif(ErtsCodeInfo *ci, Binary *match_spec, int local); +void erts_clear_call_trace_bif(ErtsCodeInfo *ci, int local); void erts_set_mtrace_break(BpFunctions *f, Binary *match_spec, ErtsTracer tracer); void erts_clear_mtrace_break(BpFunctions *f); -void erts_set_mtrace_bif(BeamInstr *pc, Binary *match_spec, +void erts_set_mtrace_bif(ErtsCodeInfo *ci, Binary *match_spec, ErtsTracer tracer); -void erts_clear_mtrace_bif(BeamInstr *pc); +void erts_clear_mtrace_bif(ErtsCodeInfo *ci); void erts_set_debug_break(BpFunctions *f); void erts_clear_debug_break(BpFunctions *f); @@ -146,32 +146,32 @@ void erts_clear_count_break(BpFunctions *f); void erts_clear_all_breaks(BpFunctions* f); int erts_clear_module_break(Module *modp); -void erts_clear_export_break(Module *modp, BeamInstr* pc); +void erts_clear_export_break(Module *modp, ErtsCodeInfo* ci); -BeamInstr erts_generic_breakpoint(Process* c_p, BeamInstr* I, Eterm* reg); -BeamInstr erts_trace_break(Process *p, BeamInstr *pc, Eterm *args, +BeamInstr erts_generic_breakpoint(Process* c_p, ErtsCodeInfo *ci, Eterm* reg); +BeamInstr erts_trace_break(Process *p, ErtsCodeInfo *ci, Eterm *args, Uint32 *ret_flags, ErtsTracer *tracer); -int erts_is_trace_break(BeamInstr *pc, Binary **match_spec_ret, int local); -int erts_is_mtrace_break(BeamInstr *pc, Binary **match_spec_ret, +int erts_is_trace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, int local); +int erts_is_mtrace_break(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_mtrace_bif(BeamInstr *pc, Binary **match_spec_ret, +int erts_is_mtrace_bif(ErtsCodeInfo *ci, Binary **match_spec_ret, ErtsTracer *tracer_ret); -int erts_is_native_break(BeamInstr *pc); -int erts_is_count_break(BeamInstr *pc, Uint *count_ret); -int erts_is_time_break(Process *p, BeamInstr *pc, Eterm *call_time); +int erts_is_native_break(ErtsCodeInfo *ci); +int erts_is_count_break(ErtsCodeInfo *ci, Uint *count_ret); +int erts_is_time_break(Process *p, ErtsCodeInfo *ci, Eterm *call_time); -void erts_trace_time_call(Process* c_p, BeamInstr* pc, BpDataTime* bdt); -void erts_trace_time_return(Process* c_p, BeamInstr* pc); +void erts_trace_time_call(Process* c_p, ErtsCodeInfo *ci, BpDataTime* bdt); +void erts_trace_time_return(Process* c_p, ErtsCodeInfo *ci); void erts_schedule_time_break(Process *p, Uint out); void erts_set_time_break(BpFunctions *f, enum erts_break_op); void erts_clear_time_break(BpFunctions *f); -int erts_is_time_trace_bif(Process *p, BeamInstr *pc, Eterm *call_time); -void erts_set_time_trace_bif(BeamInstr *pc, enum erts_break_op); -void erts_clear_time_trace_bif(BeamInstr *pc); +int erts_is_time_trace_bif(Process *p, ErtsCodeInfo *ci, Eterm *call_time); +void erts_set_time_trace_bif(ErtsCodeInfo *ci, enum erts_break_op); +void erts_clear_time_trace_bif(ErtsCodeInfo *ci); -BeamInstr *erts_find_local_func(Eterm mfa[3]); +ErtsCodeInfo *erts_find_local_func(ErtsCodeMFA *mfa); #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/beam_debug.c b/erts/emulator/beam/beam_debug.c index 21d336049f..e72d7f8de4 100644 --- a/erts/emulator/beam/beam_debug.c +++ b/erts/emulator/beam/beam_debug.c @@ -51,6 +51,7 @@ void dbg_bt(Process* p, Eterm* sp); void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg); static int print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr); +static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif); BIF_RETTYPE erts_debug_same_2(BIF_ALIST_2) @@ -115,7 +116,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) Eterm MFA = BIF_ARG_1; Eterm boolean = BIF_ARG_2; Eterm* tp; - Eterm mfa[3]; + ErtsCodeMFA mfa; int i; int specified = 0; Eterm res; @@ -131,23 +132,24 @@ erts_debug_breakpoint_2(BIF_ALIST_2) if (*tp != make_arityval(3)) { goto error; } - mfa[0] = tp[1]; - mfa[1] = tp[2]; - mfa[2] = tp[3]; - if (!is_atom(mfa[0]) || !is_atom(mfa[1]) || - (!is_small(mfa[2]) && mfa[2] != am_Underscore)) { + if (!is_atom(tp[1]) || !is_atom(tp[2]) || + (!is_small(tp[3]) && tp[3] != am_Underscore)) { goto error; } - for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) { + for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) { /* Empty loop body */ } for (i = specified; i < 3; i++) { - if (mfa[i] != am_Underscore) { + if (tp[i+1] != am_Underscore) { goto error; } } - if (is_small(mfa[2])) { - mfa[2] = signed_val(mfa[2]); + + mfa.module = tp[1]; + mfa.function = tp[2]; + + if (is_small(tp[3])) { + mfa.arity = signed_val(tp[3]); } if (!erts_try_seize_code_write_permission(BIF_P)) { @@ -157,7 +159,7 @@ erts_debug_breakpoint_2(BIF_ALIST_2) erts_smp_proc_unlock(p, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); - erts_bp_match_functions(&f, mfa, specified); + erts_bp_match_functions(&f, &mfa, specified); if (boolean == am_true) { erts_set_debug_break(&f); erts_install_breakpoints(&f); @@ -241,9 +243,9 @@ erts_debug_disassemble_1(BIF_ALIST_1) Eterm* tp; Eterm bin; Eterm mfa; - BeamInstr* funcinfo = NULL; /* Initialized to eliminate warning. */ + ErtsCodeMFA *cmfa = NULL; BeamCodeHeader* code_hdr; - BeamInstr* code_ptr = NULL; /* Initialized to eliminate warning. */ + BeamInstr *code_ptr; BeamInstr instr; BeamInstr uaddr; Uint hsz; @@ -251,7 +253,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) if (term_to_UWord(addr, &uaddr)) { code_ptr = (BeamInstr *) uaddr; - if ((funcinfo = find_function_from_pc(code_ptr)) == NULL) { + if ((cmfa = find_function_from_pc(code_ptr)) == NULL) { BIF_RET(am_false); } } else if (is_tuple(addr)) { @@ -282,24 +284,22 @@ erts_debug_disassemble_1(BIF_ALIST_1) * such as erts_debug:apply/4. Then search for it in the module. */ if ((ep = erts_find_function(mod, name, arity, code_ix)) != NULL) { - /* XXX: add "&& ep->address != ep->code+3" condition? + /* XXX: add "&& ep->address != ep->code" condition? * Consider a traced function. - * Its ep will have ep->address == ep->code+3. + * Its ep will have ep->address == ep->code. * erts_find_function() will return the non-NULL ep. * Below we'll try to derive a code_ptr from ep->address. * But this code_ptr will point to the start of the Export, * not the function's func_info instruction. BOOM !? */ - code_ptr = ((BeamInstr *) ep->addressv[code_ix]) - 5; - funcinfo = code_ptr+2; + cmfa = erts_code_to_codemfa(ep->addressv[code_ix]); } else if (modp == NULL || (code_hdr = modp->curr.code_hdr) == NULL) { BIF_RET(am_undef); } else { n = code_hdr->num_functions; for (i = 0; i < n; i++) { - code_ptr = code_hdr->functions[i]; - if (code_ptr[3] == name && code_ptr[4] == arity) { - funcinfo = code_ptr+2; + cmfa = &code_hdr->functions[i]->mfa; + if (cmfa->function == name && cmfa->arity == arity) { break; } } @@ -307,6 +307,7 @@ erts_debug_disassemble_1(BIF_ALIST_1) BIF_RET(am_undef); } } + code_ptr = erts_codemfa_to_code(cmfa); } else { goto error; } @@ -332,9 +333,10 @@ erts_debug_disassemble_1(BIF_ALIST_1) (void) erts_bld_uword(NULL, &hsz, (BeamInstr) code_ptr); hp = HAlloc(p, hsz); addr = erts_bld_uword(&hp, NULL, (BeamInstr) code_ptr); - ASSERT(is_atom(funcinfo[0]) || funcinfo[0] == NIL); - ASSERT(is_atom(funcinfo[1]) || funcinfo[1] == NIL); - mfa = TUPLE3(hp, (Eterm) funcinfo[0], (Eterm) funcinfo[1], make_small((Eterm) funcinfo[2])); + ASSERT(is_atom(cmfa->module) || is_nil(cmfa->module)); + ASSERT(is_atom(cmfa->function) || is_nil(cmfa->function)); + mfa = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); hp += 4; return TUPLE3(hp, addr, bin, mfa); } @@ -346,11 +348,12 @@ dbg_bt(Process* p, Eterm* sp) while (sp < stack) { if (is_CP(*sp)) { - BeamInstr* addr = find_function_from_pc(cp_val(*sp)); - if (addr) + ErtsCodeMFA* cmfa = find_function_from_pc(cp_val(*sp)); + if (cmfa) erts_fprintf(stderr, HEXF ": %T:%T/%bpu\n", - addr, (Eterm) addr[0], (Eterm) addr[1], addr[2]); + &cmfa->module, cmfa->module, + cmfa->function, cmfa->arity); } sp++; } @@ -359,17 +362,17 @@ dbg_bt(Process* p, Eterm* sp) void dbg_where(BeamInstr* addr, Eterm x0, Eterm* reg) { - BeamInstr* f = find_function_from_pc(addr); + ErtsCodeMFA* cmfa = find_function_from_pc(addr); - if (f == NULL) { + if (cmfa == NULL) { erts_fprintf(stderr, "???\n"); } else { int arity; int i; - addr = f; - arity = addr[2]; - erts_fprintf(stderr, HEXF ": %T:%T(", addr, (Eterm) addr[0], (Eterm) addr[1]); + arity = cmfa->arity; + erts_fprintf(stderr, HEXF ": %T:%T(", addr, + cmfa->module, cmfa->function); for (i = 0; i < arity; i++) erts_fprintf(stderr, i ? ", %T" : "%T", i ? reg[i] : x0); erts_fprintf(stderr, ")\n"); @@ -520,27 +523,49 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) break; case 'I': /* Untagged integer. */ case 't': - erts_print(to, to_arg, "%d", *ap); + switch (op) { + case op_i_gc_bif1_jIsId: + case op_i_gc_bif2_jIIssd: + case op_i_gc_bif3_jIIssd: + { + const ErtsGcBif* p; + BifFunction gcf = (BifFunction) *ap; + for (p = erts_gc_bifs; p->bif != 0; p++) { + if (p->gc_bif == gcf) { + print_bif_name(to, to_arg, p->bif); + break; + } + } + if (p->bif == 0) { + erts_print(to, to_arg, "%d", (Uint)gcf); + } + break; + } + default: + erts_print(to, to_arg, "%d", *ap); + } ap++; break; case 'f': /* Destination label */ { - BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); - if (f+3 != (BeamInstr *) *ap) { + ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); + if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "f(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); } ap++; } break; case 'p': /* Pointer (to label) */ { - BeamInstr* f = find_function_from_pc((BeamInstr *)*ap); - if (f+3 != (BeamInstr *) *ap) { + ErtsCodeMFA* cmfa = find_function_from_pc((BeamInstr *)*ap); + if (!cmfa || erts_codemfa_to_code(cmfa) != (BeamInstr *) *ap) { erts_print(to, to_arg, "p(" HEXF ")", *ap); } else { - erts_print(to, to_arg, "%T:%T/%bpu", (Eterm) f[0], (Eterm) f[1], f[2]); + erts_print(to, to_arg, "%T:%T/%bpu", cmfa->module, + cmfa->function, cmfa->arity); } ap++; } @@ -553,26 +578,16 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) { Export* ex = (Export *) *ap; erts_print(to, to_arg, - "%T:%T/%bpu", (Eterm) ex->code[0], (Eterm) ex->code[1], ex->code[2]); + "%T:%T/%bpu", (Eterm) ex->info.mfa.module, + (Eterm) ex->info.mfa.function, + ex->info.mfa.arity); ap++; } break; case 'F': /* Function definition */ break; case 'b': - for (i = 0; i < BIF_SIZE; i++) { - BifFunction bif = (BifFunction) *ap; - if (bif == bif_table[i].f) { - break; - } - } - if (i == BIF_SIZE) { - erts_print(to, to_arg, "b(%d)", (Uint) *ap); - } else { - Eterm name = bif_table[i].name; - unsigned arity = bif_table[i].arity; - erts_print(to, to_arg, "%T/%u", name, arity); - } + print_bif_name(to, to_arg, (BifFunction) *ap); ap++; break; case 'P': /* Byte offset into tuple (see beam_load.c) */ @@ -731,3 +746,21 @@ print_op(fmtfn_t to, void *to_arg, int op, int size, BeamInstr* addr) return size; } + +static void print_bif_name(fmtfn_t to, void* to_arg, BifFunction bif) +{ + int i; + + for (i = 0; i < BIF_SIZE; i++) { + if (bif == bif_table[i].f) { + break; + } + } + if (i == BIF_SIZE) { + erts_print(to, to_arg, "b(%d)", (Uint) bif); + } else { + Eterm name = bif_table[i].name; + unsigned arity = bif_table[i].arity; + erts_print(to, to_arg, "%T/%u", name, arity); + } +} diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index 3be5c0d24c..b24995ca3d 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -116,10 +116,10 @@ do { \ #define MAX(x, y) (((x) > (y)) ? (x) : (y)) #endif -#define GET_BIF_MODULE(p) ((Eterm) (((Export *) p)->code[0])) -#define GET_BIF_FUNCTION(p) ((Eterm) (((Export *) p)->code[1])) -#define GET_BIF_ARITY(p) ((Eterm) (((Export *) p)->code[2])) -#define GET_BIF_ADDRESS(p) ((BifFunction) (((Export *) p)->code[4])) +#define GET_BIF_MODULE(p) (p->info.mfa.module) +#define GET_BIF_FUNCTION(p) (p->info.mfa.function) +#define GET_BIF_ARITY(p) (p->info.mfa.arity) +#define GET_BIF_ADDRESS(p) ((BifFunction) (p->beam[1])) #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm)))) @@ -1045,7 +1045,7 @@ void** beam_ops; static BifFunction translate_gc_bif(void* gcf) NOINLINE; static BeamInstr* handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf) NOINLINE; -static BeamInstr* call_error_handler(Process* p, BeamInstr* ip, +static BeamInstr* call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) NOINLINE; static BeamInstr* fixed_apply(Process* p, Eterm* reg, Uint arity) NOINLINE; static BeamInstr* apply(Process* p, Eterm module, Eterm function, @@ -1078,7 +1078,7 @@ static Eterm make_arglist(Process* c_p, Eterm* reg, int a); void init_emulator(void) { - process_main(); + process_main(0, 0); } /* @@ -1106,98 +1106,91 @@ init_emulator(void) #ifdef USE_VM_CALL_PROBES -#define DTRACE_LOCAL_CALL(p, m, f, a) \ +#define DTRACE_LOCAL_CALL(p, mfa) \ if (DTRACE_ENABLED(local_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(local_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(local_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_GLOBAL_CALL(p, m, f, a) \ +#define DTRACE_GLOBAL_CALL(p, mfa) \ if (DTRACE_ENABLED(global_function_entry)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(global_function_entry, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(global_function_entry, process_name, mfa_buf, depth); \ } -#define DTRACE_RETURN(p, m, f, a) \ +#define DTRACE_RETURN(p, mfa) \ if (DTRACE_ENABLED(function_return)) { \ DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ int depth = STACK_START(p) - STACK_TOP(p); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE3(function_return, process_name, mfa, depth); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE3(function_return, process_name, mfa_buf, depth); \ } -#define DTRACE_BIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(bif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_entry, process_name, mfa); \ +#define DTRACE_BIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(bif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_entry, process_name, mfa_buf); \ } -#define DTRACE_BIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(bif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(bif_return, process_name, mfa); \ +#define DTRACE_BIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(bif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(bif_return, process_name, mfa_buf); \ } -#define DTRACE_NIF_ENTRY(p, m, f, a) \ - if (DTRACE_ENABLED(nif_entry)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_entry, process_name, mfa); \ +#define DTRACE_NIF_ENTRY(p, mfa) \ + if (DTRACE_ENABLED(nif_entry)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_entry, process_name, mfa_buf); \ } -#define DTRACE_NIF_RETURN(p, m, f, a) \ - if (DTRACE_ENABLED(nif_return)) { \ - DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); \ - dtrace_fun_decode(p, m, f, a, \ - process_name, mfa); \ - DTRACE2(nif_return, process_name, mfa); \ +#define DTRACE_NIF_RETURN(p, mfa) \ + if (DTRACE_ENABLED(nif_return)) { \ + DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); \ + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); \ + dtrace_fun_decode(p, mfa, process_name, mfa_buf); \ + DTRACE2(nif_return, process_name, mfa_buf); \ } #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p,e) \ do { \ if (DTRACE_ENABLED(global_function_entry)) { \ BeamInstr* fp = (BeamInstr *) (((Export *) (e))->addressv[erts_active_code_ix()]); \ - DTRACE_GLOBAL_CALL((p), (Eterm)fp[-3], (Eterm)fp[-2], fp[-1]); \ + DTRACE_GLOBAL_CALL((p), erts_code_to_codemfa(fp)); \ } \ } while(0) #define DTRACE_RETURN_FROM_PC(p) \ do { \ - BeamInstr* fp; \ - if (DTRACE_ENABLED(function_return) && (fp = find_function_from_pc((p)->cp))) { \ - DTRACE_RETURN((p), (Eterm)fp[0], (Eterm)fp[1], (Uint)fp[2]); \ + ErtsCodeMFA* cmfa; \ + if (DTRACE_ENABLED(function_return) && (cmfa = find_function_from_pc((p)->cp))) { \ + DTRACE_RETURN((p), cmfa); \ } \ } while(0) #else /* USE_VM_PROBES */ -#define DTRACE_LOCAL_CALL(p, m, f, a) do {} while (0) -#define DTRACE_GLOBAL_CALL(p, m, f, a) do {} while (0) +#define DTRACE_LOCAL_CALL(p, mfa) do {} while (0) +#define DTRACE_GLOBAL_CALL(p, mfa) do {} while (0) #define DTRACE_GLOBAL_CALL_FROM_EXPORT(p, e) do {} while (0) -#define DTRACE_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_RETURN(p, mfa) do {} while (0) #define DTRACE_RETURN_FROM_PC(p) do {} while (0) -#define DTRACE_BIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_BIF_RETURN(p, m, f, a) do {} while (0) -#define DTRACE_NIF_ENTRY(p, m, f, a) do {} while (0) -#define DTRACE_NIF_RETURN(p, m, f, a) do {} while (0) +#define DTRACE_BIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_BIF_RETURN(p, mfa) do {} while (0) +#define DTRACE_NIF_ENTRY(p, mfa) do {} while (0) +#define DTRACE_NIF_RETURN(p, mfa) do {} while (0) #endif /* USE_VM_PROBES */ #ifdef DEBUG @@ -1225,7 +1218,7 @@ init_emulator(void) * the instructions' C labels to the loader. * The second call starts execution of BEAM code. This call never returns. */ -void process_main(void) +void process_main(Eterm * x_reg_array, FloatDef* f_reg_array) { static int init_done = 0; Process* c_p = NULL; @@ -1237,7 +1230,7 @@ void process_main(void) /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC, * in all other cases x0 is used. */ - register Eterm* reg REG_xregs = NULL; + register Eterm* reg REG_xregs = x_reg_array; /* * Top of heap (next free location); grows upwards. @@ -1264,7 +1257,7 @@ void process_main(void) * X registers and floating point registers are located in * scheduler specific data. */ - register FloatDef *freg; + register FloatDef *freg = f_reg_array; /* * For keeping the negative old value of 'reds' when call saving is active. @@ -1324,8 +1317,8 @@ void process_main(void) if (start_time != 0) { Sint64 diff = erts_timestamp_millis() - start_time; if (diff > 0 && (Uint) diff > erts_system_monitor_long_schedule) { - BeamInstr *inptr = find_function_from_pc(start_time_i); - BeamInstr *outptr = find_function_from_pc(c_p->i); + ErtsCodeMFA *inptr = find_function_from_pc(start_time_i); + ErtsCodeMFA *outptr = find_function_from_pc(c_p->i); monitor_long_schedule_proc(c_p,inptr,outptr,(Uint) diff); } } @@ -1350,8 +1343,6 @@ void process_main(void) start_time_i = c_p->i; } - reg = erts_proc_sched_data(c_p)->x_reg_array; - freg = erts_proc_sched_data(c_p)->f_reg_array; ERL_BITS_RELOAD_STATEP(c_p); { int reds; @@ -1401,10 +1392,9 @@ void process_main(void) if (ERTS_PROC_IS_EXITING(c_p)) { strcpy(fun_buf, "<exiting>"); } else { - BeamInstr *fptr = find_function_from_pc(c_p->i); - if (fptr) { - dtrace_fun_decode(c_p, (Eterm)fptr[0], - (Eterm)fptr[1], (Uint)fptr[2], + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); } else { erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), @@ -1585,7 +1575,7 @@ void process_main(void) /* FALL THROUGH */ OpCase(i_call_only_f): { SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -1597,7 +1587,7 @@ void process_main(void) RESTORE_CP(E); E = ADD_BYTE_OFFSET(E, Arg(1)); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -1609,7 +1599,7 @@ void process_main(void) OpCase(i_call_f): { SET_CP(c_p, I+2); SET_I((BeamInstr *) Arg(0)); - DTRACE_LOCAL_CALL(c_p, (Eterm)I[-3], (Eterm)I[-2], I[-1]); + DTRACE_LOCAL_CALL(c_p, erts_code_to_codemfa(I)); Dispatch(); } @@ -2836,25 +2826,27 @@ do { \ Eterm result; BeamInstr *next; ErlHeapFragment *live_hf_end; + Export *export = (Export*)Arg(0); if (!((FCALLS - 1) > 0 || (FCALLS-1) > neg_o_reds)) { /* If we have run out of reductions, we do a context switch before calling the bif */ - c_p->arity = ((Export *)Arg(0))->code[2]; - c_p->current = ((Export *)Arg(0))->code; + c_p->arity = GET_BIF_ARITY(export); + c_p->current = &export->info.mfa; goto context_switch3; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X(GET_BIF_MODULE(Arg(0)), GET_BIF_ADDRESS(Arg(0))); + ERTS_MSACC_SET_BIF_STATE_CACHED_X( + GET_BIF_MODULE(export), GET_BIF_ADDRESS(export)); - bf = GET_BIF_ADDRESS(Arg(0)); + bf = GET_BIF_ADDRESS(export); PRE_BIF_SWAPOUT(c_p); ERTS_DBG_CHK_REDS(c_p, FCALLS); c_p->fcalls = FCALLS - 1; if (FCALLS <= 0) { - save_calls(c_p, (Export *) Arg(0)); + save_calls(c_p, export); } PreFetch(1, next); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -2868,7 +2860,7 @@ do { \ ERTS_HOLE_CHECK(c_p); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); if (ERTS_IS_GC_DESIRED(c_p)) { - Uint arity = ((Export *)Arg(0))->code[2]; + Uint arity = GET_BIF_ARITY(export); result = erts_gc_after_bif_call_lhf(c_p, live_hf_end, result, reg, arity); E = c_p->stop; } @@ -3361,14 +3353,15 @@ do { \ * called from I[-3], I[-2], and I[-1] respectively. */ context_switch_fun: - c_p->arity = I[-1] + 1; + /* Add one for the environment of the fun */ + c_p->arity = erts_code_to_codemfa(I)->arity + 1; goto context_switch2; context_switch: - c_p->arity = I[-1]; + c_p->arity = erts_code_to_codemfa(I)->arity; - context_switch2: /* Entry for fun calls. */ - c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + context_switch2: /* Entry for fun calls. */ + c_p->current = erts_code_to_codemfa(I); context_switch3: @@ -3509,7 +3502,8 @@ do { \ * code[4]: Not used */ HEAVY_SWAPOUT; - I = call_error_handler(c_p, I-3, reg, am_undefined_function); + I = call_error_handler(c_p, erts_code_to_codemfa(I), + reg, am_undefined_function); HEAVY_SWAPIN; if (I) { Goto(*I); @@ -3546,9 +3540,12 @@ do { \ * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct */ BifFunction vbf; ErlHeapFragment *live_hf_end; + ErtsCodeMFA *codemfa; if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { /* If we have run out of reductions, we do a context @@ -3558,12 +3555,16 @@ do { \ ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); - c_p->current = I-3; /* current and vbf set to please handle_error */ + codemfa = erts_code_to_codemfa(I); + + c_p->current = codemfa; /* current and vbf set to please handle_error */ + + DTRACE_NIF_ENTRY(c_p, codemfa); + SWAPOUT; c_p->fcalls = FCALLS - 1; PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -3590,19 +3591,19 @@ do { \ ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_NIF_RETURN(c_p, codemfa); goto apply_bif_or_nif_epilogue; OpCase(apply_bif): /* - * At this point, I points to the code[3] in the export entry for + * At this point, I points to the code[0] in the export entry for * the BIF: * - * code[0]: Module - * code[1]: Function - * code[2]: Arity - * code[3]: &&apply_bif - * code[4]: Function pointer to BIF function + * code[-3]: Module + * code[-2]: Function + * code[-1]: Arity + * code[0]: &&apply_bif + * code[1]: Function pointer to BIF function */ if (!((FCALLS - 1) > 0 || (FCALLS - 1) > neg_o_reds)) { @@ -3611,21 +3612,25 @@ do { \ goto context_switch; } - ERTS_MSACC_SET_BIF_STATE_CACHED_X((Eterm)I[-3], (BifFunction)Arg(0)); + codemfa = erts_code_to_codemfa(I); + + ERTS_MSACC_SET_BIF_STATE_CACHED_X(codemfa->module, (BifFunction)Arg(0)); + - c_p->current = I-3; /* In case we apply process_info/1,2 or load_nif/1 */ + /* In case we apply process_info/1,2 or load_nif/1 */ + c_p->current = codemfa; c_p->i = I; /* In case we apply check_process_code/2. */ c_p->arity = 0; /* To allow garbage collection on ourselves * (check_process_code/2). */ - DTRACE_BIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_BIF_ENTRY(c_p, codemfa); SWAPOUT; ERTS_DBG_CHK_REDS(c_p, FCALLS - 1); c_p->fcalls = FCALLS - 1; vbf = (BifFunction) Arg(0); PROCESS_MAIN_CHK_LOCKS(c_p); - bif_nif_arity = I[-1]; + bif_nif_arity = codemfa->arity; ASSERT(bif_nif_arity <= 4); ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p); @@ -3647,7 +3652,7 @@ do { \ if (ERTS_MSACC_IS_ENABLED_CACHED_X()) ERTS_MSACC_UPDATE_CACHE_X(); ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_EMULATOR); - DTRACE_BIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_BIF_RETURN(c_p, codemfa); apply_bif_or_nif_epilogue: ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); @@ -3714,8 +3719,9 @@ do { \ goto find_func_info; OpCase(i_func_info_IaaI): { + ErtsCodeInfo *ci = (ErtsCodeInfo*)I; c_p->freason = EXC_FUNCTION_CLAUSE; - c_p->current = I + 2; + c_p->current = &ci->mfa; goto handle_error; } @@ -4706,11 +4712,11 @@ do { \ */ OpCase(return_trace): { - BeamInstr* code = (BeamInstr *) (UWord) E[0]; + ErtsCodeMFA* mfa = (ErtsCodeMFA *)(E[0]); SWAPOUT; /* Needed for shared heap */ ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); - erts_trace_return(c_p, code, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); + erts_trace_return(c_p, mfa, r(0), ERTS_TRACER_FROM_ETERM(E+1)/* tracer */); ERTS_SMP_REQ_PROC_MAIN_LOCK(c_p); SWAPIN; c_p->cp = NULL; @@ -4721,9 +4727,8 @@ do { \ OpCase(i_generic_breakpoint): { BeamInstr real_I; - ASSERT(I[-5] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); HEAVY_SWAPOUT; - real_I = erts_generic_breakpoint(c_p, I, reg); + real_I = erts_generic_breakpoint(c_p, erts_code_to_codeinfo(I), reg); HEAVY_SWAPIN; ASSERT(VALID_INSTR(real_I)); Goto(real_I); @@ -4732,7 +4737,7 @@ do { \ OpCase(i_return_time_trace): { BeamInstr *pc = (BeamInstr *) (UWord) E[0]; SWAPOUT; - erts_trace_time_return(c_p, pc); + erts_trace_time_return(c_p, erts_code_to_codeinfo(pc)); SWAPIN; c_p->cp = NULL; SET_I((BeamInstr *) cp_val(E[1])); @@ -4917,16 +4922,18 @@ do { \ * I[ 0]: &&lb_hipe_trap_call * ... remainder of original BEAM code */ - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = (void(*)(void)) ci->native; ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (I[-1] << 8)); + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL | (ci->mfa.arity << 8)); } OpCase(hipe_trap_call_closure): { - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); - c_p->hipe.u.ncallee = (void(*)(void)) I[-4]; + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); + c_p->hipe.u.ncallee = (void(*)(void)) ci->native; ++hipe_trap_count; - HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (I[-1] << 8)); + HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_CALL_CLOSURE | (ci->mfa.arity << 8)); } OpCase(hipe_trap_return): { HIPE_MODE_SWITCH(HIPE_MODE_SWITCH_CMD_RETURN); @@ -4995,8 +5002,9 @@ do { \ * I[ 0]: &&lb_hipe_call_count * ... remainder of original BEAM code */ - struct hipe_call_count *hcc = (struct hipe_call_count*)I[-4]; - ASSERT(I[-5] == (Uint) OpCode(i_func_info_IaaI)); + ErtsCodeInfo *ci = erts_code_to_codeinfo(I); + struct hipe_call_count *hcc = (struct hipe_call_count*)ci->native; + ASSERT(ci->op == (Uint) OpCode(i_func_info_IaaI)); ASSERT(hcc != NULL); ASSERT(VALID_INSTR(hcc->opcode)); ++(hcc->count); @@ -5045,7 +5053,7 @@ do { \ } else { TestHeap(ERTS_SINT64_HEAP_SIZE(ts),0); r(0) = make_big(HTOP); -#if defined(ARCH_32) || HALFWORD_HEAP +#if defined(ARCH_32) if (ts >= (((Uint64) 1) << 32)) { *HTOP = make_pos_bignum_header(2); BIG_DIGIT(HTOP, 0) = (Uint) (ts & ((Uint) 0xffffffff)); @@ -5065,7 +5073,7 @@ do { \ OpCase(i_debug_breakpoint): { HEAVY_SWAPOUT; - I = call_error_handler(c_p, I-3, reg, am_breakpoint); + I = call_error_handler(c_p, erts_code_to_codemfa(I), reg, am_breakpoint); HEAVY_SWAPIN; if (I) { Goto(*I); @@ -5138,10 +5146,10 @@ do { \ bif_table[i].name, bif_table[i].arity); bif_export[i] = ep; - ep->code[3] = (BeamInstr) OpCode(apply_bif); - ep->code[4] = (BeamInstr) bif_table[i].f; + ep->beam[0] = (BeamInstr) OpCode(apply_bif); + ep->beam[1] = (BeamInstr) bif_table[i].f; /* XXX: set func info for bifs */ - ep->fake_op_func_info_for_hipe[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); } return; @@ -5221,8 +5229,8 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) goto do_dirty_schedule; context_switch: - c_p->arity = I[-1]; - c_p->current = I-3; /* Pointer to Mod, Func, Arity */ + c_p->current = erts_code_to_codemfa(I); /* Pointer to Mod, Func, Arity */ + c_p->arity = c_p->current->arity; { int reds_used; @@ -5347,11 +5355,9 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) if (ERTS_PROC_IS_EXITING(c_p)) { strcpy(fun_buf, "<exiting>"); } else { - BeamInstr *fptr = find_function_from_pc(c_p->i); - if (fptr) { - dtrace_fun_decode(c_p, (Eterm)fptr[0], - (Eterm)fptr[1], (Uint)fptr[2], - NULL, fun_buf); + ErtsCodeMFA *cmfa = find_function_from_pc(c_p->i); + if (cmfa) { + dtrace_fun_decode(c_p, cmfa, NULL, fun_buf); } else { erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)), "<unknown/%p>", *I); @@ -5380,16 +5386,22 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) * I[1]: Function pointer to NIF function * I[2]: Pointer to erl_module_nif * I[3]: Function pointer to dirty NIF + * + * This layout is determined by the NifExport struct */ BifFunction vbf; + ErtsCodeMFA *codemfa; ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF); - DTRACE_NIF_ENTRY(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); - c_p->current = I-3; /* current and vbf set to please handle_error */ + codemfa = erts_code_to_codemfa(I); + + DTRACE_NIF_ENTRY(c_p, codemfa); + /* current and vbf set to please handle_error */ + c_p->current = codemfa; SWAPOUT; PROCESS_MAIN_CHK_LOCKS(c_p); - arity = I[-1]; + arity = codemfa->arity; ERTS_SMP_UNREQ_PROC_MAIN_LOCK(c_p); ASSERT(!ERTS_PROC_IS_EXITING(c_p)); @@ -5424,7 +5436,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } - DTRACE_NIF_RETURN(c_p, (Eterm)I[-3], (Eterm)I[-2], (Uint)I[-1]); + DTRACE_NIF_RETURN(c_p, codemfa); ERTS_HOLE_CHECK(c_p); SWAPIN; I = c_p->i; @@ -5437,31 +5449,14 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp) static BifFunction translate_gc_bif(void* gcf) { - if (gcf == erts_gc_length_1) { - return length_1; - } else if (gcf == erts_gc_size_1) { - return size_1; - } else if (gcf == erts_gc_bit_size_1) { - return bit_size_1; - } else if (gcf == erts_gc_byte_size_1) { - return byte_size_1; - } else if (gcf == erts_gc_map_size_1) { - return map_size_1; - } else if (gcf == erts_gc_abs_1) { - return abs_1; - } else if (gcf == erts_gc_float_1) { - return float_1; - } else if (gcf == erts_gc_round_1) { - return round_1; - } else if (gcf == erts_gc_trunc_1) { - return round_1; - } else if (gcf == erts_gc_binary_part_2) { - return binary_part_2; - } else if (gcf == erts_gc_binary_part_3) { - return binary_part_3; - } else { - erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); + const ErtsGcBif* p; + + for (p = erts_gc_bifs; p->bif != 0; p++) { + if (p->gc_bif == gcf) { + return p->bif; + } } + erts_exit(ERTS_ERROR_EXIT, "bad gc bif"); } /* @@ -5621,7 +5616,8 @@ next_catch(Process* c_p, Eterm *reg) { /* Can not follow cp here - code may be unloaded */ BeamInstr *cpp = c_p->cp; if (cpp == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, reg[1], reg[2], ERTS_TRACER_FROM_ETERM(ptr+1)); /* Skip return_trace parameters */ @@ -5649,7 +5645,8 @@ next_catch(Process* c_p, Eterm *reg) { if (is_catch(*ptr) && active_catches) goto found_catch; } if (cp_val(*prev) == beam_exception_trace) { - erts_trace_exception(c_p, cp_val(ptr[0]), + ErtsCodeMFA *mfa = (ErtsCodeMFA*)cp_val(ptr[0]); + erts_trace_exception(c_p, mfa, reg[1], reg[2], ERTS_TRACER_FROM_ETERM(ptr+1)); } @@ -5855,7 +5852,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, for (i = 0; i < BIF_SIZE; i++) { if (bf == bif_table[i].f || bf == bif_table[i].traced) { Export *ep = bif_export[i]; - s->current = ep->code; + s->current = &ep->info.mfa; a = bif_table[i].arity; break; } @@ -5869,7 +5866,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, */ ASSERT(c_p->current); s->current = c_p->current; - a = s->current[2]; + a = s->current->arity; } /* Save first stack entry */ ASSERT(pc); @@ -5894,7 +5891,7 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg, BifFunction bf, (GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) ) { int a; ASSERT(s->current); - a = s->current[2]; + a = s->current->arity; args = make_arglist(c_p, reg, a); /* Overwrite CAR(c_p->ftrace) */ /* Save first stack entry */ ASSERT(c_p->cp); @@ -6065,7 +6062,7 @@ build_stacktrace(Process* c_p, Eterm exc) { erts_lookup_function_info(&fi, s->pc, 1); } else if (GET_EXC_INDEX(s->freason) == GET_EXC_INDEX(EXC_FUNCTION_CLAUSE)) { - erts_lookup_function_info(&fi, s->current, 1); + erts_lookup_function_info(&fi, erts_codemfa_to_code(s->current), 1); } else { erts_set_current_function(&fi, s->current); } @@ -6074,8 +6071,8 @@ build_stacktrace(Process* c_p, Eterm exc) { * If fi.current is still NULL, default to the initial function * (e.g. spawn_link(erlang, abs, [1])). */ - if (fi.current == NULL) { - erts_set_current_function(&fi, c_p->u.initial); + if (fi.mfa == NULL) { + erts_set_current_function(&fi, &c_p->u.initial); args = am_true; /* Just in case */ } else { args = get_args_from_exc(exc); @@ -6091,7 +6088,7 @@ build_stacktrace(Process* c_p, Eterm exc) { heap_size = fi.needed + 2; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->current) { + if (stkp->mfa) { heap_size += stkp->needed + 2; stkp++; } @@ -6115,7 +6112,7 @@ build_stacktrace(Process* c_p, Eterm exc) { } static BeamInstr* -call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) +call_error_handler(Process* p, ErtsCodeMFA* mfa, Eterm* reg, Eterm func) { Eterm* hp; Export* ep; @@ -6124,13 +6121,14 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) Uint sz; int i; + DBG_TRACE_MFA_P(mfa, "call_error_handler"); /* * Search for the error_handler module. */ ep = erts_find_function(erts_proc_get_error_handler(p), func, 3, erts_active_code_ix()); if (ep == NULL) { /* No error handler */ - p->current = fi; + p->current = mfa; p->freason = EXC_UNDEF; return 0; } @@ -6139,7 +6137,7 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) * Create a list with all arguments in the x registers. */ - arity = fi[2]; + arity = mfa->arity; sz = 2 * arity; if (HeapWordsLeft(p) < sz) { erts_garbage_collect(p, sz, reg, arity); @@ -6155,8 +6153,8 @@ call_error_handler(Process* p, BeamInstr* fi, Eterm* reg, Eterm func) /* * Set up registers for call to error_handler:<func>/3. */ - reg[0] = fi[0]; - reg[1] = fi[1]; + reg[0] = mfa->module; + reg[1] = mfa->function; reg[2] = args; return ep->addressv[erts_active_code_ix()]; } @@ -6401,11 +6399,11 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_hibernate)) { + ErtsCodeMFA cmfa = { module, function, arity}; DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); - dtrace_fun_decode(c_p, module, function, arity, - process_name, mfa); - DTRACE2(process_hibernate, process_name, mfa); + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); + dtrace_fun_decode(c_p, &cmfa, process_name, mfa_buf); + DTRACE2(process_hibernate, process_name, mfa_buf); } #endif /* @@ -6452,7 +6450,7 @@ erts_hibernate(Process* c_p, Eterm module, Eterm function, Eterm args, Eterm* re ASSERT(!ERTS_PROC_IS_EXITING(c_p)); } erts_smp_proc_unlock(c_p, ERTS_PROC_LOCK_MSGQ|ERTS_PROC_LOCK_STATUS); - c_p->current = bif_export[BIF_hibernate_3]->code; + c_p->current = &bif_export[BIF_hibernate_3]->info.mfa; c_p->flags |= F_HIBERNATE_SCHED; /* Needed also when woken! */ return 1; } @@ -6475,21 +6473,15 @@ call_fun(Process* p, /* Current process. */ if (is_fun_header(hdr)) { ErlFunThing* funp = (ErlFunThing *) fun_val(fun); - ErlFunEntry* fe; - BeamInstr* code_ptr; + ErlFunEntry* fe = funp->fe; + BeamInstr* code_ptr = fe->address; Eterm* var_ptr; - int actual_arity; - unsigned num_free; - - fe = funp->fe; - num_free = funp->num_free; - code_ptr = fe->address; - actual_arity = (int) code_ptr[-1]; + unsigned num_free = funp->num_free; + ErtsCodeMFA *mfa = erts_code_to_codemfa(code_ptr); + int actual_arity = mfa->arity; if (actual_arity == arity+num_free) { - DTRACE_LOCAL_CALL(p, (Eterm)code_ptr[-3], - (Eterm)code_ptr[-2], - code_ptr[-1]); + DTRACE_LOCAL_CALL(p, mfa); if (num_free == 0) { return code_ptr; } else { @@ -6594,10 +6586,10 @@ call_fun(Process* p, /* Current process. */ int actual_arity; ep = *((Export **) (export_val(fun) + 1)); - actual_arity = (int) ep->code[2]; + actual_arity = ep->info.mfa.arity; if (arity == actual_arity) { - DTRACE_GLOBAL_CALL(p, ep->code[0], ep->code[1], (Uint)ep->code[2]); + DTRACE_GLOBAL_CALL(p, &ep->info.mfa); return ep->addressv[erts_active_code_ix()]; } else { /* @@ -6685,9 +6677,6 @@ new_fun(Process* p, Eterm* reg, ErlFunEntry* fe, int num_free) funp->fe = fe; funp->num_free = num_free; funp->creator = p->common.id; -#ifdef HIPE - funp->native_address = fe->native_address; -#endif funp->arity = (int)fe->address[-1] - num_free; for (i = 0; i < num_free; i++) { *hp++ = reg[i]; @@ -7218,15 +7207,15 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity) return 1; } - e.code[0] = Mod; - e.code[1] = Name; - e.code[2] = arity; + e.info.mfa.module = Mod; + e.info.mfa.function = Name; + e.info.mfa.arity = arity; if ((ep = export_get(&e)) == NULL) { return 0; } - return ep->addressv[erts_active_code_ix()] == ep->code+3 - && (ep->code[3] == (BeamInstr) em_apply_bif); + return ep->addressv[erts_active_code_ix()] == ep->beam + && (ep->beam[0] == (BeamInstr) em_apply_bif); } diff --git a/erts/emulator/beam/beam_load.c b/erts/emulator/beam/beam_load.c index 0afdedf6c2..3cd395c2c1 100644 --- a/erts/emulator/beam/beam_load.c +++ b/erts/emulator/beam/beam_load.c @@ -39,11 +39,13 @@ #include "erl_binary.h" #include "erl_zlib.h" #include "erl_map.h" +#include "erl_process_dict.h" #ifdef HIPE #include "hipe_bif0.h" #include "hipe_mode_switch.h" #include "hipe_arch.h" +#include "hipe_load.h" #endif ErlDrvBinary* erts_gzinflate_buffer(char*, int); @@ -480,9 +482,12 @@ static void free_loader_state(Binary* magic); static ErlHeapFragment* new_literal_fragment(Uint size); static void free_literal_fragment(ErlHeapFragment*); static void loader_state_dtor(Binary* magic); +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code, Uint size); + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code); +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size); static int scan_iff_file(LoaderState* stp, Uint* chunk_types, Uint num_types, Uint num_mandatory); @@ -537,8 +542,6 @@ static Eterm compilation_info_for_module(Process* p, BeamCodeHeader*); static Eterm md5_of_module(Process* p, BeamCodeHeader*); static Eterm has_native(BeamCodeHeader*); static Eterm native_addresses(Process* p, BeamCodeHeader*); -int patch_funentries(Eterm Patchlist); -int patch(Eterm Addresses, Uint fe); static int safe_mul(UWord a, UWord b, UWord* resp); static int must_swap_floats; @@ -806,21 +809,21 @@ erts_finish_loading(Binary* magic, Process* c_p, for (i = 0; i < export_list_size(code_ix); i++) { Export *ep = export_list(i, code_ix); - if (ep == NULL || ep->code[0] != module) { + if (ep == NULL || ep->info.mfa.module != module) { continue; } - if (ep->addressv[code_ix] == ep->code+3) { - if (ep->code[3] == (BeamInstr) em_apply_bif) { + if (ep->addressv[code_ix] == ep->beam) { + if (ep->beam[0] == (BeamInstr) em_apply_bif) { continue; - } else if (ep->code[3] == + } else if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); ASSERT(mod_tab_p->curr.num_traced_exports > 0); - erts_clear_export_break(mod_tab_p, ep->code+3); - ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; - ep->code[4] = 0; + erts_clear_export_break(mod_tab_p, &ep->info); + ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; + ep->beam[1] = 0; } - ASSERT(ep->code[4] == 0); + ASSERT(ep->beam[1] == 0); } } ASSERT(mod_tab_p->curr.num_breakpoints == 0); @@ -842,9 +845,7 @@ erts_finish_loading(Binary* magic, Process* c_p, erts_alloc(ERTS_ALC_T_PREPARED_CODE, sizeof(struct erl_module_instance)); inst_p = mod_tab_p->on_load; - inst_p->nif = 0; - inst_p->num_breakpoints = 0; - inst_p->num_traced_exports = 0; + erts_module_instance_init(inst_p); } inst_p->code_hdr = stp->hdr; @@ -943,6 +944,13 @@ erts_module_for_prepared_code(Binary* magic) LoaderState* stp; if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != loader_state_dtor) { +#ifdef HIPE + HipeLoaderState *hipe_stp; + if ((hipe_stp = hipe_get_loader_state(magic)) + && hipe_stp->text_segment != 0) { + return hipe_stp->module; + } +#endif return NIL; } stp = ERTS_MAGIC_BIN_DATA(magic); @@ -1091,10 +1099,12 @@ loader_state_dtor(Binary* magic) ASSERT(stp->genop_blocks == 0); } +#ifdef HIPE static Eterm stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm module, - BeamCodeHeader* code_hdr, Uint size) + BeamCodeHeader* code_hdr, Uint size, + HipeModule *hipe_code) { Module* modp; Eterm retval; @@ -1117,6 +1127,9 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, modp->curr.code_hdr = code_hdr; modp->curr.code_length = size; modp->curr.catches = BEAM_CATCHES_NIL; /* Will be filled in later. */ + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "insert_new_code " + "first_hipe_ref = %p", hipe_code->first_hipe_ref); + modp->curr.hipe_code = hipe_code; /* * Update ranges (used for finding a function from a PC value). @@ -1125,6 +1138,7 @@ stub_insert_new_code(Process *c_p, ErtsProcLocks c_p_locks, erts_update_ranges((BeamInstr*)modp->curr.code_hdr, size); return NIL; } +#endif static int init_iff_file(LoaderState* stp, byte* code, Uint size) @@ -1420,8 +1434,8 @@ load_import_table(LoaderState* stp) * the BIF function. */ if ((e = erts_active_export_entry(mod, func, arity)) != NULL) { - if (e->code[3] == (BeamInstr) em_apply_bif) { - stp->import[i].bf = (BifFunction) e->code[4]; + if (e->beam[0] == (BeamInstr) em_apply_bif) { + stp->import[i].bf = (BifFunction) e->beam[1]; if (func == am_load_nif && mod == am_erlang && arity == 2) { stp->may_load_nif = 1; } @@ -1514,7 +1528,7 @@ is_bif(Eterm mod, Eterm func, unsigned arity) if (e == NULL) { return 0; } - if (e->code[3] != (BeamInstr) em_apply_bif) { + if (e->beam[0] != (BeamInstr) em_apply_bif) { return 0; } if (mod == am_erlang && func == am_apply && arity == 3) { @@ -1893,7 +1907,7 @@ load_code(LoaderState* stp) * by both the nif functionality and line instructions. */ enum { - FUNC_INFO_SZ = 5 + FUNC_INFO_SZ = sizeof(ErtsCodeInfo) / sizeof(Eterm) }; code = stp->codev; @@ -2565,8 +2579,11 @@ load_code(LoaderState* stp) stp->function = code[ci-2]; stp->arity = code[ci-1]; + /* When this assert is triggered, it is normally a sign that + the size of the ops.tab i_func_info instruction is not + the same as FUNC_INFO_SZ */ ASSERT(stp->labels[last_label].value == ci - FUNC_INFO_SZ); - stp->hdr->functions[function_number] = (BeamInstr*) stp->labels[last_label].patches; + stp->hdr->functions[function_number] = (ErtsCodeInfo*) stp->labels[last_label].patches; offset = function_number; stp->labels[last_label].patches = offset; function_number++; @@ -4031,60 +4048,52 @@ gen_make_fun2(LoaderState* stp, GenOpArg idx) op->next = NULL; return op; } + +static GenOp* +translate_gc_bif(LoaderState* stp, GenOp* op, GenOpArg Bif) +{ + const ErtsGcBif* p; + BifFunction bf; + + bf = stp->import[Bif.val].bf; + for (p = erts_gc_bifs; p->bif != 0; p++) { + if (p->bif == bf) { + op->a[1].type = TAG_u; + op->a[1].val = (BeamInstr) p->gc_bif; + return op; + } + } + + op->op = genop_unsupported_guard_bif_3; + op->arity = 3; + op->a[0].type = TAG_a; + op->a[0].val = stp->import[Bif.val].module; + op->a[1].type = TAG_a; + op->a[1].val = stp->import[Bif.val].function; + op->a[2].type = TAG_u; + op->a[2].val = stp->import[Bif.val].arity; + return op; +} + /* - * Rewrite gc_bifs with one parameter (the common case). Utilized - * in ops.tab to rewrite instructions calling bif's in guards - * to use a garbage collecting implementation. + * Rewrite gc_bifs with one parameter (the common case). */ static GenOp* gen_guard_bif1(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg Src, GenOpArg Dst) { GenOp* op; - BifFunction bf; NEW_GENOP(stp, op); op->next = NULL; - bf = stp->import[Bif.val].bf; - /* The translations here need to have a reverse counterpart in - beam_emu.c:translate_gc_bif for error handling to work properly. */ - if (bf == length_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_length_1; - } else if (bf == size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_size_1; - } else if (bf == bit_size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_bit_size_1; - } else if (bf == byte_size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_byte_size_1; - } else if (bf == map_size_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_map_size_1; - } else if (bf == abs_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_abs_1; - } else if (bf == float_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_float_1; - } else if (bf == round_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_round_1; - } else if (bf == trunc_1) { - op->a[1].val = (BeamInstr) (void *) erts_gc_trunc_1; - } else { - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; - return op; - } op->op = genop_i_gc_bif1_5; op->arity = 5; op->a[0] = Fail; - op->a[1].type = TAG_u; + /* op->a[1] is set by translate_gc_bif() */ op->a[2] = Src; op->a[3] = Live; op->a[4] = Dst; - return op; + return translate_gc_bif(stp, op, Bif); } /* @@ -4095,35 +4104,18 @@ gen_guard_bif2(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg S1, GenOpArg S2, GenOpArg Dst) { GenOp* op; - BifFunction bf; NEW_GENOP(stp, op); op->next = NULL; - bf = stp->import[Bif.val].bf; - /* The translations here need to have a reverse counterpart in - beam_emu.c:translate_gc_bif for error handling to work properly. */ - if (bf == binary_part_2) { - op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_2; - } else { - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; - return op; - } op->op = genop_i_gc_bif2_6; op->arity = 6; op->a[0] = Fail; - op->a[1].type = TAG_u; + /* op->a[1] is set by translate_gc_bif() */ op->a[2] = Live; op->a[3] = S1; op->a[4] = S2; op->a[5] = Dst; - return op; + return translate_gc_bif(stp, op, Bif); } /* @@ -4134,37 +4126,19 @@ gen_guard_bif3(LoaderState* stp, GenOpArg Fail, GenOpArg Live, GenOpArg Bif, GenOpArg S1, GenOpArg S2, GenOpArg S3, GenOpArg Dst) { GenOp* op; - BifFunction bf; NEW_GENOP(stp, op); op->next = NULL; - bf = stp->import[Bif.val].bf; - /* The translations here need to have a reverse counterpart in - beam_emu.c:translate_gc_bif for error handling to work properly. */ - if (bf == binary_part_3) { - op->a[1].val = (BeamInstr) (void *) erts_gc_binary_part_3; - } else { - op->op = genop_unsupported_guard_bif_3; - op->arity = 3; - op->a[0].type = TAG_a; - op->a[0].val = stp->import[Bif.val].module; - op->a[1].type = TAG_a; - op->a[1].val = stp->import[Bif.val].function; - op->a[2].type = TAG_u; - op->a[2].val = stp->import[Bif.val].arity; - return op; - } op->op = genop_ii_gc_bif3_7; op->arity = 7; op->a[0] = Fail; - op->a[1].type = TAG_u; + /* op->a[1] is set by translate_gc_bif() */ op->a[2] = Live; op->a[3] = S1; op->a[4] = S2; op->a[5] = S3; op->a[6] = Dst; - op->next = NULL; - return op; + return translate_gc_bif(stp, op, Bif); } static GenOp* @@ -4386,22 +4360,25 @@ gen_get_map_element(LoaderState* stp, GenOpArg Fail, GenOpArg Src, static int hash_internal_genop_arg(LoaderState* stp, GenOpArg Key, Uint32* hx) { + Eterm key_term; switch (Key.type) { case TAG_a: - *hx = atom_tab(atom_val(Key.val))->slot.bucket.hvalue; - return 1; + key_term = Key.val; + break; case TAG_i: - *hx = Key.val; - return 1; + key_term = make_small(Key.val); + break; case TAG_n: - *hx = make_internal_hash(NIL); - return 1; + key_term = NIL; + break; case TAG_q: - *hx = make_internal_hash(stp->literals[Key.val].term); - return 1; + key_term = stp->literals[Key.val].term; + break; default: return 0; } + *hx = erts_pd_make_hx(key_term); + return 1; } @@ -4556,7 +4533,7 @@ freeze_code(LoaderState* stp) * function table in the beginning of the file. */ - code_hdr->functions[stp->num_functions] = (codev + stp->ci - 1); + code_hdr->functions[stp->num_functions] = (ErtsCodeInfo*)(codev + stp->ci - 1); CHKBLK(ERTS_ALC_T_CODE,code_hdr); /* @@ -4816,16 +4793,16 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } ep = erts_export_put(stp->module, stp->export[i].function, stp->export[i].arity); - if (!on_load) { - ep->addressv[erts_staging_code_ix()] = address; - } else { + if (on_load) { /* * on_load: Don't make any of the exported functions * callable yet. Keep any function in the current * code callable. */ - ep->code[4] = (BeamInstr) address; + ep->beam[1] = (BeamInstr) address; } + else + ep->addressv[erts_staging_code_ix()] = address; } /* @@ -4871,7 +4848,7 @@ final_touch(LoaderState* stp, struct erl_module_instance* inst_p) } fe->address = code_ptr; #ifdef HIPE - hipe_set_closure_stub(fe, stp->lambdas[i].num_free); + hipe_set_closure_stub(fe); #endif } } @@ -4882,7 +4859,7 @@ transform_engine(LoaderState* st) { Uint op; int ap; /* Current argument. */ - Uint* restart; /* Where to restart if current match fails. */ + const Uint* restart; /* Where to restart if current match fails. */ GenOpArg var[TE_MAX_VARS]; /* Buffer for variables. */ GenOpArg* rest_args = NULL; int num_rest_args = 0; @@ -4891,7 +4868,7 @@ transform_engine(LoaderState* st) GenOp* instr; GenOp* first = st->genop; GenOp* keep = NULL; - Uint* pc; + const Uint* pc; static Uint restart_fail[1] = {TOP_fail}; ASSERT(gen_opc[first->op].transform != -1); @@ -5002,7 +4979,7 @@ transform_engine(LoaderState* st) if (i >= st->num_imports || st->import[i].bf == NULL) goto restart; if (bif_number != -1 && - bif_export[bif_number]->code[4] != (BeamInstr) st->import[i].bf) { + bif_export[bif_number]->beam[1] != (BeamInstr) st->import[i].bf) { goto restart; } } @@ -5627,18 +5604,16 @@ functions_in_module(Process* p, /* Process whose heap to use. */ hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = code_hdr->functions[i]; - Eterm name = (Eterm) func_info[3]; - int arity = (int) func_info[4]; + ErtsCodeInfo* ci = code_hdr->functions[i]; Eterm tuple; /* * If the function name is [], this entry is a stub for * a BIF that should be ignored. */ - ASSERT(is_atom(name) || is_nil(name)); - if (is_atom(name)) { - tuple = TUPLE2(hp, name, make_small(arity)); + ASSERT(is_atom(ci->mfa.function) || is_nil(ci->mfa.function)); + if (is_atom(ci->mfa.function)) { + tuple = TUPLE2(hp, ci->mfa.function, make_small(ci->mfa.arity)); hp += 3; result = CONS(hp, tuple, result); hp += 2; @@ -5726,17 +5701,17 @@ native_addresses(Process* p, BeamCodeHeader* code_hdr) hp = HAlloc(p, need); hp_end = hp + need; for (i = num_functions-1; i >= 0 ; i--) { - BeamInstr* func_info = code_hdr->functions[i]; - Eterm name = (Eterm) func_info[3]; - int arity = (int) func_info[4]; + ErtsCodeInfo *ci = code_hdr->functions[i]; Eterm tuple; - ASSERT(is_atom(name) || is_nil(name)); /* [] if BIF stub */ - if (func_info[1] != 0) { - Eterm addr; - ASSERT(is_atom(name)); - addr = erts_bld_uint(&hp, NULL, func_info[1]); - tuple = erts_bld_tuple(&hp, NULL, 3, name, make_small(arity), addr); + ASSERT(is_atom(ci->mfa.function) + || is_nil(ci->mfa.function)); /* [] if BIF stub */ + if (ci->native != 0) { + Eterm addr; + ASSERT(is_atom(ci->mfa.function)); + addr = erts_bld_uint(&hp, NULL, ci->native); + tuple = erts_bld_tuple(&hp, NULL, 3, ci->mfa.function, + make_small(ci->mfa.arity), addr); result = erts_bld_cons(&hp, NULL, tuple, result); } } @@ -5762,11 +5737,11 @@ exported_from_module(Process* p, /* Process whose heap to use. */ for (i = 0; i < export_list_size(code_ix); i++) { Export* ep = export_list(i,code_ix); - if (ep->code[0] == mod) { + if (ep->info.mfa.module == mod) { Eterm tuple; - if (ep->addressv[code_ix] == ep->code+3 && - ep->code[3] == (BeamInstr) em_call_error_handler) { + if (ep->addressv[code_ix] == ep->beam && + ep->beam[0] == (BeamInstr) em_call_error_handler) { /* There is a call to the function, but it does not exist. */ continue; } @@ -5776,7 +5751,8 @@ exported_from_module(Process* p, /* Process whose heap to use. */ hp = HAlloc(p, need); hend = hp + need; } - tuple = TUPLE2(hp, ep->code[1], make_small(ep->code[2])); + tuple = TUPLE2(hp, ep->info.mfa.function, + make_small(ep->info.mfa.arity)); hp += 3; result = CONS(hp, tuple, result); hp += 2; @@ -5850,7 +5826,6 @@ md5_of_module(Process* p, /* Process whose heap to use. */ Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) { - BeamInstr* current = fi->current; Eterm loc = NIL; if (fi->loc != LINE_INVALID_LOCATION) { @@ -5860,7 +5835,7 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) Eterm file_term = NIL; if (file == 0) { - Atom* ap = atom_tab(atom_val(fi->current[0])); + Atom* ap = atom_tab(atom_val(fi->mfa->module)); file_term = buf_to_intlist(&hp, ".erl", 4, NIL); file_term = buf_to_intlist(&hp, (char*)ap->name, ap->len, file_term); } else { @@ -5879,10 +5854,12 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) } if (is_list(args) || is_nil(args)) { - *mfa_p = TUPLE4(hp, current[0], current[1], args, loc); + *mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function, + args, loc); } else { - Eterm arity = make_small(current[2]); - *mfa_p = TUPLE4(hp, current[0], current[1], arity, loc); + Eterm arity = make_small(fi->mfa->arity); + *mfa_p = TUPLE4(hp, fi->mfa->module, fi->mfa->function, + arity, loc); } return hp + 5; } @@ -5893,9 +5870,9 @@ erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p) * the function. */ void -erts_set_current_function(FunctionInfo* fi, BeamInstr* current) +erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa) { - fi->current = current; + fi->mfa = mfa; fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; } @@ -5904,13 +5881,13 @@ erts_set_current_function(FunctionInfo* fi, BeamInstr* current) /* * Returns a pointer to {module, function, arity}, or NULL if not found. */ -BeamInstr* +ErtsCodeMFA* find_function_from_pc(BeamInstr* pc) { FunctionInfo fi; erts_lookup_function_info(&fi, pc, 0); - return fi.current; + return fi.mfa; } /* @@ -6027,24 +6004,21 @@ code_module_md5_1(BIF_ALIST_1) return res; } -#define WORDS_PER_FUNCTION 6 +#ifdef HIPE +#define WORDS_PER_FUNCTION (sizeof(ErtsCodeInfo) / sizeof(UWord) + 1) static BeamInstr* -make_stub(BeamInstr* fp, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) +make_stub(ErtsCodeInfo* info, Eterm mod, Eterm func, Uint arity, Uint native, BeamInstr OpCode) { - fp[0] = (BeamInstr) BeamOp(op_i_func_info_IaaI); - fp[1] = native; - fp[2] = mod; - fp[3] = func; - fp[4] = arity; -#ifdef HIPE - if (native) { - fp[5] = BeamOpCode(op_move_return_n); - hipe_mfa_save_orig_beam_op(mod, func, arity, fp+5); - } -#endif - fp[5] = OpCode; - return fp + WORDS_PER_FUNCTION; + DBG_TRACE_MFA(mod,func,arity,"make beam stub at %p", erts_codeinfo_to_code(info)); + ASSERT(WORDS_PER_FUNCTION == 6); + info->op = (BeamInstr) BeamOp(op_i_func_info_IaaI); + info->native = native; + info->mfa.module = mod; + info->mfa.function = func; + info->mfa.arity = arity; + erts_codeinfo_to_code(info)[0] = OpCode; + return erts_codeinfo_to_code(info)+1; } static byte* @@ -6103,22 +6077,17 @@ stub_read_export_table(LoaderState* stp) } static void -stub_final_touch(LoaderState* stp, BeamInstr* fp) +stub_final_touch(LoaderState* stp, ErtsCodeInfo* ci) { unsigned int i; unsigned int n = stp->num_exps; - Eterm mod = fp[2]; - Eterm function = fp[3]; - int arity = fp[4]; -#ifdef HIPE Lambda* lp; -#endif - if (is_bif(mod, function, arity)) { - fp[1] = 0; - fp[2] = 0; - fp[3] = 0; - fp[4] = 0; + if (is_bif(ci->mfa.module, ci->mfa.function, ci->mfa.arity)) { + ci->native = 0; + ci->mfa.module = 0; + ci->mfa.function = 0; + ci->mfa.arity = 0; return; } @@ -6127,9 +6096,14 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) */ for (i = 0; i < n; i++) { - if (stp->export[i].function == function && stp->export[i].arity == arity) { - Export* ep = erts_export_put(mod, function, arity); - ep->addressv[erts_staging_code_ix()] = fp+5; + if (stp->export[i].function == ci->mfa.function && + stp->export[i].arity == ci->mfa.arity) { + Export* ep = erts_export_put(ci->mfa.module, + ci->mfa.function, + ci->mfa.arity); + ep->addressv[erts_staging_code_ix()] = erts_codeinfo_to_code(ci); + DBG_TRACE_MFA_P(&ci->mfa,"set beam stub at %p in export at %p (code_ix=%d)", + erts_codeinfo_to_code(ci), ep, erts_staging_code_ix()); return; } } @@ -6139,16 +6113,14 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) * Search the lambda table to find out which. */ -#ifdef HIPE n = stp->num_lambdas; for (i = 0, lp = stp->lambdas; i < n; i++, lp++) { ErlFunEntry* fe = stp->lambdas[i].fe; - if (lp->function == function && lp->arity == arity) { - fp[5] = (Eterm) BeamOpCode(op_hipe_trap_call_closure); - fe->address = &(fp[5]); + if (lp->function == ci->mfa.function && lp->arity == ci->mfa.arity) { + *erts_codeinfo_to_code(ci) = (Eterm) BeamOpCode(op_hipe_trap_call_closure); + fe->address = erts_codeinfo_to_code(ci); } } -#endif return; } @@ -6157,10 +6129,9 @@ stub_final_touch(LoaderState* stp, BeamInstr* fp) [{Adr, Patchtyppe} | Addresses] and the address of a fun_entry. */ -int +static int patch(Eterm Addresses, Uint fe) { -#ifdef HIPE Eterm* listp; Eterm tuple; Eterm* tp; @@ -6196,15 +6167,13 @@ patch(Eterm Addresses, Uint fe) } -#endif return 1; } -int +static int patch_funentries(Eterm Patchlist) { -#ifdef HIPE while (!is_nil(Patchlist)) { Eterm Info; Eterm MFA; @@ -6293,23 +6262,23 @@ patch_funentries(Eterm Patchlist) return 0; } -#endif return 1; /* Signal that all went well */ } - /* * Do a dummy load of a module. No threaded code will be loaded. * Used for loading native code. * Will also patch all references to fun_entries to point to * the new fun_entries created. */ - Eterm -erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) +erts_make_stub_module(Process* p, Eterm hipe_magic_bin, Eterm Beam, Eterm Info) { Binary* magic; + Binary* hipe_magic; LoaderState* stp; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; BeamInstr Funcs; BeamInstr Patchlist; Eterm MD5Bin; @@ -6332,8 +6301,12 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) */ magic = erts_alloc_loader_state(); stp = ERTS_MAGIC_BIN_DATA(magic); + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); - if (is_not_atom(Mod)) { + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { goto error; } if (is_not_tuple(Info)) { @@ -6361,7 +6334,7 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Scan the Beam binary and read the interesting sections. */ - stp->module = Mod; + stp->module = hipe_stp->module; stp->group_leader = p->group_leader; stp->num_functions = n; if (!init_iff_file(stp, bytes, size)) { @@ -6469,20 +6442,17 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) * Set the pointer and make the stub. Put a return instruction * as the body until we know what kind of trap we should put there. */ - code_hdr->functions[i] = fp; -#ifdef HIPE + code_hdr->functions[i] = (ErtsCodeInfo*)fp; op = (Eterm) BeamOpCode(op_hipe_trap_call); /* Might be changed later. */ -#else - op = (Eterm) BeamOpCode(op_move_return_n); -#endif - fp = make_stub(fp, Mod, func, arity, (Uint)native_address, op); + fp = make_stub((ErtsCodeInfo*)fp, hipe_stp->module, func, arity, + (Uint)native_address, op); } /* * Insert the last pointer and the int_code_end instruction. */ - code_hdr->functions[i] = fp; + code_hdr->functions[i] = (ErtsCodeInfo*)fp; *fp++ = (BeamInstr) BeamOp(op_int_code_end); /* @@ -6515,11 +6485,20 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) } /* + * Initialise HiPE module + */ + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + /* * Insert the module in the module table. */ - rval = stub_insert_new_code(p, 0, p->group_leader, Mod, - code_hdr, code_size); + rval = stub_insert_new_code(p, 0, p->group_leader, hipe_stp->module, + code_hdr, code_size, hipe_code); if (rval != NIL) { goto error; } @@ -6530,23 +6509,74 @@ erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info) fp = code_base; for (i = 0; i < n; i++) { - stub_final_touch(stp, fp); + stub_final_touch(stp, (ErtsCodeInfo*)fp); fp += WORDS_PER_FUNCTION; } if (patch_funentries(Patchlist)) { + Eterm mod = hipe_stp->module; + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); - return Mod; + hipe_free_loader_state(hipe_stp); + + return mod; } error: + erts_free(ERTS_ALC_T_HIPE, hipe_code); erts_free_aligned_binary_bytes(temp_alloc); free_loader_state(magic); BIF_ERROR(p, BADARG); } +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin) +{ + Binary* hipe_magic; + HipeLoaderState* hipe_stp; + HipeModule *hipe_code; + Module* modp; + + if (!ERTS_TERM_IS_MAGIC_BINARY(hipe_magic_bin) || + !(hipe_magic = ((ProcBin*)binary_val(hipe_magic_bin))->val, + hipe_stp = hipe_get_loader_state(hipe_magic)) || + hipe_stp->module == NIL || hipe_stp->text_segment == 0) { + return 0; + } + + modp = erts_get_module(hipe_stp->module, erts_active_code_ix()); + if (!modp) + return 0; + + /* + * Initialise HiPE module + */ + hipe_code = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*hipe_code)); + hipe_code->text_segment = hipe_stp->text_segment; + hipe_code->text_segment_size = hipe_stp->text_segment_size; + hipe_code->data_segment = hipe_stp->data_segment; + hipe_code->first_hipe_ref = hipe_stp->new_hipe_refs; + hipe_code->first_hipe_sdesc = hipe_stp->new_hipe_sdesc; + + modp->curr.hipe_code = hipe_code; + + /* Prevent code from being freed */ + hipe_stp->text_segment = 0; + hipe_stp->data_segment = 0; + hipe_stp->new_hipe_refs = NULL; + hipe_stp->new_hipe_sdesc = NULL; + + return 1; +} + #undef WORDS_PER_FUNCTION +#endif /* HIPE */ + static int safe_mul(UWord a, UWord b, UWord* resp) { @@ -6560,3 +6590,46 @@ static int safe_mul(UWord a, UWord b, UWord* resp) } } +#ifdef ENABLE_DBG_TRACE_MFA + +#define MFA_MAX 10 +Eterm dbg_trace_m[MFA_MAX]; +Eterm dbg_trace_f[MFA_MAX]; +Uint dbg_trace_a[MFA_MAX]; +unsigned int dbg_trace_ix = 0; + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a) +{ + unsigned i = dbg_trace_ix++; + ASSERT(i < MFA_MAX); + dbg_trace_m[i] = am_atom_put(m, strlen(m)); + dbg_trace_f[i] = am_atom_put(f, strlen(f)); + dbg_trace_a[i] = a; +} + +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a) +{ + unsigned int i; + for (i = 0; i < dbg_trace_ix; ++i) { + if (m == dbg_trace_m[i] && + (!f || (f == dbg_trace_f[i] && a == dbg_trace_a[i]))) { + + return i+1; + } + } + return 0; +} + +void dbg_vtrace_mfa(unsigned ix, const char* format, ...) +{ + va_list arglist; + va_start(arglist, format); + ASSERT(--ix < MFA_MAX); + erts_fprintf(stderr, "MFA TRACE %T:%T/%u: ", + dbg_trace_m[ix], dbg_trace_f[ix], (int)dbg_trace_a[ix]); + + erts_vfprintf(stderr, format, arglist); + va_end(arglist); +} + +#endif /* ENABLE_DBG_TRACE_MFA */ diff --git a/erts/emulator/beam/beam_load.h b/erts/emulator/beam/beam_load.h index 9be5e14e40..d420a4346c 100644 --- a/erts/emulator/beam/beam_load.h +++ b/erts/emulator/beam/beam_load.h @@ -35,15 +35,7 @@ typedef struct gen_op_entry { int transform; } GenOpEntry; -extern GenOpEntry gen_opc[]; - -#ifdef NO_JUMP_TABLE -#define BeamOp(Op) (Op) -#else -extern void** beam_ops; -#define BeamOp(Op) beam_ops[(Op)] -#endif - +extern const GenOpEntry gen_opc[]; extern BeamInstr beam_debug_apply[]; extern BeamInstr* em_call_error_handler; @@ -115,7 +107,7 @@ typedef struct beam_code_header { * The actual loaded code (for the first function) start just beyond * this table. */ - BeamInstr* functions[1]; + ErtsCodeInfo* functions[1]; }BeamCodeHeader; @@ -152,4 +144,34 @@ struct BeamCodeLineTab_ { #define LOC_FILE(Loc) ((Loc) >> 24) #define LOC_LINE(Loc) ((Loc) & ((1 << 24)-1)) + +/* + * MFA event debug "tracing" usage: + * + * #define ENABLE_DBG_TRACE_MFA + * call dbg_set_traced_mfa("mymod","myfunc",arity) + * for the function(s) to trace, in some init function. + * + * Run and get stderr printouts when interesting things happen to your MFA. + */ +#ifdef ENABLE_DBG_TRACE_MFA + +void dbg_set_traced_mfa(const char* m, const char* f, Uint a); +int dbg_is_traced_mfa(Eterm m, Eterm f, Uint a); +void dbg_vtrace_mfa(unsigned ix, const char* format, ...); +#define DBG_TRACE_MFA(M,F,A,FMT, ...) do {\ + unsigned ix;\ + if ((ix=dbg_is_traced_mfa(M,F,A))) \ + dbg_vtrace_mfa(ix, FMT"\n", ##__VA_ARGS__);\ + }while(0) + +#define DBG_TRACE_MFA_P(MFA, FMT, ...) \ + DBG_TRACE_MFA((MFA)->module, (MFA)->function, (MFA)->arity, FMT, ##__VA_ARGS__) + +#else +# define dbg_set_traced_mfa(M,F,A) +# define DBG_TRACE_MFA(M,F,A,FMT, ...) +# define DBG_TRACE_MFA_P(MFA,FMT, ...) +#endif /* ENABLE_DBG_TRACE_MFA */ + #endif /* _BEAM_LOAD_H */ diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 55342a38c6..9b0335e83d 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -221,13 +221,13 @@ erts_ranges_sz(void) void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) { - BeamInstr** low; - BeamInstr** high; - BeamInstr** mid; + ErtsCodeInfo** low; + ErtsCodeInfo** high; + ErtsCodeInfo** mid; Range* rp; BeamCodeHeader* hdr; - fi->current = NULL; + fi->mfa = NULL; fi->needed = 5; fi->loc = LINE_INVALID_LOCATION; rp = find_range(pc); @@ -240,12 +240,12 @@ erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info) high = low + hdr->num_functions; while (low < high) { mid = low + (high-low) / 2; - if (pc < mid[0]) { + if (pc < (BeamInstr*)(mid[0])) { high = mid; - } else if (pc < mid[1]) { - fi->current = mid[0]+2; + } else if (pc < (BeamInstr*)(mid[1])) { + fi->mfa = &mid[0]->mfa; if (full_info) { - BeamInstr** fp = hdr->functions; + ErtsCodeInfo** fp = hdr->functions; int idx = mid - fp; lookup_loc(fi, pc, hdr, idx); } @@ -316,7 +316,7 @@ lookup_loc(FunctionInfo* fi, const BeamInstr* pc, file = LOC_FILE(fi->loc); if (file == 0) { /* Special case: Module name with ".erl" appended */ - Atom* mod_atom = atom_tab(atom_val(fi->current[0])); + Atom* mod_atom = atom_tab(atom_val(fi->mfa->module)); fi->needed += 2*(mod_atom->len+4); } else { Atom* ap = atom_tab(atom_val((fi->fname_ptr)[file-1])); diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c index d9048065c8..d886c2985e 100644 --- a/erts/emulator/beam/bif.c +++ b/erts/emulator/beam/bif.c @@ -3869,7 +3869,24 @@ BIF_RETTYPE garbage_collect_0(BIF_ALIST_0) { FLAGS(BIF_P) |= F_NEED_FULLSWEEP; erts_garbage_collect(BIF_P, 0, NULL, 0); - BIF_RET(am_true); + return am_true; +} + +/* + * Pass atom 'minor' for relaxed generational GC run. This is only + * recommendation, major run may still be chosen by VM. + * Pass atom 'major' for default behaviour - major GC run (fullsweep) + */ +BIF_RETTYPE +erts_internal_garbage_collect_1(BIF_ALIST_1) +{ + switch (BIF_ARG_1) { + case am_minor: break; + case am_major: FLAGS(BIF_P) |= F_NEED_FULLSWEEP; break; + default: BIF_ERROR(BIF_P, BADARG); + } + erts_garbage_collect(BIF_P, 0, NULL, 0); + return am_true; } /**********************************************************************/ @@ -4946,13 +4963,13 @@ void erts_init_trap_export(Export* ep, Eterm m, Eterm f, Uint a, int i; sys_memset((void *) ep, 0, sizeof(Export)); for (i=0; i<ERTS_NUM_CODE_IX; i++) { - ep->addressv[i] = &ep->code[3]; + ep->addressv[i] = ep->beam; } - ep->code[0] = m; - ep->code[1] = f; - ep->code[2] = a; - ep->code[3] = (BeamInstr) em_apply_bif; - ep->code[4] = (BeamInstr) bif; + ep->info.mfa.module = m; + ep->info.mfa.function = f; + ep->info.mfa.arity = a; + ep->beam[0] = (BeamInstr) em_apply_bif; + ep->beam[1] = (BeamInstr) bif; } void erts_init_bif(void) diff --git a/erts/emulator/beam/bif.h b/erts/emulator/beam/bif.h index 2203182a0d..0c85e19ef0 100644 --- a/erts/emulator/beam/bif.h +++ b/erts/emulator/beam/bif.h @@ -159,7 +159,7 @@ do { \ #define ERTS_BIF_ERROR_TRAPPED0(Proc, Reason, Bif) \ do { \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ return THE_NON_VALUE; \ } while (0) @@ -167,7 +167,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ return THE_NON_VALUE; \ } while (0) @@ -176,7 +176,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ return THE_NON_VALUE; \ @@ -186,7 +186,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ @@ -202,7 +202,7 @@ do { \ #define ERTS_BIF_PREP_ERROR_TRAPPED0(Ret, Proc, Reason, Bif) \ do { \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -210,7 +210,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ (Ret) = THE_NON_VALUE; \ } while (0) @@ -219,7 +219,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ (Ret) = THE_NON_VALUE; \ @@ -229,7 +229,7 @@ do { \ do { \ Eterm* reg = erts_proc_sched_data((Proc))->x_reg_array; \ (Proc)->freason = (Reason); \ - (Proc)->current = (Bif)->code; \ + (Proc)->current = &(Bif)->info.mfa; \ reg[0] = (Eterm) (A0); \ reg[1] = (Eterm) (A1); \ reg[2] = (Eterm) (A2); \ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 80db4eb6ff..32600f4338 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -23,22 +23,24 @@ # # Lines starting with '#' are ignored. # -# <bif-decl> ::= "bif" <bif> <C-name>* | "ubif" <bif> <C-name>* +# <bif-decl> ::= "bif" <bif> <C-name>* | +# "ubif" <bif> <C-name>* | +# "gcbif" <bif> <C-name>* # <bif> ::= <module> ":" <name> "/" <arity> # -# "ubif" is an unwrapped bif, i.e. a bif without a trace wrapper, -# or rather; the trace entry point in the export entry is the same -# as the normal entry point, and no trace wrapper is generated. +# ubif: Use for operators and guard BIFs that never build anything +# on the heap (such as tuple_size/1) and operators. # -# Important: Use "ubif" for guard BIFs and operators; use "bif" for ordinary BIFs. +# gcbif: Use for guard BIFs that may build on the heap (such as abs/1). +# +# bif: Use for all other BIFs. # # Add new BIFs to the end of the file. # -# Note: Guards BIFs require special support in the compiler (to be able to actually -# call them from within a guard). +# Note: Guards BIFs usually require special support in the compiler. # -ubif erlang:abs/1 +gcbif erlang:abs/1 bif erlang:adler32/1 bif erlang:adler32/2 bif erlang:adler32_combine/3 @@ -62,11 +64,12 @@ bif erlang:exit/1 bif erlang:exit/2 bif erlang:external_size/1 bif erlang:external_size/2 -ubif erlang:float/1 +gcbif erlang:float/1 bif erlang:float_to_list/1 bif erlang:float_to_list/2 bif erlang:fun_info/2 bif erlang:garbage_collect/0 +bif erts_internal:garbage_collect/1 bif erlang:get/0 bif erlang:get/1 bif erlang:get_keys/1 @@ -79,7 +82,7 @@ bif erlang:phash2/2 ubif erlang:hd/1 bif erlang:integer_to_list/1 bif erlang:is_alive/0 -ubif erlang:length/1 +gcbif erlang:length/1 bif erlang:link/1 bif erlang:list_to_atom/1 bif erlang:list_to_binary/1 @@ -126,10 +129,10 @@ bif erlang:processes/0 bif erlang:put/2 bif erlang:register/2 bif erlang:registered/0 -ubif erlang:round/1 +gcbif erlang:round/1 ubif erlang:self/0 bif erlang:setelement/3 -ubif erlang:size/1 +gcbif erlang:size/1 bif erlang:spawn/3 bif erlang:spawn_link/3 bif erlang:split_binary/2 @@ -139,7 +142,7 @@ bif erlang:term_to_binary/2 bif erlang:throw/1 bif erlang:time/0 ubif erlang:tl/1 -ubif erlang:trunc/1 +gcbif erlang:trunc/1 bif erlang:tuple_to_list/1 bif erlang:universaltime/0 bif erlang:universaltime_to_localtime/1 @@ -162,7 +165,7 @@ bif erts_internal:port_connect/2 bif erts_internal:request_system_task/3 bif erts_internal:request_system_task/4 -bif erts_internal:check_process_code/2 +bif erts_internal:check_process_code/1 bif erts_internal:map_to_tuple_keys/1 bif erts_internal:term_type/1 @@ -464,8 +467,8 @@ bif erlang:list_to_existing_atom/1 # ubif erlang:is_bitstring/1 ubif erlang:tuple_size/1 -ubif erlang:byte_size/1 -ubif erlang:bit_size/1 +gcbif erlang:byte_size/1 +gcbif erlang:bit_size/1 bif erlang:list_to_bitstring/1 bif erlang:bitstring_to_list/1 @@ -517,8 +520,8 @@ bif erlang:binary_to_term/2 # # The searching/splitting/substituting thingies # -ubif erlang:binary_part/2 -ubif erlang:binary_part/3 +gcbif erlang:binary_part/2 +gcbif erlang:binary_part/3 bif binary:compile_pattern/1 bif binary:match/2 @@ -610,7 +613,7 @@ bif os:unsetenv/1 bif re:inspect/2 ubif erlang:is_map/1 -ubif erlang:map_size/1 +gcbif erlang:map_size/1 bif maps:to_list/1 bif maps:find/2 bif maps:get/2 @@ -657,6 +660,16 @@ bif erlang:has_prepared_code_on_load/1 bif maps:take/2 # +# New in 20.0 +# + +gcbif erlang:floor/1 +gcbif erlang:ceil/1 +bif math:floor/1 +bif math:ceil/1 +bif math:fmod/2 + +# # Obsolete # diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 33433df68a..013f2e4c01 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -230,9 +230,9 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) * Display the initial function name */ erts_print(to, to_arg, "Spawned as: %T:%T/%bpu\n", - p->u.initial[INITIAL_MOD], - p->u.initial[INITIAL_FUN], - p->u.initial[INITIAL_ARI]); + p->u.initial.module, + p->u.initial.function, + p->u.initial.arity); if (p->current != NULL) { if (running) { @@ -241,9 +241,9 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "Current call: "); } erts_print(to, to_arg, "%T:%T/%bpu\n", - p->current[0], - p->current[1], - p->current[2]); + p->current->module, + p->current->function, + p->current->arity); } erts_print(to, to_arg, "Spawned by: %T\n", p->parent); @@ -290,9 +290,9 @@ print_process_info(fmtfn_t to, void *to_arg, Process *p) erts_print(to, to_arg, "timeout"); else erts_print(to, to_arg, "%T:%T/%bpu\n", - scb->ct[j]->code[0], - scb->ct[j]->code[1], - scb->ct[j]->code[2]); + scb->ct[j]->info.mfa.module, + scb->ct[j]->info.mfa.function, + scb->ct[j]->info.mfa.arity); } erts_print(to, to_arg, "\n"); } diff --git a/erts/emulator/beam/code_ix.h b/erts/emulator/beam/code_ix.h index 584a605771..1b451bf921 100644 --- a/erts/emulator/beam/code_ix.h +++ b/erts/emulator/beam/code_ix.h @@ -56,12 +56,49 @@ # endif # include "sys.h" #endif + +#include "beam_opcodes.h" + struct process; #define ERTS_NUM_CODE_IX 3 typedef unsigned ErtsCodeIndex; +typedef struct ErtsCodeMFA_ { + Eterm module; + Eterm function; + Uint arity; +} ErtsCodeMFA; + +/* + * The ErtsCodeInfo structure is used both in the Export entry + * and in the code as the function header. + */ + +/* If you change the size of this, you also have to update the code + in ops.tab to reflect the new func_info size */ +typedef struct ErtsCodeInfo_ { + BeamInstr op; /* OpCode(i_func_info) */ + BeamInstr native; /* Used by hipe and trace to store extra data */ + ErtsCodeMFA mfa; +} ErtsCodeInfo; + +/* Get the code associated with a ErtsCodeInfo ptr. */ +ERTS_GLB_INLINE +BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci); + +/* Get the ErtsCodeInfo for from a code ptr. */ +ERTS_GLB_INLINE +ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I); + +/* Get the code associated with a ErtsCodeMFA ptr. */ +ERTS_GLB_INLINE +BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa); + +/* Get the ErtsCodeMFA from a code ptr. */ +ERTS_GLB_INLINE +ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I); /* Called once at emulator initialization. */ @@ -121,10 +158,47 @@ void erts_abort_staging_code_ix(void); int erts_has_code_write_permission(void); #endif - +/* module/function/arity can be NIL/NIL/-1 when the MFA is pointing to some + invalid code, for instance unloaded_fun. */ +#define ASSERT_MFA(MFA) \ + ASSERT((is_atom((MFA)->module) || is_nil((MFA)->module)) && \ + (is_atom((MFA)->function) || is_nil((MFA)->function)) && \ + (((MFA)->arity >= 0 && (MFA)->arity < 1024) || (MFA)->arity == -1)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF +ERTS_GLB_INLINE +BeamInstr *erts_codeinfo_to_code(ErtsCodeInfo *ci) +{ + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT_MFA(&ci->mfa); + return (BeamInstr*)(ci + 1); +} + +ERTS_GLB_INLINE +ErtsCodeInfo *erts_code_to_codeinfo(BeamInstr *I) +{ + ErtsCodeInfo *ci = ((ErtsCodeInfo *)(((char *)(I)) - sizeof(ErtsCodeInfo))); + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI) || !ci->op); + ASSERT_MFA(&ci->mfa); + return ci; +} + +ERTS_GLB_INLINE +BeamInstr *erts_codemfa_to_code(ErtsCodeMFA *mfa) +{ + ASSERT_MFA(mfa); + return (BeamInstr*)(mfa + 1); +} + +ERTS_GLB_INLINE +ErtsCodeMFA *erts_code_to_codemfa(BeamInstr *I) +{ + ErtsCodeMFA *mfa = ((ErtsCodeMFA *)(((char *)(I)) - sizeof(ErtsCodeMFA))); + ASSERT_MFA(mfa); + return mfa; +} + extern erts_smp_atomic32_t the_active_code_index; extern erts_smp_atomic32_t the_staging_code_index; diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index ccc4cbad43..c4dcd6a3cc 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -40,7 +40,8 @@ static void move_one_frag(Eterm** hpp, ErlHeapFragment*, ErlOffHeap*, int); /* * Copy object "obj" to process p. */ -Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { +Eterm copy_object_x(Eterm obj, Process* to, Uint extra) +{ if (!is_immed(obj)) { Uint size = size_object(obj); Eterm* hp = HAllocX(to, size, extra); @@ -70,33 +71,46 @@ Eterm copy_object_x(Eterm obj, Process* to, Uint extra) { * Return the "flat" size of the object. */ -Uint size_object(Eterm obj) +#define in_literal_purge_area(PTR) \ + (lit_purge_ptr && ( \ + (lit_purge_ptr <= (PTR) && \ + (PTR) < (lit_purge_ptr + lit_purge_sz)))) + +Uint size_object_x(Eterm obj, erts_literal_area_t *litopt) { Uint sum = 0; Eterm* ptr; int arity; + Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL; + Uint lit_purge_sz = litopt ? litopt->lit_purge_sz : 0; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif - DECLARE_ESTACK(s); - VERBOSE(DEBUG_SHCOPY, ("[pid=%T] size_object %p\n", mypid, obj)); for (;;) { switch (primary_tag(obj)) { case TAG_PRIMARY_LIST: - sum += 2; ptr = list_val(obj); + if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) { + goto pop_next; + } + sum += 2; obj = *ptr++; if (!IS_CONST(obj)) { ESTACK_PUSH(s, obj); - } + } obj = *ptr; break; case TAG_PRIMARY_BOXED: { - Eterm hdr = *boxed_val(obj); + Eterm hdr; + ptr = boxed_val(obj); + if (litopt && erts_is_literal(obj,ptr) && !in_literal_purge_area(ptr)) { + goto pop_next; + } + hdr = *ptr; ASSERT(is_header(hdr)); switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -279,10 +293,6 @@ do { \ #define COUNT_OFF_HEAP (0) -#define IN_LITERAL_PURGE_AREA(info, ptr) \ - ((info)->range_ptr && ( \ - (info)->range_ptr <= (ptr) && \ - (ptr) < ((info)->range_ptr + (info)->range_sz))) /* * Return the real size of an object and find sharing information * This currently returns the same as erts_debug:size/1. @@ -599,7 +609,7 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, erts_literal_area_t *litopt) { char* hstart; Uint hsize; @@ -616,6 +626,8 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint Eterm hdr; Eterm *hend; int i; + Eterm *lit_purge_ptr = litopt ? litopt->lit_purge_ptr : NULL; + Uint lit_purge_sz = litopt ? litopt->lit_purge_sz : 0; #ifdef DEBUG Eterm org_obj = obj; Uint org_sz = sz; @@ -651,7 +663,6 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy: while (hp != htop) { obj = *hp; - switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: hp++; @@ -667,6 +678,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy_list: tailp = argp; + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *tailp = obj; + goto L_copy; + } for (;;) { tp = tailp; elem = CAR(objp); @@ -674,18 +689,23 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint hbot -= 2; CAR(hbot) = elem; tailp = &CDR(hbot); - } - else { + } else { CAR(htop) = elem; tailp = &CDR(htop); htop += 2; } *tp = make_list(tailp - 1); obj = CDR(objp); + if (!is_list(obj)) { break; } objp = list_val(obj); + + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *tailp = obj; + goto L_copy; + } } switch (primary_tag(obj)) { case TAG_PRIMARY_IMMED1: *tailp = obj; goto L_copy; @@ -695,7 +715,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint "%s, line %d: Internal error in copy_struct: 0x%08x\n", __FILE__, __LINE__,obj); } - + case TAG_PRIMARY_BOXED: if (ErtsInArea(boxed_val(obj),hstart,hsize)) { hp++; @@ -705,6 +725,10 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint L_copy_boxed: objp = boxed_val(obj); + if (litopt && erts_is_literal(obj,objp) && !in_literal_purge_area(objp)) { + *argp = obj; + break; + } hdr = *objp; switch (hdr & _TAG_HEADER_MASK) { case ARITYVAL_SUBTAG: @@ -765,7 +789,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint extra_bytes = 1; } else { extra_bytes = 0; - } + } real_size = size+extra_bytes; objp = binary_val(real_bin); if (thing_subtag(*objp) == HEAP_BINARY_SUBTAG) { @@ -780,7 +804,7 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint } else { ProcBin* from = (ProcBin *) objp; ProcBin* to; - + ASSERT(thing_subtag(*objp) == REFC_BINARY_SUBTAG); if (from->flags) { erts_emasculate_writable_binary(from); @@ -900,6 +924,12 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz = hend - hbot; } else { #ifdef DEBUG + if (!eq(org_obj, res)) { + erts_exit(ERTS_ABORT_EXIT, + "Internal error in copy_struct() when copying %T:" + " not equal to copy %T\n", + org_obj, res); + } if (htop != hbot) erts_exit(ERTS_ABORT_EXIT, "Internal error in copy_struct() when copying %T:" @@ -1036,6 +1066,8 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) Uint e; unsigned sz; Eterm* ptr; + Eterm *lit_purge_ptr = info->lit_purge_ptr; + Uint lit_purge_sz = info->lit_purge_sz; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); #endif @@ -1081,7 +1113,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (IN_LITERAL_PURGE_AREA(info,ptr)) + if (in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1132,7 +1164,7 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { VERBOSE(DEBUG_SHCOPY, ("[pid=%T] bypassed copying %p is %T\n", mypid, ptr, obj)); - if (IN_LITERAL_PURGE_AREA(info,ptr)) + if (in_literal_purge_area(ptr)) info->literal_size += size_object(obj); goto pop_next; } @@ -1298,6 +1330,8 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, Eterm* resp; Eterm *hbot, *hend; unsigned remaining; + Eterm *lit_purge_ptr = info->lit_purge_ptr; + Uint lit_purge_sz = info->lit_purge_sz; #ifdef DEBUG Eterm mypid = erts_get_current_pid(); Eterm saved_obj = obj; @@ -1347,11 +1381,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = list_val(obj); /* off heap list pointers are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + if (!in_literal_purge_area(ptr)) { *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1415,11 +1449,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, ptr = boxed_val(obj); /* off heap pointers to boxes are copied verbatim */ if (erts_is_literal(obj,ptr)) { - if (!IN_LITERAL_PURGE_AREA(info,ptr)) { + if (!in_literal_purge_area(ptr)) { *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz); + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1923,7 +1957,7 @@ move_one_frag(Eterm** hpp, ErlHeapFragment* frag, ErlOffHeap* off_heap, int lite if (is_header(val)) { struct erl_off_heap_header* hdr = (struct erl_off_heap_header*)hp; ASSERT(ptr + header_arity(val) < end); - MOVE_BOXED(ptr, val, hp, &dummy_ref); + MOVE_BOXED(ptr, val, hp, &dummy_ref); switch (val & _HEADER_SUBTAG_MASK) { case REFC_BINARY_SUBTAG: case FUN_SUBTAG: diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index d79245e0e6..990edb274e 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -25,6 +25,7 @@ /* define this to get a lot of debug output */ /* #define ERTS_DIST_MSG_DBG */ +/* #define ERTS_RAW_DIST_MSG_DBG */ #ifdef HAVE_CONFIG_H # include "config.h" diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 214fb1f2af..40a45c961f 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -373,10 +373,16 @@ set_default_exec_alloc_opts(struct au_init *ip) ip->init.util.rmbcmt = 0; ip->init.util.acul = 0; +# ifdef ERTS_HAVE_EXEC_MMAPPER ip->init.util.mseg_alloc = &erts_alcu_mmapper_mseg_alloc; ip->init.util.mseg_realloc = &erts_alcu_mmapper_mseg_realloc; ip->init.util.mseg_dealloc = &erts_alcu_mmapper_mseg_dealloc; ip->init.util.mseg_mmapper = &erts_exec_mmapper; +# else + ip->init.util.mseg_alloc = &erts_alcu_exec_mseg_alloc; + ip->init.util.mseg_realloc = &erts_alcu_exec_mseg_realloc; + ip->init.util.mseg_dealloc = &erts_alcu_exec_mseg_dealloc; +# endif } #endif /* ERTS_ALC_A_EXEC */ @@ -1571,7 +1577,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init) break; case 'X': if (has_prefix("scs", argv[i]+3)) { -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER init->mseg.exec_mmap.scs = #endif get_mb_value(argv[i]+6, argv, &i); @@ -2852,7 +2858,7 @@ erts_allocator_info(fmtfn_t to, void *arg) erts_print(to, arg, "=allocator:erts_mmap.literal_mmap\n"); erts_mmap_info(&erts_literal_mmapper, &to, arg, NULL, NULL, &emis); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER erts_print(to, arg, "=allocator:erts_mmap.exec_mmap\n"); erts_mmap_info(&erts_exec_mmapper, &to, arg, NULL, NULL, &emis); #endif @@ -3010,7 +3016,7 @@ erts_allocator_options(void *proc) #if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) terms[length++] = ERTS_MAKE_AM("literal_mmap"); #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER terms[length++] = ERTS_MAKE_AM("exec_mmap"); #endif features = length ? erts_bld_list(hpp, szp, length, terms) : NIL; @@ -3102,7 +3108,7 @@ reply_alloc_info(void *vair) # if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) struct erts_mmap_info_struct mmap_info_literal; # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER struct erts_mmap_info_struct mmap_info_exec; # endif #endif @@ -3232,7 +3238,7 @@ reply_alloc_info(void *vair) erts_bld_atom(hpp,szp,"literal_mmap"), ainfo); # endif -# ifdef ERTS_ALC_A_EXEC +# ifdef ERTS_HAVE_EXEC_MMAPPER ai_list = erts_bld_cons(hpp, szp, ainfo, ai_list); ainfo = (air->only_sz ? NIL : @@ -4129,12 +4135,20 @@ debug_free(ErtsAlcType_t n, void *extra, void *ptr) ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra; void *dptr; Uint size; + int free_pattern = n; ASSERT(ERTS_ALC_N_MIN <= n && n <= ERTS_ALC_N_MAX); dptr = check_memory_fence(ptr, &size, n, ERTS_ALC_O_FREE); - sys_memset((void *) dptr, n, size + FENCE_SZ); +#ifdef ERTS_ALC_A_EXEC +# if defined(__i386__) || defined(__x86_64__) + if (ERTS_ALC_T2A(ERTS_ALC_N2T(n)) == ERTS_ALC_A_EXEC) { + free_pattern = 0x0f; /* Illegal instruction */ + } +# endif +#endif + sys_memset((void *) dptr, free_pattern, size + FENCE_SZ); (*real_af->free)(n, real_af->extra, dptr); diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index f4ca689040..b4ee9cba33 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -907,7 +907,9 @@ erts_alcu_literal_32_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, #elif defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) -/* Used by literal allocator that has its own mmapper (super carrier) */ +/* For allocators that have their own mmapper (super carrier), + * like literal_alloc and exec_alloc on amd64 + */ void* erts_alcu_mmapper_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) { @@ -948,6 +950,50 @@ erts_alcu_mmapper_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, } #endif /* ARCH_64 && ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ +#if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) + +/* + * For exec_alloc on non-amd64 that just need memory with PROT_EXEC + */ +void* +erts_alcu_exec_mseg_alloc(Allctr_t *allctr, Uint *size_p, Uint flags) +{ + void* res = erts_alcu_mseg_alloc(allctr, size_p, flags); + + if (res) { + int r = mprotect(res, *size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void* +erts_alcu_exec_mseg_realloc(Allctr_t *allctr, void *seg, + Uint old_size, Uint *new_size_p) +{ + void *res; + + if (seg && old_size) { + int r = mprotect(seg, old_size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + res = erts_alcu_mseg_realloc(allctr, seg, old_size, new_size_p); + if (res) { + int r = mprotect(res, *new_size_p, PROT_EXEC | PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + } + return res; +} + +void +erts_alcu_exec_mseg_dealloc(Allctr_t *allctr, void *seg, Uint size, Uint flags) +{ + int r = mprotect(seg, size, PROT_READ | PROT_WRITE); + ASSERT(r == 0); (void)r; + erts_alcu_mseg_dealloc(allctr, seg, size, flags); +} +#endif /* ERTS_ALC_A_EXEC && !ERTS_HAVE_EXEC_MMAPPER */ + #endif /* HAVE_ERTS_MSEG */ void* @@ -1361,6 +1407,7 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); res = fix->list; if (res) { @@ -1372,8 +1419,6 @@ fix_cpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) fix_cpool_check_shrink(allctr, type, fix, NULL); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (size >= allctr->sbc_threshold) { Block_t *blk; blk = create_carrier(allctr, size, CFLG_SBC); @@ -1493,6 +1538,7 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) && type <= ERTS_ALC_N_MAX_A_FIXED_SIZE); fix = &allctr->fix[type - ERTS_ALC_N_MIN_A_FIXED_SIZE]; + ASSERT(size == fix->type_size); ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 1); fix->u.nocpool.used++; @@ -1515,8 +1561,6 @@ fix_nocpool_alloc(Allctr_t *allctr, ErtsAlcType_t type, Uint size) ERTS_DBG_CHK_FIX_LIST(allctr, fix, ix, 0); return res; } - if (size < 2*sizeof(UWord)) - size += sizeof(UWord); if (fix->u.nocpool.limit < fix->u.nocpool.used) fix->u.nocpool.limit = fix->u.nocpool.used; if (fix->u.nocpool.max_used < fix->u.nocpool.used) @@ -6039,7 +6083,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) goto error; allctr->min_block_size = UNIT_CEILING(allctr->min_block_size + sizeof(FreeBlkFtr_t)); -#if ERTS_SMP +#ifdef ERTS_SMP if (init->tpref) { Uint sz = ABLK_HDR_SZ; sz += (init->fix ? diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 81180382af..f570703f25 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -210,6 +210,12 @@ void* erts_alcu_mmapper_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); void* erts_alcu_mmapper_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); void erts_alcu_mmapper_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); # endif + +# if defined(ERTS_ALC_A_EXEC) && !defined(ERTS_HAVE_EXEC_MMAPPER) +void* erts_alcu_exec_mseg_alloc(Allctr_t*, Uint *size_p, Uint flags); +void* erts_alcu_exec_mseg_realloc(Allctr_t*, void *seg, Uint old_size, Uint *new_size_p); +void erts_alcu_exec_mseg_dealloc(Allctr_t*, void *seg, Uint size, Uint flags); +# endif #endif /* HAVE_ERTS_MSEG */ void* erts_alcu_sys_alloc(Allctr_t*, Uint *size_p, int superalign); diff --git a/erts/emulator/beam/erl_bif_ddll.c b/erts/emulator/beam/erl_bif_ddll.c index ef77201544..587859e413 100644 --- a/erts/emulator/beam/erl_bif_ddll.c +++ b/erts/emulator/beam/erl_bif_ddll.c @@ -1099,7 +1099,7 @@ void erts_ddll_increment_port_count(DE_Handle *dh) void erts_ddll_decrement_port_count(DE_Handle *dh) { assert_drv_list_locked(); -#if DEBUG +#ifdef DEBUG ASSERT(erts_smp_atomic32_dec_read_nob(&dh->port_count) >= 0); #else erts_smp_atomic32_dec_nob(&dh->port_count); diff --git a/erts/emulator/beam/erl_bif_guard.c b/erts/emulator/beam/erl_bif_guard.c index b42d2dc28b..458315f293 100644 --- a/erts/emulator/beam/erl_bif_guard.c +++ b/erts/emulator/beam/erl_bif_guard.c @@ -141,6 +141,39 @@ BIF_RETTYPE trunc_1(BIF_ALIST_1) BIF_RET(res); } +BIF_RETTYPE floor_1(BIF_ALIST_1) +{ + Eterm res; + FloatDef f; + + if (is_not_float(BIF_ARG_1)) { + if (is_integer(BIF_ARG_1)) + BIF_RET(BIF_ARG_1); + BIF_ERROR(BIF_P, BADARG); + } + GET_DOUBLE(BIF_ARG_1, f); + res = double_to_integer(BIF_P, floor(f.fd)); + BIF_RET(res); +} + +BIF_RETTYPE ceil_1(BIF_ALIST_1) +{ + Eterm res; + FloatDef f; + + /* check arg */ + if (is_not_float(BIF_ARG_1)) { + if (is_integer(BIF_ARG_1)) + BIF_RET(BIF_ARG_1); + BIF_ERROR(BIF_P, BADARG); + } + /* get the float */ + GET_DOUBLE(BIF_ARG_1, f); + + res = double_to_integer(BIF_P, ceil(f.fd)); + BIF_RET(res); +} + BIF_RETTYPE round_1(BIF_ALIST_1) { Eterm res; @@ -621,6 +654,38 @@ Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live) reg, live); } +Eterm erts_gc_floor_1(Process* p, Eterm* reg, Uint live) +{ + Eterm arg; + FloatDef f; + + arg = reg[live]; + if (is_not_float(arg)) { + if (is_integer(arg)) { + return arg; + } + BIF_ERROR(p, BADARG); + } + GET_DOUBLE(arg, f); + return gc_double_to_integer(p, floor(f.fd), reg, live); +} + +Eterm erts_gc_ceil_1(Process* p, Eterm* reg, Uint live) +{ + Eterm arg; + FloatDef f; + + arg = reg[live]; + if (is_not_float(arg)) { + if (is_integer(arg)) { + return arg; + } + BIF_ERROR(p, BADARG); + } + GET_DOUBLE(arg, f); + return gc_double_to_integer(p, ceil(f.fd), reg, live); +} + static Eterm gc_double_to_integer(Process* p, double x, Eterm* reg, Uint live) { diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 735aabbee3..0e0842e139 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -1114,9 +1114,9 @@ process_info_aux(Process *BIF_P, case am_initial_call: hp = HAlloc(BIF_P, 3+4); res = TUPLE3(hp, - rp->u.initial[INITIAL_MOD], - rp->u.initial[INITIAL_FUN], - make_small(rp->u.initial[INITIAL_ARI])); + rp->u.initial.module, + rp->u.initial.function, + make_small(rp->u.initial.arity)); hp += 4; break; @@ -1563,9 +1563,9 @@ process_info_aux(Process *BIF_P, term = am_timeout; else { term = TUPLE3(hp, - scb->ct[j]->code[0], - scb->ct[j]->code[1], - make_small(scb->ct[j]->code[2])); + scb->ct[j]->info.mfa.module, + scb->ct[j]->info.mfa.function, + make_small(scb->ct[j]->info.mfa.arity)); hp += 4; } list = CONS(hp, term, list); @@ -1614,10 +1614,10 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) if (rp->current == NULL) { erts_lookup_function_info(&fi, rp->i, full_info); - rp->current = fi.current; + rp->current = fi.mfa; } else if (full_info) { erts_lookup_function_info(&fi, rp->i, full_info); - if (fi.current == NULL) { + if (fi.mfa == NULL) { /* Use the current function without location info */ erts_set_current_function(&fi, rp->current); } @@ -1633,9 +1633,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) * instead if it can be looked up. */ erts_lookup_function_info(&fi2, rp->cp, full_info); - if (fi2.current) { + if (fi2.mfa) { fi = fi2; - rp->current = fi2.current; + rp->current = fi2.mfa; } } @@ -1650,8 +1650,9 @@ current_function(Process* BIF_P, Process* rp, Eterm** hpp, int full_info) hp = erts_build_mfa_item(&fi, hp, am_true, &res); } else { hp = HAlloc(BIF_P, 3+4); - res = TUPLE3(hp, rp->current[0], - rp->current[1], make_small(rp->current[2])); + res = TUPLE3(hp, rp->current->module, + rp->current->function, + make_small(rp->current->arity)); hp += 4; } *hpp = hp; @@ -1692,7 +1693,7 @@ current_stacktrace(Process* p, Process* rp, Eterm** hpp) heap_size = 3; for (i = 0; i < depth; i++) { erts_lookup_function_info(stkp, s->trace[i], 1); - if (stkp->current) { + if (stkp->mfa) { heap_size += stkp->needed + 2; stkp++; } @@ -2391,7 +2392,7 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) ERTS_ATOM_ENC_LATIN1, 1), erts_bld_uint(hpp, hszp, - opc[i].count)), + erts_instr_count[i])), res); } @@ -2859,6 +2860,9 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("ets_limit",BIF_ARG_1)) { BIF_RET(make_small(erts_db_get_max_tabs())); } + else if (ERTS_IS_ATOM_STR("atom_limit",BIF_ARG_1)) { + BIF_RET(make_small(erts_get_atom_limit())); + } else if (ERTS_IS_ATOM_STR("tolerant_timeofday",BIF_ARG_1)) { if (erts_has_time_correction() && erts_time_offset_state() == ERTS_TIME_OFFSET_FINAL) { @@ -2883,27 +2887,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1) BIF_RET(AM_tag); #endif } - else if (ERTS_IS_ATOM_STR("check_process_code",BIF_ARG_1)) { - Eterm terms[3]; - Sint length = 1; - Uint sz = 0; - Eterm *hp, res; - DECL_AM(direct_references); - - terms[0] = AM_direct_references; -#if !defined(ERTS_NEW_PURGE_STRATEGY) - { - DECL_AM(indirect_references); - terms[1] = AM_indirect_references; - terms[2] = am_copy_literals; - length = 3; - } -#endif - erts_bld_list(NULL, &sz, length, terms); - hp = HAlloc(BIF_P, sz); - res = erts_bld_list(&hp, NULL, length, terms); - BIF_RET(res); - } BIF_ERROR(BIF_P, BADARG); } @@ -3248,7 +3231,7 @@ fun_info_2(BIF_ALIST_2) break; case am_module: hp = HAlloc(p, 3); - val = exp->code[0]; + val = exp->info.mfa.module; break; case am_new_index: hp = HAlloc(p, 3); @@ -3276,11 +3259,11 @@ fun_info_2(BIF_ALIST_2) break; case am_arity: hp = HAlloc(p, 3); - val = make_small(exp->code[2]); + val = make_small(exp->info.mfa.arity); break; case am_name: hp = HAlloc(p, 3); - val = exp->code[1]; + val = exp->info.mfa.function; break; default: goto error; @@ -3306,7 +3289,9 @@ fun_info_mfa_1(BIF_ALIST_1) } else if (is_export(fun)) { Export* exp = (Export *) ((UWord) (export_val(fun))[1]); hp = HAlloc(p, 4); - BIF_RET(TUPLE3(hp,exp->code[0],exp->code[1],make_small(exp->code[2]))); + BIF_RET(TUPLE3(hp,exp->info.mfa.module, + exp->info.mfa.function, + make_small(exp->info.mfa.arity))); } BIF_ERROR(p, BADARG); } @@ -3563,8 +3548,14 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1) else if (ERTS_IS_ATOM_STR("DbTable_words", BIF_ARG_1)) { /* Used by ets_SUITE (stdlib) */ size_t words = (sizeof(DbTable) + sizeof(Uint) - 1)/sizeof(Uint); - BIF_RET(make_small((Uint) words)); + Eterm* hp = HAlloc(BIF_P ,3); + BIF_RET(TUPLE2(hp, make_small((Uint) words), + erts_ets_hash_sizeof_ext_segtab())); } + else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { + /* Used by ets_SUITE (stdlib) */ + BIF_RET(erts_ets_get_meta_state(BIF_P)); + } else if (ERTS_IS_ATOM_STR("check_io_debug", BIF_ARG_1)) { /* Used by driver_SUITE (emulator) */ Uint sz, *szp; @@ -4298,6 +4289,10 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } BIF_RET(am_ok); } + else if (ERTS_IS_ATOM_STR("DbTable_meta", BIF_ARG_1)) { + /* Used by ets_SUITE (stdlib) */ + BIF_RET(erts_ets_restore_meta_state(BIF_P, BIF_ARG_2)); + } } BIF_ERROR(BIF_P, BADARG); diff --git a/erts/emulator/beam/erl_bif_op.c b/erts/emulator/beam/erl_bif_op.c index aecb8bf0c1..a594ec1493 100644 --- a/erts/emulator/beam/erl_bif_op.c +++ b/erts/emulator/beam/erl_bif_op.c @@ -260,7 +260,7 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2) } else if (is_export(arg1)) { Export* exp = (Export *) (export_val(arg1)[1]); - if (exp->code[2] == (Uint) arity) { + if (exp->info.mfa.arity == (Uint) arity) { BIF_RET(am_true); } } diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 96275eb228..b1c2c427b0 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -125,7 +125,6 @@ erts_internal_trace_pattern_3(BIF_ALIST_3) static Eterm trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) { - DeclareTmpHeap(mfa,3,p); /* Not really heap here, but might be when setting pattern */ int i; int matches = -1; int specified = 0; @@ -308,30 +307,30 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) } matches = 0; } else if (is_tuple(MFA)) { + ErtsCodeMFA mfa; Eterm *tp = tuple_val(MFA); if (tp[0] != make_arityval(3)) { goto error; } - mfa[0] = tp[1]; - mfa[1] = tp[2]; - mfa[2] = tp[3]; - if (!is_atom(mfa[0]) || !is_atom(mfa[1]) || - (!is_small(mfa[2]) && mfa[2] != am_Underscore)) { + if (!is_atom(tp[1]) || !is_atom(tp[2]) || + (!is_small(tp[3]) && tp[3] != am_Underscore)) { goto error; } - for (i = 0; i < 3 && mfa[i] != am_Underscore; i++, specified++) { + for (i = 0; i < 3 && tp[i+1] != am_Underscore; i++, specified++) { /* Empty loop body */ } for (i = specified; i < 3; i++) { - if (mfa[i] != am_Underscore) { + if (tp[i+1] != am_Underscore) { goto error; } } - if (is_small(mfa[2])) { - mfa[2] = signed_val(mfa[2]); + mfa.module = tp[1]; + mfa.function = tp[2]; + if (specified == 3) { + mfa.arity = signed_val(tp[3]); } - matches = erts_set_trace_pattern(p, mfa, specified, + matches = erts_set_trace_pattern(p, &mfa, specified, match_prog_set, match_prog_set, on, flags, meta_tracer, 0); } else if (is_atom(MFA)) { @@ -343,7 +342,6 @@ trace_pattern(Process* p, Eterm MFA, Eterm Pattern, Eterm flaglist) error: MatchSetUnref(match_prog_set); - UnUseTmpHeap(3,p); ERTS_TRACER_CLEAR(&meta_tracer); @@ -975,13 +973,14 @@ static int function_is_traced(Process *p, Export e; Export* ep; BeamInstr* pc; + ErtsCodeInfo *ci; /* First look for an export entry */ - e.code[0] = mfa[0]; - e.code[1] = mfa[1]; - e.code[2] = mfa[2]; + e.info.mfa.module = mfa[0]; + e.info.mfa.function = mfa[1]; + e.info.mfa.arity = mfa[2]; if ((ep = export_get(&e)) != NULL) { - pc = ep->code+3; + pc = ep->beam; if (ep->addressv[erts_active_code_ix()] == pc && *pc != (BeamInstr) em_call_error_handler) { @@ -990,17 +989,17 @@ static int function_is_traced(Process *p, ASSERT(*pc == (BeamInstr) em_apply_bif || *pc == (BeamInstr) BeamOp(op_i_generic_breakpoint)); - if (erts_is_trace_break(pc, ms, 0)) { + if (erts_is_trace_break(&ep->info, ms, 0)) { return FUNC_TRACE_GLOBAL_TRACE; } - if (erts_is_trace_break(pc, ms, 1)) { + if (erts_is_trace_break(&ep->info, ms, 1)) { r |= FUNC_TRACE_LOCAL_TRACE; } - if (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta)) { + if (erts_is_mtrace_break(&ep->info, ms_meta, tracer_pid_meta)) { r |= FUNC_TRACE_META_TRACE; } - if (erts_is_time_break(p, pc, call_time)) { + if (erts_is_time_break(p, &ep->info, call_time)) { r |= FUNC_TRACE_TIME_TRACE; } return r ? r : FUNC_TRACE_UNTRACED; @@ -1008,15 +1007,15 @@ static int function_is_traced(Process *p, } /* OK, now look for breakpoint tracing */ - if ((pc = erts_find_local_func(mfa)) != NULL) { + if ((ci = erts_find_local_func(&e.info.mfa)) != NULL) { int r = - (erts_is_trace_break(pc, ms, 1) + (erts_is_trace_break(ci, ms, 1) ? FUNC_TRACE_LOCAL_TRACE : 0) - | (erts_is_mtrace_break(pc, ms_meta, tracer_pid_meta) + | (erts_is_mtrace_break(ci, ms_meta, tracer_pid_meta) ? FUNC_TRACE_META_TRACE : 0) - | (erts_is_count_break(pc, count) + | (erts_is_count_break(ci, count) ? FUNC_TRACE_COUNT_TRACE : 0) - | (erts_is_time_break(p, pc, call_time) + | (erts_is_time_break(p, ci, call_time) ? FUNC_TRACE_TIME_TRACE : 0); return r ? r : FUNC_TRACE_UNTRACED; @@ -1350,7 +1349,7 @@ trace_info_event(Process* p, Eterm event, Eterm key) #undef FUNC_TRACE_LOCAL_TRACE int -erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, +erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, Binary* match_prog_set, Binary *meta_match_prog_set, int on, struct trace_pattern_flags flags, ErtsTracer meta_tracer, int is_blocking) @@ -1370,22 +1369,23 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, n = finish_bp.e.matched; for (i = 0; i < n; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + ErtsCodeInfo *ci = fp[i].ci; + BeamInstr* pc = erts_codeinfo_to_code(ci); + Export* ep = ErtsContainerStruct(ci, Export, info); if (on && !flags.breakpoint) { /* Turn on global call tracing */ if (ep->addressv[code_ix] != pc) { fp[i].mod->curr.num_traced_exports++; #ifdef DEBUG - pc[-5] = (BeamInstr) BeamOp(op_i_func_info_IaaI); + ep->info.op = (BeamInstr) BeamOp(op_i_func_info_IaaI); #endif - pc[0] = (BeamInstr) BeamOp(op_jump_f); - pc[1] = (BeamInstr) ep->addressv[code_ix]; + ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); + ep->beam[1] = (BeamInstr) ep->addressv[code_ix]; } - erts_set_call_trace_bif(pc, match_prog_set, 0); + erts_set_call_trace_bif(ci, match_prog_set, 0); if (ep->addressv[code_ix] != pc) { - pc[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); + ep->beam[0] = (BeamInstr) BeamOp(op_i_generic_breakpoint); } } else if (!on && flags.breakpoint) { /* Turn off breakpoint tracing -- nothing to do here. */ @@ -1394,9 +1394,9 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, * Turn off global tracing, either explicitly or implicitly * before turning on breakpoint tracing. */ - erts_clear_call_trace_bif(pc, 0); - if (pc[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { - pc[0] = (BeamInstr) BeamOp(op_jump_f); + erts_clear_call_trace_bif(ci, 0); + if (ep->beam[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)) { + ep->beam[0] = (BeamInstr) BeamOp(op_jump_f); } } } @@ -1406,68 +1406,76 @@ erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, */ for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - int j; - + if (!ExportIsBuiltIn(ep)) { continue; } - + if (bif_table[i].f == bif_table[i].traced) { /* Trace wrapper same as regular function - untraceable */ continue; } - - for (j = 0; j < specified && mfa[j] == ep->code[j]; j++) { - /* Empty loop body */ - } - if (j == specified) { - BeamInstr* pc = (BeamInstr *)bif_export[i]->code + 3; - if (! flags.breakpoint) { /* Export entry call trace */ - if (on) { - erts_clear_call_trace_bif(pc, 1); - erts_clear_mtrace_bif(pc); - erts_set_call_trace_bif(pc, match_prog_set, 0); - } else { /* off */ - erts_clear_call_trace_bif(pc, 0); - } - matches++; - } else { /* Breakpoint call trace */ - int m = 0; - - if (on) { - if (flags.local) { - erts_clear_call_trace_bif(pc, 0); - erts_set_call_trace_bif(pc, match_prog_set, 1); - m = 1; - } - if (flags.meta) { - erts_set_mtrace_bif(pc, meta_match_prog_set, - meta_tracer); - m = 1; - } - if (flags.call_time) { - erts_set_time_trace_bif(pc, on); - /* I don't want to remove any other tracers */ - m = 1; - } - } else { /* off */ - if (flags.local) { - erts_clear_call_trace_bif(pc, 1); - m = 1; - } - if (flags.meta) { - erts_clear_mtrace_bif(pc); - m = 1; - } - if (flags.call_time) { - erts_clear_time_trace_bif(pc); - m = 1; - } - } - matches += m; - } - } + switch (specified) { + case 3: + if (mfa->arity != ep->info.mfa.arity) + continue; + case 2: + if (mfa->function != ep->info.mfa.function) + continue; + case 1: + if (mfa->module != ep->info.mfa.module) + continue; + case 0: + break; + default: + ASSERT(0); + } + + if (! flags.breakpoint) { /* Export entry call trace */ + if (on) { + erts_clear_call_trace_bif(&ep->info, 1); + erts_clear_mtrace_bif(&ep->info); + erts_set_call_trace_bif(&ep->info, match_prog_set, 0); + } else { /* off */ + erts_clear_call_trace_bif(&ep->info, 0); + } + matches++; + } else { /* Breakpoint call trace */ + int m = 0; + + if (on) { + if (flags.local) { + erts_clear_call_trace_bif(&ep->info, 0); + erts_set_call_trace_bif(&ep->info, match_prog_set, 1); + m = 1; + } + if (flags.meta) { + erts_set_mtrace_bif(&ep->info, meta_match_prog_set, + meta_tracer); + m = 1; + } + if (flags.call_time) { + erts_set_time_trace_bif(&ep->info, on); + /* I don't want to remove any other tracers */ + m = 1; + } + } else { /* off */ + if (flags.local) { + erts_clear_call_trace_bif(&ep->info, 1); + m = 1; + } + if (flags.meta) { + erts_clear_mtrace_bif(&ep->info); + m = 1; + } + if (flags.call_time) { + erts_clear_time_trace_bif(&ep->info); + m = 1; + } + } + matches += m; + } } /* @@ -1669,10 +1677,9 @@ install_exp_breakpoints(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - ep->addressv[code_ix] = pc; + ep->addressv[code_ix] = ep->beam; } } @@ -1685,14 +1692,13 @@ uninstall_exp_breakpoints(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] != pc) { + if (ep->addressv[code_ix] != ep->beam) { continue; } - ASSERT(*pc == (BeamInstr) BeamOp(op_jump_f)); - ep->addressv[code_ix] = (BeamInstr *) ep->code[4]; + ASSERT(ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)); + ep->addressv[code_ix] = (BeamInstr *) ep->beam[1]; } } @@ -1705,15 +1711,14 @@ clean_export_entries(BpFunctions* f) Uint i; for (i = 0; i < ne; i++) { - BeamInstr* pc = fp[i].pc; - Export* ep = ErtsContainerStruct(pc, Export, code[3]); + Export* ep = ErtsContainerStruct(fp[i].ci, Export, info); - if (ep->addressv[code_ix] == pc) { + if (ep->addressv[code_ix] == ep->beam) { continue; } - if (*pc == (BeamInstr) BeamOp(op_jump_f)) { - ep->code[3] = (BeamInstr) 0; - ep->code[4] = (BeamInstr) 0; + if (ep->beam[0] == (BeamInstr) BeamOp(op_jump_f)) { + ep->beam[0] = (BeamInstr) 0; + ep->beam[1] = (BeamInstr) 0; } } } @@ -1725,11 +1730,11 @@ setup_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - GenericBp* g = (GenericBp *) ep->fake_op_func_info_for_hipe[1]; + GenericBp* g = (GenericBp *) ep->info.native; if (g) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[4]); - ep->code[4] = (BeamInstr) bif_table[i].traced; + ASSERT(ep->beam[1]); + ep->beam[1] = (BeamInstr) bif_table[i].traced; } } } @@ -1743,12 +1748,11 @@ reset_bif_trace(void) for (i = 0; i < BIF_SIZE; ++i) { Export *ep = bif_export[i]; - BeamInstr* pc = ep->code+3; - GenericBp* g = (GenericBp *) pc[-4]; + GenericBp* g = (GenericBp *) ep->info.native; if (g && g->data[active].flags == 0) { if (ExportIsBuiltIn(ep)) { - ASSERT(ep->code[4]); - ep->code[4] = (BeamInstr) bif_table[i].f; + ASSERT(ep->beam[1]); + ep->beam[1] = (BeamInstr) bif_table[i].f; } } } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index dceadc46f4..fd092015bd 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -3978,6 +3978,36 @@ erts_ets_colliding_names(Process* p, Eterm name, Uint cnt) return list; } +/* + * For testing only + * Retreive meta table size state + */ +Eterm erts_ets_get_meta_state(Process* p) +{ + Eterm* hp = HAlloc(p, 3); + return TUPLE2(hp, + erts_ets_hash_get_memstate(p, &meta_pid_to_tab->hash), + erts_ets_hash_get_memstate(p, &meta_pid_to_fixed_tab->hash)); +} +/* + * For testing only + * Restore a previously retrieved meta table size state. + * We do this to suppress failed memory checks + * caused by the hysteresis of meta tables grow/shrink limits. + */ +Eterm erts_ets_restore_meta_state(Process* p, Eterm meta_state) +{ + Eterm* tv; + Eterm* hp; + if (!is_tuple_arity(meta_state, 2)) + return am_badarg; + + tv = tuple_val(meta_state); + hp = HAlloc(p, 3); + return TUPLE2(hp, + erts_ets_hash_restore_memstate(&meta_pid_to_tab->hash, tv[1]), + erts_ets_hash_restore_memstate(&meta_pid_to_fixed_tab->hash, tv[2])); +} #ifdef HARDDEBUG /* Here comes some debug functions */ diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index b0508f2e74..852440ff31 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -90,6 +90,8 @@ extern Export ets_select_continue_exp; extern erts_smp_atomic_t erts_ets_misc_mem_size; Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt); +Eterm erts_ets_get_meta_state(Process* p); +Eterm erts_ets_restore_meta_state(Process* p, Eterm target_state); Uint erts_db_get_max_tabs(void); @@ -267,7 +269,5 @@ erts_db_free_nt(ErtsAlcType_t type, void *ptr, Uint size) #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ -#undef ERTS_DB_ALC_MEM_UPDATE_ - #endif /* #if defined(ERTS_WANT_DB_INTERNAL__) && !defined(ERTS_HAVE_DB_INTERNAL__) */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 390369fdb9..b2a231467b 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -84,25 +84,28 @@ #include "erl_db_hash.h" -#ifdef MYDEBUG /* Will fail test case ets_SUITE:memory */ -# define IF_DEBUG(x) x -# define MY_ASSERT(x) ASSERT(x) -#else -# define IF_DEBUG(x) -# define MY_ASSERT(x) -#endif - /* * The following symbols can be manipulated to "tune" the linear hash array */ -#define CHAIN_LEN 6 /* Medium bucket chain len */ +#define GROW_LIMIT(NACTIVE) ((NACTIVE)*1) +#define SHRINK_LIMIT(NACTIVE) ((NACTIVE) / 2) -/* Number of slots per segment */ -#define SEGSZ_EXP 8 -#define SEGSZ (1 << SEGSZ_EXP) -#define SEGSZ_MASK (SEGSZ-1) +/* +** We want the first mandatory segment to be small (to reduce minimal footprint) +** and larger extra segments (to reduce number of alloc/free calls). +*/ -#define NSEG_1 2 /* Size of first segment table (must be at least 2) */ +/* Number of slots in first segment */ +#define FIRST_SEGSZ_EXP 8 +#define FIRST_SEGSZ (1 << FIRST_SEGSZ_EXP) +#define FIRST_SEGSZ_MASK (FIRST_SEGSZ - 1) + +/* Number of slots per extra segment */ +#define EXT_SEGSZ_EXP 11 +#define EXT_SEGSZ (1 << EXT_SEGSZ_EXP) +#define EXT_SEGSZ_MASK (EXT_SEGSZ-1) + +#define NSEG_1 (ErtsSizeofMember(DbTableHash,first_segtab) / sizeof(struct segment*)) #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ @@ -123,7 +126,9 @@ #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) -#define BUCKET(tb, i) SEGTAB(tb)[(i) >> SEGSZ_EXP]->buckets[(i) & SEGSZ_MASK] +#define SLOT_IX_TO_SEG_IX(i) (((i)+(EXT_SEGSZ-FIRST_SEGSZ)) >> EXT_SEGSZ_EXP) + +#define BUCKET(tb, i) SEGTAB(tb)[SLOT_IX_TO_SEG_IX(i)]->buckets[(i) & EXT_SEGSZ_MASK] /* * When deleting a table, the number of records to delete. @@ -189,6 +194,7 @@ static ERTS_INLINE int add_fixed_deletion(DbTableHash* tb, int ix, #ifdef ERTS_SMP # define DB_HASH_LOCK_MASK (DB_HASH_LOCK_CNT-1) # define GET_LOCK(tb,hval) (&(tb)->locks->lck_vec[(hval) & DB_HASH_LOCK_MASK].lck) +# define GET_LOCK_MAYBE(tb,hval) ((tb)->common.is_thread_safe ? NULL : GET_LOCK(tb,hval)) /* Fine grained read lock */ static ERTS_INLINE erts_smp_rwmtx_t* RLOCK_HASH(DbTableHash* tb, HashValue hval) @@ -318,27 +324,23 @@ struct mp_info { /* A table segment */ struct segment { - HashDbTerm* buckets[SEGSZ]; -#ifdef MYDEBUG - int is_ext_segment; -#endif + HashDbTerm* buckets[1]; }; +#define SIZEOF_SEGMENT(N) \ + (offsetof(struct segment,buckets) + sizeof(HashDbTerm*)*(N)) -/* A segment that also contains a segment table */ -struct ext_segment { - struct segment s; /* The segment itself. Must be first */ - +/* An extended segment table */ +struct ext_segtab { +#ifdef ERTS_SMP + ErtsThrPrgrLaterOp lop; +#endif struct segment** prev_segtab; /* Used when table is shrinking */ - int nsegs; /* Size of segtab */ + int prev_nsegs; /* Size of prev_segtab */ + int nsegs; /* Size of this segtab */ struct segment* segtab[1]; /* The segment table */ }; -#define SIZEOF_EXTSEG(NSEGS) \ - (offsetof(struct ext_segment,segtab) + sizeof(struct segment*)*(NSEGS)) - -#if defined(DEBUG) || defined(VALGRIND) -# define EXTSEG(SEGTAB_PTR) \ - ((struct ext_segment*) (((char*)(SEGTAB_PTR)) - offsetof(struct ext_segment,segtab))) -#endif +#define SIZEOF_EXT_SEGTAB(NSEGS) \ + (offsetof(struct ext_segtab,segtab) + sizeof(struct segment*)*(NSEGS)) static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, @@ -348,49 +350,22 @@ static ERTS_INLINE void SET_SEGTAB(DbTableHash* tb, erts_smp_atomic_set_wb(&tb->segtab, (erts_aint_t) segtab); else erts_smp_atomic_set_nob(&tb->segtab, (erts_aint_t) segtab); -#ifdef VALGRIND - tb->top_ptr_to_segment_with_active_segtab = EXTSEG(segtab); -#endif } -/* How the table segments relate to each other: - - ext_segment: ext_segment: "plain" segment - #=================# #================# #=============# - | bucket[0] |<--+ +------->| bucket[256] | +->| bucket[512] | - | bucket[1] | | | | [257] | | | [513] | - : : | | : : | : : - | bucket[255] | | | | [511] | | | [767] | - |-----------------| | | |----------------| | #=============# - | prev_segtab=NULL| | | +--<---prev_segtab | | - | nsegs = 2 | | | | | nsegs = 256 | | -+->| segtab[0] -->-------+---|---|--<---segtab[0] |<-+ | -| | segtab[1] -->-----------+---|--<---segtab[1] | | | -| #=================# | | segtab[2] -->-----|--+ ext_segment: -| | : : | #================# -+----------------<---------------+ | segtab[255] ->----|----->| bucket[255*256]| - #================# | | | - | : : - | |----------------| - +----<---prev_segtab | - : : -*/ - /* ** Forward decl's (static functions) */ -static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix, - struct segment** old_segtab); -static int alloc_seg(DbTableHash *tb); +static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix); +static void alloc_seg(DbTableHash *tb); static int free_seg(DbTableHash *tb, int free_records); static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_smp_rwmtx_t** lck_ptr, HashDbTerm *list); static HashDbTerm* search_list(DbTableHash* tb, Eterm key, HashValue hval, HashDbTerm *list); -static void shrink(DbTableHash* tb, int nactive); -static void grow(DbTableHash* tb, int nactive); +static void shrink(DbTableHash* tb, int nitems); +static void grow(DbTableHash* tb, int nitems); static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, Uint sz, DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, @@ -463,9 +438,10 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle); static ERTS_INLINE void try_shrink(DbTableHash* tb) { int nactive = NACTIVE(tb); - if (nactive > SEGSZ && NITEMS(tb) < (nactive * CHAIN_LEN) + int nitems = NITEMS(tb); + if (nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive) && !IS_FIXED(tb)) { - shrink(tb, nactive); + shrink(tb, nitems); } } @@ -662,16 +638,20 @@ int db_create_hash(Process *p, DbTable *tbl) { DbTableHash *tb = &tbl->hash; - erts_smp_atomic_init_nob(&tb->szm, SEGSZ_MASK); - erts_smp_atomic_init_nob(&tb->nactive, SEGSZ); + erts_smp_atomic_init_nob(&tb->szm, FIRST_SEGSZ_MASK); + erts_smp_atomic_init_nob(&tb->nactive, FIRST_SEGSZ); erts_smp_atomic_init_nob(&tb->fixdel, (erts_aint_t)NULL); erts_smp_atomic_init_nob(&tb->segtab, (erts_aint_t)NULL); - SET_SEGTAB(tb, alloc_ext_seg(tb,0,NULL)->segtab); + SET_SEGTAB(tb, tb->first_segtab); tb->nsegs = NSEG_1; - tb->nslots = SEGSZ; + tb->nslots = FIRST_SEGSZ; + tb->first_segtab[0] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_SEGMENT(FIRST_SEGSZ)); + sys_memset(tb->first_segtab[0], 0, SIZEOF_SEGMENT(FIRST_SEGSZ)); - erts_smp_atomic_init_nob(&tb->is_resizing, 0); #ifdef ERTS_SMP + erts_smp_atomic_init_nob(&tb->is_resizing, 0); if (tb->common.type & DB_FINE_LOCKED) { erts_smp_rwmtx_opt_t rwmtx_opt = ERTS_SMP_RWMTX_OPT_DEFAULT_INITER; int i; @@ -686,7 +666,7 @@ int db_create_hash(Process *p, DbTable *tbl) erts_smp_rwmtx_init_opt_x(&tb->locks->lck_vec[i].lck, &rwmtx_opt, "db_hash_slot", make_small(i)); } - /* This important property is needed to guarantee that the buckets + /* This important property is needed to guarantee the two buckets * involved in a grow/shrink operation it protected by the same lock: */ ASSERT(erts_smp_atomic_read_nob(&tb->nactive) % DB_HASH_LOCK_CNT == 0); @@ -862,8 +842,8 @@ Lnew: WUNLOCK_HASH(lck); { int nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN+1) && !IS_FIXED(tb)) { - grow(tb, nactive); + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { + grow(tb, nitems); } } CHECK_TABLES(); @@ -1514,6 +1494,7 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, match_list = CONS(hp, match_res, match_list); ++got; } + --num_left; } current = current->next; } @@ -1531,7 +1512,6 @@ static int db_select_chunk_hash(Process *p, DbTable *tbl, } } else { /* Key is variable */ - --num_left; if ((slot_ix=next_slot(tb,slot_ix,&lck)) == 0) { slot_ix = -1; @@ -2250,12 +2230,12 @@ static int db_free_table_continue_hash(DbTable *tbl) done /= 2; while(tb->nslots != 0) { - free_seg(tb, 1); + done += 1 + EXT_SEGSZ/64 + free_seg(tb, 1); /* * If we have done enough work, get out here. */ - if (++done >= (DELETE_RECORD_LIMIT / CHAIN_LEN / SEGSZ)) { + if (done >= DELETE_RECORD_LIMIT) { return 0; /* Not done */ } } @@ -2414,90 +2394,81 @@ static int analyze_pattern(DbTableHash *tb, Eterm pattern, return DB_ERROR_NONE; } -static struct ext_segment* alloc_ext_seg(DbTableHash* tb, unsigned seg_ix, - struct segment** old_segtab) +static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix) { - int nsegs; - struct ext_segment* eseg; + struct segment** old_segtab = SEGTAB(tb); + int nsegs = 0; + struct ext_segtab* est; + ASSERT(seg_ix >= NSEG_1); switch (seg_ix) { - case 0: nsegs = NSEG_1; break; - case 1: nsegs = NSEG_2; break; - default: nsegs = seg_ix + NSEG_INC; break; - } - eseg = (struct ext_segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, - (DbTable *) tb, - SIZEOF_EXTSEG(nsegs)); - ASSERT(eseg != NULL); - sys_memset(&eseg->s, 0, sizeof(struct segment)); - IF_DEBUG(eseg->s.is_ext_segment = 1); - eseg->prev_segtab = old_segtab; - eseg->nsegs = nsegs; - if (old_segtab) { - ASSERT(nsegs > tb->nsegs); - sys_memcpy(eseg->segtab, old_segtab, tb->nsegs*sizeof(struct segment*)); - } + case NSEG_1: nsegs = NSEG_2; break; + default: nsegs = seg_ix + NSEG_INC; break; + } + ASSERT(nsegs > tb->nsegs); + est = (struct ext_segtab*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_EXT_SEGTAB(nsegs)); + est->nsegs = nsegs; + est->prev_segtab = old_segtab; + est->prev_nsegs = tb->nsegs; + sys_memcpy(est->segtab, old_segtab, tb->nsegs*sizeof(struct segment*)); #ifdef DEBUG - sys_memset(&eseg->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*)); + sys_memset(&est->segtab[seg_ix], 0, (nsegs-seg_ix)*sizeof(struct segment*)); #endif - eseg->segtab[seg_ix] = &eseg->s; - return eseg; + return est; } /* Extend table with one new segment */ -static int alloc_seg(DbTableHash *tb) +static void alloc_seg(DbTableHash *tb) { - int seg_ix = tb->nslots >> SEGSZ_EXP; - - if (seg_ix+1 == tb->nsegs) { /* New segtab needed (extended segment) */ - struct segment** segtab = SEGTAB(tb); - struct ext_segment* seg = alloc_ext_seg(tb, seg_ix, segtab); - if (seg == NULL) return 0; - segtab[seg_ix] = &seg->s; - /* We don't use the new segtab until next call (see "shrink race") */ - } - else { /* Just a new plain segment */ - struct segment** segtab; - if (seg_ix == tb->nsegs) { /* Time to start use segtab from last call */ - struct ext_segment* eseg; - eseg = (struct ext_segment*) SEGTAB(tb)[seg_ix-1]; - MY_ASSERT(eseg!=NULL && eseg->s.is_ext_segment); - SET_SEGTAB(tb, eseg->segtab); - tb->nsegs = eseg->nsegs; - } - ASSERT(seg_ix < tb->nsegs); - segtab = SEGTAB(tb); - ASSERT(segtab[seg_ix] == NULL); - segtab[seg_ix] = (struct segment*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, - (DbTable *) tb, - sizeof(struct segment)); - if (segtab[seg_ix] == NULL) return 0; - sys_memset(segtab[seg_ix], 0, sizeof(struct segment)); - } - tb->nslots += SEGSZ; - return 1; + int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots); + struct segment** segtab; + + ASSERT(seg_ix > 0); + if (seg_ix == tb->nsegs) { /* New segtab needed */ + struct ext_segtab* est = alloc_ext_segtab(tb, seg_ix); + SET_SEGTAB(tb, est->segtab); + tb->nsegs = est->nsegs; + } + ASSERT(seg_ix < tb->nsegs); + segtab = SEGTAB(tb); + segtab[seg_ix] = (struct segment*) erts_db_alloc(ERTS_ALC_T_DB_SEG, + (DbTable *) tb, + SIZEOF_SEGMENT(EXT_SEGSZ)); + sys_memset(segtab[seg_ix], 0, SIZEOF_SEGMENT(EXT_SEGSZ)); + tb->nslots += EXT_SEGSZ; +} + +#ifdef ERTS_SMP +static void dealloc_ext_segtab(void* lop_data) +{ + struct ext_segtab* est = (struct ext_segtab*) lop_data; + + erts_free(ERTS_ALC_T_DB_SEG, est); } +#endif /* Shrink table by freeing the top segment ** free_records: 1=free any records in segment, 0=assume segment is empty */ static int free_seg(DbTableHash *tb, int free_records) { - const int seg_ix = (tb->nslots >> SEGSZ_EXP) - 1; + const int seg_ix = SLOT_IX_TO_SEG_IX(tb->nslots) - 1; struct segment** const segtab = SEGTAB(tb); - struct ext_segment* const top = (struct ext_segment*) segtab[seg_ix]; - int bytes; + struct segment* const segp = segtab[seg_ix]; + Uint seg_sz; int nrecords = 0; - ASSERT(top != NULL); + ASSERT(segp != NULL); #ifndef DEBUG if (free_records) #endif { - int i; - for (i=0; i<SEGSZ; ++i) { - HashDbTerm* p = top->s.buckets[i]; + int i = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; + while (i--) { + HashDbTerm* p = segp->buckets[i]; while(p != 0) { HashDbTerm* nxt = p->next; ASSERT(free_records); /* segment not empty as assumed? */ @@ -2507,55 +2478,46 @@ static int free_seg(DbTableHash *tb, int free_records) } } } - - /* The "shrink race": - * We must avoid deallocating an extended segment while its segtab may - * still be used by other threads. - * The trick is to stop use a segtab one call earlier. That is, stop use - * a segtab when the segment above it is deallocated. When the segtab is - * later deallocated, it has not been used for a very long time. - * It is even theoretically safe as we have by then rehashed the entire - * segment, seizing *all* locks, so there cannot exist any retarded threads - * still hanging in BUCKET macro with an old segtab pointer. - * For this to work, we must of course allocate a new segtab one call - * earlier in alloc_seg() as well. And this is also the reason why - * the minimum size of the first segtab is 2 and not 1 (NSEG_1). - */ - if (seg_ix == tb->nsegs-1 || seg_ix==0) { /* Dealloc extended segment */ - MY_ASSERT(top->s.is_ext_segment); - ASSERT(segtab != top->segtab || seg_ix==0); - bytes = SIZEOF_EXTSEG(top->nsegs); - } - else { /* Dealloc plain segment */ - struct ext_segment* newtop = (struct ext_segment*) segtab[seg_ix-1]; - MY_ASSERT(!top->s.is_ext_segment); - - if (segtab == newtop->segtab) { /* New top segment is extended */ - MY_ASSERT(newtop->s.is_ext_segment); - if (newtop->prev_segtab != NULL) { - /* Time to use a smaller segtab */ - SET_SEGTAB(tb, newtop->prev_segtab); - tb->nsegs = seg_ix; - ASSERT(tb->nsegs == EXTSEG(SEGTAB(tb))->nsegs); - } - else { - ASSERT(NSEG_1 > 2 && seg_ix==1); - } - } - bytes = sizeof(struct segment); + if (seg_ix >= NSEG_1) { + struct ext_segtab* est = ErtsContainerStruct_(segtab,struct ext_segtab,segtab); + + if (seg_ix == est->prev_nsegs) { /* Dealloc extended segtab */ + ASSERT(est->prev_segtab != NULL); + SET_SEGTAB(tb, est->prev_segtab); + tb->nsegs = est->prev_nsegs; + +#ifdef ERTS_SMP + if (!tb->common.is_thread_safe) { + /* + * Table is doing a graceful shrink operation and we must avoid + * deallocating this segtab while it may still be read by other + * threads. Schedule deallocation with thread progress to make + * sure no lingering threads are still hanging in BUCKET macro + * with an old segtab pointer. + */ + Uint sz = SIZEOF_EXT_SEGTAB(est->nsegs); + ASSERT(sz == ERTS_ALC_DBG_BLK_SZ(est)); + ERTS_DB_ALC_MEM_UPDATE_(tb, sz, 0); + erts_schedule_thr_prgr_later_cleanup_op(dealloc_ext_segtab, + est, + &est->lop, + sz); + } + else +#endif + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable*)tb, est, + SIZEOF_EXT_SEGTAB(est->nsegs)); + } } + seg_sz = (seg_ix == 0) ? FIRST_SEGSZ : EXT_SEGSZ; + erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, segp, SIZEOF_SEGMENT(seg_sz)); - erts_db_free(ERTS_ALC_T_DB_SEG, (DbTable *)tb, - (void*)top, bytes); #ifdef DEBUG - if (seg_ix > 0) { - segtab[seg_ix] = NULL; - } else { - SET_SEGTAB(tb, NULL); - } + if (seg_ix < tb->nsegs) + SEGTAB(tb)[seg_ix] = NULL; #endif - tb->nslots -= SEGSZ; + tb->nslots -= seg_sz; ASSERT(tb->nslots >= 0); return nrecords; } @@ -2604,104 +2566,111 @@ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, static ERTS_INLINE int begin_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - return !erts_smp_atomic_xchg_acqb(&tb->is_resizing, 1); - else { - if (erts_smp_atomic_read_nob(&tb->is_resizing)) - return 0; - erts_smp_atomic_set_nob(&tb->is_resizing, 1); - return 1; - } + return !erts_atomic_xchg_acqb(&tb->is_resizing, 1); + else + ERTS_LC_ASSERT(erts_lc_rwmtx_is_rwlocked(&tb->common.rwlock)); +#endif + return 1; } static ERTS_INLINE void done_resizing(DbTableHash* tb) { +#ifdef ERTS_SMP if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->is_resizing, 0); - else - erts_smp_atomic_set_nob(&tb->is_resizing, 0); + erts_atomic_set_relb(&tb->is_resizing, 0); +#endif } -/* Grow table with one new bucket. +/* Grow table with one or more new buckets. ** Allocate new segment if needed. */ -static void grow(DbTableHash* tb, int nactive) +static void grow(DbTableHash* tb, int nitems) { HashDbTerm** pnext; HashDbTerm** to_pnext; HashDbTerm* p; erts_smp_rwmtx_t* lck; - int from_ix; + int nactive; + int from_ix, to_ix; int szm; + int loop_limit = 5; - if (!begin_resizing(tb)) - return; /* already in progress */ - if (NACTIVE(tb) != nactive) { - goto abort; /* already done (race) */ - } - - /* Ensure that the slot nactive exists */ - if (nactive == tb->nslots) { - /* Time to get a new segment */ - ASSERT((nactive & SEGSZ_MASK) == 0); - if (!alloc_seg(tb)) goto abort; - } - ASSERT(nactive < tb->nslots); + do { + if (!begin_resizing(tb)) + return; /* already in progress */ + nactive = NACTIVE(tb); + if (nitems <= GROW_LIMIT(nactive)) { + goto abort; /* already done (race) */ + } - szm = erts_smp_atomic_read_nob(&tb->szm); - if (nactive <= szm) { - from_ix = nactive & (szm >> 1); - } else { - ASSERT(nactive == szm+1); - from_ix = 0; - szm = (szm<<1) | 1; - } + /* Ensure that the slot nactive exists */ + if (nactive == tb->nslots) { + /* Time to get a new segment */ + ASSERT(((nactive-FIRST_SEGSZ) & EXT_SEGSZ_MASK) == 0); + alloc_seg(tb); + } + ASSERT(nactive < tb->nslots); - lck = WLOCK_HASH(tb, from_ix); - /* Now a final double check (with the from_ix lock held) - * that we did not get raced by a table fixer. - */ - if (IS_FIXED(tb)) { - WUNLOCK_HASH(lck); - goto abort; - } - erts_smp_atomic_inc_nob(&tb->nactive); - if (from_ix == 0) { - if (DB_USING_FINE_LOCKING(tb)) - erts_smp_atomic_set_relb(&tb->szm, szm); - else - erts_smp_atomic_set_nob(&tb->szm, szm); - } - done_resizing(tb); + szm = erts_smp_atomic_read_nob(&tb->szm); + if (nactive <= szm) { + from_ix = nactive & (szm >> 1); + } else { + ASSERT(nactive == szm+1); + from_ix = 0; + szm = (szm<<1) | 1; + } + to_ix = nactive; + + lck = WLOCK_HASH(tb, from_ix); + ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,to_ix)); + /* Now a final double check (with the from_ix lock held) + * that we did not get raced by a table fixer. + */ + if (IS_FIXED(tb)) { + WUNLOCK_HASH(lck); + goto abort; + } + erts_smp_atomic_set_nob(&tb->nactive, ++nactive); + if (from_ix == 0) { + if (DB_USING_FINE_LOCKING(tb)) + erts_smp_atomic_set_relb(&tb->szm, szm); + else + erts_smp_atomic_set_nob(&tb->szm, szm); + } + done_resizing(tb); + + /* Finally, let's split the bucket. We try to do it in a smart way + to keep link order and avoid unnecessary updates of next-pointers */ + pnext = &BUCKET(tb, from_ix); + p = *pnext; + to_pnext = &BUCKET(tb, to_ix); + while (p != NULL) { + if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */ + *pnext = p->next; + free_term(tb, p); + p = *pnext; + } + else { + int ix = p->hvalue & szm; + if (ix != from_ix) { + ASSERT(ix == (from_ix ^ ((szm+1)>>1))); + *to_pnext = p; + /* Swap "from" and "to": */ + from_ix = ix; + to_pnext = pnext; + } + pnext = &p->next; + p = *pnext; + } + } + *to_pnext = NULL; + WUNLOCK_HASH(lck); - /* Finally, let's split the bucket. We try to do it in a smart way - to keep link order and avoid unnecessary updates of next-pointers */ - pnext = &BUCKET(tb, from_ix); - p = *pnext; - to_pnext = &BUCKET(tb, nactive); - while (p != NULL) { - if (p->hvalue == INVALID_HASH) { /* rare but possible with fine locking */ - *pnext = p->next; - free_term(tb, p); - p = *pnext; - } - else { - int ix = p->hvalue & szm; - if (ix != from_ix) { - ASSERT(ix == (from_ix ^ ((szm+1)>>1))); - *to_pnext = p; - /* Swap "from" and "to": */ - from_ix = ix; - to_pnext = pnext; - } - pnext = &p->next; - p = *pnext; - } - } - *to_pnext = NULL; + }while (--loop_limit && nitems > GROW_LIMIT(nactive)); - WUNLOCK_HASH(lck); return; abort: @@ -2712,60 +2681,78 @@ abort: /* Shrink table by joining top bucket. ** Remove top segment if it gets empty. */ -static void shrink(DbTableHash* tb, int nactive) -{ - if (!begin_resizing(tb)) - return; /* already in progress */ - if (NACTIVE(tb) == nactive) { - erts_smp_rwmtx_t* lck; - int src_ix = nactive - 1; - int low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; - int dst_ix = src_ix & low_szm; - - ASSERT(dst_ix < src_ix); - ASSERT(nactive > SEGSZ); - lck = WLOCK_HASH(tb, dst_ix); - /* Double check for racing table fixers */ - if (!IS_FIXED(tb)) { - HashDbTerm** src_bp = &BUCKET(tb, src_ix); - HashDbTerm** dst_bp = &BUCKET(tb, dst_ix); - HashDbTerm** bp = src_bp; - - /* Q: Why join lists by appending "dst" at the end of "src"? - A: Must step through "src" anyway to purge pseudo deleted. */ - while(*bp != NULL) { - if ((*bp)->hvalue == INVALID_HASH) { - HashDbTerm* deleted = *bp; - *bp = deleted->next; - free_term(tb, deleted); - } else { - bp = &(*bp)->next; - } - } - *bp = *dst_bp; - *dst_bp = *src_bp; - *src_bp = NULL; - - erts_smp_atomic_set_nob(&tb->nactive, src_ix); - if (dst_ix == 0) { - erts_smp_atomic_set_relb(&tb->szm, low_szm); - } - WUNLOCK_HASH(lck); - - if (tb->nslots - src_ix >= SEGSZ) { - free_seg(tb, 0); - } - } - else { - WUNLOCK_HASH(lck); - } +static void shrink(DbTableHash* tb, int nitems) +{ + HashDbTerm** src_bp; + HashDbTerm** dst_bp; + HashDbTerm** bp; + erts_smp_rwmtx_t* lck; + int src_ix, dst_ix, low_szm; + int nactive; + int loop_limit = 5; - } - /*else already done */ + do { + if (!begin_resizing(tb)) + return; /* already in progress */ + nactive = NACTIVE(tb); + if (!(nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive))) { + goto abort; /* already done (race) */ + } + src_ix = nactive - 1; + low_szm = erts_smp_atomic_read_nob(&tb->szm) >> 1; + dst_ix = src_ix & low_szm; + + ASSERT(dst_ix < src_ix); + ASSERT(nactive > FIRST_SEGSZ); + lck = WLOCK_HASH(tb, dst_ix); + ERTS_SMP_ASSERT(lck == GET_LOCK_MAYBE(tb,src_ix)); + /* Double check for racing table fixers */ + if (IS_FIXED(tb)) { + WUNLOCK_HASH(lck); + goto abort; + } + + src_bp = &BUCKET(tb, src_ix); + dst_bp = &BUCKET(tb, dst_ix); + bp = src_bp; + + /* + * We join lists by appending "dst" at the end of "src" + * as we must step through "src" anyway to purge pseudo deleted. + */ + while(*bp != NULL) { + if ((*bp)->hvalue == INVALID_HASH) { + HashDbTerm* deleted = *bp; + *bp = deleted->next; + free_term(tb, deleted); + } else { + bp = &(*bp)->next; + } + } + *bp = *dst_bp; + *dst_bp = *src_bp; + *src_bp = NULL; + + nactive = src_ix; + erts_smp_atomic_set_nob(&tb->nactive, nactive); + if (dst_ix == 0) { + erts_smp_atomic_set_relb(&tb->szm, low_szm); + } + WUNLOCK_HASH(lck); + + if (tb->nslots - src_ix >= EXT_SEGSZ) { + free_seg(tb, 0); + } + done_resizing(tb); + + } while (--loop_limit + && nactive > FIRST_SEGSZ && nitems < SHRINK_LIMIT(nactive)); + return; + +abort: done_resizing(tb); } - /* Search a list of tuples for a matching key */ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, @@ -2931,8 +2918,8 @@ db_finalize_dbterm_hash(int cret, DbUpdateHandle* handle) WUNLOCK_HASH(lck); nactive = NACTIVE(tb); - if (nitems > nactive * (CHAIN_LEN + 1) && !IS_FIXED(tb)) { - grow(tb, nactive); + if (nitems > GROW_LIMIT(nactive) && !IS_FIXED(tb)) { + grow(tb, nitems); } } else { WUNLOCK_HASH(lck); @@ -3016,6 +3003,51 @@ void db_calc_stats_hash(DbTableHash* tb, DbHashStats* stats) stats->std_dev_expected = sqrt(stats->avg_chain_len * (1 - 1.0/NACTIVE(tb))); stats->kept_items = kept_items; } + +/* For testing only */ +Eterm erts_ets_hash_sizeof_ext_segtab(void) +{ + return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1); +} +/* For testing only */ +Eterm erts_ets_hash_get_memstate(Process* p, DbTableHash* tb) +{ + Eterm seg_cnt; + while (!begin_resizing(tb)) + /*spinn*/; + + seg_cnt = make_small(SLOT_IX_TO_SEG_IX(tb->nslots)); + done_resizing(tb); + return seg_cnt; +} +/* For testing only */ +Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate) +{ + int seg_cnt, target; + + if (!is_small(memstate)) + return make_small(__LINE__); + + target = signed_val(memstate); + if (target < 1) + return make_small(__LINE__); + while (1) { + while (!begin_resizing(tb)) + /*spin*/; + seg_cnt = SLOT_IX_TO_SEG_IX(tb->nslots); + done_resizing(tb); + + if (target == seg_cnt) + return am_ok; + if (IS_FIXED(tb)) + return make_small(__LINE__); + if (target < seg_cnt) + shrink(tb, 0); + else + grow(tb, INT_MAX); + } +} + #ifdef HARDDEBUG void db_check_table_hash(DbTable *tbl) @@ -3024,7 +3056,7 @@ void db_check_table_hash(DbTable *tbl) HashDbTerm* list; int j; - for (j = 0; j < tb->nactive; j++) { + for (j = 0; j < NACTIVE(tb); j++) { if ((list = BUCKET(tb,j)) != 0) { while (list != 0) { if (!is_tuple(make_tuple(list->dbterm.tpl))) { diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index e654363cd5..6d25c73549 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -50,23 +50,23 @@ typedef struct db_table_hash_fine_locks { typedef struct db_table_hash { DbTableCommon common; - erts_smp_atomic_t segtab; /* The segment table (struct segment**) */ + /* SMP: szm and nactive are write-protected by is_resizing or table write lock */ erts_smp_atomic_t szm; /* current size mask. */ - + erts_smp_atomic_t nactive; /* Number of "active" slots */ + + erts_smp_atomic_t segtab; /* The segment table (struct segment**) */ + struct segment* first_segtab[1]; + /* SMP: nslots and nsegs are protected by is_resizing or table write lock */ int nslots; /* Total number of slots */ int nsegs; /* Size of segment table */ /* List of slots where elements have been deleted while table was fixed */ erts_smp_atomic_t fixdel; /* (FixedDeletion*) */ - erts_smp_atomic_t nactive; /* Number of "active" slots */ - erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ #ifdef ERTS_SMP + erts_smp_atomic_t is_resizing; /* grow/shrink in progress */ DbTableHashFineLocks* locks; #endif -#ifdef VALGRIND - struct ext_segment* top_ptr_to_segment_with_active_segtab; -#endif } DbTableHash; @@ -109,5 +109,8 @@ typedef struct { }DbHashStats; void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); +Eterm erts_ets_hash_sizeof_ext_segtab(void); +Eterm erts_ets_hash_get_memstate(Process*, DbTableHash* tb); +Eterm erts_ets_hash_restore_memstate(DbTableHash* tb, Eterm memstate); #endif /* _DB_HASH_H */ diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index 6732b708a8..4e987d5bee 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -1769,7 +1769,7 @@ Eterm db_prog_match(Process *c_p, Eterm t; Eterm *esp; MatchVariable* variables; - BeamInstr *cp; + ErtsCodeMFA *cp; const UWord *pc = prog->text; Eterm *ehp; Eterm ret; @@ -2408,9 +2408,9 @@ restart: ehp = HAllocX(build_proc, 4, HEAP_XTRA); *esp++ = make_tuple(ehp); ehp[0] = make_arityval(3); - ehp[1] = cp[0]; - ehp[2] = cp[1]; - ehp[3] = make_small((Uint) cp[2]); + ehp[1] = cp->module; + ehp[2] = cp->function; + ehp[3] = make_small((Uint) cp->arity); } break; case matchSilent: diff --git a/erts/emulator/beam/erl_drv_nif.h b/erts/emulator/beam/erl_drv_nif.h index 6ec5fbb895..2489099b5c 100644 --- a/erts/emulator/beam/erl_drv_nif.h +++ b/erts/emulator/beam/erl_drv_nif.h @@ -69,10 +69,6 @@ typedef enum { # define SIZEOF_LONG_LONG_SAVED__ SIZEOF_LONG_LONG # undef SIZEOF_LONG_LONG #endif -#ifdef HALFWORD_HEAP_EMULATOR -# define HALFWORD_HEAP_EMULATOR_SAVED__ HALFWORD_HEAP_EMULATOR -# undef HALFWORD_HEAP_EMULATOR -#endif #include "erl_int_sizes_config.h" #if defined(SIZEOF_CHAR_SAVED__) && SIZEOF_CHAR_SAVED__ != SIZEOF_CHAR # error SIZEOF_CHAR mismatch diff --git a/erts/emulator/beam/erl_fun.c b/erts/emulator/beam/erl_fun.c index d0a57f0ad0..41097203d4 100644 --- a/erts/emulator/beam/erl_fun.c +++ b/erts/emulator/beam/erl_fun.c @@ -31,6 +31,9 @@ static Hash erts_fun_table; #include "erl_smp.h" +#ifdef HIPE +# include "hipe_mode_switch.h" +#endif static erts_smp_rwmtx_t erts_fun_table_lock; @@ -49,8 +52,8 @@ static void fun_free(ErlFunEntry* obj); * to unloaded_fun[]. The -1 in unloaded_fun[0] will be interpreted * as an illegal arity when attempting to call a fun. */ -static BeamInstr unloaded_fun_code[3] = {NIL, -1, 0}; -static BeamInstr* unloaded_fun = unloaded_fun_code + 2; +static BeamInstr unloaded_fun_code[4] = {NIL, NIL, -1, 0}; +static BeamInstr* unloaded_fun = unloaded_fun_code + 3; void erts_init_fun_table(void) @@ -219,6 +222,10 @@ erts_fun_purge_prepare(BeamInstr* start, BeamInstr* end) fe->pend_purge_address = addr; ERTS_SMP_WRITE_MEMORY_BARRIER; fe->address = unloaded_fun; +#ifdef HIPE + fe->pend_purge_native_address = fe->native_address; + hipe_set_closure_stub(fe); +#endif erts_purge_state_add_fun(fe); } b = b->next; @@ -234,8 +241,12 @@ erts_fun_purge_abort_prepare(ErlFunEntry **funs, Uint no) for (ix = 0; ix < no; ix++) { ErlFunEntry *fe = funs[ix]; - if (fe->address == unloaded_fun) + if (fe->address == unloaded_fun) { fe->address = fe->pend_purge_address; +#ifdef HIPE + fe->native_address = fe->pend_purge_native_address; +#endif + } } } @@ -244,8 +255,12 @@ erts_fun_purge_abort_finalize(ErlFunEntry **funs, Uint no) { Uint ix; - for (ix = 0; ix < no; ix++) + for (ix = 0; ix < no; ix++) { funs[ix]->pend_purge_address = NULL; +#ifdef HIPE + funs[ix]->pend_purge_native_address = NULL; +#endif + } } void @@ -256,6 +271,9 @@ erts_fun_purge_complete(ErlFunEntry **funs, Uint no) for (ix = 0; ix < no; ix++) { ErlFunEntry *fe = funs[ix]; fe->pend_purge_address = NULL; +#ifdef HIPE + fe->pend_purge_native_address = NULL; +#endif if (erts_refc_dectest(&fe->refc, 0) == 0) erts_erase_fun_entry(fe); } @@ -324,6 +342,7 @@ fun_alloc(ErlFunEntry* template) obj->pend_purge_address = NULL; #ifdef HIPE obj->native_address = NULL; + obj->pend_purge_native_address = NULL; #endif return obj; } diff --git a/erts/emulator/beam/erl_fun.h b/erts/emulator/beam/erl_fun.h index caa55c730c..83899f24a6 100644 --- a/erts/emulator/beam/erl_fun.h +++ b/erts/emulator/beam/erl_fun.h @@ -45,6 +45,9 @@ typedef struct erl_fun_entry { erts_refc_t refc; /* Reference count: One for code + one for each fun object in each process. */ BeamInstr *pend_purge_address; /* address stored during a pending purge */ +#ifdef HIPE + UWord* pend_purge_native_address; +#endif } ErlFunEntry; /* @@ -57,9 +60,6 @@ typedef struct erl_fun_thing { Eterm thing_word; /* Subtag FUN_SUBTAG. */ ErlFunEntry* fe; /* Pointer to fun entry. */ struct erl_off_heap_header* next; -#ifdef HIPE - UWord* native_address; /* Native code for the fun. */ -#endif Uint arity; /* The arity of the fun. */ Uint num_free; /* Number of free variables (in env). */ /* -- The following may be compound Erlang terms ---------------------- */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 6f641a1ea7..6c752b9741 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -35,7 +35,7 @@ #include "error.h" #include "big.h" #include "erl_gc.h" -#if HIPE +#ifdef HIPE #include "hipe_stack.h" #include "hipe_mode_switch.h" #endif @@ -395,11 +395,11 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end, if (is_non_value(result)) { if (p->freason == TRAP) { - #if HIPE +#ifdef HIPE if (regs == NULL) { regs = erts_proc_sched_data(p)->x_reg_array; } - #endif +#endif cost = garbage_collect(p, live_hf_end, 0, regs, p->arity, p->fcalls); } else { cost = garbage_collect(p, live_hf_end, 0, regs, arity, p->fcalls); @@ -2182,26 +2182,18 @@ copy_one_frag(Eterm** hpp, ErlOffHeap* off_heap, *hp++ = val; break; case TAG_PRIMARY_LIST: -#ifdef SHCOPY_SEND if (erts_is_literal(val,list_val(val))) { *hp++ = val; } else { *hp++ = offset_ptr(val, offs); } -#else - *hp++ = offset_ptr(val, offs); -#endif break; case TAG_PRIMARY_BOXED: -#ifdef SHCOPY_SEND if (erts_is_literal(val,boxed_val(val))) { *hp++ = val; } else { *hp++ = offset_ptr(val, offs); } -#else - *hp++ = offset_ptr(val, offs); -#endif break; case TAG_PRIMARY_HEADER: *hp++ = val; diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c index 362239f765..165c440f71 100644 --- a/erts/emulator/beam/erl_init.c +++ b/erts/emulator/beam/erl_init.c @@ -2264,7 +2264,6 @@ erl_start(int argc, char **argv) ASSERT(erts_code_purger && erts_code_purger->common.id == pid); erts_proc_inc_refc(erts_code_purger); -#ifdef ERTS_NEW_PURGE_STRATEGY pid = erl_system_process_otp(otp_ring0_pid, "erts_literal_area_collector"); erts_literal_area_collector = (Process *) erts_ptab_pix2intptr_ddrb(&erts_proc, @@ -2272,7 +2271,6 @@ erl_start(int argc, char **argv) ASSERT(erts_literal_area_collector && erts_literal_area_collector->common.id == pid); erts_proc_inc_refc(erts_literal_area_collector); -#endif #ifdef ERTS_DIRTY_SCHEDULERS pid = erl_system_process_otp(otp_ring0_pid, "erts_dirty_process_code_checker"); @@ -2302,7 +2300,7 @@ erl_start(int argc, char **argv) #endif set_main_stack_size(); erts_sched_init_time_sup(esdp); - process_main(); + process_main(esdp->x_reg_array, esdp->f_reg_array); } #endif } diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c index 06266363b5..356f5ca71e 100644 --- a/erts/emulator/beam/erl_lock_check.c +++ b/erts/emulator/beam/erl_lock_check.c @@ -113,10 +113,8 @@ static erts_lc_lock_order_t erts_lock_order[] = { { "export_tab", NULL }, { "fun_tab", NULL }, { "environ", NULL }, -#ifdef ERTS_NEW_PURGE_STRATEGY { "release_literal_areas", NULL }, #endif -#endif { "efile_drv", "address" }, { "drv_ev_state_grow", NULL, }, { "drv_ev_state", "address" }, diff --git a/erts/emulator/beam/erl_math.c b/erts/emulator/beam/erl_math.c index fc0aaed18a..1f270eb55f 100644 --- a/erts/emulator/beam/erl_math.c +++ b/erts/emulator/beam/erl_math.c @@ -247,6 +247,17 @@ BIF_RETTYPE math_pow_2(BIF_ALIST_2) return math_call_2(BIF_P, pow, BIF_ARG_1, BIF_ARG_2); } +BIF_RETTYPE math_ceil_1(BIF_ALIST_1) +{ + return math_call_1(BIF_P, ceil, BIF_ARG_1); +} +BIF_RETTYPE math_floor_1(BIF_ALIST_1) +{ + return math_call_1(BIF_P, floor, BIF_ARG_1); +} - +BIF_RETTYPE math_fmod_2(BIF_ALIST_2) +{ + return math_call_2(BIF_P, fmod, BIF_ARG_1, BIF_ARG_2); +} diff --git a/erts/emulator/beam/erl_message.c b/erts/emulator/beam/erl_message.c index e4c696ae3b..118adc0c1b 100644 --- a/erts/emulator/beam/erl_message.c +++ b/erts/emulator/beam/erl_message.c @@ -696,6 +696,9 @@ erts_send_message(Process* sender, erts_aint32_t receiver_state; #ifdef SHCOPY_SEND erts_shcopy_t info; +#else + erts_literal_area_t litarea; + INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif #ifdef USE_VM_PROBES @@ -725,7 +728,7 @@ erts_send_message(Process* sender, */ if (have_seqtrace(stoken)) { seq_trace_update_send(sender); - seq_trace_output(stoken, message, SEQ_TRACE_SEND, + seq_trace_output(stoken, message, SEQ_TRACE_SEND, receiver->common.id, sender); seq_trace_size = 6; /* TUPLE5 */ } @@ -741,7 +744,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object(message); + msize = size_object_litopt(message, &litarea); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -760,7 +763,7 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); + message = copy_struct_litopt(message, msize, &hp, ohp, &litarea); #endif if (is_immed(stoken)) token = stoken; @@ -796,7 +799,7 @@ erts_send_message(Process* sender, INITIALIZE_SHCOPY(info); msize = copy_shared_calculate(message, &info); #else - msize = size_object(message); + msize = size_object_litopt(message, &litarea); #endif mp = erts_alloc_message_heap_state(receiver, &receiver_state, @@ -810,7 +813,7 @@ erts_send_message(Process* sender, DESTROY_SHCOPY(info); #else if (is_not_immed(message)) - message = copy_struct(message, msize, &hp, ohp); + message = copy_struct_litopt(message, msize, &hp, ohp, &litarea); #endif } #ifdef USE_VM_PROBES @@ -998,8 +1001,8 @@ erts_move_messages_off_heap(Process *c_p) hp = hfrag->mem; if (is_not_immed(ERL_MESSAGE_TERM(mp))) ERL_MESSAGE_TERM(mp) = copy_struct(ERL_MESSAGE_TERM(mp), - msg_sz, &hp, - &hfrag->off_heap); + msg_sz, &hp, + &hfrag->off_heap); if (is_not_immed(ERL_MESSAGE_TOKEN(mp))) ERL_MESSAGE_TOKEN(mp) = copy_struct(ERL_MESSAGE_TOKEN(mp), token_sz, &hp, diff --git a/erts/emulator/beam/erl_msacc.h b/erts/emulator/beam/erl_msacc.h index ad7c8c5eee..7e02d8e101 100644 --- a/erts/emulator/beam/erl_msacc.h +++ b/erts/emulator/beam/erl_msacc.h @@ -22,7 +22,7 @@ #define ERL_MSACC_H__ /* Can be enabled/disabled via configure */ -#if ERTS_ENABLE_MSACC == 2 +#if defined(ERTS_ENABLE_MSACC) && ERTS_ENABLE_MSACC == 2 #define ERTS_MSACC_EXTENDED_STATES 1 #endif @@ -66,7 +66,7 @@ #define ERTS_MSACC_STATE_COUNT 7 -#if ERTS_MSACC_STATE_STRINGS && ERTS_ENABLE_MSACC +#if defined(ERTS_MSACC_STATE_STRINGS) && defined(ERTS_ENABLE_MSACC) static char *erts_msacc_states[] = { "aux", "check_io", @@ -104,7 +104,7 @@ static char *erts_msacc_states[] = { #define ERTS_MSACC_STATE_COUNT ERTS_MSACC_STATIC_STATE_COUNT #endif -#if ERTS_MSACC_STATE_STRINGS +#ifdef ERTS_MSACC_STATE_STRINGS static char *erts_msacc_states[] = { "alloc", "aux", @@ -157,7 +157,7 @@ struct erl_msacc_t_ { }; -#if ERTS_ENABLE_MSACC +#ifdef ERTS_ENABLE_MSACC #ifdef USE_THREADS extern erts_tsd_key_t erts_msacc_key; diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c index 3fe8bdb491..19adff610b 100644 --- a/erts/emulator/beam/erl_nif.c +++ b/erts/emulator/beam/erl_nif.c @@ -687,9 +687,12 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, MBUF(&menv->phony_proc) = NULL; } } else { - Uint sz = size_object(msg); + erts_literal_area_t litarea; ErlOffHeap *ohp; Eterm *hp; + Uint sz; + INITIALIZE_LITERAL_PURGE_AREA(litarea); + sz = size_object_litopt(msg, &litarea); if (env && !env->tracee) { full_flush_env(env); mp = erts_alloc_message_heap(rp, &rp_locks, sz, &hp, &ohp); @@ -709,7 +712,7 @@ int enif_send(ErlNifEnv* env, const ErlNifPid* to_pid, ohp = &bp->off_heap; } } - msg = copy_struct(msg, sz, &hp, ohp); + msg = copy_struct_litopt(msg, sz, &hp, ohp, &litarea); } ERL_MESSAGE_TERM(mp) = msg; @@ -2282,7 +2285,7 @@ typedef struct { Export exp; struct erl_module_nif* m; NativeFunPtr fp; - BeamInstr *saved_current; + ErtsCodeMFA *saved_current; int exception_thrown; int saved_argc; int rootset_extra; @@ -2325,9 +2328,9 @@ allocate_nif_sched_data(Process* proc, int argc) ep->rootset_extra = argc; ep->rootset[0] = NIL; for (i=0; i<ERTS_NUM_CODE_IX; i++) { - ep->exp.addressv[i] = &ep->exp.code[3]; + ep->exp.addressv[i] = &ep->exp.beam[0]; } - ep->exp.code[3] = (BeamInstr) em_call_nif; + ep->exp.beam[0] = (BeamInstr) em_call_nif; (void) ERTS_PROC_SET_NIF_TRAP_EXPORT(proc, ep); return ep; } @@ -2398,10 +2401,10 @@ init_nif_sched_data(ErlNifEnv* env, NativeFunPtr direct_fp, NativeFunPtr indirec ep->saved_argc = argc; } proc->i = (BeamInstr*) ep->exp.addressv[0]; - ep->exp.code[0] = (BeamInstr) proc->current[0]; - ep->exp.code[1] = (BeamInstr) proc->current[1]; - ep->exp.code[2] = argc; - ep->exp.code[4] = (BeamInstr) direct_fp; + ep->exp.info.mfa.module = proc->current->module; + ep->exp.info.mfa.function = proc->current->function; + ep->exp.info.mfa.arity = argc; + ep->exp.beam[1] = (BeamInstr) direct_fp; ep->m = env->mod_nif; ep->fp = indirect_fp; proc->freason = TRAP; @@ -2423,7 +2426,7 @@ restore_nif_mfa(Process* proc, NifExport* ep, int exception) ERTS_SMP_LC_ASSERT(erts_proc_lc_my_proc_locks(proc) & ERTS_PROC_LOCK_MAIN); - ASSERT(ep->saved_current != &ep->exp.code[0]); + ASSERT(ep->saved_current != &ep->exp.info.mfa); proc->current = ep->saved_current; ep->saved_current = NULL; if (exception) { @@ -2497,7 +2500,8 @@ execute_dirty_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) execution_state(env, &proc, NULL); - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(ERTS_SCHEDULER_IS_DIRTY(erts_proc_sched_data(proc))); @@ -2566,7 +2570,8 @@ schedule_dirty_nif(ErlNifEnv* env, int flags, int argc, const ERL_NIF_TERM argv[ erts_smp_proc_lock(proc, ERTS_PROC_LOCK_MAIN); } - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(fp); @@ -2626,7 +2631,8 @@ execute_nif(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) ERL_NIF_TERM result; execution_state(env, &proc, NULL); - fp = (NativeFunPtr) proc->current[6]; + ep = ErtsContainerStruct(proc->current, NifExport, exp.info.mfa); + fp = ep->fp; ASSERT(!env->exception_thrown); ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); @@ -2694,7 +2700,7 @@ enif_schedule_nif(ErlNifEnv* env, const char* fun_name, int flags, ep = (NifExport*) ERTS_PROC_GET_NIF_TRAP_EXPORT(proc); ASSERT(ep); - ep->exp.code[1] = (BeamInstr) fun_name_atom; + ep->exp.info.mfa.function = (BeamInstr) fun_name_atom; done: if (scheduler < 0) @@ -3008,17 +3014,16 @@ int enif_map_iterator_get_pair(ErlNifEnv *env, ***************************************************************************/ -static BeamInstr** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) +static ErtsCodeInfo** get_func_pp(BeamCodeHeader* mod_code, Eterm f_atom, unsigned arity) { int n = (int) mod_code->num_functions; int j; for (j = 0; j < n; ++j) { - BeamInstr* code_ptr = (BeamInstr*) mod_code->functions[j]; - ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_func_info_IaaI)); - if (f_atom == ((Eterm) code_ptr[3]) - && arity == ((unsigned) code_ptr[4])) { - - return (BeamInstr**) &mod_code->functions[j]; + ErtsCodeInfo* ci = mod_code->functions[j]; + ASSERT(ci->op == (BeamInstr) BeamOp(op_i_func_info_IaaI)); + if (f_atom == ci->mfa.function + && arity == ci->mfa.arity) { + return mod_code->functions+j; } } return NULL; @@ -3142,7 +3147,6 @@ static ErlNifFunc* next_func(ErlNifEntry* entry, int* incrp, ErlNifFunc* func) BIF_RETTYPE load_nif_2(BIF_ALIST_2) { static const char bad_lib[] = "bad_lib"; - static const char reload[] = "reload"; static const char upgrade[] = "upgrade"; char* lib_name = NULL; void* handle = NULL; @@ -3154,15 +3158,20 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) Eterm mod_atom; const Atom* mod_atomp; Eterm f_atom; - BeamInstr* caller; + ErtsCodeMFA* caller; ErtsSysDdllError errdesc = ERTS_SYS_DDLL_ERROR_INIT; Eterm ret = am_ok; int veto; struct erl_module_nif* lib = NULL; - int reload_warning = 0; struct erl_module_instance* this_mi; struct erl_module_instance* prev_mi; + if (BIF_P->flags & F_HIPE_MODE) { + ret = load_nif_error(BIF_P, "notsup", "Calling load_nif from HiPE compiled " + "modules not supported"); + BIF_RET(ret); + } + encoding = erts_get_native_filename_encoding(); if (encoding == ERL_FILENAME_WIN_WCHAR) { /* Do not convert the lib name to utf-16le yet, do that in win32 specific code */ @@ -3188,12 +3197,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) /* Find calling module */ ASSERT(BIF_P->current != NULL); - ASSERT(BIF_P->current[0] == am_erlang - && BIF_P->current[1] == am_load_nif - && BIF_P->current[2] == 2); + ASSERT(BIF_P->current->module == am_erlang + && BIF_P->current->function == am_load_nif + && BIF_P->current->arity == 2); caller = find_function_from_pc(BIF_P->cp); ASSERT(caller != NULL); - mod_atom = caller[0]; + mod_atom = caller->module; ASSERT(is_atom(mod_atom)); module_p = erts_get_module(mod_atom, erts_active_code_ix()); ASSERT(module_p != NULL); @@ -3219,8 +3228,12 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) this_mi = module_p->on_load; } - if (init_func == NULL && - (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { + if (this_mi->nif != NULL) { + ret = load_nif_error(BIF_P,"reload","NIF library already loaded" + " (reload disallowed since OTP 20)."); + } + else if (init_func == NULL && + (err=erts_sys_ddll_open(lib_name, &handle, &errdesc)) != ERL_DE_NO_ERROR) { const char slogan[] = "Failed to load NIF library"; if (strstr(errdesc.str, lib_name) != NULL) { ret = load_nif_error(BIF_P, "load_failed", "%s: '%s'", slogan, errdesc.str); @@ -3266,9 +3279,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) int incr = 0; ErlNifFunc* f = entry->funcs; for (i=0; i < entry->num_of_funcs && ret==am_ok; i++) { - BeamInstr** code_pp; + ErtsCodeInfo** ci_pp; if (!erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1) - || (code_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { + || (ci_pp = get_func_pp(this_mi->code_hdr, f_atom, f->arity))==NULL) { ret = load_nif_error(BIF_P,bad_lib,"Function not found %T:%s/%u", mod_atom, f->name, f->arity); } @@ -3290,9 +3303,9 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) #endif } #ifdef ERTS_DIRTY_SCHEDULERS - else if (code_pp[1] - code_pp[0] < (5+4)) + else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (4)) #else - else if (code_pp[1] - code_pp[0] < (5+3)) + else if (erts_codeinfo_to_code(ci_pp[1]) - erts_codeinfo_to_code(ci_pp[0]) < (3)) #endif { ret = load_nif_error(BIF_P,bad_lib,"No explicit call to load_nif" @@ -3309,7 +3322,7 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) goto error; } - /* Call load, reload or upgrade: + /* Call load or upgrade: */ @@ -3321,82 +3334,33 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) ASSERT(opened_rt_list == NULL); lib->mod = module_p; env.mod_nif = lib; - if (this_mi->nif != NULL) { /*************** Reload ******************/ - /* - * Repeated load_nif calls from same Erlang module instance ("reload") - * is deprecated and was only ment as a development feature not to - * be used in production systems. (See warning below) - */ - int k, old_incr = 0; - ErlNifFunc* old_func; - lib->priv_data = this_mi->nif->priv_data; - - ASSERT(this_mi->nif->entry != NULL); - if (entry->reload == NULL) { - ret = load_nif_error(BIF_P,reload,"Reload not supported by this NIF library."); - goto error; - } - /* Check that no NIF is removed */ - old_func = this_mi->nif->entry->funcs; - for (k=0; k < this_mi->nif->entry->num_of_funcs; k++) { - int incr = 0; - ErlNifFunc* f = entry->funcs; - for (i=0; i < entry->num_of_funcs; i++) { - if (old_func->arity == f->arity - && sys_strcmp(old_func->name, f->name) == 0) { - break; - } - f = next_func(entry, &incr, f); - } - if (i == entry->num_of_funcs) { - ret = load_nif_error(BIF_P,reload,"Reloaded library missing " - "function %T:%s/%u\r\n", mod_atom, - old_func->name, old_func->arity); - goto error; - } - old_func = next_func(this_mi->nif->entry, &old_incr, old_func); - } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->reload(&env, &lib->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - ret = load_nif_error(BIF_P, reload, "Library reload-call unsuccessful (%d).", veto); - } - else { - commit_opened_resource_types(lib); - this_mi->nif->entry = NULL; /* to prevent 'unload' callback */ - erts_unload_nif(this_mi->nif); - reload_warning = 1; - } + + lib->priv_data = NULL; + if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ + void* prev_old_data = prev_mi->nif->priv_data; + if (entry->upgrade == NULL) { + ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); + goto error; + } + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); + erts_post_nif(&env); + if (veto) { + prev_mi->nif->priv_data = prev_old_data; + ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto); + } + else + commit_opened_resource_types(lib); } - else { - lib->priv_data = NULL; - if (prev_mi->nif != NULL) { /**************** Upgrade ***************/ - void* prev_old_data = prev_mi->nif->priv_data; - if (entry->upgrade == NULL) { - ret = load_nif_error(BIF_P, upgrade, "Upgrade not supported by this NIF library."); - goto error; - } - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->upgrade(&env, &lib->priv_data, &prev_mi->nif->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - prev_mi->nif->priv_data = prev_old_data; - ret = load_nif_error(BIF_P, upgrade, "Library upgrade-call unsuccessful (%d).", veto); - } - else - commit_opened_resource_types(lib); - } - else if (entry->load != NULL) { /********* Initial load ***********/ - erts_pre_nif(&env, BIF_P, lib, NULL); - veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); - erts_post_nif(&env); - if (veto) { - ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto); - } - else - commit_opened_resource_types(lib); - } + else if (entry->load != NULL) { /********* Initial load ***********/ + erts_pre_nif(&env, BIF_P, lib, NULL); + veto = entry->load(&env, &lib->priv_data, BIF_ARG_2); + erts_post_nif(&env); + if (veto) { + ret = load_nif_error(BIF_P, "load", "Library load-call unsuccessful (%d).", veto); + } + else + commit_opened_resource_types(lib); } if (ret == am_ok) { /* @@ -3409,31 +3373,33 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) this_mi->nif = lib; for (i=0; i < entry->num_of_funcs; i++) { - BeamInstr* code_ptr; + ErtsCodeInfo* ci; + BeamInstr *code_ptr; erts_atom_get(f->name, sys_strlen(f->name), &f_atom, ERTS_ATOM_ENC_LATIN1); - code_ptr = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); + ci = *get_func_pp(this_mi->code_hdr, f_atom, f->arity); + code_ptr = erts_codeinfo_to_code(ci); - if (code_ptr[1] == 0) { - code_ptr[5+0] = (BeamInstr) BeamOp(op_call_nif); + if (ci->native == 0) { + code_ptr[0] = (BeamInstr) BeamOp(op_call_nif); } else { /* Function traced, patch the original instruction word */ - GenericBp* g = (GenericBp *) code_ptr[1]; - ASSERT(code_ptr[5+0] == + GenericBp* g = (GenericBp *) ci->native; + ASSERT(code_ptr[0] == (BeamInstr) BeamOp(op_i_generic_breakpoint)); g->orig_instr = (BeamInstr) BeamOp(op_call_nif); } #ifdef ERTS_DIRTY_SCHEDULERS if ((entry->major > 2 || (entry->major == 2 && entry->minor >= 7)) && (entry->options & ERL_NIF_DIRTY_NIF_OPTION) && f->flags) { - code_ptr[5+3] = (BeamInstr) f->fptr; - code_ptr[5+1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? + code_ptr[3] = (BeamInstr) f->fptr; + code_ptr[1] = (f->flags == ERL_NIF_DIRTY_JOB_IO_BOUND) ? (BeamInstr) schedule_dirty_io_nif : (BeamInstr) schedule_dirty_cpu_nif; } else #endif - code_ptr[5+1] = (BeamInstr) f->fptr; - code_ptr[5+2] = (BeamInstr) lib; + code_ptr[1] = (BeamInstr) f->fptr; + code_ptr[2] = (BeamInstr) lib; f = next_func(entry, &incr, f); } } @@ -3455,14 +3421,6 @@ BIF_RETTYPE load_nif_2(BIF_ALIST_2) erts_release_code_write_permission(); erts_free(ERTS_ALC_T_TMP, lib_name); - if (reload_warning) { - erts_dsprintf_buf_t* dsbufp = erts_create_logger_dsbuf(); - erts_dsprintf(dsbufp, - "Repeated calls to erlang:load_nif from module '%T'.\n\n" - "The NIF reload mechanism is deprecated and must not " - "be used in production systems.\n", mod_atom); - erts_send_warning_to_logger(BIF_P->group_leader, dsbufp); - } BIF_RET(ret); } diff --git a/erts/emulator/beam/erl_port.h b/erts/emulator/beam/erl_port.h index c59b42cdae..206078903d 100644 --- a/erts/emulator/beam/erl_port.h +++ b/erts/emulator/beam/erl_port.h @@ -733,7 +733,7 @@ erts_thr_drvport2port(ErlDrvPort drvport, int lock_pdl) if (lock_pdl && prt->port_data_lock) driver_pdl_lock(prt->port_data_lock); -#if ERTS_ENABLE_LOCK_CHECK +#ifdef ERTS_ENABLE_LOCK_CHECK if (!ERTS_IS_CRASH_DUMPING) { if (erts_lc_is_emu_thr()) { ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 1a579704a8..b43a1b0190 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -526,8 +526,8 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { case EXPORT_DEF: { Export* ep = *((Export **) (export_val(wobj) + 1)); - Atom* module = atom_tab(atom_val(ep->code[0])); - Atom* name = atom_tab(atom_val(ep->code[1])); + Atom* module = atom_tab(atom_val(ep->info.mfa.module)); + Atom* name = atom_tab(atom_val(ep->info.mfa.function)); PRINT_STRING(res, fn, arg, "#Fun<"); PRINT_BUF(res, fn, arg, module->name, module->len); @@ -535,7 +535,7 @@ print_term(fmtfn_t fn, void* arg, Eterm obj, long *dcount) { PRINT_BUF(res, fn, arg, name->name, name->len); PRINT_CHAR(res, fn, arg, '.'); PRINT_SWORD(res, fn, arg, 'd', 0, 1, - (ErlPfSWord) ep->code[2]); + (ErlPfSWord) ep->info.mfa.arity); PRINT_CHAR(res, fn, arg, '>'); } break; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b989d3be7a..22c6b3310a 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -446,11 +446,10 @@ int erts_system_profile_ts_type = ERTS_TRACE_FLG_NOW_TIMESTAMP; #endif typedef enum { - ERTS_PSTT_GC, /* Garbage Collect */ + ERTS_PSTT_GC_MAJOR, /* Garbage Collect: Fullsweep */ + ERTS_PSTT_GC_MINOR, /* Garbage Collect: Generational */ ERTS_PSTT_CPC, /* Check Process Code */ -#ifdef ERTS_NEW_PURGE_STRATEGY ERTS_PSTT_CLA, /* Copy Literal Area */ -#endif ERTS_PSTT_COHMQ, /* Change off heap message queue */ ERTS_PSTT_FTMQ /* Flush trace msg queue */ } ErtsProcSysTaskType; @@ -8169,7 +8168,8 @@ sched_thread_func(void *vesdp) ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL); #endif - process_main(); + process_main(esdp->x_reg_array, esdp->f_reg_array); + /* No schedulers should *ever* terminate */ erts_exit(ERTS_ABORT_EXIT, "Scheduler thread number %beu terminated\n", @@ -10414,7 +10414,8 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) break; switch (st->type) { - case ERTS_PSTT_GC: + case ERTS_PSTT_GC_MAJOR: + case ERTS_PSTT_GC_MINOR: if (c_p->flags & F_DISABLE_GC) { save_gc_task(c_p, st, st_prio); st = NULL; @@ -10422,7 +10423,9 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } else { if (!garbage_collected) { - FLAGS(c_p) |= F_NEED_FULLSWEEP; + if (st->type == ERTS_PSTT_GC_MAJOR) { + FLAGS(c_p) |= F_NEED_FULLSWEEP; + } reds -= scheduler_gc_proc(c_p, reds); garbage_collected = 1; } @@ -10438,7 +10441,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) fcalls = reds - CONTEXT_REDS; st_res = erts_check_process_code(c_p, st->arg[0], - unsigned_val(st->arg[1]), &cpc_reds, fcalls); reds -= cpc_reds; @@ -10449,7 +10451,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } -#ifdef ERTS_NEW_PURGE_STRATEGY case ERTS_PSTT_CLA: { int fcalls; int cla_reds = 0; @@ -10469,7 +10470,6 @@ execute_sys_tasks(Process *c_p, erts_aint32_t *statep, int in_reds) } break; } -#endif case ERTS_PSTT_COHMQ: reds -= erts_complete_off_heap_message_queue_change(c_p); st_res = am_true; @@ -10519,16 +10519,15 @@ cleanup_sys_tasks(Process *c_p, erts_aint32_t in_state, int in_reds) break; switch (st->type) { - case ERTS_PSTT_GC: + case ERTS_PSTT_GC_MAJOR: + case ERTS_PSTT_GC_MINOR: case ERTS_PSTT_CPC: case ERTS_PSTT_COHMQ: st_res = am_false; break; -#ifdef ERTS_NEW_PURGE_STRATEGY case ERTS_PSTT_CLA: st_res = am_ok; break; -#endif #ifdef ERTS_SMP case ERTS_PSTT_FTMQ: reds -= erts_flush_trace_messages(c_p, ERTS_PROC_LOCK_MAIN); @@ -10694,17 +10693,19 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, switch (req_type) { case am_garbage_collect: - st->type = ERTS_PSTT_GC; - noproc_res = am_false; - if (!rp) + switch (st->arg[0]) { + case am_minor: st->type = ERTS_PSTT_GC_MINOR; break; + case am_major: st->type = ERTS_PSTT_GC_MAJOR; break; + default: goto badarg; + } + noproc_res = am_false; + if (!rp) goto noproc; break; case am_check_process_code: if (is_not_atom(st->arg[0])) goto badarg; - if (is_not_small(st->arg[1]) || (unsigned_val(st->arg[1]) & ~ERTS_CPC_ALL)) - goto badarg; noproc_res = am_false; st->type = ERTS_PSTT_CPC; if (!rp) @@ -10720,7 +10721,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, #endif break; -#ifdef ERTS_NEW_PURGE_STRATEGY case am_copy_literals: if (st->arg[0] != am_true && st->arg[0] != am_false) goto badarg; @@ -10729,7 +10729,6 @@ request_system_task(Process *c_p, Eterm requester, Eterm target, if (!rp) goto noproc; break; -#endif default: goto badarg; @@ -11379,6 +11378,9 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN erts_shcopy_t info; INITIALIZE_SHCOPY(info); +#else + erts_literal_area_t litarea; + INITIALIZE_LITERAL_PURGE_AREA(litarea); #endif erts_smp_proc_lock(parent, ERTS_PROC_LOCKS_ALL_MINOR); @@ -11437,7 +11439,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef SHCOPY_SPAWN arg_size = copy_shared_calculate(args, &info); #else - arg_size = size_object(args); + arg_size = size_object_litopt(args, &litarea); #endif heap_need = arg_size; @@ -11461,10 +11463,10 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). } p->schedule_count = 0; ASSERT(p->min_heap_size == erts_next_heap_size(p->min_heap_size, 0)); - - p->u.initial[INITIAL_MOD] = mod; - p->u.initial[INITIAL_FUN] = func; - p->u.initial[INITIAL_ARI] = (Uint) arity; + + p->u.initial.module = mod; + p->u.initial.function = func; + p->u.initial.arity = (Uint) arity; /* * Must initialize binary lists here before copying binaries to process. @@ -11506,7 +11508,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). /* No need to initialize p->fcalls. */ - p->current = p->u.initial+INITIAL_MOD; + p->current = &p->u.initial; p->i = (BeamInstr *) beam_apply; p->cp = (BeamInstr *) beam_apply+1; @@ -11519,7 +11521,7 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). p->arg_reg[2] = copy_shared_perform(args, arg_size, &info, &p->htop, &p->off_heap); DESTROY_SHCOPY(info); #else - p->arg_reg[2] = copy_struct(args, arg_size, &p->htop, &p->off_heap); + p->arg_reg[2] = copy_struct_litopt(args, arg_size, &p->htop, &p->off_heap, &litarea); #endif p->arity = 3; @@ -11688,11 +11690,12 @@ erl_create_process(Process* parent, /* Parent of process (default group leader). #ifdef USE_VM_PROBES if (DTRACE_ENABLED(process_spawn)) { + ErtsCodeMFA cmfa = {mod, func, arity}; DTRACE_CHARBUF(process_name, DTRACE_TERM_BUF_SIZE); - DTRACE_CHARBUF(mfa, DTRACE_TERM_BUF_SIZE); + DTRACE_CHARBUF(mfa_buf, DTRACE_TERM_BUF_SIZE); - dtrace_fun_decode(p, mod, func, arity, process_name, mfa); - DTRACE2(process_spawn, process_name, mfa); + dtrace_fun_decode(p, &cmfa, process_name, mfa_buf); + DTRACE2(process_spawn, process_name, mfa_buf); } #endif return res; @@ -11767,9 +11770,9 @@ void erts_init_empty_process(Process *p) p->seq_trace_clock = 0; p->seq_trace_lastcnt = 0; p->seq_trace_token = NIL; - p->u.initial[0] = 0; - p->u.initial[1] = 0; - p->u.initial[2] = 0; + p->u.initial.module = 0; + p->u.initial.function = 0; + p->u.initial.arity = 0; p->catches = 0; p->cp = NULL; p->i = NULL; @@ -13218,8 +13221,8 @@ erts_program_counter_info(fmtfn_t to, void *to_arg, Process *p) static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) { - BeamInstr* addr = find_function_from_pc(x); - if (addr == NULL) { + ErtsCodeMFA *cmfa = find_function_from_pc(x); + if (cmfa == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); } else if (x == beam_continue_exit) { @@ -13233,7 +13236,8 @@ print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%d + %d", - addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm)); + cmfa->module, cmfa->function, cmfa->arity, + (x-(BeamInstr*)cmfa) * sizeof(Eterm)); } } diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 9e5121abd5..4d0093d0d8 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -21,6 +21,8 @@ #ifndef __PROCESS_H__ #define __PROCESS_H__ +#include "sys.h" + #undef ERTS_INCLUDE_SCHEDULER_INTERNALS #if (defined(ERL_PROCESS_C__) \ || defined(ERL_PORT_TASK_C__) \ @@ -37,8 +39,6 @@ typedef struct process Process; -#include "sys.h" - #define ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ #include "erl_process_lock.h" /* Only pull out important types... */ #undef ERTS_PROCESS_LOCK_ONLY_PROC_LOCK_TYPE__ @@ -93,10 +93,6 @@ struct ErtsNodesMonitor_; #define ERTS_HEAP_FREE(Type, Ptr, Size) \ erts_free((Type), (Ptr)) -#define INITIAL_MOD 0 -#define INITIAL_FUN 1 -#define INITIAL_ARI 2 - #include "export.h" struct saved_calls { @@ -1022,15 +1018,16 @@ struct process { #endif union { void *terminate; - BeamInstr initial[3]; /* Initial module(0), function(1), arity(2), often used instead - of pointer to funcinfo instruction, hence the BeamInstr datatype */ + ErtsCodeMFA initial; /* Initial module(0), function(1), arity(2), + often used instead of pointer to funcinfo + instruction. */ } u; - BeamInstr* current; /* Current Erlang function, part of the funcinfo: + ErtsCodeMFA* current; /* Current Erlang function, part of the funcinfo: * module(0), function(1), arity(2) * (module and functions are tagged atoms; - * arity an untagged integer). BeamInstr * because it references code + * arity an untagged integer). */ - + /* * Information mainly for post-mortem use (erl crash dump). */ diff --git a/erts/emulator/beam/erl_process_dict.c b/erts/emulator/beam/erl_process_dict.c index 42654604cb..8311fde025 100644 --- a/erts/emulator/beam/erl_process_dict.c +++ b/erts/emulator/beam/erl_process_dict.c @@ -56,7 +56,7 @@ #define MAKE_HASH(Term) \ ((is_small(Term)) ? unsigned_val(Term) : \ ((is_atom(Term)) ? \ - (atom_tab(atom_val(Term))->slot.bucket.hvalue) : \ + atom_val(Term) : \ make_internal_hash(Term))) #define PD_SZ2BYTES(Sz) (sizeof(ProcDict) + ((Sz) - 1)*sizeof(Eterm)) @@ -408,6 +408,11 @@ static void pd_hash_erase_all(Process *p) } } +Uint32 erts_pd_make_hx(Eterm key) +{ + return MAKE_HASH(key); +} + Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id) { unsigned int hval; diff --git a/erts/emulator/beam/erl_process_dict.h b/erts/emulator/beam/erl_process_dict.h index b50a2af72c..ab58f3c239 100644 --- a/erts/emulator/beam/erl_process_dict.h +++ b/erts/emulator/beam/erl_process_dict.h @@ -43,6 +43,7 @@ void erts_deep_dictionary_dump(fmtfn_t to, void *to_arg, Eterm erts_dictionary_copy(struct process *p, ProcDict *pd); Eterm erts_pd_hash_get(struct process *p, Eterm id); +Uint32 erts_pd_make_hx(Eterm key); Eterm erts_pd_hash_get_with_hx(Process *p, Uint32 hx, Eterm id); #endif diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index d8bb00e8c6..e534c33637 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -334,8 +334,8 @@ stack_element_dump(fmtfn_t to, void *to_arg, Eterm* sp, int yreg) static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) { - BeamInstr* addr = find_function_from_pc(x); - if (addr == NULL) { + ErtsCodeMFA* cmfa = find_function_from_pc(x); + if (cmfa == NULL) { if (x == beam_exit) { erts_print(to, to_arg, "<terminate process>"); } else if (x == beam_continue_exit) { @@ -347,7 +347,8 @@ print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x) } } else { erts_print(to, to_arg, "%T:%T/%bpu + %bpu", - addr[0], addr[1], addr[2], ((x-addr)-2) * sizeof(Eterm)); + cmfa->module, cmfa->function, cmfa->arity, + (x-(BeamInstr*)cmfa) * sizeof(Eterm)); } } diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 8c84303997..9be4741ec8 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -781,7 +781,8 @@ trace_sched_aux(Process *p, ErtsProcLocks locks, Eterm what) tmp = make_small(0); } else { hp = HAlloc(p, 4); - tmp = TUPLE3(hp,p->current[0],p->current[1],make_small(p->current[2])); + tmp = TUPLE3(hp,p->current->module,p->current->function, + make_small(p->current->arity)); hp += 4; } @@ -1027,14 +1028,14 @@ erts_trace_return_to(Process *p, BeamInstr *pc) { Eterm mfa; - BeamInstr *code_ptr = find_function_from_pc(pc); + ErtsCodeMFA *cmfa = find_function_from_pc(pc); - - if (!code_ptr) { + if (!cmfa) { mfa = am_undefined; } else { Eterm *hp = HAlloc(p, 4); - mfa = TUPLE3(hp, code_ptr[0], code_ptr[1], make_small(code_ptr[2])); + mfa = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); } send_to_tracer_nif(p, &p->common, p->common.id, NULL, TRACE_FUN_T_CALL, @@ -1046,11 +1047,11 @@ erts_trace_return_to(Process *p, BeamInstr *pc) * or {trace, Pid, return_from, {Mod, Name, Arity}, Retval} */ void -erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) +erts_trace_return(Process* p, ErtsCodeMFA *mfa, + Eterm retval, ErtsTracer *tracer) { Eterm* hp; - Eterm mfa, mod, name; - int arity; + Eterm mfa_tuple; Uint meta_flags, *tracee_flags; ASSERT(tracer); @@ -1084,15 +1085,13 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) tracee_flags = &meta_flags; } - mod = fi[0]; - name = fi[1]; - arity = fi[2]; - hp = HAlloc(p, 4); - mfa = TUPLE3(hp, mod, name, make_small(arity)); + mfa_tuple = TUPLE3(hp, mfa->module, mfa->function, + make_small(mfa->arity)); hp += 4; send_to_tracer_nif_raw(p, NULL, *tracer, *tracee_flags, p->common.id, - NULL, TRACE_FUN_T_CALL, am_return_from, mfa, retval, am_true); + NULL, TRACE_FUN_T_CALL, am_return_from, mfa_tuple, + retval, am_true); } /* Send {trace_ts, Pid, exception_from, {Mod, Name, Arity}, {Class,Value}, @@ -1103,7 +1102,7 @@ erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, ErtsTracer *tracer) * Where Class is atomic but Value is any term. */ void -erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, +erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value, ErtsTracer *tracer) { Eterm* hp; @@ -1142,7 +1141,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, } hp = HAlloc(p, 7);; - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], make_small((Eterm)mfa[2])); + mfa_tuple = TUPLE3(hp, mfa->module, mfa->function, make_small(mfa->arity)); hp += 4; cv = TUPLE2(hp, class, value); hp += 3; @@ -1165,7 +1164,7 @@ erts_trace_exception(Process* p, BeamInstr mfa[3], Eterm class, Eterm value, * if it is a pid or port we do a meta trace. */ Uint32 -erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, +erts_call_trace(Process* p, ErtsCodeInfo *info, Binary *match_spec, Eterm* args, int local, ErtsTracer *tracer) { Eterm* hp; @@ -1244,7 +1243,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, * such as size_object() and copy_struct(), we must make sure that we * temporarily convert any match contexts to sub binaries. */ - arity = (Eterm) mfa[2]; + arity = info->mfa.arity; for (i = 0; i < arity; i++) { Eterm arg = args[i]; if (is_boxed(arg) && header_is_bin_matchstate(*boxed_val(arg))) { @@ -1339,7 +1338,7 @@ erts_call_trace(Process* p, BeamInstr mfa[3], Binary *match_spec, hp += 2; } } - mfa_tuple = TUPLE3(hp, (Eterm) mfa[0], (Eterm) mfa[1], mfa_tuple); + mfa_tuple = TUPLE3(hp, info->mfa.module, info->mfa.function, mfa_tuple); hp += 4; /* @@ -1459,7 +1458,8 @@ trace_gc(Process *p, Eterm what, Uint size, Eterm msg) } void -monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint time) +monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_fp, + ErtsCodeMFA *out_fp, Uint time) { ErlHeapFragment *bp; ErlOffHeap *off_heap; @@ -1490,11 +1490,13 @@ monitor_long_schedule_proc(Process *p, BeamInstr *in_fp, BeamInstr *out_fp, Uint hp = ERTS_ALLOC_SYSMSG_HEAP(hsz, &bp, &off_heap, monitor_p); tmo = erts_bld_uint(&hp, NULL, time); if (in_fp != NULL) { - in_mfa = TUPLE3(hp,(Eterm) in_fp[0], (Eterm) in_fp[1], make_small(in_fp[2])); + in_mfa = TUPLE3(hp, in_fp->module, in_fp->function, + make_small(in_fp->arity)); hp +=4; } if (out_fp != NULL) { - out_mfa = TUPLE3(hp,(Eterm) out_fp[0], (Eterm) out_fp[1], make_small(out_fp[2])); + out_mfa = TUPLE3(hp, out_fp->module, out_fp->function, + make_small(out_fp->arity)); hp +=4; } tmo_tpl = TUPLE2(hp,am_timeout, tmo); @@ -2129,7 +2131,7 @@ profile_runnable_proc(Process *p, Eterm status){ Eterm *hp, msg; Eterm where = am_undefined; ErlHeapFragment *bp = NULL; - BeamInstr *current = NULL; + ErtsCodeMFA *cmfa = NULL; #ifndef ERTS_SMP #define LOCAL_HEAP_SIZE (4 + 6 + ERTS_TRACE_PATCH_TS_MAX_SIZE) @@ -2151,14 +2153,14 @@ profile_runnable_proc(Process *p, Eterm status){ if (!ERTS_PROC_IS_EXITING(p)) { if (p->current) { - current = p->current; + cmfa = p->current; } else { - current = find_function_from_pc(p->i); + cmfa = find_function_from_pc(p->i); } } #ifdef ERTS_SMP - if (!current) { + if (!cmfa) { hsz -= 4; } @@ -2166,8 +2168,10 @@ profile_runnable_proc(Process *p, Eterm status){ hp = bp->mem; #endif - if (current) { - where = TUPLE3(hp, current[0], current[1], make_small(current[2])); hp += 4; + if (cmfa) { + where = TUPLE3(hp, cmfa->module, cmfa->function, + make_small(cmfa->arity)); + hp += 4; } else { where = make_small(0); } diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index 0095d4386b..378b6de49c 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -101,11 +101,11 @@ void erts_send_sys_msg_proc(Eterm, Eterm, Eterm, ErlHeapFragment *); void trace_send(Process*, Eterm, Eterm); void trace_receive(Process*, Eterm, Eterm, ErtsTracingEvent*); -Uint32 erts_call_trace(Process *p, BeamInstr mfa[], struct binary *match_spec, +Uint32 erts_call_trace(Process *p, ErtsCodeInfo *info, struct binary *match_spec, Eterm* args, int local, ErtsTracer *tracer); -void erts_trace_return(Process* p, BeamInstr* fi, Eterm retval, +void erts_trace_return(Process* p, ErtsCodeMFA *mfa, Eterm retval, ErtsTracer *tracer); -void erts_trace_exception(Process* p, BeamInstr mfa[], Eterm class, Eterm value, +void erts_trace_exception(Process* p, ErtsCodeMFA *mfa, Eterm class, Eterm value, ErtsTracer *tracer); void erts_trace_return_to(Process *p, BeamInstr *pc); void trace_sched(Process*, ErtsProcLocks, Eterm); @@ -134,7 +134,8 @@ void erts_system_profile_setup_active_schedulers(void); /* system_monitor */ void monitor_long_gc(Process *p, Uint time); -void monitor_long_schedule_proc(Process *p, BeamInstr *in_i, BeamInstr *out_i, Uint time); +void monitor_long_schedule_proc(Process *p, ErtsCodeMFA *in_i, + ErtsCodeMFA *out_i, Uint time); void monitor_long_schedule_port(Port *pp, ErtsPortTaskType type, Uint time); void monitor_large_heap(Process *p); void monitor_generic(Process *p, Eterm type, Eterm spec); @@ -176,7 +177,7 @@ struct trace_pattern_flags { }; extern const struct trace_pattern_flags erts_trace_pattern_flags_off; extern int erts_call_time_breakpoint_tracing; -int erts_set_trace_pattern(Process*p, Eterm* mfa, int specified, +int erts_set_trace_pattern(Process*p, ErtsCodeMFA *mfa, int specified, struct binary* match_prog_set, struct binary *meta_match_prog_set, int on, struct trace_pattern_flags, diff --git a/erts/emulator/beam/erl_vm.h b/erts/emulator/beam/erl_vm.h index f97716d030..93cfe08105 100644 --- a/erts/emulator/beam/erl_vm.h +++ b/erts/emulator/beam/erl_vm.h @@ -146,11 +146,12 @@ typedef struct op_entry { int sz; /* Number of loaded words. */ char* pack; /* Instructions for packing engine. */ char* sign; /* Signature string. */ - unsigned count; /* Number of times executed. */ } OpEntry; -extern OpEntry opc[]; /* Description of all instructions. */ -extern int num_instructions; /* Number of instruction in opc[]. */ +extern const OpEntry opc[]; /* Description of all instructions. */ +extern const int num_instructions; /* Number of instruction in opc[]. */ + +extern Uint erts_instr_count[]; /* some constants for various table sizes etc */ @@ -185,4 +186,11 @@ extern int erts_pd_initial_size;/* Initial Process dictionary table size */ #include "erl_term.h" +#ifdef NO_JUMP_TABLE +#define BeamOp(Op) (Op) +#else +extern void** beam_ops; +#define BeamOp(Op) beam_ops[(Op)] +#endif + #endif /* __ERL_VM_H__ */ diff --git a/erts/emulator/beam/error.h b/erts/emulator/beam/error.h index 6c33b12dd0..e431c3051b 100644 --- a/erts/emulator/beam/error.h +++ b/erts/emulator/beam/error.h @@ -21,6 +21,8 @@ #ifndef __ERROR_H__ #define __ERROR_H__ +#include "code_ix.h" + /* * There are three primary exception classes: * @@ -197,7 +199,7 @@ struct StackTrace { Eterm header; /* bignum header - must be first in struct */ Eterm freason; /* original exception reason is saved in the struct */ BeamInstr* pc; - BeamInstr* current; + ErtsCodeMFA* current; int depth; /* number of saved pointers in trace[] */ BeamInstr *trace[1]; /* varying size - must be last in struct */ }; diff --git a/erts/emulator/beam/export.c b/erts/emulator/beam/export.c index 7f043e191b..f397ab6b00 100644 --- a/erts/emulator/beam/export.c +++ b/erts/emulator/beam/export.c @@ -103,7 +103,8 @@ static HashValue export_hash(struct export_entry* ee) { Export* x = ee->ep; - return EXPORT_HASH(x->code[0], x->code[1], x->code[2]); + return EXPORT_HASH(x->info.mfa.module, x->info.mfa.function, + x->info.mfa.arity); } static int @@ -111,9 +112,9 @@ export_cmp(struct export_entry* tmpl_e, struct export_entry* obj_e) { Export* tmpl = tmpl_e->ep; Export* obj = obj_e->ep; - return !(tmpl->code[0] == obj->code[0] && - tmpl->code[1] == obj->code[1] && - tmpl->code[2] == obj->code[2]); + return !(tmpl->info.mfa.module == obj->info.mfa.module && + tmpl->info.mfa.function == obj->info.mfa.function && + tmpl->info.mfa.arity == obj->info.mfa.arity); } @@ -130,21 +131,23 @@ export_alloc(struct export_entry* tmpl_e) blob = (struct export_blob*) erts_alloc(ERTS_ALC_T_EXPORT, sizeof(*blob)); erts_smp_atomic_add_nob(&total_entries_bytes, sizeof(*blob)); obj = &blob->exp; - obj->fake_op_func_info_for_hipe[0] = 0; - obj->fake_op_func_info_for_hipe[1] = 0; - obj->code[0] = tmpl->code[0]; - obj->code[1] = tmpl->code[1]; - obj->code[2] = tmpl->code[2]; - obj->code[3] = (BeamInstr) em_call_error_handler; - obj->code[4] = 0; + obj->info.op = 0; + obj->info.native = 0; + obj->info.mfa.module = tmpl->info.mfa.module; + obj->info.mfa.function = tmpl->info.mfa.function; + obj->info.mfa.arity = tmpl->info.mfa.arity; + obj->beam[0] = (BeamInstr) em_call_error_handler; + obj->beam[1] = 0; for (ix=0; ix<ERTS_NUM_CODE_IX; ix++) { - obj->addressv[ix] = obj->code+3; + obj->addressv[ix] = obj->beam; blob->entryv[ix].slot.index = -1; blob->entryv[ix].ep = &blob->exp; } ix = 0; + + DBG_TRACE_MFA_P(&obj->info.mfa, "export allocation at %p", obj); } else { /* Existing entry in another table, use free entry in blob */ blob = entry_to_blob(tmpl_e); @@ -163,9 +166,12 @@ export_free(struct export_entry* obj) obj->slot.index = -1; for (i=0; i < ERTS_NUM_CODE_IX; i++) { if (blob->entryv[i].slot.index >= 0) { + DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export entry slot %u freed for %p", + (obj - blob->entryv), &blob->exp); return; } } + DBG_TRACE_MFA_P(&blob->exp.info.mfa, "export blob deallocation at %p", &blob->exp); erts_free(ERTS_ALC_T_EXPORT, blob); erts_smp_atomic_add_nob(&total_entries_bytes, -sizeof(*blob)); } @@ -224,7 +230,9 @@ erts_find_export_entry(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) while (b != (HashBucket*) 0) { Export* ep = ((struct export_entry*) b)->ep; - if (ep->code[0] == m && ep->code[1] == f && ep->code[2] == a) { + if (ep->info.mfa.module == m && + ep->info.mfa.function == f && + ep->info.mfa.arity == a) { return ep; } b = b->next; @@ -237,9 +245,9 @@ static struct export_entry* init_template(struct export_templ* templ, { templ->entry.ep = &templ->exp; templ->entry.slot.index = -1; - templ->exp.code[0] = m; - templ->exp.code[1] = f; - templ->exp.code[2] = a; + templ->exp.info.mfa.module = m; + templ->exp.info.mfa.function = f; + templ->exp.info.mfa.arity = a; return &templ->entry; } @@ -263,8 +271,8 @@ erts_find_function(Eterm m, Eterm f, unsigned int a, ErtsCodeIndex code_ix) ee = hash_get(&export_tables[code_ix].htable, init_template(&templ, m, f, a)); if (ee == NULL || - (ee->ep->addressv[code_ix] == ee->ep->code+3 && - ee->ep->code[3] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { + (ee->ep->addressv[code_ix] == ee->ep->beam && + ee->ep->beam[0] != (BeamInstr) BeamOp(op_i_generic_breakpoint))) { return NULL; } return ee->ep; diff --git a/erts/emulator/beam/export.h b/erts/emulator/beam/export.h index 17fc4828ca..7c812b306c 100644 --- a/erts/emulator/beam/export.h +++ b/erts/emulator/beam/export.h @@ -33,22 +33,20 @@ typedef struct export { void* addressv[ERTS_NUM_CODE_IX]; /* Pointer to code for function. */ - BeamInstr fake_op_func_info_for_hipe[2]; /* MUST be just before code[] */ + ErtsCodeInfo info; /* MUST be just before beam[] */ + /* - * code[0]: Tagged atom for module. - * code[1]: Tagged atom for function. - * code[2]: Arity (untagged integer). - * code[3]: This entry is 0 unless the 'address' field points to it. + * beam[0]: This entry is 0 unless the 'addressv' field points to it. * Threaded code instruction to load function * (em_call_error_handler), execute BIF (em_apply_bif), * or a breakpoint instruction (op_i_generic_breakpoint). - * code[4]: Function pointer to BIF function (for BIFs only), + * beam[1]: Function pointer to BIF function (for BIFs only), * or pointer to threaded code if the module has an * on_load function that has not been run yet, or pointer - * to code for function code[3] is a breakpont instruction. + * to code if function beam[0] is a breakpoint instruction. * Otherwise: 0. */ - BeamInstr code[5]; + BeamInstr beam[2]; } Export; @@ -74,8 +72,8 @@ extern erts_smp_mtx_t export_staging_lock; #include "beam_load.h" /* For em_* extern declarations */ #define ExportIsBuiltIn(EntryPtr) \ -(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->code + 3) && \ - ((EntryPtr)->code[3] == (BeamInstr) em_apply_bif)) +(((EntryPtr)->addressv[erts_active_code_ix()] == (EntryPtr)->beam) && \ + ((EntryPtr)->beam[0] == (BeamInstr) em_apply_bif)) #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/external.c b/erts/emulator/beam/external.c index beed847578..a49b242d7c 100644 --- a/erts/emulator/beam/external.c +++ b/erts/emulator/beam/external.c @@ -2826,9 +2826,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, Export* exp = *((Export **) (export_val(obj) + 1)); if ((dflags & DFLAG_EXPORT_PTR_TAG) != 0) { *ep++ = EXPORT_EXT; - ep = enc_atom(acmp, exp->code[0], ep, dflags); - ep = enc_atom(acmp, exp->code[1], ep, dflags); - ep = enc_term(acmp, make_small(exp->code[2]), ep, dflags, off_heap); + ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); + ep = enc_term(acmp, make_small(exp->info.mfa.arity), + ep, dflags, off_heap); } else { /* Tag, arity */ *ep++ = SMALL_TUPLE_EXT; @@ -2836,10 +2837,10 @@ enc_term_int(TTBEncodeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, byte* ep, ep += 1; /* Module name */ - ep = enc_atom(acmp, exp->code[0], ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.module, ep, dflags); /* Function name */ - ep = enc_atom(acmp, exp->code[1], ep, dflags); + ep = enc_atom(acmp, exp->info.mfa.function, ep, dflags); } break; } @@ -3747,9 +3748,8 @@ dec_term_atom_common: funp->arity = arity; #ifdef HIPE if (funp->fe->native_address == NULL) { - hipe_set_closure_stub(funp->fe, num_free); + hipe_set_closure_stub(funp->fe); } - funp->native_address = funp->fe->native_address; #endif hp = factory->hp; @@ -3821,9 +3821,6 @@ dec_term_atom_common: funp->fe = erts_put_fun_entry(module, old_uniq, old_index); funp->arity = funp->fe->address[-1] - num_free; -#ifdef HIPE - funp->native_address = funp->fe->native_address; -#endif hp = factory->hp; /* Environment */ @@ -4262,9 +4259,9 @@ encode_size_struct_int(TTBSizeContext* ctx, ErtsAtomCacheMap *acmp, Eterm obj, { Export* ep = *((Export **) (export_val(obj) + 1)); result += 1; - result += encode_size_struct2(acmp, ep->code[0], dflags); - result += encode_size_struct2(acmp, ep->code[1], dflags); - result += encode_size_struct2(acmp, make_small(ep->code[2]), dflags); + result += encode_size_struct2(acmp, ep->info.mfa.module, dflags); + result += encode_size_struct2(acmp, ep->info.mfa.function, dflags); + result += encode_size_struct2(acmp, make_small(ep->info.mfa.arity), dflags); } break; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index d6df85034c..2b2f3c5cdc 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -773,8 +773,8 @@ do { \ typedef struct ErtsPStack_ { byte* pstart; - byte* psp; - byte* pend; + int offs; /* "stack pointer" as byte offset from pstart */ + int size; /* allocated size in bytes */ ErtsAlcType_t alloc_type; }ErtsPStack; @@ -785,8 +785,8 @@ void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes); #define PSTACK_DECLARE(s, DEF_PSTACK_SIZE) \ PSTACK_TYPE PSTK_DEF_STACK(s)[DEF_PSTACK_SIZE]; \ ErtsPStack s = { (byte*)PSTK_DEF_STACK(s), /* pstart */ \ - (byte*)(PSTK_DEF_STACK(s) - 1), /* psp */ \ - (byte*)(PSTK_DEF_STACK(s) + (DEF_PSTACK_SIZE)), /* pend */\ + -(int)sizeof(PSTACK_TYPE), /* offs */ \ + DEF_PSTACK_SIZE*sizeof(PSTACK_TYPE), /* size */ \ ERTS_ALC_T_ESTACK /* alloc_type */ \ } @@ -806,19 +806,21 @@ do { \ } \ } while(0) -#define PSTACK_IS_EMPTY(s) (s.psp < s.pstart) +#define PSTACK_IS_EMPTY(s) (s.offs < 0) -#define PSTACK_COUNT(s) (((PSTACK_TYPE*)s.psp + 1) - (PSTACK_TYPE*)s.pstart) +#define PSTACK_COUNT(s) ((s.offs + sizeof(PSTACK_TYPE)) / sizeof(PSTACK_TYPE)) -#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), (PSTACK_TYPE*)(s.psp)) +#define PSTACK_TOP(s) (ASSERT(!PSTACK_IS_EMPTY(s)), \ + (PSTACK_TYPE*)(s.pstart + s.offs)) -#define PSTACK_PUSH(s) \ - (s.psp += sizeof(PSTACK_TYPE), \ - ((s.psp == s.pend) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ - sizeof(PSTACK_TYPE)) : (void)0), \ - ((PSTACK_TYPE*) s.psp)) +#define PSTACK_PUSH(s) \ + (s.offs += sizeof(PSTACK_TYPE), \ + ((s.offs == s.size) ? erl_grow_pstack(&s, PSTK_DEF_STACK(s), \ + sizeof(PSTACK_TYPE)) : (void)0), \ + ((PSTACK_TYPE*) (s.pstart + s.offs))) -#define PSTACK_POP(s) ((PSTACK_TYPE*) (s.psp -= sizeof(PSTACK_TYPE))) +#define PSTACK_POP(s) ((s.offs -= sizeof(PSTACK_TYPE)), \ + (PSTACK_TYPE*)(s.pstart + s.offs)) /* * Do not free the stack after this, it may have pointers into what @@ -831,8 +833,8 @@ do {\ (dst)->pstart = erts_alloc(s.alloc_type,\ sizeof(PSTK_DEF_STACK(s)));\ sys_memcpy((dst)->pstart, s.pstart, _pbytes);\ - (dst)->psp = (dst)->pstart + _pbytes - sizeof(PSTACK_TYPE);\ - (dst)->pend = (dst)->pstart + sizeof(PSTK_DEF_STACK(s));\ + (dst)->offs = s.offs;\ + (dst)->size = s.size;\ (dst)->alloc_type = s.alloc_type;\ } else\ *(dst) = s;\ @@ -847,8 +849,8 @@ do { \ ASSERT(s.pstart == (byte*)PSTK_DEF_STACK(s)); \ s = *(src); /* struct copy */ \ (src)->pstart = NULL; \ - ASSERT(s.psp >= (s.pstart - sizeof(PSTACK_TYPE))); \ - ASSERT(s.psp < s.pend); \ + ASSERT(s.offs >= -(int)sizeof(PSTACK_TYPE)); \ + ASSERT(s.offs < s.size); \ } while (0) #define PSTACK_DESTROY_SAVED(pstack)\ @@ -1000,12 +1002,8 @@ Eterm erl_send(Process *p, Eterm to, Eterm msg); Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ -#define ERTS_CPC_ALLOW_GC (1 << 0) -#define ERTS_CPC_ALL ERTS_CPC_ALLOW_GC -Eterm erts_check_process_code(Process *c_p, Eterm module, Uint flags, int *redsp, int fcalls); -#ifdef ERTS_NEW_PURGE_STRATEGY +Eterm erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls); Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed); -#endif typedef struct ErtsLiteralArea_ { struct erl_off_heap_header *off_heap; @@ -1019,10 +1017,7 @@ typedef struct ErtsLiteralArea_ { extern erts_smp_atomic_t erts_copy_literal_area__; #define ERTS_COPY_LITERAL_AREA() \ ((ErtsLiteralArea *) erts_smp_atomic_read_nob(&erts_copy_literal_area__)) - -#ifdef ERTS_NEW_PURGE_STRATEGY extern Process *erts_literal_area_collector; -#endif #ifdef ERTS_DIRTY_SCHEDULERS extern Process *erts_dirty_process_code_checker; #endif @@ -1031,7 +1026,7 @@ extern Process *erts_code_purger; /* beam_load.c */ typedef struct { - BeamInstr* current; /* Pointer to: Mod, Name, Arity */ + ErtsCodeMFA* mfa; /* Pointer to: Mod, Name, Arity */ Uint needed; /* Heap space needed for entire tuple */ Uint32 loc; /* Location in source code */ Eterm* fname_ptr; /* Pointer to fname table */ @@ -1048,13 +1043,14 @@ Eterm erts_finish_loading(Binary* loader_state, Process* c_p, Eterm erts_preload_module(Process *c_p, ErtsProcLocks c_p_locks, Eterm group_leader, Eterm* mod, byte* code, Uint size); void init_load(void); -BeamInstr* find_function_from_pc(BeamInstr* pc); +ErtsCodeMFA* find_function_from_pc(BeamInstr* pc); Eterm* erts_build_mfa_item(FunctionInfo* fi, Eterm* hp, Eterm args, Eterm* mfa_p); -void erts_set_current_function(FunctionInfo* fi, BeamInstr* current); +void erts_set_current_function(FunctionInfo* fi, ErtsCodeMFA* mfa); Eterm erts_module_info_0(Process* p, Eterm module); Eterm erts_module_info_1(Process* p, Eterm module, Eterm what); Eterm erts_make_stub_module(Process* p, Eterm Mod, Eterm Beam, Eterm Info); +int erts_commit_hipe_patch_load(Eterm hipe_magic_bin); /* beam_ranges.c */ void erts_init_ranges(void); @@ -1105,26 +1101,26 @@ typedef struct { Eterm* shtable_start; ErtsAlcType_t shtable_alloc_type; Uint literal_size; - Eterm *range_ptr; - Uint range_sz; + Eterm *lit_purge_ptr; + Uint lit_purge_sz; } erts_shcopy_t; -#define INITIALIZE_SHCOPY(info) \ -do { \ - ErtsLiteralArea *larea__ = ERTS_COPY_LITERAL_AREA();\ - info.queue_start = info.queue_default; \ - info.bitstore_start = info.bitstore_default; \ - info.shtable_start = info.shtable_default; \ - info.literal_size = 0; \ - if (larea__) { \ - info.range_ptr = &larea__->start[0]; \ - info.range_sz = larea__->end - info.range_ptr; \ - } \ - else { \ - info.range_ptr = NULL; \ - info.range_sz = 0; \ - } \ -} while(0) +#define INITIALIZE_SHCOPY(info) \ + do { \ + ErtsLiteralArea *larea__ = ERTS_COPY_LITERAL_AREA(); \ + info.queue_start = info.queue_default; \ + info.bitstore_start = info.bitstore_default; \ + info.shtable_start = info.shtable_default; \ + info.literal_size = 0; \ + if (larea__) { \ + info.lit_purge_ptr = &larea__->start[0]; \ + info.lit_purge_sz = larea__->end - info.lit_purge_ptr; \ + } \ + else { \ + info.lit_purge_ptr = NULL; \ + info.lit_purge_sz = 0; \ + } \ + } while(0) #define DESTROY_SHCOPY(info) \ do { \ @@ -1140,18 +1136,42 @@ do { \ } while(0) /* copy.c */ +typedef struct { + Eterm *lit_purge_ptr; + Uint lit_purge_sz; +} erts_literal_area_t; + +#define INITIALIZE_LITERAL_PURGE_AREA(Area) \ + do { \ + ErtsLiteralArea *larea__ = ERTS_COPY_LITERAL_AREA(); \ + if (larea__) { \ + (Area).lit_purge_ptr = &larea__->start[0]; \ + (Area).lit_purge_sz = larea__->end - (Area).lit_purge_ptr; \ + } \ + else { \ + (Area).lit_purge_ptr = NULL; \ + (Area).lit_purge_sz = 0; \ + } \ + } while(0) + Eterm copy_object_x(Eterm, Process*, Uint); #define copy_object(Term, Proc) copy_object_x(Term,Proc,0) -Uint size_object(Eterm); +Uint size_object_x(Eterm, erts_literal_area_t*); +#define size_object(Term) size_object_x(Term,NULL) +#define size_object_litopt(Term,LitArea) size_object_x(Term,LitArea) + Uint copy_shared_calculate(Eterm, erts_shcopy_t*); Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); Uint size_shared(Eterm); -Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint* bsz); +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_t*); #define copy_struct(Obj,Sz,HPP,OH) \ - copy_struct_x(Obj,Sz,HPP,OH,NULL) + copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL) +#define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) + Eterm copy_shallow(Eterm*, Uint, Eterm**, ErlOffHeap*); void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, @@ -1180,7 +1200,7 @@ void print_pass_through(int, byte*, int); /* beam_emu.c */ int catchlevel(Process*); void init_emulator(void); -void process_main(void); +void process_main(Eterm* x_reg_array, FloatDef* f_reg_array); void erts_dirty_process_main(ErtsSchedulerData *); Eterm build_stacktrace(Process* c_p, Eterm exc); Eterm expand_error_value(Process* c_p, Uint freason, Eterm Value); @@ -1453,18 +1473,6 @@ Eterm erts_gc_bor(Process* p, Eterm* reg, Uint live); Eterm erts_gc_bxor(Process* p, Eterm* reg, Uint live); Eterm erts_gc_bnot(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_length_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_bit_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_byte_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_map_size_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_abs_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_float_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_round_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_trunc_1(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_binary_part_3(Process* p, Eterm* reg, Uint live); -Eterm erts_gc_binary_part_2(Process* p, Eterm* reg, Uint live); - Uint erts_current_reductions(Process* current, Process *p); int erts_print_system_version(fmtfn_t to, void *arg, Process *c_p); @@ -1573,8 +1581,7 @@ int erts_beam_jump_table(void); ERTS_GLB_INLINE void dtrace_pid_str(Eterm pid, char *process_buf); ERTS_GLB_INLINE void dtrace_proc_str(Process *process, char *process_buf); ERTS_GLB_INLINE void dtrace_port_str(Port *port, char *port_buf); -ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, - Eterm module, Eterm function, int arity, +ERTS_GLB_INLINE void dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa, char *process_buf, char *mfa_buf); #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -1608,8 +1615,7 @@ dtrace_port_str(Port *port, char *port_buf) } ERTS_GLB_INLINE void -dtrace_fun_decode(Process *process, - Eterm module, Eterm function, int arity, +dtrace_fun_decode(Process *process, ErtsCodeMFA *mfa, char *process_buf, char *mfa_buf) { if (process_buf) { @@ -1617,7 +1623,7 @@ dtrace_fun_decode(Process *process, } erts_snprintf(mfa_buf, DTRACE_TERM_BUF_SIZE, "%T:%T/%d", - module, function, arity); + mfa->module, mfa->function, mfa->arity); } #endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */ diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 5978478323..8ab6c713d6 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -26,6 +26,7 @@ #include "erl_vm.h" #include "global.h" #include "module.h" +#include "beam_catches.h" #ifdef DEBUG # define IF_DEBUG(x) x @@ -67,6 +68,18 @@ static int module_cmp(Module* tmpl, Module* obj) return tmpl->module != obj->module; } +void erts_module_instance_init(struct erl_module_instance* modi) +{ + modi->code_hdr = 0; + modi->code_length = 0; + modi->catches = BEAM_CATCHES_NIL; + modi->nif = NULL; + modi->num_breakpoints = 0; + modi->num_traced_exports = 0; +#ifdef HIPE + modi->hipe_code = NULL; +#endif +} static Module* module_alloc(Module* tmpl) { @@ -74,18 +87,11 @@ static Module* module_alloc(Module* tmpl) erts_smp_atomic_add_nob(&tot_module_bytes, sizeof(Module)); obj->module = tmpl->module; - obj->curr.code_hdr = 0; - obj->old.code_hdr = 0; - obj->curr.code_length = 0; - obj->old.code_length = 0; obj->slot.index = -1; - obj->curr.nif = NULL; - obj->old.nif = NULL; - obj->curr.num_breakpoints = 0; - obj->old.num_breakpoints = 0; - obj->curr.num_traced_exports = 0; - obj->old.num_traced_exports = 0; + erts_module_instance_init(&obj->curr); + erts_module_instance_init(&obj->old); obj->on_load = 0; + DBG_TRACE_MFA(make_atom(obj->module), 0, 0, "module_alloc"); return obj; } @@ -119,6 +125,7 @@ void init_module_table(void) erts_smp_atomic_init_nob(&tot_module_bytes, 0); } + Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix) { @@ -139,19 +146,14 @@ erts_get_module(Eterm mod, ErtsCodeIndex code_ix) } } -Module* -erts_put_module(Eterm mod) + +static Module* put_module(Eterm mod, IndexTable* mod_tab) { Module e; - IndexTable* mod_tab; int oldsz, newsz; Module* res; ASSERT(is_atom(mod)); - ERTS_SMP_LC_ASSERT(erts_initialized == 0 - || erts_has_code_write_permission()); - - mod_tab = &module_tables[erts_staging_code_ix()]; e.module = atom_val(mod); oldsz = index_table_sz(mod_tab); res = (Module*) index_put_entry(mod_tab, (void*) &e); @@ -160,6 +162,15 @@ erts_put_module(Eterm mod) return res; } +Module* +erts_put_module(Eterm mod) +{ + ERTS_SMP_LC_ASSERT(erts_initialized == 0 + || erts_has_code_write_permission()); + + return put_module(mod, &module_tables[erts_staging_code_ix()]); +} + Module *module_code(int i, ErtsCodeIndex code_ix) { return (Module*) erts_index_lookup(&module_tables[code_ix], i); @@ -181,6 +192,13 @@ static ErtsCodeIndex dbg_load_code_ix = 0; static int entries_at_start_staging = 0; +static ERTS_INLINE void copy_module(Module* dst_mod, Module* src_mod) +{ + dst_mod->curr = src_mod->curr; + dst_mod->old = src_mod->old; + dst_mod->on_load = src_mod->on_load; +} + void module_start_staging(void) { IndexTable* src = &module_tables[erts_active_code_ix()]; @@ -199,10 +217,7 @@ void module_start_staging(void) src_mod = (Module*) erts_index_lookup(src, i); dst_mod = (Module*) erts_index_lookup(dst, i); ASSERT(src_mod->module == dst_mod->module); - - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } /* @@ -214,9 +229,7 @@ void module_start_staging(void) dst_mod = (Module*) index_put_entry(dst, src_mod); ASSERT(dst_mod != src_mod); - dst_mod->curr = src_mod->curr; - dst_mod->old = src_mod->old; - dst_mod->on_load = src_mod->on_load; + copy_module(dst_mod, src_mod); } newsz = index_table_sz(dst); erts_smp_atomic_add_nob(&tot_module_bytes, (newsz - oldsz)); diff --git a/erts/emulator/beam/module.h b/erts/emulator/beam/module.h index f105b3f401..9d258d5dbf 100644 --- a/erts/emulator/beam/module.h +++ b/erts/emulator/beam/module.h @@ -23,6 +23,10 @@ #include "index.h" +#ifdef HIPE +#include "hipe_module.h" +#endif + struct erl_module_instance { BeamCodeHeader* code_hdr; int code_length; /* Length of loaded code in bytes. */ @@ -30,6 +34,9 @@ struct erl_module_instance { struct erl_module_nif* nif; int num_breakpoints; int num_traced_exports; +#ifdef HIPE + HipeModule *hipe_code; +#endif }; typedef struct erl_module { @@ -42,6 +49,7 @@ typedef struct erl_module { struct erl_module_instance* on_load; } Module; +void erts_module_instance_init(struct erl_module_instance* modi); Module* erts_get_module(Eterm mod, ErtsCodeIndex code_ix); Module* erts_put_module(Eterm mod); diff --git a/erts/emulator/beam/ops.tab b/erts/emulator/beam/ops.tab index 879daaca0a..4ef04d020a 100644 --- a/erts/emulator/beam/ops.tab +++ b/erts/emulator/beam/ops.tab @@ -792,14 +792,14 @@ allocate_init t I y ################################################################# # -# The BIFs erts_internal:check_process_code/2 must be called like a function, +# The BIFs erts_internal:check_process_code/1 must be called like a function, # to ensure that c_p->i (program counter) is set correctly (an ordinary # BIF call doesn't set it). # -call_ext u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext Bif -call_ext_last u==2 Bif=u$bif:erts_internal:check_process_code/2 D => i_call_ext_last Bif D -call_ext_only u==2 Bif=u$bif:erts_internal:check_process_code/2 => i_call_ext_only Bif +call_ext u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext Bif +call_ext_last u==1 Bif=u$bif:erts_internal:check_process_code/1 D => i_call_ext_last Bif D +call_ext_only u==1 Bif=u$bif:erts_internal:check_process_code/1 => i_call_ext_only Bif # # The BIFs erlang:garbage_collect/0 must be called like a function, @@ -811,6 +811,10 @@ call_ext u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext Bif call_ext_last u==0 Bif=u$bif:erlang:garbage_collect/0 D => i_call_ext_last Bif D call_ext_only u==0 Bif=u$bif:erlang:garbage_collect/0 => i_call_ext_only Bif +call_ext u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext Bif +call_ext_last u==1 Bif=u$bif:erts_internal:garbage_collect/1 D => i_call_ext_last Bif D +call_ext_only u==1 Bif=u$bif:erts_internal:garbage_collect/1 => i_call_ext_only Bif + # # put/2 and erase/1 must be able to do garbage collection, so we must call # them like functions. diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 7740dd4373..4b3ac594a0 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -99,6 +99,12 @@ #define ErtsContainerStruct(ptr, type, member) \ ((type *)((char *)(1 ? (ptr) : &((type *)0)->member) - offsetof(type, member))) +/* Use this variant when the member is an array */ +#define ErtsContainerStruct_(ptr, type, memberv) \ + ((type *)((char *)(1 ? (ptr) : ((type *)0)->memberv) - offsetof(type, memberv))) + +#define ErtsSizeofMember(type, member) sizeof(((type *)0)->member) + #if defined (__WIN32__) # include "erl_win_sys.h" #else diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 87ea4f05a1..ec502d5a78 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -204,9 +204,8 @@ erl_grow_wstack(ErtsWStack* s, Uint need) void erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) { - Uint old_size = s->pend - s->pstart; + Uint old_size = s->size; Uint new_size; - Uint sp_offs = s->psp - s->pstart; if (need_bytes < old_size) new_size = 2 * old_size; @@ -220,8 +219,7 @@ erl_grow_pstack(ErtsPStack* s, void* default_pstack, unsigned need_bytes) sys_memcpy(new_ptr, s->pstart, old_size); s->pstart = new_ptr; } - s->pend = s->pstart + new_size; - s->psp = s->pstart + sp_offs; + s->size = new_size; } /* @@ -894,11 +892,11 @@ tail_recur: { Export* ep = *((Export **) (export_val(term) + 1)); - hash = hash * FUNNY_NUMBER11 + ep->code[2]; + hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity; hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue); break; } @@ -1331,11 +1329,11 @@ make_hash2(Eterm term) { Export* ep = *((Export **) (export_val(term) + 1)); UINT32_HASH_2 - (ep->code[2], - atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue, + (ep->info.mfa.arity, + atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue, HCONST); UINT32_HASH - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue, + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue, HCONST_14); goto hash2_common; } @@ -2026,11 +2024,11 @@ tail_recur: { Export* ep = *((Export **) (export_val(term) + 1)); - hash = hash * FUNNY_NUMBER11 + ep->code[2]; + hash = hash * FUNNY_NUMBER11 + ep->info.mfa.arity; hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[0]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.module))->slot.bucket.hvalue); hash = hash*FUNNY_NUMBER1 + - (atom_tab(atom_val(ep->code[1]))->slot.bucket.hvalue); + (atom_tab(atom_val(ep->info.mfa.function))->slot.bucket.hvalue); break; } @@ -3307,13 +3305,15 @@ tailrecur_ne: Export* a_exp = *((Export **) (export_val(a) + 1)); Export* b_exp = *((Export **) (export_val(b) + 1)); - if ((j = erts_cmp_atoms(a_exp->code[0], b_exp->code[0])) != 0) { + if ((j = erts_cmp_atoms(a_exp->info.mfa.module, + b_exp->info.mfa.module)) != 0) { RETURN_NEQ(j); } - if ((j = erts_cmp_atoms(a_exp->code[1], b_exp->code[1])) != 0) { + if ((j = erts_cmp_atoms(a_exp->info.mfa.function, + b_exp->info.mfa.function)) != 0) { RETURN_NEQ(j); } - ON_CMP_GOTO((Sint) a_exp->code[2] - (Sint) b_exp->code[2]); + ON_CMP_GOTO((Sint) a_exp->info.mfa.arity - (Sint) b_exp->info.mfa.arity); } break; case (_TAG_HEADER_FUN >> _TAG_PRIMARY_SIZE): diff --git a/erts/emulator/drivers/common/zlib_drv.c b/erts/emulator/drivers/common/zlib_drv.c index 440ba956d8..066cf87c9d 100644 --- a/erts/emulator/drivers/common/zlib_drv.c +++ b/erts/emulator/drivers/common/zlib_drv.c @@ -44,30 +44,34 @@ #define INFLATE_INIT 8 #define INFLATE_INIT2 9 #define INFLATE_SETDICT 10 -#define INFLATE_SYNC 11 -#define INFLATE_RESET 12 -#define INFLATE_END 13 -#define INFLATE 14 +#define INFLATE_GETDICT 11 +#define INFLATE_SYNC 12 +#define INFLATE_RESET 13 +#define INFLATE_END 14 +#define INFLATE 15 -#define CRC32_0 15 -#define CRC32_1 16 -#define CRC32_2 17 +#define CRC32_0 16 +#define CRC32_1 17 +#define CRC32_2 18 -#define SET_BUFSZ 18 -#define GET_BUFSZ 19 -#define GET_QSIZE 20 +#define SET_BUFSZ 19 +#define GET_BUFSZ 20 +#define GET_QSIZE 21 -#define ADLER32_1 21 -#define ADLER32_2 22 +#define ADLER32_1 22 +#define ADLER32_2 23 -#define CRC32_COMBINE 23 -#define ADLER32_COMBINE 24 +#define CRC32_COMBINE 24 +#define ADLER32_COMBINE 25 -#define INFLATE_CHUNK 25 +#define INFLATE_CHUNK 26 #define DEFAULT_BUFSZ 4000 +/* According to zlib documentation, it can never exceed this */ +#define INFL_DICT_SZ 32768 + /* This flag is used in the same places, where zlib return codes * (Z_OK, Z_STREAM_END, Z_NEED_DICT) are. So, we need to set it to * relatively large value to avoid possible value clashes in future. @@ -248,6 +252,20 @@ static int zlib_output(ZLibData* d) return zlib_output_init(d); } +#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY +static int zlib_inflate_get_dictionary(ZLibData* d) +{ + ErlDrvBinary* dbin = driver_alloc_binary(INFL_DICT_SZ); + uInt dlen = 0; + int res = inflateGetDictionary(&d->s, (unsigned char*)dbin->orig_bytes, &dlen); + if ((res == Z_OK) && (driver_output_binary(d->port, NULL, 0, dbin, 0, dlen) < 0)) { + res = Z_ERRNO; + } + driver_free_binary(dbin); + return res; +} +#endif + static int zlib_inflate(ZLibData* d, int flush) { int res = Z_OK; @@ -586,6 +604,16 @@ static ErlDrvSSizeT zlib_ctl(ErlDrvData drv_data, unsigned int command, char *bu res = inflateSetDictionary(&d->s, (unsigned char*)buf, len); return zlib_return(res, rbuf, rlen); + case INFLATE_GETDICT: +#ifdef HAVE_ZLIB_INFLATEGETDICTIONARY + if (d->state != ST_INFLATE) goto badarg; + res = zlib_inflate_get_dictionary(d); + return zlib_return(res, rbuf, rlen); +#else + errno = ENOTSUP; + return zlib_return(Z_ERRNO, rbuf, rlen); +#endif + case INFLATE_SYNC: if (d->state != ST_INFLATE) goto badarg; if (len != 0) goto badarg; diff --git a/erts/emulator/hipe/hipe_amd64.c b/erts/emulator/hipe/hipe_amd64.c index 62739d2a78..e3cff4a4ba 100644 --- a/erts/emulator/hipe/hipe_amd64.c +++ b/erts/emulator/hipe/hipe_amd64.c @@ -73,8 +73,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Sint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Sint)destAddress - (Sint)callAddress - 4; if ((Sint)(Sint32)rel32 != rel32) return -1; @@ -83,29 +83,6 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - /* * Memory allocator for executable code. * @@ -116,9 +93,6 @@ static void atexit_alloc_code_stats(void) */ static void *alloc_code(unsigned int alloc_bytes) { - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } @@ -130,6 +104,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + /* Make stub for native code calling exported beam function. */ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) @@ -234,6 +213,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_arch.h b/erts/emulator/hipe/hipe_arch.h index 6f959815bb..059b8e7f29 100644 --- a/erts/emulator/hipe/hipe_arch.h +++ b/erts/emulator/hipe/hipe_arch.h @@ -30,23 +30,31 @@ extern void hipe_patch_load_fe(Uint *address, Uint value); extern int hipe_patch_insn(void *address, Uint value, Eterm type); extern int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline); -extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p); -extern void *hipe_make_native_stub(void *beamAddress, unsigned int beamArity); +extern void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, struct process *p); +extern void hipe_free_code(void*, unsigned int); +extern void *hipe_make_native_stub(void *exp, unsigned int beamArity); +extern void hipe_free_native_stub(void*); + #if defined(__sparc__) #include "hipe_sparc.h" +#include "hipe_sparc_asm.h" #endif #if defined(__i386__) #include "hipe_x86.h" +#include "hipe_x86_asm.h" #endif #if defined(__x86_64__) #include "hipe_amd64.h" +#include "hipe_amd64_asm.h" #endif #if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) #include "hipe_ppc.h" +#include "hipe_ppc_asm.h" #endif #if defined(__arm__) #include "hipe_arm.h" +#include "hipe_arm_asm.h" #endif #if !defined(AEXTERN) diff --git a/erts/emulator/hipe/hipe_arm.c b/erts/emulator/hipe/hipe_arm.c index f8ef468341..b61939724c 100644 --- a/erts/emulator/hipe/hipe_arm.c +++ b/erts/emulator/hipe/hipe_arm.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -57,30 +56,6 @@ void hipe_flush_icache_word(void *address) hipe_flush_icache_range(address, 4); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ - /* On ARM we always allocate a trampoline at base+32MB-8 for - nbif_callemu, so tramp_pos <= base+32MB-8. */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} static int check_callees(Eterm callees) { @@ -107,126 +82,53 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) -{ - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; - Eterm mfa, m, f; - unsigned int a, *trampoline; - - m = NIL; f = NIL; a = 0; /* silence stupid compiler warning */ - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; +#define TRAMPOLINE_WORDS 2 - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - mfa = tuple_val(callees)[trampnr]; - if (is_atom(mfa)) - trampoline = hipe_primop_get_trampoline(mfa); - else { - m = tuple_val(mfa)[1]; - f = tuple_val(mfa)[2]; - a = unsigned_val(tuple_val(mfa)[3]); - trampoline = hipe_mfa_get_trampoline(m, f, a); - } - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { - if (nrfreewords < 2) - return NULL; - nrfreewords -= 2; - tramp_pos = trampoline = tramp_pos - 2; - trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - trampoline[1] = 0; /* callee's address */ - hipe_flush_icache_range(trampoline, 2*sizeof(int)); - if (is_atom(mfa)) - hipe_primop_set_trampoline(mfa, trampoline); - else - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) +{ + Uint32* trampoline = address; + int i; + + for (i = 0; i < nrcallees; ++i) { + trampoline[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ + trampoline[1] = 0; /* callee's address */ + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*2*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } -static unsigned int *alloc_stub(Uint nrwords, unsigned int **tramp_callemu) +void hipe_free_code(void* code, unsigned int bytes) { - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - curseg.tramp_pos -= 2; - curseg.tramp_pos[0] = 0xE51FF004; /* ldr pc, [pc,#-4] */ - curseg.tramp_pos[1] = (unsigned int)&nbif_callemu; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - *tramp_callemu = (unsigned int*)((char*)curseg.base + SEGMENT_NRBYTES) - 2; - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); } /* @@ -266,8 +168,8 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; - unsigned int *tramp_callemu; int callemu_offset; + int is_short_jmp; /* * Native code calls BEAM via a stub looking as follows: @@ -277,36 +179,57 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) * b nbif_callemu * .long callee_exp * + * or if nbif_callemu is too far away: + * + * mov r0, #beamArity + * ldr r8, [pc,#0] // callee_exp + * ldr pc, [pc,#0] // nbif_callemu + * .long callee_exp + * .long nbif_callemu + * * I'm using r0 and r8 since they aren't used for - * parameter passing in native code. The branch to - * nbif_callemu may need to go via a trampoline. - * (Trampolines are allowed to modify r12, but they don't.) + * parameter passing in native code. */ - code = alloc_stub(4, &tramp_callemu); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 5*sizeof(Uint32)); if (!code) return NULL; callemu_offset = ((int)&nbif_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) { - callemu_offset = ((int)tramp_callemu - ((int)&code[2] + 8)) >> 2; - if (!(callemu_offset >= -0x00800000 && callemu_offset <= 0x007FFFFF)) - abort(); + is_short_jmp = (callemu_offset >= -0x00800000 && + callemu_offset <= 0x007FFFFF); +#ifdef DEBUG + if (is_short_jmp && (callemu_offset % 3)==0) { + is_short_jmp = 0; } +#endif /* mov r0, #beamArity */ code[0] = 0xE3A00000 | (beamArity & 0xFF); /* ldr r8, [pc,#0] // callee_exp */ code[1] = 0xE59F8000; - /* b nbif_callemu */ - code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + if (is_short_jmp) { + /* b nbif_callemu */ + code[2] = 0xEA000000 | (callemu_offset & 0x00FFFFFF); + } + else { + /* ldr pc, [pc,#0] // nbif_callemu */ + code[2] = 0xE59FF000; + /* .long nbif_callemu */ + code[4] = (unsigned int)&nbif_callemu; + } /* .long callee_exp */ code[3] = (unsigned int)callee_exp; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 5*sizeof(Uint32)); return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + static void patch_b(Uint32 *address, Sint32 offset, Uint32 AA) { Uint32 oldI = *address; diff --git a/erts/emulator/hipe/hipe_bif0.c b/erts/emulator/hipe/hipe_bif0.c index dcb6c35bfa..57fbbd9403 100644 --- a/erts/emulator/hipe/hipe_bif0.c +++ b/erts/emulator/hipe/hipe_bif0.c @@ -44,6 +44,7 @@ #include "hipe_mode_switch.h" #include "hipe_native_bif.h" #include "hipe_bif0.h" +#include "hipe_load.h" /* We need hipe_literals.h for HIPE_SYSTEM_CRC, but it redefines a few constants. #undef them here to avoid warnings. */ #undef F_TIMO @@ -54,6 +55,7 @@ #define BeamOpCode(Op) ((Uint)BeamOp(Op)) + int term_to_Sint32(Eterm term, Sint *sp) { Sint val; @@ -374,15 +376,31 @@ BIF_RETTYPE hipe_bifs_ref_set_2(BIF_ALIST_2) } /* + * BIFs for loading code. + */ + +static HipeLoaderState *get_loader_state(Eterm term) +{ + ProcBin *pb; + + if (!ERTS_TERM_IS_MAGIC_BINARY(term)) return NULL; + + pb = (ProcBin*) binary_val(term); + return hipe_get_loader_state(pb->val); +} + + +/* * Allocate memory and copy machine code to it. */ -BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_enter_code_3(BIF_ALIST_3) { Uint nrbytes; void *bytes; void *address; Eterm trampolines; Eterm *hp; + HipeLoaderState *stp; #ifndef DEBUG ERTS_DECLARE_DUMMY(Uint bitoffs); ERTS_DECLARE_DUMMY(Uint bitsize); @@ -391,7 +409,8 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) Uint bitsize; #endif - if (is_not_binary(BIF_ARG_1)) + if (is_not_binary(BIF_ARG_1) || + (!(stp = get_loader_state(BIF_ARG_3)))) BIF_ERROR(BIF_P, BADARG); nrbytes = binary_size(BIF_ARG_1); ERTS_GET_BINARY_BYTES(BIF_ARG_1, bytes, bitoffs, bitsize); @@ -406,11 +425,15 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) nrcallees = arityval(tuple_val(BIF_ARG_2)[0]); else nrcallees = 0; + // XXX: Is there any reason to not just BIF_ERROR, so that the runtime + // survives? erts_exit(ERTS_ERROR_EXIT, "%s: failed to allocate %lu bytes and %lu trampolines\r\n", __func__, (unsigned long)nrbytes, (unsigned long)nrcallees); } memcpy(address, bytes, nrbytes); hipe_flush_icache_range(address, nrbytes); + stp->text_segment = address; + stp->text_segment_size = nrbytes; hp = HAlloc(BIF_P, 3); hp[0] = make_arityval(2); hp[1] = address_to_term(address, BIF_P); @@ -423,25 +446,34 @@ BIF_RETTYPE hipe_bifs_enter_code_2(BIF_ALIST_2) /* * Allocate memory for arbitrary non-Erlang data. */ -BIF_RETTYPE hipe_bifs_alloc_data_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_alloc_data_3(BIF_ALIST_3) { - Uint align, nrbytes; - void *block; + Uint align; + HipeLoaderState *stp; if (is_not_small(BIF_ARG_1) || is_not_small(BIF_ARG_2) || + (!(stp = get_loader_state(BIF_ARG_3))) || (align = unsigned_val(BIF_ARG_1), !IS_POWER_OF_TWO(align))) BIF_ERROR(BIF_P, BADARG); - nrbytes = unsigned_val(BIF_ARG_2); - if (nrbytes == 0) + + if (stp->data_segment_size || stp->data_segment) + BIF_ERROR(BIF_P, BADARG); + + stp->data_segment_size = unsigned_val(BIF_ARG_2); + if (stp->data_segment_size == 0) BIF_RET(make_small(0)); - block = erts_alloc(ERTS_ALC_T_HIPE, nrbytes); - if ((unsigned long)block & (align-1)) { - fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte aligned\r\n", - __FUNCTION__, (unsigned long)nrbytes, block, (unsigned long)align); - erts_free(ERTS_ALC_T_HIPE, block); + stp->data_segment = erts_alloc(ERTS_ALC_T_HIPE, stp->data_segment_size); + if ((unsigned long)stp->data_segment & (align-1)) { + fprintf(stderr, "%s: erts_alloc(%lu) returned %p which is not %lu-byte " + "aligned\r\n", + __FUNCTION__, (unsigned long)stp->data_segment_size, + stp->data_segment, (unsigned long)align); + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; BIF_ERROR(BIF_P, EXC_NOTSUP); } - BIF_RET(address_to_term(block, BIF_P)); + BIF_RET(address_to_term(stp->data_segment, BIF_P)); } /* @@ -537,13 +569,13 @@ BIF_RETTYPE hipe_bifs_merge_term_1(BIF_ALIST_1) BIF_RET(val); } -struct mfa_t { +struct hipe_mfa { Eterm mod; Eterm fun; Uint ari; }; -static int term_to_mfa(Eterm term, struct mfa_t *mfa) +static int term_to_mfa(Eterm term, struct hipe_mfa *mfa) { Eterm mod, fun, a; Uint ari; @@ -601,7 +633,7 @@ static Uint *hipe_find_emu_address(Eterm mod, Eterm name, unsigned int arity) Uint *hipe_bifs_find_pc_from_mfa(Eterm term) { - struct mfa_t mfa; + struct hipe_mfa mfa; if (!term_to_mfa(term, &mfa)) return NULL; @@ -616,12 +648,20 @@ BIF_RETTYPE hipe_bifs_fun_to_address_1(BIF_ALIST_1) BIF_RET(address_to_term(pc, BIF_P)); } +BIF_RETTYPE hipe_bifs_commit_patch_load_1(BIF_ALIST_1) +{ + if (!erts_commit_hipe_patch_load(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + BIF_RET(am_ok); +} + BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) { Eterm *pc; void *address; int is_closure; - struct mfa_t mfa; + struct hipe_mfa mfa; switch (BIF_ARG_3) { case am_false: @@ -644,28 +684,22 @@ BIF_RETTYPE hipe_bifs_set_native_address_3(BIF_ALIST_3) pc = hipe_find_emu_address(mfa.mod, mfa.fun, mfa.ari); if (pc) { - hipe_mfa_save_orig_beam_op(mfa.mod, mfa.fun, mfa.ari, pc); -#if HIPE -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": planting call trap to %p at BEAM pc %p\r\n", address, pc); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "set beam call trap at %p -> %p", pc, address); hipe_set_call_trap(pc, address, is_closure); BIF_RET(am_true); -#endif } -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no BEAM pc found\r\n"); -#endif + DBG_TRACE_MFA(mfa.mod,mfa.fun,mfa.ari, "failed set call trap to %p, no beam code found", address); BIF_RET(am_false); } -BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) +BIF_RETTYPE hipe_bifs_enter_sdesc_2(BIF_ALIST_2) { - struct sdesc *sdesc; + struct hipe_sdesc *sdesc; + HipeLoaderState* stp; + + stp = get_loader_state(BIF_ARG_2); + if (!stp) + BIF_ERROR(BIF_P, BADARG); sdesc = hipe_decode_sdesc(BIF_ARG_1); if (!sdesc) { @@ -676,6 +710,13 @@ BIF_RETTYPE hipe_bifs_enter_sdesc_1(BIF_ALIST_1) fprintf(stderr, "%s: duplicate entry!\r\n", __FUNCTION__); BIF_ERROR(BIF_P, BADARG); } + + /* + * Link into list of sdesc's in same module instance + */ + sdesc->next_in_modi = stp->new_hipe_sdesc; + stp->new_hipe_sdesc = sdesc; + BIF_RET(NIL); } @@ -778,9 +819,6 @@ BIF_RETTYPE hipe_bifs_bif_address_3(BIF_ALIST_3) struct primop { HashBucket bucket; /* bucket.hvalue == atom_val(name) */ const void *address; -#if defined(__arm__) - void *trampoline; -#endif }; static struct primop primops[] = { @@ -839,29 +877,6 @@ static struct primop *primop_table_get(Eterm name) return hash_get(&primop_table, &tmpl); } -#if defined(__arm__) -static struct primop *primop_table_put(Eterm name) -{ - struct primop tmpl; - - init_primop_table(); - tmpl.bucket.hvalue = atom_val(name); - return hash_put(&primop_table, &tmpl); -} - -void *hipe_primop_get_trampoline(Eterm name) -{ - struct primop *primop = primop_table_get(name); - return primop ? primop->trampoline : NULL; -} - -void hipe_primop_set_trampoline(Eterm name, void *trampoline) -{ - struct primop *primop = primop_table_put(name); - primop->trampoline = trampoline; -} -#endif - /* * hipe_bifs_primop_address(Atom) -> address or false */ @@ -970,7 +985,7 @@ BIF_RETTYPE hipe_bifs_get_fe_2(BIF_ALIST_2) atom_buf[0] = '\0'; strncat(atom_buf, (char*)atom_tab(i)->name, atom_tab(i)->len); - printf("no fun entry for %s %ld:%ld\n", atom_buf, uniq, index); + printf("no fun entry for %s %ld:%ld\n", atom_buf, (unsigned long)uniq, (unsigned long)index); BIF_ERROR(BIF_P, BADARG); } BIF_RET(address_to_term((void *)fe, BIF_P)); @@ -997,30 +1012,35 @@ BIF_RETTYPE hipe_bifs_set_native_address_in_fe_2(BIF_ALIST_2) BIF_RET(am_true); } +struct hipe_ref_head { + struct hipe_ref_head* next; + struct hipe_ref_head* prev; +}; + /* - * MFA info hash table: + * An exported function called from or implemented by native code * - maps MFA to native code entry point - * - the MFAs it calls (refers_to) - * - the references to it (referred_from) + * - all references to it (callers) * - maps MFA to most recent trampoline [if powerpc or arm] */ struct hipe_mfa_info { + HashBucket mod2mfa; struct { unsigned long hvalue; struct hipe_mfa_info *next; } bucket; Eterm m; /* atom */ Eterm f; /* atom */ - unsigned int a; + unsigned int a : sizeof(int)*8 - 1; + unsigned int is_stub : 1; /* if beam or not (yet) loaded */ void *remote_address; - void *local_address; - Eterm *beam_code; - Uint orig_beam_op; - struct hipe_mfa_info_list *refers_to; - struct ref *referred_from; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - void *trampoline; + void *new_address; + struct hipe_ref_head callers; /* sentinel in list of hipe_ref's */ + struct hipe_mfa_info* next_in_mod; +#ifdef DEBUG + Export* dbg_export; #endif + }; static struct { @@ -1038,6 +1058,82 @@ static struct { erts_smp_rwmtx_t lock; } hipe_mfa_info_table; +Hash mod2mfa_tab; /* map from module atom to list of hipe_mfa_info */ + +static HashValue mod2mfa_hash(struct hipe_mfa_info* mfa) +{ + return mfa->mod2mfa.hvalue; +} + +static int mod2mfa_cmp(HashBucket* tmpl, struct hipe_mfa_info* mfa) +{ + return tmpl->hvalue != mfa->mod2mfa.hvalue; +} + +static struct hipe_mfa_info* mod2mfa_alloc(struct hipe_mfa_info* tmpl) +{ + return tmpl; /* hash_put always use mfa itself at template */ +} + +static void mod2mfa_free(struct hipe_mfa_info* mfa) +{ +} + +static void mod2mfa_tab_init(void) +{ + HashFunctions f; + static int init_done = 0; + + if (init_done) + return; + init_done = 1; + + f.hash = (H_FUN) mod2mfa_hash; + f.cmp = (HCMP_FUN) mod2mfa_cmp; + f.alloc = (HALLOC_FUN) mod2mfa_alloc; + f.free = (HFREE_FUN) mod2mfa_free; + f.meta_alloc = (HMALLOC_FUN) erts_alloc; + f.meta_free = (HMFREE_FUN) erts_free; + f.meta_print = (HMPRINT_FUN) erts_print; + + hash_init(ERTS_ALC_T_HIPE, &mod2mfa_tab, "mod2mfa_tab", 50, f); +} + +static struct hipe_mfa_info* mod2mfa_get(Module* modp) +{ + HashBucket tmpl; + tmpl.hvalue = modp->module; + return hash_get(&mod2mfa_tab, &tmpl); +} + +static struct hipe_mfa_info* mod2mfa_put(struct hipe_mfa_info* mfa) +{ + mfa->mod2mfa.hvalue = atom_val(mfa->m); + return hash_put(&mod2mfa_tab, mfa); +} + + + +/* + * An external native call site M:F(...) + * to be patched when the callee changes. + */ +struct hipe_ref { + struct hipe_ref_head head; /* list of refs to same calleee */ + void *address; +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void *trampoline; +#endif + unsigned int flags; + struct hipe_ref* next_from_modi; /* list of refs from same module instance */ +#if defined(DEBUG) + struct hipe_mfa_info* callee; + Eterm caller_m, caller_f, caller_a; +#endif +}; +#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ + + static inline void hipe_mfa_info_table_init_lock(void) { erts_smp_rwmtx_init(&hipe_mfa_info_table.lock, "hipe_mfait_lock"); @@ -1063,6 +1159,16 @@ static inline void hipe_mfa_info_table_rwunlock(void) erts_smp_rwmtx_rwunlock(&hipe_mfa_info_table.lock); } +static ERTS_INLINE +struct hipe_mfa_info* mod2mfa_get_safe(Module* modp) +{ + struct hipe_mfa_info* mfa; + hipe_mfa_info_table_rlock(); + mfa = mod2mfa_get(modp); + hipe_mfa_info_table_runlock(); + return mfa; +} + #define HIPE_MFA_HASH(M,F,A) (atom_val(M) ^ atom_val(F) ^ (A)) static struct hipe_mfa_info **hipe_mfa_info_table_alloc_bucket(unsigned int size) @@ -1108,14 +1214,14 @@ static struct hipe_mfa_info *hipe_mfa_info_table_alloc(Eterm m, Eterm f, unsigne res->m = m; res->f = f; res->a = arity; + res->is_stub = 0; res->remote_address = NULL; - res->local_address = NULL; - res->beam_code = NULL; - res->orig_beam_op = 0; - res->refers_to = NULL; - res->referred_from = NULL; -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) - res->trampoline = NULL; + res->new_address = NULL; + res->callers.next = &res->callers; + res->callers.prev = &res->callers; + res->next_in_mod = NULL; +#ifdef DEBUG + res->dbg_export = NULL; #endif return res; @@ -1133,6 +1239,8 @@ void hipe_mfa_info_table_init(void) hipe_mfa_info_table.bucket = hipe_mfa_info_table_alloc_bucket(size); hipe_mfa_info_table_init_lock(); + + mod2mfa_tab_init(); } static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eterm f, unsigned int arity) @@ -1154,21 +1262,12 @@ static inline struct hipe_mfa_info *hipe_mfa_info_table_get_locked(Eterm m, Eter return NULL; } -#if 0 /* XXX: unused */ -void *hipe_mfa_find_na(Eterm m, Eterm f, unsigned int arity) -{ - const struct hipe_mfa_info *p; - - p = hipe_mfa_info_table_get(m, f, arity); - return p ? p->address : NULL; -} -#endif - static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, unsigned int arity) { unsigned long h; unsigned int i; struct hipe_mfa_info *p; + struct hipe_mfa_info *first_in_mod; unsigned int size; h = HIPE_MFA_HASH(m, f, arity); @@ -1189,216 +1288,137 @@ static struct hipe_mfa_info *hipe_mfa_info_table_put_rwlocked(Eterm m, Eterm f, size = 1 << hipe_mfa_info_table.log2size; if (hipe_mfa_info_table.used > (4*size/5)) /* rehash at 80% */ hipe_mfa_info_table_grow(); - return p; -} -static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address, int is_exported) -{ - struct hipe_mfa_info *p; + first_in_mod = mod2mfa_put(p); + if (p != first_in_mod) { + p->next_in_mod = first_in_mod->next_in_mod; + first_in_mod->next_in_mod = p; + } + else { + p->next_in_mod = NULL; + } - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(m, f, arity); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(m, f, arity); - printf(": changing address from %p to %p\r\n", p->local_address, address); -#endif - p->local_address = address; - if (is_exported) - p->remote_address = address; - hipe_mfa_info_table_rwunlock(); + DBG_TRACE_MFA(m,f,arity, "hipe_mfa_info allocated at %p", p); + + return p; } -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int arity) +static void remove_mfa_info(struct hipe_mfa_info* rm) { + unsigned int i; struct hipe_mfa_info *p; - void *trampoline; - - hipe_mfa_info_table_rlock(); - p = hipe_mfa_info_table_get_locked(m, f, arity); - trampoline = p ? p->trampoline : NULL; - hipe_mfa_info_table_runlock(); - return trampoline; + struct hipe_mfa_info **prevp; + + i = rm->bucket.hvalue & hipe_mfa_info_table.mask; + prevp = &hipe_mfa_info_table.bucket[i]; + for (;;) { + p = *prevp; + ASSERT(p); + if (p == rm) { + *prevp = p->bucket.next; + ASSERT(hipe_mfa_info_table.used > 0); + hipe_mfa_info_table.used--; + return; + } + prevp = &p->bucket.next; + } } -void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int arity, void *trampoline) +static void hipe_mfa_set_na(Eterm m, Eterm f, unsigned int arity, void *address) { struct hipe_mfa_info *p; hipe_mfa_info_table_rwlock(); p = hipe_mfa_info_table_put_rwlocked(m, f, arity); - p->trampoline = trampoline; + DBG_TRACE_MFA(m,f,arity,"set native address in hipe_mfa_info at %p", p); + p->new_address = address; + hipe_mfa_info_table_rwunlock(); } -#endif BIF_RETTYPE hipe_bifs_set_funinfo_native_address_3(BIF_ALIST_3) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; - int is_exported; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - address = term_to_address(BIF_ARG_2); - if (!address) - BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_3 == am_true) - is_exported = 1; - else if (BIF_ARG_3 == am_false) - is_exported = 0; - else + switch (BIF_ARG_3) { + case am_true: /* is_exported */ + if (!term_to_mfa(BIF_ARG_1, &mfa)) + BIF_ERROR(BIF_P, BADARG); + address = term_to_address(BIF_ARG_2); + if (!address) + BIF_ERROR(BIF_P, BADARG); + hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address); + break; + case am_false: + break; /* ignore local functions */ + default: BIF_ERROR(BIF_P, BADARG); - hipe_mfa_set_na(mfa.mod, mfa.fun, mfa.ari, address, is_exported); - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_invalidate_funinfo_native_addresses_1(BIF_ALIST_1) -{ - Eterm lst; - struct mfa_t mfa; - struct hipe_mfa_info *p; - - hipe_mfa_info_table_rwlock(); - lst = BIF_ARG_1; - while (is_list(lst)) { - if (!term_to_mfa(CAR(list_val(lst)), &mfa)) - break; - lst = CDR(list_val(lst)); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - p->remote_address = NULL; - p->local_address = NULL; - if (p->beam_code) { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": removing call trap from BEAM pc %p (new op %#lx)\r\n", - p->beam_code, p->orig_beam_op); -#endif - p->beam_code[0] = p->orig_beam_op; - p->beam_code = NULL; - p->orig_beam_op = 0; - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mfa.mod, mfa.fun, mfa.ari); - printf(": no call trap to remove\r\n"); -#endif - } - } } - hipe_mfa_info_table_rwunlock(); - if (is_not_nil(lst)) - BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } -void hipe_mfa_save_orig_beam_op(Eterm mod, Eterm fun, unsigned int ari, Eterm *pc) + +/* Ask if we need to block all threads + * while loading/deleting code for this module? + */ +int hipe_need_blocking(Module* modp) { - Uint orig_beam_op; struct hipe_mfa_info *p; - orig_beam_op = pc[0]; - if (orig_beam_op != BeamOpCode(op_hipe_trap_call_closure) && - orig_beam_op != BeamOpCode(op_hipe_trap_call)) { - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_put_rwlocked(mod, fun, ari); -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": saving orig op %#lx from BEAM pc %p\r\n", orig_beam_op, pc); -#endif - p->beam_code = pc; - p->orig_beam_op = orig_beam_op; - hipe_mfa_info_table_rwunlock(); - } else { -#ifdef DEBUG_LINKER - printf("%s: ", __FUNCTION__); - print_mfa(mod, fun, ari); - printf(": orig op %#lx already saved\r\n", orig_beam_op); -#endif + /* Need to block if we have at least one native caller to this module + * or native code to make unaccessible. + */ + hipe_mfa_info_table_rlock(); + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { + ASSERT(!p->new_address); + if (p->callers.next != &p->callers || !p->is_stub) { + break; + } } + hipe_mfa_info_table_runlock(); + return (p != NULL); } -static void *hipe_make_stub(Eterm m, Eterm f, unsigned int arity, int is_remote) -{ - Export *export_entry; - void *StubAddress; - - ASSERT(is_remote); - - export_entry = erts_export_get_or_make_stub(m, f, arity); - StubAddress = hipe_make_native_stub(export_entry, arity); - if (!StubAddress) - erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - return StubAddress; -} - -static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info **pp) +static void *hipe_get_na_try_locked(Eterm m, Eterm f, unsigned int a) { struct hipe_mfa_info *p; - void *address; p = hipe_mfa_info_table_get_locked(m, f, a); - if (p) { - /* find address, predicting for a runtime apply call */ - address = p->remote_address; - if (!is_remote) - address = p->local_address; - if (address) - return address; - - /* bummer, install stub, checking if one already existed */ - address = p->remote_address; - if (address) - return address; - } - /* Caller must take the slow path with the write lock held, but allow - it to avoid some work if it already holds the write lock. */ - if (pp) - *pp = p; - return NULL; -} - -static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote, struct hipe_mfa_info *p) -{ - void *address; - - if (!p) - p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_make_stub(m, f, a, is_remote); - /* XXX: how to tell if a BEAM MFA is exported or not? */ - p->remote_address = address; - return address; + return p ? p->remote_address : NULL; } -static void *hipe_get_na_nofail_rwlocked(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_slow_rwlocked(Eterm m, Eterm f, unsigned int a) { - struct hipe_mfa_info *p; - void *address; + struct hipe_mfa_info *p = hipe_mfa_info_table_put_rwlocked(m, f, a); - address = hipe_get_na_try_locked(m, f, a, is_remote, &p); - if (address) - return address; + if (!p->remote_address) { + Export* export_entry = erts_export_get_or_make_stub(m, f, a); + void* stubAddress = hipe_make_native_stub(export_entry, a); + if (!stubAddress) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, p); - return address; + p->remote_address = stubAddress; + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = export_entry; +#endif + } + return p->remote_address; } -static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a, int is_remote) +static void *hipe_get_na_nofail(Eterm m, Eterm f, unsigned int a) { void *address; hipe_mfa_info_table_rlock(); - address = hipe_get_na_try_locked(m, f, a, is_remote, NULL); + address = hipe_get_na_try_locked(m, f, a); hipe_mfa_info_table_runlock(); if (address) return address; hipe_mfa_info_table_rwlock(); - address = hipe_get_na_slow_rwlocked(m, f, a, is_remote, NULL); + address = hipe_get_na_slow_rwlocked(m, f, a); hipe_mfa_info_table_rwunlock(); return address; } @@ -1408,7 +1428,7 @@ void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a) { if (is_not_atom(m) || is_not_atom(f) || a > 255) return NULL; - return hipe_get_na_nofail(m, f, a, 1); + return hipe_get_na_nofail(m, f, a); } /* primop, but called like a BIF for error handling purposes */ @@ -1420,25 +1440,19 @@ BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3) if (is_not_atom(BIF_ARG_1) || is_not_atom(BIF_ARG_2)) BIF_ERROR(BIF_P, BADARG); arity = unsigned_val(BIF_ARG_3); /* no error check */ - address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity, 1); + address = hipe_get_na_nofail(BIF_ARG_1, BIF_ARG_2, arity); BIF_RET((Eterm)address); /* semi-Ok */ } -BIF_RETTYPE hipe_bifs_find_na_or_make_stub_2(BIF_ALIST_2) +BIF_RETTYPE hipe_bifs_find_na_or_make_stub_1(BIF_ALIST_1) { - struct mfa_t mfa; + struct hipe_mfa mfa; void *address; - int is_remote; if (!term_to_mfa(BIF_ARG_1, &mfa)) BIF_ERROR(BIF_P, BADARG); - if (BIF_ARG_2 == am_true) - is_remote = 1; - else if (BIF_ARG_2 == am_false) - is_remote = 0; - else - BIF_ERROR(BIF_P, BADARG); - address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari, is_remote); + + address = hipe_get_na_nofail(mfa.mod, mfa.fun, mfa.ari); BIF_RET(address_to_term(address, BIF_P)); } @@ -1453,14 +1467,14 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) hdr = *boxed_val(BIF_ARG_1); if (is_export_header(hdr)) { Export *ep = (Export*)(export_val(BIF_ARG_1)[1]); - unsigned int actual_arity = ep->code[2]; + unsigned int actual_arity = ep->info.mfa.arity; if (actual_arity != BIF_ARG_2) goto badfun; - m = ep->code[0]; - f = ep->code[1]; + m = ep->info.mfa.module; + f = ep->info.mfa.function; } else goto badfun; - address = hipe_get_na_nofail(m, f, BIF_ARG_2, 1); + address = hipe_get_na_nofail(m, f, BIF_ARG_2); BIF_RET((Eterm)address); badfun: @@ -1471,77 +1485,31 @@ BIF_RETTYPE hipe_nonclosure_address(BIF_ALIST_2) int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a) { - struct hipe_mfa_info *mfa; - long mfa_offset, ra_offset; - struct hipe_mfa_info **bucket; - unsigned int i, nrbuckets; + const struct hipe_sdesc* sdesc = hipe_find_sdesc((unsigned long)ra); - if (hipe_is_ra_mode_switch(ra)) { + if (!sdesc || sdesc->m_aix == atom_val(am_Empty)) return 0; - } - /* Note about locking: the table is only updated from the - loader, which runs with the rest of the system suspended. */ - /* XXX: alas not true; see comment at hipe_mfa_info_table.lock */ - hipe_mfa_info_table_rlock(); - bucket = hipe_mfa_info_table.bucket; - nrbuckets = 1 << hipe_mfa_info_table.log2size; - mfa = NULL; - mfa_offset = LONG_MAX; - for (i = 0; i < nrbuckets; ++i) { - struct hipe_mfa_info *b = bucket[i]; - while (b != NULL) { - ra_offset = (char*)ra - (char*)b->local_address; - if (ra_offset > 0 && ra_offset < mfa_offset) { - mfa_offset = ra_offset; - mfa = b; - } - b = b->bucket.next; - } - } - if (mfa) { - *m = mfa->m; - *f = mfa->f; - *a = mfa->a; - } - hipe_mfa_info_table_runlock(); - return mfa ? 1 : 0; + *m = make_atom(sdesc->m_aix); + *f = make_atom(sdesc->f_aix); + *a = sdesc->a; + return 1; } -/* - * Patch Reference Handling. - */ -struct hipe_mfa_info_list { - struct hipe_mfa_info *mfa; - struct hipe_mfa_info_list *next; -}; - -struct ref { - struct hipe_mfa_info *caller_mfa; - void *address; - void *trampoline; - unsigned int flags; - struct ref *next; -}; -#define REF_FLAG_IS_LOAD_MFA 1 /* bit 0: 0 == call, 1 == load_mfa */ -#define REF_FLAG_IS_REMOTE 2 /* bit 1: 0 == local, 1 == remote */ -#define REF_FLAG_PENDING_REDIRECT 4 /* bit 2: 1 == pending redirect */ -#define REF_FLAG_PENDING_REMOVE 8 /* bit 3: 1 == pending remove */ -/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,'remote'|'local'}) +/* add_ref(CalleeMFA, {CallerMFA,Address,'call'|'load_mfa',Trampoline,LoaderState}) */ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) { - struct mfa_t callee; + struct hipe_mfa callee; Eterm *tuple; - struct mfa_t caller; + struct hipe_mfa caller; void *address; void *trampoline; unsigned int flags; struct hipe_mfa_info *callee_mfa; - struct hipe_mfa_info *caller_mfa; - struct hipe_mfa_info_list *refers_to; - struct ref *ref; + struct hipe_ref *ref; + HipeLoaderState* stp; if (!term_to_mfa(BIF_ARG_1, &callee)) goto badarg; @@ -1572,63 +1540,90 @@ BIF_RETTYPE hipe_bifs_add_ref_2(BIF_ALIST_2) if (!trampoline) goto badarg; } - switch (tuple[5]) { - case am_local: - break; - case am_remote: - flags |= REF_FLAG_IS_REMOTE; - break; - default: - goto badarg; - } + stp = get_loader_state(tuple[5]); + if (!stp) + goto badarg; + hipe_mfa_info_table_rwlock(); callee_mfa = hipe_mfa_info_table_put_rwlocked(callee.mod, callee.fun, callee.ari); - caller_mfa = hipe_mfa_info_table_put_rwlocked(caller.mod, caller.fun, caller.ari); - - refers_to = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*refers_to)); - refers_to->mfa = callee_mfa; - refers_to->next = caller_mfa->refers_to; - caller_mfa->refers_to = refers_to; - ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(*ref)); - ref->caller_mfa = caller_mfa; + ref = erts_alloc(ERTS_ALC_T_HIPE, sizeof(struct hipe_ref)); ref->address = address; +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) ref->trampoline = trampoline; +#endif ref->flags = flags; - ref->next = callee_mfa->referred_from; - callee_mfa->referred_from = ref; + + /* + * Link into list of refs to same callee + */ + ASSERT(callee_mfa->callers.next->prev == &callee_mfa->callers); + ASSERT(callee_mfa->callers.prev->next == &callee_mfa->callers); + ref->head.next = callee_mfa->callers.next; + ref->head.prev = &callee_mfa->callers; + ref->head.next->prev = &ref->head; + ref->head.prev->next = &ref->head; + + /* + * Link into list of refs from same module instance + */ + ref->next_from_modi = stp->new_hipe_refs; + stp->new_hipe_refs = ref; + +#if defined(DEBUG) + ref->callee = callee_mfa; + ref->caller_m = caller.mod; + ref->caller_f = caller.fun; + ref->caller_a = caller.ari; +#endif hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + DBG_TRACE_MFA(caller.mod, caller.fun, caller.ari, "add_ref at %p TO %T:%T/%u (from %p)", + ref, callee.mod, callee.fun, callee.ari, ref->address); + DBG_TRACE_MFA(callee.mod, callee.fun, callee.ari, "add_ref at %p FROM %T:%T/%u (from %p)", + ref, caller.mod, caller.fun, caller.ari, ref->address); + BIF_RET(am_ok); badarg: BIF_ERROR(BIF_P, BADARG); } -/* Given a CalleeMFA, mark each ref to it as pending-redirect. - * This ensures that remove_refs_from() won't remove them: any - * removal is instead done at the end of redirect_referred_from(). - */ -BIF_RETTYPE hipe_bifs_mark_referred_from_1(BIF_ALIST_1) /* get_refs_from */ + +static void unlink_mfa_from_mod(struct hipe_mfa_info* unlink_me) { - struct mfa_t mfa; - const struct hipe_mfa_info *p; - struct ref *ref; + struct hipe_mfa_info* p; - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) - for (ref = p->referred_from; ref != NULL; ref = ref->next) - ref->flags |= REF_FLAG_PENDING_REDIRECT; - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); + p = hash_get(&mod2mfa_tab, unlink_me); + ASSERT(p); + if (p == unlink_me) { + hash_erase(&mod2mfa_tab, p); + if (p->next_in_mod) + mod2mfa_put(p->next_in_mod); + } + else { + struct hipe_mfa_info** prevp; + + do { + prevp = &p->next_in_mod; + p = *prevp; + ASSERT(p && p->m == unlink_me->m); + } while (p != unlink_me); + + *prevp = p->next_in_mod; + } +} + +static void purge_mfa(struct hipe_mfa_info* p) +{ + ASSERT(p->is_stub); + remove_mfa_info(p); + hipe_free_native_stub(p->remote_address); + erts_free(ERTS_ALC_T_HIPE, p); } /* Called by init:restart after unloading all hipe compiled modules - * to work around bug causing execution of deallocated beam code. - * Can be removed when delete/purge of native modules works better. + * to work around old bug that caused execution of deallocated beam code. + * Can be removed now when delete/purge of native modules works better. * Test: Do init:restart in debug compiled vm with hipe compiled kernel. */ static void hipe_purge_all_refs(void) @@ -1638,126 +1633,249 @@ static void hipe_purge_all_refs(void) hipe_mfa_info_table_rwlock(); + ASSERT(hipe_mfa_info_table.used == 0); bucket = hipe_mfa_info_table.bucket; nrbuckets = 1 << hipe_mfa_info_table.log2size; for (i = 0; i < nrbuckets; ++i) { + ASSERT(bucket[i] == NULL); while (bucket[i] != NULL) { struct hipe_mfa_info* mfa = bucket[i]; bucket[i] = mfa->bucket.next; - while (mfa->refers_to) { - struct hipe_mfa_info_list *to = mfa->refers_to; - mfa->refers_to = to->next; - erts_free(ERTS_ALC_T_HIPE, to); - } - while (mfa->referred_from) { - struct ref* from = mfa->referred_from; - mfa->referred_from = from->next; - erts_free(ERTS_ALC_T_HIPE, from); - } - erts_free(ERTS_ALC_T_HIPE, mfa); + hash_erase(&mod2mfa_tab, mfa); + erts_free(ERTS_ALC_T_HIPE, mfa); } } + hipe_mfa_info_table.used = 0; hipe_mfa_info_table_rwunlock(); } BIF_RETTYPE hipe_bifs_remove_refs_from_1(BIF_ALIST_1) { - struct mfa_t mfa; - struct hipe_mfa_info *caller_mfa, *callee_mfa; - struct hipe_mfa_info_list *refers_to, *tmp_refers_to; - struct ref **prev, *ref; - if (BIF_ARG_1 == am_all) { hipe_purge_all_refs(); BIF_RET(am_ok); } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - caller_mfa = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (caller_mfa) { - refers_to = caller_mfa->refers_to; - while (refers_to) { - callee_mfa = refers_to->mfa; - prev = &callee_mfa->referred_from; - ref = *prev; - while (ref) { - if (ref->caller_mfa == caller_mfa) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - ref->flags |= REF_FLAG_PENDING_REMOVE; - prev = &ref->next; - ref = ref->next; - } else { - struct ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } - } else { - prev = &ref->next; - ref = ref->next; - } - } - tmp_refers_to = refers_to; - refers_to = refers_to->next; - erts_free(ERTS_ALC_T_HIPE, tmp_refers_to); - } - caller_mfa->refers_to = NULL; + ASSERT(!"hipe_bifs_remove_refs_from_1() called"); + BIF_ERROR(BIF_P, BADARG); +} + +int hipe_purge_need_blocking(Module* modp) +{ + /* SVERK: Verify if this is really necessary */ + if (modp->old.hipe_code) { + if (modp->old.hipe_code->first_hipe_ref || + modp->old.hipe_code->first_hipe_sdesc) + return 1; + } + if (!modp->curr.code_hdr) { + return mod2mfa_get_safe(modp) != NULL; + } + return 0; +} + +void hipe_purge_refs(struct hipe_ref* first_ref, Eterm caller_module, + int is_blocking) +{ + struct hipe_ref* ref = first_ref; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + while (ref) { + struct hipe_ref* free_ref = ref; + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, "PURGE ref at %p to %T:%T/%u", ref, + ref->callee->m, ref->callee->f, ref->callee->a); + DBG_TRACE_MFA(ref->callee->m, ref->callee->f, ref->callee->a, "PURGE ref at %p from %T:%T/%u", ref, + ref->caller_m, ref->caller_f, ref->caller_a); + ASSERT(ref->caller_m == caller_module); + + /* + * Unlink from other refs to same callee + */ + ASSERT(ref->head.next->prev == &ref->head); + ASSERT(ref->head.prev->next == &ref->head); + ASSERT(ref->head.next != &ref->head); + ASSERT(ref->head.prev != &ref->head); + ref->head.next->prev = ref->head.prev; + ref->head.prev->next = ref->head.next; + + /* + * Was this the last ref to that callee? + */ + if (ref->head.next == ref->head.prev) { + struct hipe_mfa_info* p = ErtsContainerStruct(ref->head.next, struct hipe_mfa_info, callers); + if (p->is_stub) { + if (!is_blocking) + hipe_mfa_info_table_rwlock(); + unlink_mfa_from_mod(p); + purge_mfa(p); + if (!is_blocking) + hipe_mfa_info_table_rwunlock(); + } + } + + ref = ref->next_from_modi; + erts_free(ERTS_ALC_T_HIPE, free_ref); } - hipe_mfa_info_table_rwunlock(); - BIF_RET(am_ok); } +void hipe_purge_sdescs(struct hipe_sdesc* first_sdesc, Eterm module, + int is_blocking) +{ + struct hipe_sdesc* sdesc = first_sdesc; + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + ERTS_SMP_LC_ASSERT(is_blocking); /*XXX Fix safe sdesc destruction */ + + while (sdesc) { + struct hipe_sdesc* free_sdesc = sdesc; + + DBG_TRACE_MFA(make_atom(sdesc->m_aix), make_atom(sdesc->f_aix), sdesc->a, "PURGE sdesc at %p", (void*)sdesc->bucket.hvalue); + ASSERT(make_atom(sdesc->m_aix) == module); + + sdesc = sdesc->next_in_modi; + hipe_destruct_sdesc(free_sdesc); + } +} + + +void hipe_purge_module(Module* modp, int is_blocking) +{ + ASSERT(modp); + + ERTS_SMP_LC_ASSERT(is_blocking == erts_smp_thr_progress_is_blocking()); + + DBG_TRACE_MFA(make_atom(modp->module), 0, 0, "hipe_purge_module"); + + if (modp->old.hipe_code) { + /* + * Remove all hipe_ref's (external calls) from the old module instance + */ + if (modp->old.hipe_code->first_hipe_ref) { + ERTS_SMP_LC_ASSERT(is_blocking); + + hipe_purge_refs(modp->old.hipe_code->first_hipe_ref, + make_atom(modp->module), is_blocking); + modp->old.hipe_code->first_hipe_ref = NULL; + } -/* redirect_referred_from(CalleeMFA) - * Redirect all pending-redirect refs in CalleeMFA's referred_from. - * Then remove any pending-redirect && pending-remove refs from CalleeMFA's referred_from. + /* + * Remove all hipe_sdesc's for the old module instance + */ + if (modp->old.hipe_code->first_hipe_sdesc) { + ERTS_SMP_LC_ASSERT(is_blocking); + + hipe_purge_sdescs(modp->old.hipe_code->first_hipe_sdesc, + make_atom(modp->module), is_blocking); + modp->old.hipe_code->first_hipe_sdesc = NULL; + } + + hipe_free_module(modp->old.hipe_code); + modp->old.hipe_code = NULL; + } + + + /* + * Remove unreferred hipe_mfa_info's + * when all module instances are removed (like in init:restart) + */ + if (is_blocking && modp->curr.code_hdr == NULL) { + struct hipe_mfa_info* was_first = mod2mfa_get(modp); + struct hipe_mfa_info* is_first = was_first; + struct hipe_mfa_info** prevp = &is_first; + struct hipe_mfa_info *p; + + if (was_first) { + for (p = was_first ; p; p = *prevp) { + if (p->callers.next == &p->callers) { + *prevp = p->next_in_mod; + if (p != was_first) + purge_mfa(p); + } + else + prevp = &p->next_in_mod; + } + if (was_first != is_first) { + hash_erase(&mod2mfa_tab, was_first); + purge_mfa(was_first); + if (is_first) + mod2mfa_put(is_first); + } + } + } +} + + +/* + * Redirect all existing native calls to this module */ -BIF_RETTYPE hipe_bifs_redirect_referred_from_1(BIF_ALIST_1) +void hipe_redirect_to_module(Module* modp) { - struct mfa_t mfa; struct hipe_mfa_info *p; - struct ref **prev, *ref; - int is_remote, res; - void *new_address; + struct hipe_ref_head* refh; + + ERTS_SMP_LC_ASSERT(erts_smp_thr_progress_is_blocking()); + + for (p = mod2mfa_get(modp); p; p = p->next_in_mod) { + if (p->new_address) { + if (p->is_stub) { + hipe_free_native_stub(p->remote_address); + p->is_stub = 0; + } + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit new_address %p", p->new_address); + p->remote_address = p->new_address; + p->new_address = NULL; +#ifdef DEBUG + p->dbg_export = NULL; +#endif + } + else if (!p->is_stub) { + Export* exp = erts_export_get_or_make_stub(p->m, p->f, p->a); + p->remote_address = hipe_make_native_stub(exp, p->a); + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit stub %p", p->remote_address); + if (!p->remote_address) + erts_exit(ERTS_ERROR_EXIT, "hipe_make_stub: code allocation failed\r\n"); + p->is_stub = 1; +#ifdef DEBUG + p->dbg_export = exp; +#endif + } + else { + DBG_TRACE_MFA(p->m, p->f, p->a, "Commit no-op, already stub"); + ASSERT(p->remote_address && p->dbg_export); + } - if (!term_to_mfa(BIF_ARG_1, &mfa)) - BIF_ERROR(BIF_P, BADARG); - hipe_mfa_info_table_rwlock(); - p = hipe_mfa_info_table_get_locked(mfa.mod, mfa.fun, mfa.ari); - if (p) { - prev = &p->referred_from; - ref = *prev; - while (ref) { - if (ref->flags & REF_FLAG_PENDING_REDIRECT) { - is_remote = ref->flags & REF_FLAG_IS_REMOTE; - new_address = hipe_get_na_nofail_rwlocked(p->m, p->f, p->a, is_remote); - if (ref->flags & REF_FLAG_IS_LOAD_MFA) - res = hipe_patch_insn(ref->address, (Uint)new_address, am_load_mfa); - else - res = hipe_patch_call(ref->address, new_address, ref->trampoline); - if (res) - fprintf(stderr, "%s: patch failed\r\n", __FUNCTION__); - ref->flags &= ~REF_FLAG_PENDING_REDIRECT; - if (ref->flags & REF_FLAG_PENDING_REMOVE) { - struct ref *tmp = ref; - ref = ref->next; - *prev = ref; - erts_free(ERTS_ALC_T_HIPE, tmp); - } else { - prev = &ref->next; - ref = ref->next; - } - } else { - prev = &ref->next; - ref = ref->next; - } + DBG_TRACE_MFA(p->m,p->f,p->a,"START REDIRECT towards hipe_mfa_info at %p", p); + for (refh = p->callers.next; refh != &p->callers; refh = refh->next) { + struct hipe_ref* ref = (struct hipe_ref*) refh; + int res; + + DBG_TRACE_MFA(p->m,p->f,p->a, " REDIRECT ref at %p FROM %T:%T/%u (%p -> %p)", + ref, ref->caller_m, ref->caller_f, ref->caller_a, + ref->address, p->remote_address); + + DBG_TRACE_MFA(ref->caller_m, ref->caller_f, ref->caller_a, + " REDIRECT ref at %p TO %T:%T/%u (%p -> %p)", + ref, p->m,p->f,p->a, ref->address, p->remote_address); + + if (ref->flags & REF_FLAG_IS_LOAD_MFA) + res = hipe_patch_insn(ref->address, (Uint)p->remote_address, am_load_mfa); + else { +#if defined(arm) || defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) + void* trampoline = ref->trampoline; +#else + void* trampoline = NULL; +#endif + res = hipe_patch_call(ref->address, p->remote_address, trampoline); + } + if (res) + fprintf(stderr, "%s: patch failed", __FUNCTION__); } + DBG_TRACE_MFA(p->m,p->f,p->a,"DONE REDIRECT towards hipe_mfa_info at %p", p); } - hipe_mfa_info_table_rwunlock(); - BIF_RET(NIL); } BIF_RETTYPE hipe_bifs_check_crc_1(BIF_ALIST_1) @@ -1811,89 +1929,6 @@ void hipe_patch_address(Uint *address, Eterm patchtype, Uint value) } } -struct modinfo { - HashBucket bucket; /* bucket.hvalue == atom_val(the module name) */ - unsigned int code_size; -}; - -static Hash modinfo_table; - -static HashValue modinfo_hash(void *tmpl) -{ - Eterm mod = (Eterm)tmpl; - return atom_val(mod); -} - -static int modinfo_cmp(void *tmpl, void *bucket) -{ - /* bucket->hvalue == modinfo_hash(tmpl), so just return 0 (match) */ - return 0; -} - -static void *modinfo_alloc(void *tmpl) -{ - struct modinfo *p; - - p = (struct modinfo*)erts_alloc(ERTS_ALC_T_HIPE, sizeof(*p)); - p->code_size = 0; - return &p->bucket; -} - -static void init_modinfo_table(void) -{ - HashFunctions f; - static int init_done = 0; - - if (init_done) - return; - init_done = 1; - f.hash = (H_FUN) modinfo_hash; - f.cmp = (HCMP_FUN) modinfo_cmp; - f.alloc = (HALLOC_FUN) modinfo_alloc; - f.free = (HFREE_FUN) NULL; - f.meta_alloc = (HMALLOC_FUN) erts_alloc; - f.meta_free = (HMFREE_FUN) erts_free; - f.meta_print = (HMPRINT_FUN) erts_print; - hash_init(ERTS_ALC_T_HIPE, &modinfo_table, "modinfo_table", 11, f); -} - -BIF_RETTYPE hipe_bifs_update_code_size_3(BIF_ALIST_3) -{ - struct modinfo *p; - Sint code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1) || - is_not_small(BIF_ARG_3) || - (code_size = signed_val(BIF_ARG_3)) < 0) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_put(&modinfo_table, (void*)BIF_ARG_1); - - if (is_nil(BIF_ARG_2)) /* some MFAs, not whole module */ - p->code_size += code_size; - else /* whole module */ - p->code_size = code_size; - BIF_RET(NIL); -} - -BIF_RETTYPE hipe_bifs_code_size_1(BIF_ALIST_1) -{ - struct modinfo *p; - unsigned int code_size; - - init_modinfo_table(); - - if (is_not_atom(BIF_ARG_1)) - BIF_ERROR(BIF_P, BADARG); - - p = (struct modinfo*)hash_get(&modinfo_table, (void*)BIF_ARG_1); - - code_size = p ? p->code_size : 0; - BIF_RET(make_small(code_size)); -} - BIF_RETTYPE hipe_bifs_patch_insn_3(BIF_ALIST_3) { Uint *address, value; @@ -1929,3 +1964,23 @@ BIF_RETTYPE hipe_bifs_patch_call_3(BIF_ALIST_3) BIF_ERROR(BIF_P, BADARG); BIF_RET(NIL); } + +BIF_RETTYPE hipe_bifs_alloc_loader_state_1(BIF_ALIST_1) +{ + Binary *magic; + Eterm *hp; + Eterm res; + + if (is_not_atom(BIF_ARG_1)) + BIF_ERROR(BIF_P, BADARG); + + magic = hipe_alloc_loader_state(BIF_ARG_1); + + if (!magic) + BIF_ERROR(BIF_P, BADARG); + + hp = HAlloc(BIF_P, PROC_BIN_SIZE); + res = erts_mk_magic_binary_term(&hp, &MSO(BIF_P), magic); + erts_refc_dec(&magic->refc, 1); + BIF_RET(res); +} diff --git a/erts/emulator/hipe/hipe_bif0.h b/erts/emulator/hipe/hipe_bif0.h index c9a8216368..4a59bacc6e 100644 --- a/erts/emulator/hipe/hipe_bif0.h +++ b/erts/emulator/hipe/hipe_bif0.h @@ -32,17 +32,14 @@ extern void hipe_mfa_info_table_init(void); extern void *hipe_get_remote_na(Eterm m, Eterm f, unsigned int a); extern BIF_RETTYPE hipe_find_na_or_make_stub(BIF_ALIST_3); extern int hipe_find_mfa_from_ra(const void *ra, Eterm *m, Eterm *f, unsigned int *a); -#if defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__) || defined(__arm__) -extern void *hipe_mfa_get_trampoline(Eterm m, Eterm f, unsigned int a); -extern void hipe_mfa_set_trampoline(Eterm m, Eterm f, unsigned int a, void *trampoline); -#endif -#if defined(__arm__) -extern void *hipe_primop_get_trampoline(Eterm name); -extern void hipe_primop_set_trampoline(Eterm name, void *trampoline); -#endif /* needed in beam_load.c */ -void hipe_mfa_save_orig_beam_op(Eterm m, Eterm f, unsigned int a, Eterm *pc); +int hipe_need_blocking(Module*); +int hipe_purge_need_blocking(Module*); +void hipe_purge_refs(struct hipe_ref*, Eterm, int is_blocking); +void hipe_purge_sdescs(struct hipe_sdesc*, Eterm, int is_blocking); +void hipe_purge_module(Module*, int is_blocking); +void hipe_redirect_to_module(Module* modp); /* these are also needed in hipe_amd64.c */ extern void *term_to_address(Eterm); diff --git a/erts/emulator/hipe/hipe_bif0.tab b/erts/emulator/hipe/hipe_bif0.tab index 99237aae05..264ea2c34a 100644 --- a/erts/emulator/hipe/hipe_bif0.tab +++ b/erts/emulator/hipe/hipe_bif0.tab @@ -44,22 +44,20 @@ bif hipe_bifs:ref/1 bif hipe_bifs:ref_get/1 bif hipe_bifs:ref_set/2 -bif hipe_bifs:enter_code/2 -bif hipe_bifs:alloc_data/2 +bif hipe_bifs:enter_code/3 +bif hipe_bifs:alloc_data/3 bif hipe_bifs:constants_size/0 bif hipe_bifs:merge_term/1 bif hipe_bifs:fun_to_address/1 +bif hipe_bifs:commit_patch_load/1 bif hipe_bifs:set_native_address/3 #bif hipe_bifs:address_to_fun/1 bif hipe_bifs:set_funinfo_native_address/3 -bif hipe_bifs:invalidate_funinfo_native_addresses/1 +#bif hipe_bifs:invalidate_funinfo_native_addresses/1 -bif hipe_bifs:update_code_size/3 -bif hipe_bifs:code_size/1 - -bif hipe_bifs:enter_sdesc/1 +bif hipe_bifs:enter_sdesc/2 bif hipe_bifs:bif_address/3 bif hipe_bifs:primop_address/1 @@ -72,7 +70,7 @@ bif hipe_bifs:term_to_word/1 bif hipe_bifs:get_fe/2 bif hipe_bifs:set_native_address_in_fe/2 -bif hipe_bifs:find_na_or_make_stub/2 +bif hipe_bifs:find_na_or_make_stub/1 bif hipe_bifs:check_crc/1 bif hipe_bifs:system_crc/0 @@ -84,9 +82,9 @@ bif hipe_bifs:patch_insn/3 bif hipe_bifs:patch_call/3 bif hipe_bifs:add_ref/2 -bif hipe_bifs:mark_referred_from/1 bif hipe_bifs:remove_refs_from/1 -bif hipe_bifs:redirect_referred_from/1 + +bif hipe_bifs:alloc_loader_state/1 # atoms used by add_ref/2 atom call diff --git a/erts/emulator/hipe/hipe_bif1.c b/erts/emulator/hipe/hipe_bif1.c index 5e127755c6..0c66eb6abe 100644 --- a/erts/emulator/hipe/hipe_bif1.c +++ b/erts/emulator/hipe/hipe_bif1.c @@ -45,7 +45,7 @@ BIF_RETTYPE hipe_bifs_call_count_on_1(BIF_ALIST_1) pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!pc) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); if (pc[0] == BeamOpCode(op_hipe_trap_call)) BIF_ERROR(BIF_P, BADARG); if (pc[0] == BeamOpCode(op_hipe_call_count)) @@ -67,7 +67,7 @@ BIF_RETTYPE hipe_bifs_call_count_off_1(BIF_ALIST_1) pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!pc) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); hcc = (struct hipe_call_count*)pc[-4]; @@ -86,7 +86,7 @@ BIF_RETTYPE hipe_bifs_call_count_get_1(BIF_ALIST_1) pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!pc) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); hcc = (struct hipe_call_count*)pc[-4]; @@ -102,7 +102,7 @@ BIF_RETTYPE hipe_bifs_call_count_clear_1(BIF_ALIST_1) pc = hipe_bifs_find_pc_from_mfa(BIF_ARG_1); if (!pc) BIF_ERROR(BIF_P, BADARG); - ASSERT(pc[-5] == BeamOpCode(op_i_func_info_IaaI)); + ASSERT(pc[-6] == BeamOpCode(op_i_func_info_IaaI)); if (pc[0] != BeamOpCode(op_hipe_call_count)) BIF_RET(am_false); hcc = (struct hipe_call_count*)pc[-4]; diff --git a/erts/emulator/hipe/hipe_bif_list.m4 b/erts/emulator/hipe/hipe_bif_list.m4 index dcf3447af9..bb328b5915 100644 --- a/erts/emulator/hipe/hipe_bif_list.m4 +++ b/erts/emulator/hipe/hipe_bif_list.m4 @@ -153,7 +153,7 @@ standard_bif_interface_0(nbif_ports_0, ports_0) * BIFs and primops that may do a GC (change heap limit and walk the native stack). * XXX: erase/1 and put/2 cannot fail */ -gc_bif_interface_2(nbif_erts_internal_check_process_code_2, hipe_erts_internal_check_process_code_2) +gc_bif_interface_1(nbif_erts_internal_check_process_code_1, hipe_erts_internal_check_process_code_1) gc_bif_interface_1(nbif_erase_1, erase_1) gc_bif_interface_0(nbif_garbage_collect_0, garbage_collect_0) gc_nofail_primop_interface_1(nbif_gc_1, hipe_gc) diff --git a/erts/emulator/hipe/hipe_debug.c b/erts/emulator/hipe/hipe_debug.c index ace489452f..222a11db3d 100644 --- a/erts/emulator/hipe/hipe_debug.c +++ b/erts/emulator/hipe/hipe_debug.c @@ -62,10 +62,12 @@ static void print_beam_pc(BeamInstr *pc) } else if (pc == &beam_apply[1]) { printf("normal-process-exit"); } else { - BeamInstr *mfa = find_function_from_pc(pc); - if (mfa) + ErtsCodeMFA *cmfa = find_function_from_pc(pc); + if (cmfa) erts_printf("%T:%T/%bpu + 0x%bpx", - mfa[0], mfa[1], mfa[2], pc - &mfa[3]); + cmfa->module, cmfa->function, + cmfa->arity, + pc - erts_codemfa_to_code(cmfa)); else printf("?"); } @@ -214,10 +216,10 @@ void hipe_print_pcb(Process *p) U("seq..clock ", seq_trace_clock); U("seq..astcnt", seq_trace_lastcnt); U("seq..token ", seq_trace_token); - U("intial[0] ", u.initial[0]); - U("intial[1] ", u.initial[1]); - U("intial[2] ", u.initial[2]); - P("current ", current); + U("intial.mod ", u.initial.module); + U("intial.fun ", u.initial.function); + U("intial.ari ", u.initial.arity); + U("current ", current); P("cp ", cp); P("i ", i); U("catches ", catches); diff --git a/erts/emulator/hipe/hipe_gc.c b/erts/emulator/hipe/hipe_gc.c index 68c65dea27..e6ce7ce628 100644 --- a/erts/emulator/hipe/hipe_gc.c +++ b/erts/emulator/hipe/hipe_gc.c @@ -38,7 +38,7 @@ Eterm *fullsweep_nstack(Process *p, Eterm *n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -123,7 +123,7 @@ void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop) /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int sdesc_size; unsigned long ra; unsigned int i; @@ -244,7 +244,7 @@ Eterm *sweep_literals_nstack(Process *p, Eterm *old_htop, char *area, /* known nstack walk state */ Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; @@ -311,7 +311,7 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; /* arch-specific nstack walk state */ struct nstack_walk_state walk_state; @@ -356,3 +356,36 @@ nstack_any_heap_ref_ptrs(Process *rp, char* mod_start, Uint mod_size) } return 0; } + +int +nstack_any_cps_in_segment(Process *p, char* seg_start, Uint seg_size) +{ + Eterm *nsp; + Eterm *nsp_end; + const struct hipe_sdesc *sdesc; + /* arch-specific nstack walk state */ + struct nstack_walk_state walk_state; + + if (!p->hipe.nstack || !nstack_walk_init_check(p)) + return 0; + ASSERT(p->hipe.nsp && p->hipe.nstend); + nsp = nstack_walk_nsp_begin(p); + nsp_end = nstack_walk_nsp_end(p); + sdesc = nstack_walk_init_sdesc_ignore_trap(p, &walk_state); + + /* Check the topmost frame */ + if (ErtsInArea(sdesc->bucket.hvalue, seg_start, seg_size)) + return 1; + + while (!nstack_walk_nsp_reached_end(nsp, nsp_end)) { + unsigned sdesc_size = nstack_walk_frame_size(sdesc); + unsigned long ra = nstack_walk_frame_ra(nsp, sdesc); + if (ra == (unsigned long)nbif_stack_trap_ra) + ra = (unsigned long)p->hipe.ngra; + if (ErtsInArea(ra, seg_start, seg_size)) + return 1; + sdesc = hipe_find_sdesc(ra); + nsp = nstack_walk_next_frame(nsp, sdesc_size); + } + return 0; +} diff --git a/erts/emulator/hipe/hipe_load.c b/erts/emulator/hipe/hipe_load.c new file mode 100644 index 0000000000..2998ed87a2 --- /dev/null +++ b/erts/emulator/hipe/hipe_load.c @@ -0,0 +1,105 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.c + * + * HiPE atomic code loader + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "global.h" +#include "erl_binary.h" +#include "hipe_load.h" +#include "hipe_bif0.h" + +void hipe_free_loader_state(HipeLoaderState *stp) +{ + if (stp->module == NIL) return; + + // TODO: Needs to be freed separately. We'd like have a unified executable + // code allocator, so postpone this for now. + /* if (stp->text_segment) */ + /* erts_free(ERTS_ALC_T_HIPE, stp->text_segment); */ + stp->text_segment = NULL; + stp->text_segment_size = 0; + + if (stp->data_segment) + erts_free(ERTS_ALC_T_HIPE, stp->data_segment); + stp->data_segment = NULL; + stp->data_segment_size = 0; + + if (stp->new_hipe_refs) { + hipe_purge_refs(stp->new_hipe_refs, stp->module, 0); + stp->new_hipe_refs = NULL; + } + if (stp->new_hipe_sdesc) { + hipe_purge_sdescs(stp->new_hipe_sdesc, stp->module, 0); + stp->new_hipe_sdesc = NULL; + } + + stp->module = NIL; +} + +static void +hipe_loader_state_dtor(Binary* magic) +{ + HipeLoaderState* stp = ERTS_MAGIC_BIN_DATA(magic); + + ASSERT(ERTS_MAGIC_BIN_DESTRUCTOR(magic) == hipe_loader_state_dtor); + + hipe_free_loader_state(stp); +} + +Binary *hipe_alloc_loader_state(Eterm module) +{ + HipeLoaderState *stp; + Binary *magic; + + if (is_not_atom(module)) return NULL; + + magic = erts_create_magic_binary(sizeof(HipeLoaderState), + hipe_loader_state_dtor); + erts_refc_inc(&magic->refc, 1); + stp = ERTS_MAGIC_BIN_DATA(magic); + + stp->module = module; + stp->text_segment = NULL; + stp->text_segment_size = 0; + stp->data_segment = NULL; + stp->data_segment_size = 0; + + stp->new_hipe_refs = NULL; + stp->new_hipe_sdesc = NULL; + + return magic; +} + +HipeLoaderState * +hipe_get_loader_state(Binary *magic) +{ + if (ERTS_MAGIC_BIN_DESTRUCTOR(magic) != hipe_loader_state_dtor) + return NULL; + + return (HipeLoaderState*) ERTS_MAGIC_BIN_DATA(magic); +} diff --git a/erts/emulator/hipe/hipe_load.h b/erts/emulator/hipe/hipe_load.h new file mode 100644 index 0000000000..40c8a8aa2a --- /dev/null +++ b/erts/emulator/hipe/hipe_load.h @@ -0,0 +1,48 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_load.h + * + * HiPE atomic code loader + */ +#ifndef HIPE_LOAD_H +#define HIPE_LOAD_H + +#include "global.h" + +typedef struct hipe_loader_state { + Eterm module; /* Module name, atom */ + + void *text_segment; + Uint text_segment_size; + + void *data_segment; + Uint data_segment_size; + + struct hipe_ref* new_hipe_refs; + struct hipe_sdesc* new_hipe_sdesc; + +} HipeLoaderState; + +extern Binary *hipe_alloc_loader_state(Eterm module); +extern void hipe_free_loader_state(HipeLoaderState*); +extern HipeLoaderState *hipe_get_loader_state(Binary *binary); + +#endif /* HIPE_LOAD_H */ diff --git a/erts/emulator/hipe/hipe_mkliterals.c b/erts/emulator/hipe/hipe_mkliterals.c index b9d4226705..4573980e1e 100644 --- a/erts/emulator/hipe/hipe_mkliterals.c +++ b/erts/emulator/hipe/hipe_mkliterals.c @@ -435,9 +435,6 @@ static const struct rts_param rts_params[] = { presence or absence of struct erl_fun_thing's "next" field. */ { 5, "EFT_CREATOR", 1, offsetof(struct erl_fun_thing, creator) }, { 6, "EFT_FE", 1, offsetof(struct erl_fun_thing, fe) }, -#ifdef HIPE - { 7, "EFT_NATIVE_ADDRESS", 1, offsetof(struct erl_fun_thing, native_address) }, -#endif { 8, "EFT_ARITY", 1, offsetof(struct erl_fun_thing, arity) }, { 9, "EFT_NUM_FREE", 1, offsetof(struct erl_fun_thing, num_free) }, { 10, "EFT_ENV", 1, offsetof(struct erl_fun_thing, env[0]) }, diff --git a/erts/emulator/hipe/hipe_mode_switch.c b/erts/emulator/hipe/hipe_mode_switch.c index ed95045292..0706f8d2c9 100644 --- a/erts/emulator/hipe/hipe_mode_switch.c +++ b/erts/emulator/hipe/hipe_mode_switch.c @@ -718,12 +718,9 @@ void hipe_empty_nstack(Process *p) p->hipe.nstend = NULL; } -void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free) +void hipe_set_closure_stub(ErlFunEntry *fe) { - unsigned arity; - - arity = fe->arity; - fe->native_address = (Eterm*) hipe_closure_stub_address(arity); + fe->native_address = (Eterm*) hipe_closure_stub_address(fe->arity); } Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s) diff --git a/erts/emulator/hipe/hipe_mode_switch.h b/erts/emulator/hipe/hipe_mode_switch.h index e54b81cf78..334e978307 100644 --- a/erts/emulator/hipe/hipe_mode_switch.h +++ b/erts/emulator/hipe/hipe_mode_switch.h @@ -59,12 +59,11 @@ void hipe_set_call_trap(Uint *bfun, void *nfun, int is_closure); Process *hipe_mode_switch(Process*, unsigned, Eterm*); void hipe_inc_nstack(Process *p); void hipe_empty_nstack(Process *p); -void hipe_set_closure_stub(ErlFunEntry *fe, unsigned num_free); +void hipe_set_closure_stub(ErlFunEntry *fe); Eterm hipe_build_stacktrace(Process *p, struct StackTrace *s); ERTS_GLB_INLINE void hipe_reserve_beam_trap_frame(Process*, Eterm reg[], unsigned arity); ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process*); -ERTS_GLB_INLINE int hipe_is_ra_mode_switch(const void* ra); extern Uint hipe_beam_pc_return[]; extern Uint hipe_beam_pc_throw[]; @@ -113,11 +112,6 @@ ERTS_GLB_INLINE void hipe_unreserve_beam_trap_frame(Process *p) p->stop += 2; } -ERTS_GLB_INLINE int hipe_is_ra_mode_switch(const void* ra) -{ - return ra == nbif_return; -} - #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* ASM */ diff --git a/erts/emulator/hipe/hipe_module.c b/erts/emulator/hipe/hipe_module.c new file mode 100644 index 0000000000..469f077dd2 --- /dev/null +++ b/erts/emulator/hipe/hipe_module.c @@ -0,0 +1,35 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include "sys.h" +#include "hipe_arch.h" +#include "hipe_module.h" + +void hipe_free_module(HipeModule *mod) +{ + hipe_free_code(mod->text_segment, mod->text_segment_size); + if (mod->data_segment) /* Some modules lack data segments */ + erts_free(ERTS_ALC_T_HIPE, mod->data_segment); + + erts_free(ERTS_ALC_T_HIPE, mod); +} diff --git a/erts/emulator/hipe/hipe_module.h b/erts/emulator/hipe/hipe_module.h new file mode 100644 index 0000000000..b489f567cb --- /dev/null +++ b/erts/emulator/hipe/hipe_module.h @@ -0,0 +1,45 @@ +/* + * %CopyrightBegin% + + * + * Copyright Ericsson AB 2016. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ +/* + * hipe_module.h + * + * + */ +#ifndef HIPE_MODULE_H +#define HIPE_MODULE_H + +/* Forward-declare type to resolve circular dependency with module.h */ +typedef struct hipe_module HipeModule; + +#include "global.h" + +struct hipe_module { + void *text_segment; + Uint text_segment_size; + void *data_segment; + + struct hipe_ref* first_hipe_ref; /* all external hipe calls from this module */ + struct hipe_sdesc* first_hipe_sdesc; /* all stack descriptors for this module */ +}; + +extern void hipe_free_module(HipeModule *mod); + +#endif /* HIPE_MODULE_H */ diff --git a/erts/emulator/hipe/hipe_native_bif.c b/erts/emulator/hipe/hipe_native_bif.c index 9c03b3811c..9439b823ab 100644 --- a/erts/emulator/hipe/hipe_native_bif.c +++ b/erts/emulator/hipe/hipe_native_bif.c @@ -42,7 +42,7 @@ */ /* for -Wmissing-prototypes :-( */ -extern Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2); +extern Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1); extern Eterm hipe_show_nstack_1(BIF_ALIST_1); /* Used when a BIF can trigger a stack walk. */ @@ -51,12 +51,12 @@ static __inline__ void hipe_set_narity(Process *p, unsigned int arity) p->hipe.narity = arity; } -Eterm hipe_erts_internal_check_process_code_2(BIF_ALIST_2) +Eterm hipe_erts_internal_check_process_code_1(BIF_ALIST_1) { Eterm ret; - hipe_set_narity(BIF_P, 2); - ret = erts_internal_check_process_code_2(BIF_P, BIF__ARGS); + hipe_set_narity(BIF_P, 1); + ret = erts_internal_check_process_code_1(BIF_P, BIF__ARGS); hipe_set_narity(BIF_P, 0); return ret; } diff --git a/erts/emulator/hipe/hipe_ppc.c b/erts/emulator/hipe/hipe_ppc.c index 9b2048c457..4413748936 100644 --- a/erts/emulator/hipe/hipe_ppc.c +++ b/erts/emulator/hipe/hipe_ppc.c @@ -25,7 +25,6 @@ #endif #include "global.h" #include "erl_binary.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -68,34 +67,6 @@ void hipe_flush_icache_range(void *address, unsigned int nbytes) asm volatile("sync\n\tisync"); } -/* - * Management of 32MB code segments for regular code and trampolines. - */ - -#define SEGMENT_NRBYTES (32*1024*1024) /* named constant, _not_ a tunable */ - -static struct segment { - unsigned int *base; /* [base,base+32MB[ */ - unsigned int *code_pos; /* INV: base <= code_pos <= tramp_pos */ - unsigned int *tramp_pos; /* INV: tramp_pos <= base+32MB */ -} curseg; - -#define in_area(ptr,start,nbytes) \ - ((UWord)((char*)(ptr) - (char*)(start)) < (nbytes)) - -/* Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static void *new_code_mapping(void) -{ - return mmap(0, SEGMENT_NRBYTES, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS, - -1, 0); -} - static int check_callees(Eterm callees) { Eterm *tuple; @@ -119,126 +90,71 @@ static int check_callees(Eterm callees) return arity; } -static unsigned int *try_alloc(Uint nrwords, int nrcallees, Eterm callees, unsigned int **trampvec) + +static void generate_trampolines(Uint32* address, + int nrcallees, Eterm callees, + Uint32** trampvec) { - unsigned int *base, *address, *tramp_pos, nrfreewords; - int trampnr; + Uint32* trampoline = address; + int i; - tramp_pos = curseg.tramp_pos; - address = curseg.code_pos; - nrfreewords = tramp_pos - address; - if (nrwords > nrfreewords) - return NULL; - curseg.code_pos = address + nrwords; - nrfreewords -= nrwords; - - base = curseg.base; - for (trampnr = 1; trampnr <= nrcallees; ++trampnr) { - Eterm mfa = tuple_val(callees)[trampnr]; - Eterm m = tuple_val(mfa)[1]; - Eterm f = tuple_val(mfa)[2]; - unsigned int a = unsigned_val(tuple_val(mfa)[3]); - unsigned int *trampoline = hipe_mfa_get_trampoline(m, f, a); - if (!in_area(trampoline, base, SEGMENT_NRBYTES)) { + for (i = 0; i < nrcallees; ++i) { #if defined(__powerpc64__) - if (nrfreewords < 7) - return NULL; - nrfreewords -= 7; - tramp_pos = trampoline = tramp_pos - 7; - trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ - trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ - trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ - trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ - trampoline[5] = 0x7D6903A6; /* mtctr r11 */ - trampoline[6] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 7*sizeof(int)); +# define TRAMPOLINE_WORDS 7 + trampoline[0] = 0x3D600000; /* addis r11,r0,0 */ + trampoline[1] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[2] = 0x796B07C6; /* rldicr r11,r11,32,31 */ + trampoline[3] = 0x656B0000; /* oris r11,r11,0 */ + trampoline[4] = 0x616B0000; /* ori r11,r11,0 */ + trampoline[5] = 0x7D6903A6; /* mtctr r11 */ + trampoline[6] = 0x4E800420; /* bctr */ #else - if (nrfreewords < 4) - return NULL; - nrfreewords -= 4; - tramp_pos = trampoline = tramp_pos - 4; - trampoline[0] = 0x39600000; /* addi r11,r0,0 */ - trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ - trampoline[2] = 0x7D6903A6; /* mtctr r11 */ - trampoline[3] = 0x4E800420; /* bctr */ - hipe_flush_icache_range(trampoline, 4*sizeof(int)); +# define TRAMPOLINE_WORDS 4 + trampoline[0] = 0x39600000; /* addi r11,r0,0 */ + trampoline[1] = 0x3D6B0000; /* addis r11,r11,0 */ + trampoline[2] = 0x7D6903A6; /* mtctr r11 */ + trampoline[3] = 0x4E800420; /* bctr */ #endif - hipe_mfa_set_trampoline(m, f, a, trampoline); - } - trampvec[trampnr-1] = trampoline; + trampvec[i] = trampoline; + trampoline += TRAMPOLINE_WORDS; } - curseg.tramp_pos = tramp_pos; - return address; + hipe_flush_icache_range(address, nrcallees*TRAMPOLINE_WORDS*sizeof(Uint32)); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) { - Uint nrwords; + Uint code_words; int nrcallees; Eterm trampvecbin; - unsigned int **trampvec; - unsigned int *address; - unsigned int *base; - struct segment oldseg; + Uint32 **trampvec; + Uint32 *address; if (nrbytes & 0x3) return NULL; - nrwords = nrbytes >> 2; + code_words = nrbytes / sizeof(Uint32); nrcallees = check_callees(callees); if (nrcallees < 0) return NULL; - trampvecbin = new_binary(p, NULL, nrcallees*sizeof(unsigned int*)); - trampvec = (unsigned int**)binary_bytes(trampvecbin); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, nrcallees, callees, trampvec); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } + trampvecbin = new_binary(p, NULL, nrcallees*sizeof(Uint32*)); + trampvec = (Uint32**)binary_bytes(trampvecbin); + + address = erts_alloc(ERTS_ALC_T_HIPE_EXEC, + (code_words + nrcallees*TRAMPOLINE_WORDS)*sizeof(Uint32)); + + generate_trampolines(address + code_words, nrcallees, callees, trampvec); *trampolines = trampvecbin; return address; } -static unsigned int *alloc_stub(Uint nrwords) +void hipe_free_code(void* code, unsigned int bytes) { - unsigned int *address; - unsigned int *base; - struct segment oldseg; - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - base = new_code_mapping(); - if (base == MAP_FAILED) - return NULL; - oldseg = curseg; - curseg.base = base; - curseg.code_pos = base; - curseg.tramp_pos = (unsigned int*)((char*)base + SEGMENT_NRBYTES); - - address = try_alloc(nrwords, 0, NIL, NULL); - if (!address) { - munmap(base, SEGMENT_NRBYTES); - curseg = oldseg; - return NULL; - } - /* commit to new segment, ignore leftover space in old segment */ - } - return address; + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); } static void patch_imm16(Uint32 *address, unsigned int imm16) @@ -288,12 +204,12 @@ int hipe_patch_insn(void *address, Uint64 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(7); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 7*sizeof(Uint32)); if (!code) return NULL; @@ -312,7 +228,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[6] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 7*sizeof(int)); + hipe_flush_icache_range(code, 7*sizeof(Uint32)); return code; } @@ -360,7 +276,7 @@ int hipe_patch_insn(void *address, Uint32 value, Eterm type) void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { - unsigned int *code; + Uint32 *code; /* * Native code calls BEAM via a stub looking as follows: @@ -383,7 +299,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) if ((unsigned long)&nbif_callemu & ~0x01FFFFFCUL) abort(); - code = alloc_stub(4); + code = erts_alloc(ERTS_ALC_T_HIPE_EXEC, 4*sizeof(Uint32)); if (!code) return NULL; @@ -396,7 +312,7 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) /* ba nbif_callemu */ code[3] = 0x48000002 | (unsigned long)&nbif_callemu; - hipe_flush_icache_range(code, 4*sizeof(int)); + hipe_flush_icache_range(code, 4*sizeof(Uint32)); return code; } diff --git a/erts/emulator/hipe/hipe_risc_gc.h b/erts/emulator/hipe/hipe_risc_gc.h index 09568c140e..f019434f67 100644 --- a/erts/emulator/hipe/hipe_risc_gc.h +++ b/erts/emulator/hipe/hipe_risc_gc.h @@ -27,7 +27,7 @@ /* arch wrapper includes hipe_${arch}_asm.h to define NR_ARG_REGS */ struct nstack_walk_state { - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ }; static inline int nstack_walk_init_check(const Process *p) @@ -43,20 +43,20 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) return p->hipe.nsp + nstkarity; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)p->hipe.nra); state->sdesc0 = sdesc; return sdesc; } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { unsigned long ra = (unsigned long)p->hipe.nra; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)&nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -64,7 +64,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, return sdesc; } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { Eterm *nsp = p->hipe.nsp; p->hipe.nsp = nstack_walk_nsp_begin(p); @@ -103,7 +103,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -114,7 +114,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_risc_glue.h b/erts/emulator/hipe/hipe_risc_glue.h index 0284265307..1369b392fe 100644 --- a/erts/emulator/hipe/hipe_risc_glue.h +++ b/erts/emulator/hipe/hipe_risc_glue.h @@ -66,13 +66,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)&nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)&nbif_return }, - .summary = (1<<8), - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_risc_stack.c b/erts/emulator/hipe/hipe_risc_stack.c index dc98c96b8f..4001bedeb6 100644 --- a/erts/emulator/hipe/hipe_risc_stack.c +++ b/erts/emulator/hipe/hipe_risc_stack.c @@ -56,8 +56,8 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; @@ -175,7 +175,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -216,7 +216,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -237,7 +237,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; @@ -277,7 +277,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; int i; if (depth < 1) @@ -292,7 +292,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) ra = (unsigned long)p->hipe.nra; prev_ra = 0; i = 0; - for (;;) { + while (nsp < nsp_end) { if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; if (ra != prev_ra) { @@ -302,8 +302,6 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) break; prev_ra = ra; } - if (nsp >= nsp_end) - break; sdesc = hipe_find_sdesc(ra); nsp += arity + sdesc_fsize(sdesc); arity = sdesc_arity(sdesc); diff --git a/erts/emulator/hipe/hipe_sparc.c b/erts/emulator/hipe/hipe_sparc.c index 23020f34ee..876b20bb15 100644 --- a/erts/emulator/hipe/hipe_sparc.c +++ b/erts/emulator/hipe/hipe_sparc.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -88,8 +87,8 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint32 relDest, newI; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + relDest = (Uint32)((Sint32)destAddress - (Sint32)callAddress); newI = (1 << 30) | (relDest >> 2); *(Uint32*)callAddress = newI; @@ -97,105 +96,9 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -206,6 +109,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int nrbytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { unsigned int *code; @@ -235,6 +143,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_stack.c b/erts/emulator/hipe/hipe_stack.c index e2e6eb74b1..b80e44bc37 100644 --- a/erts/emulator/hipe/hipe_stack.c +++ b/erts/emulator/hipe/hipe_stack.c @@ -43,10 +43,10 @@ */ struct hipe_sdesc_table hipe_sdesc_table; -static struct sdesc **alloc_bucket(unsigned int size) +static struct hipe_sdesc **alloc_bucket(unsigned int size) { - unsigned long nbytes = size * sizeof(struct sdesc*); - struct sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); + unsigned long nbytes = size * sizeof(struct hipe_sdesc*); + struct hipe_sdesc **bucket = erts_alloc(ERTS_ALC_T_HIPE, nbytes); sys_memzero(bucket, nbytes); return bucket; } @@ -54,7 +54,7 @@ static struct sdesc **alloc_bucket(unsigned int size) static void hipe_grow_sdesc_table(void) { unsigned int old_size, new_size, new_mask; - struct sdesc **old_bucket, **new_bucket; + struct hipe_sdesc **old_bucket, **new_bucket; unsigned int i; old_size = 1 << hipe_sdesc_table.log2size; @@ -66,9 +66,9 @@ static void hipe_grow_sdesc_table(void) new_bucket = alloc_bucket(new_size); hipe_sdesc_table.bucket = new_bucket; for (i = 0; i < old_size; ++i) { - struct sdesc *b = old_bucket[i]; + struct hipe_sdesc *b = old_bucket[i]; while (b != NULL) { - struct sdesc *next = b->bucket.next; + struct hipe_sdesc *next = b->bucket.next; unsigned int j = (b->bucket.hvalue >> HIPE_RA_LSR_COUNT) & new_mask; b->bucket.next = new_bucket[j]; new_bucket[j] = b; @@ -78,11 +78,11 @@ static void hipe_grow_sdesc_table(void) erts_free(ERTS_ALC_T_HIPE, old_bucket); } -struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) +struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc *sdesc) { unsigned long ra; unsigned int i; - struct sdesc *chain; + struct hipe_sdesc *chain; unsigned int size; ra = sdesc->bucket.hvalue; @@ -102,7 +102,29 @@ struct sdesc *hipe_put_sdesc(struct sdesc *sdesc) return sdesc; } -void hipe_init_sdesc_table(struct sdesc *sdesc) +void hipe_destruct_sdesc(struct hipe_sdesc *sdesc) +{ + unsigned int i; + struct hipe_sdesc** prevp; + void* free_me; + + i = (sdesc->bucket.hvalue >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; + prevp = &hipe_sdesc_table.bucket[i]; + + for (; *prevp != sdesc; prevp = &(*prevp)->bucket.next) + ASSERT(*prevp); + + *prevp = sdesc->bucket.next; + hipe_sdesc_table.used -= 1; + + if (sdesc->has_exnra) + free_me = ErtsContainerStruct(sdesc, struct hipe_sdesc_with_exnra, sdesc); + else + free_me = sdesc; + erts_free(ERTS_ALC_T_HIPE, free_me); +} + +void hipe_init_sdesc_table(struct hipe_sdesc *sdesc) { unsigned int log2size, size; @@ -121,31 +143,46 @@ void hipe_init_sdesc_table(struct sdesc *sdesc) * representation. If different representations are needed in * the future, this code has to be made target dependent. */ -struct sdesc *hipe_decode_sdesc(Eterm arg) +struct hipe_sdesc *hipe_decode_sdesc(Eterm arg) { Uint ra, exnra; Eterm *live; - Uint fsize, arity, nlive, i, nslots, off; + Uint fsize, nargs, stk_nargs, nlive, i, nslots, off; Uint livebitswords, sdescbytes; void *p; - struct sdesc *sdesc; - - if (is_not_tuple(arg) || - (tuple_val(arg))[0] != make_arityval(6) || - term_to_Uint((tuple_val(arg))[1], &ra) == 0 || - term_to_Uint((tuple_val(arg))[2], &exnra) == 0 || - is_not_small((tuple_val(arg))[3]) || - (fsize = unsigned_val((tuple_val(arg))[3])) > 65535 || - is_not_small((tuple_val(arg))[4]) || - (arity = unsigned_val((tuple_val(arg))[4])) > 255 || - is_not_tuple((tuple_val(arg))[5])) + struct hipe_sdesc *sdesc; + Eterm* mfa_tpl; + Eterm* tp; + + if (is_not_tuple(arg)) + return 0; + + tp = tuple_val(arg); + if (tp[0] != make_arityval(6) || + term_to_Uint(tp[1], &ra) == 0 || + term_to_Uint(tp[2], &exnra) == 0 || + is_not_small(tp[3]) || + (fsize = unsigned_val(tp[3])) > 65535 || + is_not_small(tp[4]) || + (stk_nargs = unsigned_val(tp[4])) > 255 || + is_not_tuple(tp[5]) || + is_not_tuple(tp[6]) || + (mfa_tpl = tuple_val(tp[6]))[0] != make_arityval(3) || + is_not_atom(mfa_tpl[1]) || + is_not_atom(mfa_tpl[2]) || + is_not_small(mfa_tpl[3]) || + (nargs = unsigned_val(mfa_tpl[3])) > 255) return 0; + + if (stk_nargs > nargs) + return 0; + /* Get tuple with live slots */ - live = tuple_val((tuple_val(arg))[5]) + 1; + live = tuple_val(tp[5]) + 1; /* Get number of live slots */ nlive = arityval(live[-1]); - /* Calculate size of frame = locals + ra + arguments */ - nslots = fsize + 1 + arity; + /* Calculate size of frame = locals + ra + stack arguments */ + nslots = fsize + 1 + stk_nargs; /* Check that only valid slots are given. */ for (i = 0; i < nlive; ++i) { if (is_not_small(live[i]) || @@ -155,27 +192,34 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) } /* Calculate number of words for the live bitmap. */ - livebitswords = (fsize + arity + 1 + 31) / 32; + livebitswords = (fsize + stk_nargs + 1 + 31) / 32; /* Calculate number of bytes needed for the stack descriptor. */ sdescbytes = (exnra - ? offsetof(struct sdesc_with_exnra, sdesc.livebits) - : offsetof(struct sdesc, livebits)) + ? offsetof(struct hipe_sdesc_with_exnra, sdesc.livebits) + : offsetof(struct hipe_sdesc, livebits)) + livebitswords * sizeof(int); p = erts_alloc(ERTS_ALC_T_HIPE, sdescbytes); /* If we have an exception handler use the special sdesc_with_exnra structure. */ if (exnra) { - struct sdesc_with_exnra *sdesc_we = p; + struct hipe_sdesc_with_exnra *sdesc_we = p; sdesc_we->exnra = exnra; sdesc = &(sdesc_we->sdesc); } else sdesc = p; + sdesc->m_aix = atom_val(mfa_tpl[1]); + sdesc->f_aix = atom_val(mfa_tpl[2]); + sdesc->a = nargs; + + /* Initialise head of sdesc. */ sdesc->bucket.next = 0; sdesc->bucket.hvalue = ra; - sdesc->summary = (fsize << 9) | (exnra ? (1<<8) : 0) | arity; + sdesc->fsize = fsize; + sdesc->has_exnra = (exnra ? 1 : 0); + sdesc->stk_nargs = stk_nargs; /* Clear all live-bits */ for (i = 0; i < livebitswords; ++i) sdesc->livebits[i] = 0; @@ -184,13 +228,5 @@ struct sdesc *hipe_decode_sdesc(Eterm arg) off = unsigned_val(live[i]); sdesc->livebits[off / 32] |= (1 << (off & 31)); } -#ifdef DEBUG - { - Eterm mfa_tpl = tuple_val(arg)[6]; - sdesc->dbg_M = tuple_val(mfa_tpl)[1]; - sdesc->dbg_F = tuple_val(mfa_tpl)[2]; - sdesc->dbg_A = tuple_val(mfa_tpl)[3]; - } -#endif return sdesc; } diff --git a/erts/emulator/hipe/hipe_stack.h b/erts/emulator/hipe/hipe_stack.h index afa0ed4256..7e30358767 100644 --- a/erts/emulator/hipe/hipe_stack.h +++ b/erts/emulator/hipe/hipe_stack.h @@ -30,40 +30,43 @@ #include <stddef.h> /* offsetof() */ -struct sdesc { +struct hipe_sdesc { struct { unsigned long hvalue; /* return address */ - struct sdesc *next; /* hash collision chain */ + struct hipe_sdesc *next; /* hash collision chain */ } bucket; - unsigned int summary; /* frame size, exn handler presence flag, arity */ -#ifdef DEBUG - Eterm dbg_M, dbg_F; - unsigned dbg_A; -#endif - unsigned int livebits[1]; /* size depends on arch & data in summary field */ + unsigned int fsize : 23; /* frame size */ + unsigned int has_exnra : 1; /* exn handler presence flag */ + unsigned int stk_nargs : 8; /* arguments on stack */ + Uint32 m_aix; + Uint32 f_aix; + Uint32 a; + struct hipe_sdesc* next_in_modi; + Uint32 livebits[1]; /* size depends on arch & data in summary field */ }; -struct sdesc_with_exnra { +struct hipe_sdesc_with_exnra { unsigned long exnra; - struct sdesc sdesc; + struct hipe_sdesc sdesc; }; -static __inline__ unsigned int sdesc_fsize(const struct sdesc *sdesc) +static __inline__ unsigned int sdesc_fsize(const struct hipe_sdesc *sdesc) { - return sdesc->summary >> 9; + return sdesc->fsize; } -static __inline__ unsigned int sdesc_arity(const struct sdesc *sdesc) +/* Nr of arguments pushed on stack */ +static __inline__ unsigned int sdesc_arity(const struct hipe_sdesc *sdesc) { - return sdesc->summary & 0xFF; + return sdesc->stk_nargs; } -static __inline__ unsigned long sdesc_exnra(const struct sdesc *sdesc) +static __inline__ unsigned long sdesc_exnra(const struct hipe_sdesc *sdesc) { - if ((sdesc->summary & (1<<8))) { + if (sdesc->has_exnra) { const char *tmp; - tmp = (const char*)sdesc - offsetof(struct sdesc_with_exnra, sdesc); - return ((const struct sdesc_with_exnra*)tmp)->exnra; + tmp = (const char*)sdesc - offsetof(struct hipe_sdesc_with_exnra, sdesc); + return ((const struct hipe_sdesc_with_exnra*)tmp)->exnra; } return 0; } @@ -72,13 +75,14 @@ struct hipe_sdesc_table { unsigned int log2size; unsigned int mask; /* INV: mask == (1 << log2size)-1 */ unsigned int used; - struct sdesc **bucket; + struct hipe_sdesc **bucket; }; extern struct hipe_sdesc_table hipe_sdesc_table; -extern struct sdesc *hipe_put_sdesc(struct sdesc*); -extern void hipe_init_sdesc_table(struct sdesc*); -extern struct sdesc *hipe_decode_sdesc(Eterm); +extern struct hipe_sdesc *hipe_put_sdesc(struct hipe_sdesc*); +extern void hipe_destruct_sdesc(struct hipe_sdesc*); +extern void hipe_init_sdesc_table(struct hipe_sdesc*); +extern struct hipe_sdesc *hipe_decode_sdesc(Eterm); #if !defined(__GNUC__) || (__GNUC__ < 2) || (__GNUC__ == 2 && __GNUC_MINOR__ < 96) #define __builtin_expect(x, expected_value) (x) @@ -86,10 +90,10 @@ extern struct sdesc *hipe_decode_sdesc(Eterm); #define likely(x) __builtin_expect((x),1) #define unlikely(x) __builtin_expect((x),0) -static __inline__ const struct sdesc *hipe_find_sdesc(unsigned long ra) +static __inline__ const struct hipe_sdesc *hipe_find_sdesc(unsigned long ra) { unsigned int i = (ra >> HIPE_RA_LSR_COUNT) & hipe_sdesc_table.mask; - const struct sdesc *sdesc = hipe_sdesc_table.bucket[i]; + const struct hipe_sdesc *sdesc = hipe_sdesc_table.bucket[i]; if (likely(sdesc->bucket.hvalue == ra)) return sdesc; do { @@ -103,7 +107,7 @@ AEXTERN(void,nbif_stack_trap_ra,(void)); extern void hipe_print_nstack(Process*); extern void hipe_find_handler(Process*); extern void (*hipe_handle_stack_trap(Process*))(void); -extern void hipe_update_stack_trap(Process*, const struct sdesc*); +extern void hipe_update_stack_trap(Process*, const struct hipe_sdesc*); extern int hipe_fill_stacktrace(Process*, int, Eterm**); #if 0 && defined(HIPE_NSTACK_GROWS_UP) @@ -134,5 +138,7 @@ extern void gensweep_nstack(Process *p, Eterm **ptr_old_htop, Eterm **ptr_n_htop extern Eterm *sweep_literals_nstack(Process *p, Eterm *n_htop, char *area, Uint area_size); extern int nstack_any_heap_ref_ptrs(Process *, char* mod_start, Uint mod_size); +extern int nstack_any_cps_in_segment(Process *, char* seg_start, Uint seg_size); + #endif /* HIPE_STACK_H */ diff --git a/erts/emulator/hipe/hipe_x86.c b/erts/emulator/hipe/hipe_x86.c index 5f6c8c200e..c7e24673ac 100644 --- a/erts/emulator/hipe/hipe_x86.c +++ b/erts/emulator/hipe/hipe_x86.c @@ -24,7 +24,6 @@ #include "config.h" #endif #include "global.h" -#include <sys/mman.h> #include "hipe_arch.h" #include "hipe_native_bif.h" /* nbif_callemu() */ @@ -62,118 +61,17 @@ int hipe_patch_call(void *callAddress, void *destAddress, void *trampoline) { Uint rel32; - if (trampoline) - return -1; + ASSERT(trampoline == NULL); + rel32 = (Uint)destAddress - (Uint)callAddress - 4; *(Uint32*)callAddress = rel32; hipe_flush_icache_word(callAddress); return 0; } -/* - * Memory allocator for executable code. - * - * This is required on x86 because some combinations - * of Linux kernels and CPU generations default to - * non-executable memory mappings, causing ordinary - * malloc() memory to be non-executable. - */ -static unsigned int code_bytes; -static char *code_next; - -#if 0 /* change to non-zero to get allocation statistics at exit() */ -static unsigned int total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs, nr_large, total_lost; -static unsigned int atexit_done; - -static void alloc_code_stats(void) -{ - printf("\r\nalloc_code_stats: %u bytes mapped, %u joins, %u splits, %u bytes allocated, %u average alloc, %u large allocs, %u bytes lost\r\n", - total_mapped, nr_joins, nr_splits, total_alloc, nr_allocs ? total_alloc/nr_allocs : 0, nr_large, total_lost); -} - -static void atexit_alloc_code_stats(void) -{ - if (!atexit_done) { - atexit_done = 1; - (void)atexit(alloc_code_stats); - } -} - -#define ALLOC_CODE_STATS(X) do{X;}while(0) -#else -#define ALLOC_CODE_STATS(X) do{}while(0) -#endif - -/* FreeBSD 6.1 and Darwin breakage */ -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif - -static int morecore(unsigned int alloc_bytes) -{ - unsigned int map_bytes; - char *map_hint, *map_start; - - /* Page-align the amount to allocate. */ - map_bytes = (alloc_bytes + 4095) & ~4095; - - /* Round up small allocations. */ - if (map_bytes < 1024*1024) - map_bytes = 1024*1024; - else - ALLOC_CODE_STATS(++nr_large); - - /* Create a new memory mapping, ensuring it is executable - and in the low 2GB of the address space. Also attempt - to make it adjacent to the previous mapping. */ - map_hint = code_next + code_bytes; - if ((unsigned long)map_hint & 4095) - abort(); - map_start = mmap(map_hint, map_bytes, - PROT_EXEC|PROT_READ|PROT_WRITE, - MAP_PRIVATE|MAP_ANONYMOUS -#ifdef __x86_64__ - |MAP_32BIT -#endif - , - -1, 0); - if (map_start == MAP_FAILED) - return -1; - - ALLOC_CODE_STATS(total_mapped += map_bytes); - - /* Merge adjacent mappings, so the trailing portion of the previous - mapping isn't lost. In practice this is quite successful. */ - if (map_start == map_hint) { - ALLOC_CODE_STATS(++nr_joins); - code_bytes += map_bytes; - } else { - ALLOC_CODE_STATS(++nr_splits); - ALLOC_CODE_STATS(total_lost += code_bytes); - code_next = map_start; - code_bytes = map_bytes; - } - - ALLOC_CODE_STATS(atexit_alloc_code_stats()); - - return 0; -} - static void *alloc_code(unsigned int alloc_bytes) { - void *res; - - /* Align function entries. */ - alloc_bytes = (alloc_bytes + 3) & ~3; - - if (code_bytes < alloc_bytes && morecore(alloc_bytes) != 0) - return NULL; - ALLOC_CODE_STATS(++nr_allocs); - ALLOC_CODE_STATS(total_alloc += alloc_bytes); - res = code_next; - code_next += alloc_bytes; - code_bytes -= alloc_bytes; - return res; + return erts_alloc(ERTS_ALC_T_HIPE_EXEC, alloc_bytes); } void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process *p) @@ -184,6 +82,11 @@ void *hipe_alloc_code(Uint nrbytes, Eterm callees, Eterm *trampolines, Process * return alloc_code(nrbytes); } +void hipe_free_code(void* code, unsigned int bytes) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, code); +} + void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) { /* @@ -264,6 +167,11 @@ void *hipe_make_native_stub(void *callee_exp, unsigned int beamArity) return code; } +void hipe_free_native_stub(void* stub) +{ + erts_free(ERTS_ALC_T_HIPE_EXEC, stub); +} + void hipe_arch_print_pcb(struct hipe_process_state *p) { #define U(n,x) \ diff --git a/erts/emulator/hipe/hipe_x86_gc.h b/erts/emulator/hipe/hipe_x86_gc.h index 00fe03d8f9..a703e24b8c 100644 --- a/erts/emulator/hipe/hipe_x86_gc.h +++ b/erts/emulator/hipe/hipe_x86_gc.h @@ -30,9 +30,9 @@ struct nstack_walk_state { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ + const struct hipe_sdesc *sdesc0; /* .sdesc0 must be a pointer rvalue */ #else - struct sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ + struct hipe_sdesc sdesc0[1]; /* .sdesc0 must be a pointer rvalue */ #endif }; @@ -57,37 +57,36 @@ static inline Eterm *nstack_walk_nsp_begin(const Process *p) #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME - const struct sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); + const struct hipe_sdesc *sdesc = hipe_find_sdesc(p->hipe.nsp[0]); state->sdesc0 = sdesc; return sdesc; #else - unsigned int nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; - state->sdesc0[0].summary = (0 << 9) | (0 << 8) | nstkarity; + state->sdesc0[0].bucket.hvalue = 0; /* for nstack_any_cps_in_segment */ + state->sdesc0[0].fsize = 0; + state->sdesc0[0].has_exnra = 0; + state->sdesc0[0].stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); state->sdesc0[0].livebits[0] = 0; -# ifdef DEBUG - state->sdesc0[0].dbg_M = 0; - state->sdesc0[0].dbg_F = am_undefined; - state->sdesc0[0].dbg_A = 0; -# endif + state->sdesc0[0].m_aix = 0; + state->sdesc0[0].f_aix = atom_val(am_undefined); + state->sdesc0[0].a = 0; /* XXX: this appears to prevent a gcc-4.1.1 bug on x86 */ __asm__ __volatile__("" : : "m"(*state) : "memory"); return &state->sdesc0[0]; #endif } -static inline const struct sdesc* +static inline const struct hipe_sdesc* nstack_walk_init_sdesc_ignore_trap(const Process *p, struct nstack_walk_state *state) { #ifdef SKIP_YOUNGEST_FRAME unsigned long ra = p->hipe.nsp[0]; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; if (ra == (unsigned long)nbif_stack_trap_ra) ra = (unsigned long)p->hipe.ngra; sdesc = hipe_find_sdesc(ra); @@ -98,7 +97,7 @@ nstack_walk_init_sdesc_ignore_trap(const Process *p, #endif } -static inline void nstack_walk_update_trap(Process *p, const struct sdesc *sdesc0) +static inline void nstack_walk_update_trap(Process *p, const struct hipe_sdesc *sdesc0) { #ifdef SKIP_YOUNGEST_FRAME Eterm *nsp = p->hipe.nsp; @@ -137,7 +136,7 @@ static inline int nstack_walk_nsp_reached_end(const Eterm *nsp, const Eterm *nsp return nsp >= nsp_end; } -static inline unsigned int nstack_walk_frame_size(const struct sdesc *sdesc) +static inline unsigned int nstack_walk_frame_size(const struct hipe_sdesc *sdesc) { return sdesc_fsize(sdesc) + 1 + sdesc_arity(sdesc); } @@ -148,7 +147,7 @@ static inline Eterm *nstack_walk_frame_index(Eterm *nsp, unsigned int i) } static inline unsigned long -nstack_walk_frame_ra(const Eterm *nsp, const struct sdesc *sdesc) +nstack_walk_frame_ra(const Eterm *nsp, const struct hipe_sdesc *sdesc) { return nsp[sdesc_fsize(sdesc)]; } diff --git a/erts/emulator/hipe/hipe_x86_glue.h b/erts/emulator/hipe/hipe_x86_glue.h index 818d7444e2..de2b061706 100644 --- a/erts/emulator/hipe/hipe_x86_glue.h +++ b/erts/emulator/hipe/hipe_x86_glue.h @@ -58,16 +58,17 @@ static __inline__ unsigned int max(unsigned int x, unsigned int y) static __inline__ void hipe_arch_glue_init(void) { - static struct sdesc_with_exnra nbif_return_sdesc = { - .exnra = (unsigned long)nbif_fail, - .sdesc = { - .bucket = { .hvalue = (unsigned long)nbif_return }, - .summary = (1<<8), - #ifdef DEBUG - .dbg_F = am_return, - #endif - }, - }; + static struct hipe_sdesc_with_exnra nbif_return_sdesc; + + nbif_return_sdesc.exnra = (unsigned long)nbif_fail; + nbif_return_sdesc.sdesc.bucket.hvalue = (unsigned long)nbif_return; + nbif_return_sdesc.sdesc.fsize = 0; + nbif_return_sdesc.sdesc.has_exnra = 1; + nbif_return_sdesc.sdesc.stk_nargs = 0; + nbif_return_sdesc.sdesc.m_aix = atom_val(am_Empty); + nbif_return_sdesc.sdesc.f_aix = atom_val(am_return); + nbif_return_sdesc.sdesc.a = 0; + hipe_init_sdesc_table(&nbif_return_sdesc.sdesc); } diff --git a/erts/emulator/hipe/hipe_x86_signal.c b/erts/emulator/hipe/hipe_x86_signal.c index 50d08b96d3..1a34ce786c 100644 --- a/erts/emulator/hipe/hipe_x86_signal.c +++ b/erts/emulator/hipe/hipe_x86_signal.c @@ -51,7 +51,7 @@ #endif #include "hipe_signal.h" -#if __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3) +#if defined(__GLIBC__) && __GLIBC__ == 2 && (__GLIBC_MINOR__ >= 3) /* * __libc_sigaction() is the core routine. * Without libpthread, sigaction() and __sigaction() are both aliases diff --git a/erts/emulator/hipe/hipe_x86_stack.c b/erts/emulator/hipe/hipe_x86_stack.c index f1559b1451..31582b3a2e 100644 --- a/erts/emulator/hipe/hipe_x86_stack.c +++ b/erts/emulator/hipe/hipe_x86_stack.c @@ -52,15 +52,14 @@ void hipe_print_nstack(Process *p) { Eterm *nsp; Eterm *nsp_end; - struct sdesc sdesc0; - const struct sdesc *sdesc1; - const struct sdesc *sdesc; + struct hipe_sdesc sdesc0; + const struct hipe_sdesc *sdesc1; + const struct hipe_sdesc *sdesc; unsigned long ra; unsigned long exnra; unsigned int mask; unsigned int sdesc_size; unsigned int i; - unsigned int nstkarity; static const char dashes[2*sizeof(long)+5] = { [0 ... 2*sizeof(long)+3] = '-' }; @@ -68,10 +67,10 @@ void hipe_print_nstack(Process *p) nsp = p->hipe.nsp; nsp_end = p->hipe.nstend; - nstkarity = p->hipe.narity - NR_ARG_REGS; - if ((int)nstkarity < 0) - nstkarity = 0; - sdesc0.summary = nstkarity; + sdesc0.fsize = 0; + sdesc0.has_exnra = 0; + sdesc0.stk_nargs = (p->hipe.narity < NR_ARG_REGS ? 0 : + p->hipe.narity - NR_ARG_REGS); sdesc0.livebits[0] = ~1; sdesc = &sdesc0; @@ -158,7 +157,7 @@ void hipe_print_nstack(Process *p) #define MINSTACK 128 #define NSKIPFRAMES 4 -void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) +void hipe_update_stack_trap(Process *p, const struct hipe_sdesc *sdesc) { Eterm *nsp; Eterm *nsp_end; @@ -199,7 +198,7 @@ void hipe_update_stack_trap(Process *p, const struct sdesc *sdesc) void (*hipe_handle_stack_trap(Process *p))(void) { void (*ngra)(void) = p->hipe.ngra; - const struct sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); + const struct hipe_sdesc *sdesc = hipe_find_sdesc((unsigned long)ngra); hipe_update_stack_trap(p, sdesc); return ngra; } @@ -220,7 +219,7 @@ void hipe_find_handler(Process *p) unsigned long ra; unsigned long exnra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; nsp = p->hipe.nsp; @@ -262,7 +261,7 @@ int hipe_fill_stacktrace(Process *p, int depth, Eterm **trace) Eterm *nsp_end; unsigned long ra, prev_ra; unsigned int arity; - const struct sdesc *sdesc; + const struct hipe_sdesc *sdesc; unsigned int nstkarity; int i; diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c index a5714f8325..bb930ff03b 100644 --- a/erts/emulator/sys/common/erl_mmap.c +++ b/erts/emulator/sys/common/erl_mmap.c @@ -21,6 +21,7 @@ # include "config.h" #endif +#define ERTS_WANT_MEM_MAPPERS #include "sys.h" #include "erl_process.h" #include "erl_smp.h" @@ -358,12 +359,11 @@ char* erts_literals_start; UWord erts_literals_size; #endif -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER ErtsMemMapper erts_exec_mmapper; #endif - #define ERTS_MMAP_SIZE_SC_SA_INC(SZ) \ do { \ mm->size.supercarrier.used.total += (SZ); \ @@ -1878,7 +1878,7 @@ erts_mremap(ErtsMemMapper* mm, return NULL; } -#if ERTS_HAVE_OS_MREMAP || ERTS_HAVE_GENUINE_OS_MMAP +#if defined(ERTS_HAVE_OS_MREMAP) || defined(ERTS_HAVE_GENUINE_OS_MMAP) superaligned = (ERTS_MMAPFLG_SUPERALIGNED & flags); if (superaligned) { @@ -1898,7 +1898,7 @@ erts_mremap(ErtsMemMapper* mm, } } -#if ERTS_HAVE_GENUINE_OS_MMAP +#ifdef ERTS_HAVE_GENUINE_OS_MMAP if (asize < old_size && (!superaligned || ERTS_IS_SUPERALIGNED(ptr))) { @@ -1913,7 +1913,7 @@ erts_mremap(ErtsMemMapper* mm, return ptr; } #endif -#if ERTS_HAVE_OS_MREMAP +#ifdef ERTS_HAVE_OS_MREMAP if (superaligned) return remap_move(mm, flags, new_ptr, old_size, sizep); else { diff --git a/erts/emulator/sys/common/erl_mmap.h b/erts/emulator/sys/common/erl_mmap.h index 623b5188ac..2a07d93c8c 100644 --- a/erts/emulator/sys/common/erl_mmap.h +++ b/erts/emulator/sys/common/erl_mmap.h @@ -158,12 +158,23 @@ Eterm erts_mmap_info_options(ErtsMemMapper*, # include "erl_alloc_types.h" extern ErtsMemMapper erts_dflt_mmapper; -# if defined(ARCH_64) && defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + +# if defined(ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION) + +# if defined(ARCH_64) extern ErtsMemMapper erts_literal_mmapper; # endif -# ifdef ERTS_ALC_A_EXEC + +# if defined(ERTS_ALC_A_EXEC) && defined(__x86_64__) + /* + * On x86_64, exec_alloc employs its own super carrier 'erts_exec_mmaper' + * to ensure low memory for HiPE AMD64 small code model. + */ +# define ERTS_HAVE_EXEC_MMAPPER extern ErtsMemMapper erts_exec_mmapper; # endif + +# endif /* ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION */ #endif /* ERTS_WANT_MEM_MAPPERS */ /*#define HARD_DEBUG_MSEG*/ diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c index 882c93a83c..4dd986f4c8 100644 --- a/erts/emulator/sys/common/erl_mseg.c +++ b/erts/emulator/sys/common/erl_mseg.c @@ -1414,7 +1414,7 @@ erts_mseg_init(ErtsMsegInit_t *init) erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms"); -#ifdef ERTS_ALC_A_EXEC +#ifdef ERTS_HAVE_EXEC_MMAPPER /* Initialize erts_exec_mapper *FIRST*, to increase probability * of getting low memory for HiPE AMD64's small code model. */ diff --git a/erts/emulator/sys/unix/erl_unix_sys.h b/erts/emulator/sys/unix/erl_unix_sys.h index b64b0d87f6..bd554238b7 100644 --- a/erts/emulator/sys/unix/erl_unix_sys.h +++ b/erts/emulator/sys/unix/erl_unix_sys.h @@ -46,10 +46,10 @@ #include <signal.h> #include <setjmp.h> -#if HAVE_SYS_SOCKETIO_H +#ifdef HAVE_SYS_SOCKETIO_H # include <sys/socketio.h> #endif -#if HAVE_SYS_SOCKIO_H +#ifdef HAVE_SYS_SOCKIO_H # include <sys/sockio.h> #endif diff --git a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h index f0ca91bd06..2b742dd7e3 100644 --- a/erts/emulator/test/alloc_SUITE_data/testcase_driver.h +++ b/erts/emulator/test/alloc_SUITE_data/testcase_driver.h @@ -20,7 +20,7 @@ #ifndef TESTCASE_DRIVER_H__ #define TESTCASE_DRIVER_H__ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdlib.h> typedef struct { diff --git a/erts/emulator/test/code_SUITE.erl b/erts/emulator/test/code_SUITE.erl index 34515efa3d..774461c525 100644 --- a/erts/emulator/test/code_SUITE.erl +++ b/erts/emulator/test/code_SUITE.erl @@ -22,10 +22,9 @@ -export([all/0, suite/0, init_per_suite/1, end_per_suite/1, versions/1,new_binary_types/1, call_purged_fun_code_gone/1, call_purged_fun_code_reload/1, call_purged_fun_code_there/1, - t_check_process_code/1,t_check_old_code/1, - t_check_process_code_ets/1, - external_fun/1,get_chunk/1,module_md5/1,make_stub/1, - make_stub_many_funs/1,constant_pools/1,constant_refc_binaries/1, + multi_proc_purge/1, t_check_old_code/1, + external_fun/1,get_chunk/1,module_md5/1, + constant_pools/1,constant_refc_binaries/1, false_dependency/1,coverage/1,fun_confusion/1, t_copy_literals/1, t_copy_literals_frags/1]). @@ -36,9 +35,9 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [versions, new_binary_types, call_purged_fun_code_gone, - call_purged_fun_code_reload, call_purged_fun_code_there, t_check_process_code, - t_check_process_code_ets, t_check_old_code, external_fun, get_chunk, - module_md5, make_stub, make_stub_many_funs, + call_purged_fun_code_reload, call_purged_fun_code_there, + multi_proc_purge, t_check_old_code, external_fun, get_chunk, + module_md5, constant_pools, constant_refc_binaries, false_dependency, coverage, fun_confusion, t_copy_literals, t_copy_literals_frags]. @@ -154,218 +153,80 @@ call_purged_fun_code_there(Config) when is_list(Config) -> ok. call_purged_fun_test(Priv, Data, Type) -> - File = filename:join(Data, "my_code_test2"), - Code = filename:join(Priv, "my_code_test2"), - - catch erlang:purge_module(my_code_test2), - catch erlang:delete_module(my_code_test2), - catch erlang:purge_module(my_code_test2), - - {ok,my_code_test2} = c:c(File, [{outdir,Priv}]), - - T = ets:new(my_code_test2_fun_table, []), - ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}), - ets:insert(T, {my_fun2,my_code_test2:make_fun2()}), - - spawn(fun () -> - [{my_fun2,F2}] = ets:lookup(T, my_fun2), - F2(fun () -> - receive after infinity -> ok end - end, - fun () -> ok end), - exit(completed) - end), - - PurgeType = case Type of - code_gone -> - ok = file:delete(Code++".beam"), - true; - code_reload -> - true; - code_there -> - false - end, - - true = erlang:delete_module(my_code_test2), - - Purge = start_purge(my_code_test2, PurgeType), - - {P0, M0} = spawn_monitor(fun () -> - [{my_fun,F}] = ets:lookup(T, my_fun), - 4712 = F(1), - exit(completed) - end), - - wait_until(fun () -> - {status, suspended} - == process_info(P0, status) - end), - - ok = continue_purge(Purge), - - {P1, M1} = spawn_monitor(fun () -> - [{my_fun,F}] = ets:lookup(T, my_fun), - 4713 = F(2), - exit(completed) - end), - {P2, M2} = spawn_monitor(fun () -> - [{my_fun,F}] = ets:lookup(T, my_fun), - 4714 = F(3), - exit(completed) - end), - - wait_until(fun () -> - {status, suspended} - == process_info(P1, status) - end), - wait_until(fun () -> - {status, suspended} - == process_info(P2, status) - end), - - {current_function, - {erts_code_purger, - pending_purge_lambda, - 3}} = process_info(P0, current_function), - {current_function, - {erts_code_purger, - pending_purge_lambda, - 3}} = process_info(P1, current_function), - {current_function, - {erts_code_purger, - pending_purge_lambda, - 3}} = process_info(P2, current_function), - - case Type of - code_there -> - false = complete_purge(Purge); - _ -> - {true, true} = complete_purge(Purge) - end, - - case Type of - code_gone -> - receive - {'DOWN', M0, process, P0, Reason0} -> - {undef, _} = Reason0 - end, - receive - {'DOWN', M1, process, P1, Reason1} -> - {undef, _} = Reason1 - end, - receive - {'DOWN', M2, process, P2, Reason2} -> - {undef, _} = Reason2 - end; - _ -> - receive - {'DOWN', M0, process, P0, Reason0} -> - completed = Reason0 - end, - receive - {'DOWN', M1, process, P1, Reason1} -> - completed = Reason1 - end, - receive - {'DOWN', M2, process, P2, Reason2} -> - completed = Reason2 - end, - catch erlang:purge_module(my_code_test2), - catch erlang:delete_module(my_code_test2), - catch erlang:purge_module(my_code_test2) - end, - ok. - -t_check_process_code(Config) when is_list(Config) -> - case check_process_code_handle(indirect_references) of - false -> {skipped, "check_process_code() ignores funs"}; - true -> t_check_process_code_test(Config) - end. - -t_check_process_code_test(Config) -> - Priv = proplists:get_value(priv_dir, Config), - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "my_code_test"), - Code = filename:join(Priv, "my_code_test"), + OptsList = case erlang:system_info(hipe_architecture) of + undefined -> [[]]; + _ -> [[], [native]] + end, + [call_purged_fun_test_do(Priv, Data, Type, CO, FO) + || CO <- OptsList, FO <- OptsList]. - catch erlang:purge_module(my_code_test), - catch erlang:delete_module(my_code_test), - catch erlang:purge_module(my_code_test), - - {ok,my_code_test} = c:c(File, [{outdir,Priv}]), - - MyFun = fun(X, Y) -> X + Y end, %Confuse things. - F = my_code_test:make_fun(42), - 2 = fun_refc(F), - MyFun2 = fun(X, Y) -> X * Y end, %Confuse things. - 44 = F(2), - %% Delete the module and call the fun again. - true = erlang:delete_module(my_code_test), - 2 = fun_refc(F), - 45 = F(3), - {'EXIT',{undef,_}} = (catch my_code_test:make_fun(33)), - - %% The fun should still be there, preventing purge. - true = erlang:check_process_code(self(), my_code_test), - gc(), - gc(), %Place funs on the old heap. - true = erlang:check_process_code(self(), my_code_test), - - %% Using the funs here guarantees that they will not be prematurely garbed. - 48 = F(6), - 3 = MyFun(1, 2), - 12 = MyFun2(3, 4), - - %% Kill all funs. - t_check_process_code1(Code, []). - -%% The real fun was killed, but we have some fakes which look similar. - -t_check_process_code1(Code, Fakes) -> - MyFun = fun(X, Y) -> X + Y + 1 end, %Confuse things. - false = erlang:check_process_code(self(), my_code_test), - 4 = MyFun(1, 2), - t_check_process_code2(Code, Fakes). - -t_check_process_code2(Code, _) -> - false = erlang:check_process_code(self(), my_code_test), - true = erlang:purge_module(my_code_test), - - %% In the next test we will load the same module twice. - {module,my_code_test} = code:load_abs(Code), - F = my_code_test:make_fun(37), - 2 = fun_refc(F), - false = erlang:check_process_code(self(), my_code_test), - {module,my_code_test} = code:load_abs(Code), - 2 = fun_refc(F), +call_purged_fun_test_do(Priv, Data, Type, CallerOpts, FunOpts) -> + io:format("Compile caller as ~p and funs as ~p\n", [CallerOpts, FunOpts]), + SrcFile = filename:join(Data, "call_purged_fun_tester.erl"), + ObjFile = filename:join(Priv, "call_purged_fun_tester.beam"), + {ok,Mod,Code} = compile:file(SrcFile, [binary, report | CallerOpts]), + {module,Mod} = code:load_binary(Mod, ObjFile, Code), - %% Still false because the fun with the same identify is found - %% in the current code. - false = erlang:check_process_code(self(), my_code_test), + call_purged_fun_tester:do(Priv, Data, Type, FunOpts). - %% Some fake funs in the same module should not do any difference. - false = erlang:check_process_code(self(), my_code_test), - 38 = F(1), - t_check_process_code3(Code, F, []). +multi_proc_purge(Config) when is_list(Config) -> + %% + %% Make sure purge requests aren't lost when + %% purger process is working. + %% + Priv = proplists:get_value(priv_dir, Config), + Data = proplists:get_value(data_dir, Config), + File1 = filename:join(Data, "my_code_test"), + File2 = filename:join(Data, "my_code_test2"), + + {ok,my_code_test} = c:c(File1, [{outdir,Priv}]), + {ok,my_code_test2} = c:c(File2, [{outdir,Priv}]), + erlang:delete_module(my_code_test), + erlang:delete_module(my_code_test2), -t_check_process_code3(Code, F, Fakes) -> - Pid = spawn_link(fun() -> body(F, Fakes) end), - true = erlang:purge_module(my_code_test), - false = erlang:check_process_code(self(), my_code_test), - false = erlang:check_process_code(Pid, my_code_test), + Self = self(), - true = erlang:delete_module(my_code_test), - true = erlang:check_process_code(self(), my_code_test), - true = erlang:check_process_code(Pid, my_code_test), - 39 = F(2), - t_check_process_code4(Code, Pid). - -t_check_process_code4(_Code, Pid) -> - Pid ! drop_funs, - receive after 1 -> ok end, - false = erlang:check_process_code(Pid, my_code_test), + Fun1 = fun () -> + erts_code_purger:purge(my_code_test), + Self ! {self(), done} + end, + Fun2 = fun () -> + erts_code_purger:soft_purge(my_code_test2), + Self ! {self(), done} + end, + Fun3 = fun () -> + erts_code_purger:purge('__nonexisting_module__'), + Self ! {self(), done} + end, + Fun4 = fun () -> + erts_code_purger:soft_purge('__another_nonexisting_module__'), + Self ! {self(), done} + end, + + Pid1 = spawn_link(Fun1), + Pid2 = spawn_link(Fun2), + Pid3 = spawn_link(Fun3), + Pid4 = spawn_link(Fun4), + Pid5 = spawn_link(Fun1), + Pid6 = spawn_link(Fun2), + Pid7 = spawn_link(Fun3), + receive after 50 -> ok end, + Pid8 = spawn_link(Fun4), + Pid9 = spawn_link(Fun1), + Pid10 = spawn_link(Fun2), + Pid11 = spawn_link(Fun3), + Pid12 = spawn_link(Fun4), + Pid13 = spawn_link(Fun1), + receive after 50 -> ok end, + Pid14 = spawn_link(Fun2), + Pid15 = spawn_link(Fun3), + Pid16 = spawn_link(Fun4), + + lists:foreach(fun (P) -> receive {P, done} -> ok end end, + [Pid1, Pid2, Pid3, Pid4, Pid5, Pid6, Pid7, Pid8, + Pid9, Pid10, Pid11, Pid12, Pid13, Pid14, Pid15, Pid16]), ok. body(F, Fakes) -> @@ -388,72 +249,6 @@ gc() -> gc1(). gc1() -> ok. -%% Test check_process_code/2 in combination with a fun obtained from an ets table. -t_check_process_code_ets(Config) when is_list(Config) -> - case check_process_code_handle(indirect_references) of - false -> - {skipped, "check_process_code() ignores funs"}; - true -> - case test_server:is_native(?MODULE) of - true -> - {skip,"Native code"}; - false -> - do_check_process_code_ets(Config) - end - end. - -do_check_process_code_ets(Config) -> - Priv = proplists:get_value(priv_dir, Config), - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "my_code_test"), - - catch erlang:purge_module(my_code_test), - catch erlang:delete_module(my_code_test), - catch erlang:purge_module(my_code_test), - {ok,my_code_test} = c:c(File, [{outdir,Priv}]), - - T = ets:new(my_code_test, []), - ets:insert(T, {7,my_code_test:make_fun(107)}), - ets:insert(T, {8,my_code_test:make_fun(108)}), - erlang:delete_module(my_code_test), - false = erlang:check_process_code(self(), my_code_test), - Body = fun() -> - [{7,F1}] = ets:lookup(T, 7), - [{8,F2}] = ets:lookup(T, 8), - IdleLoop = fun() -> receive _X -> ok end end, - RecLoop = fun(Again) -> - receive - call -> 110 = F1(3), - 100 = F2(-8), - Again(Again); - {drop_funs,To} -> - To ! funs_dropped, - IdleLoop() - end - end, - true = erlang:check_process_code(self(), my_code_test), - RecLoop(RecLoop) - end, - Pid = spawn_link(Body), - receive after 1 -> ok end, - true = erlang:check_process_code(Pid, my_code_test), - Pid ! call, - Pid ! {drop_funs,self()}, - - receive - funs_dropped -> ok; - Other -> ct:fail({unexpected,Other}) - after 10000 -> - ct:fail(no_funs_dropped_answer) - end, - - false = erlang:check_process_code(Pid, my_code_test), - ok. - -fun_refc(F) -> - {refc,Count} = erlang:fun_info(F, refc), - Count. - %% Test the erlang:check_old_code/1 BIF. t_check_old_code(Config) when is_list(Config) -> @@ -543,67 +338,6 @@ module_md5_ok(Code) -> end. -make_stub(Config) when is_list(Config) -> - catch erlang:purge_module(my_code_test), - MD5 = erlang:md5(<<>>), - - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "my_code_test"), - {ok,my_code_test,Code} = compile:file(File, [binary]), - - my_code_test = code:make_stub_module(my_code_test, Code, {[],[],MD5}), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - my_code_test = code:make_stub_module(my_code_test, - make_unaligned_sub_binary(Code), - {[],[],MD5}), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - my_code_test = code:make_stub_module(my_code_test, zlib:gzip(Code), - {[],[],MD5}), - true = erlang:delete_module(my_code_test), - true = erlang:purge_module(my_code_test), - - %% Should fail. - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, <<"bad">>, {[],[],MD5})), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test, - bit_sized_binary(Code), - {[],[],MD5})), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(my_code_test_with_wrong_name, - Code, {[],[],MD5})), - ok. - -make_stub_many_funs(Config) when is_list(Config) -> - catch erlang:purge_module(many_funs), - MD5 = erlang:md5(<<>>), - - Data = proplists:get_value(data_dir, Config), - File = filename:join(Data, "many_funs"), - {ok,many_funs,Code} = compile:file(File, [binary]), - - many_funs = code:make_stub_module(many_funs, Code, {[],[],MD5}), - true = erlang:delete_module(many_funs), - true = erlang:purge_module(many_funs), - many_funs = code:make_stub_module(many_funs, - make_unaligned_sub_binary(Code), - {[],[],MD5}), - true = erlang:delete_module(many_funs), - true = erlang:purge_module(many_funs), - - %% Should fail. - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, <<"bad">>, {[],[],MD5})), - {'EXIT',{badarg,_}} = - (catch code:make_stub_module(many_funs, - bit_sized_binary(Code), - {[],[],MD5})), - ok. - constant_pools(Config) when is_list(Config) -> Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "literals"), @@ -1137,38 +871,3 @@ flush() -> id(I) -> I. -check_process_code_handle(What) -> - lists:member(What, erlang:system_info(check_process_code)). - -wait_until(Fun) -> - case Fun() of - true -> - ok; - false -> - receive after 100 -> ok end, - wait_until(Fun) - end. - -start_purge(Mod, Type) when is_atom(Mod) - andalso ((Type == true) - orelse (Type == false)) -> - Ref = make_ref(), - erts_code_purger ! {test_purge, Mod, self(), Type, Ref}, - receive - {started, Ref} -> - Ref - end. - -continue_purge(Ref) when is_reference(Ref) -> - erts_code_purger ! {continue, Ref}, - receive - {continued, Ref} -> - ok - end. - -complete_purge(Ref) when is_reference(Ref) -> - erts_code_purger ! {complete, Ref}, - receive - {test_purge, Res, Ref} -> - Res - end. diff --git a/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl new file mode 100644 index 0000000000..5e031abca8 --- /dev/null +++ b/erts/emulator/test/code_SUITE_data/call_purged_fun_tester.erl @@ -0,0 +1,167 @@ +-module(call_purged_fun_tester). + +-export([do/4]). + +do(Priv, Data, Type, Opts) -> + File = filename:join(Data, "my_code_test2"), + Code = filename:join(Priv, "my_code_test2"), + + catch erlang:purge_module(my_code_test2), + catch erlang:delete_module(my_code_test2), + catch erlang:purge_module(my_code_test2), + + {ok,my_code_test2} = c:c(File, [{outdir,Priv} | Opts]), + + IsNative = lists:member(native,Opts), + IsNative = code:is_module_native(my_code_test2), + + T = ets:new(my_code_test2_fun_table, []), + ets:insert(T, {my_fun,my_code_test2:make_fun(4711)}), + ets:insert(T, {my_fun2,my_code_test2:make_fun2()}), + + spawn(fun () -> + [{my_fun2,F2}] = ets:lookup(T, my_fun2), + F2(fun () -> + receive after infinity -> ok end + end, + fun () -> ok end), + exit(completed) + end), + + PurgeType = case Type of + code_gone -> + ok = file:delete(Code++".beam"), + true; + code_reload -> + true; + code_there -> + false + end, + + true = erlang:delete_module(my_code_test2), + + Purge = start_purge(my_code_test2, PurgeType), + + {P0, M0} = spawn_monitor(fun () -> + [{my_fun,F}] = ets:lookup(T, my_fun), + 4712 = F(1), + exit(completed) + end), + + wait_until(fun () -> + {status, suspended} + == process_info(P0, status) + end), + + ok = continue_purge(Purge), + + {P1, M1} = spawn_monitor(fun () -> + [{my_fun,F}] = ets:lookup(T, my_fun), + 4713 = F(2), + exit(completed) + end), + {P2, M2} = spawn_monitor(fun () -> + [{my_fun,F}] = ets:lookup(T, my_fun), + 4714 = F(3), + exit(completed) + end), + + wait_until(fun () -> + {status, suspended} + == process_info(P1, status) + end), + wait_until(fun () -> + {status, suspended} + == process_info(P2, status) + end), + + {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P0, current_function), + {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P1, current_function), + {current_function, + {erts_code_purger, + pending_purge_lambda, + 3}} = process_info(P2, current_function), + + case Type of + code_there -> + false = complete_purge(Purge); + _ -> + {true, true} = complete_purge(Purge) + end, + + case Type of + code_gone -> + receive + {'DOWN', M0, process, P0, Reason0} -> + {undef, _} = Reason0 + end, + receive + {'DOWN', M1, process, P1, Reason1} -> + {undef, _} = Reason1 + end, + receive + {'DOWN', M2, process, P2, Reason2} -> + {undef, _} = Reason2 + end; + _ -> + receive + {'DOWN', M0, process, P0, Reason0} -> + completed = Reason0 + end, + receive + {'DOWN', M1, process, P1, Reason1} -> + completed = Reason1 + end, + receive + {'DOWN', M2, process, P2, Reason2} -> + completed = Reason2 + end, + catch erlang:purge_module(my_code_test2), + catch erlang:delete_module(my_code_test2), + catch erlang:purge_module(my_code_test2) + end, + ok. + +wait_until(Fun) -> + ok = wait_until(Fun, 20). + +wait_until(Fun, N) -> + case {Fun(),N} of + {true, _} -> + ok; + {false, 0} -> + timeout; + {false, _} -> + receive after 100 -> ok end, + wait_until(Fun, N-1) + end. + +start_purge(Mod, Type) when is_atom(Mod) + andalso ((Type == true) + orelse (Type == false)) -> + Ref = make_ref(), + erts_code_purger ! {test_purge, Mod, self(), Type, Ref}, + receive + {started, Ref} -> + Ref + end. + +continue_purge(Ref) when is_reference(Ref) -> + erts_code_purger ! {continue, Ref}, + receive + {continued, Ref} -> + ok + end. + +complete_purge(Ref) when is_reference(Ref) -> + erts_code_purger ! {complete, Ref}, + receive + {test_purge, Res, Ref} -> + Res + end. diff --git a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c index a0019e5d95..daaff955bc 100644 --- a/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c +++ b/erts/emulator/test/dirty_nif_SUITE_data/dirty_nif_SUITE.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <assert.h> #ifdef __WIN32__ #include <windows.h> diff --git a/erts/emulator/test/gc_SUITE.erl b/erts/emulator/test/gc_SUITE.erl index 8a600b7d9f..35dd147550 100644 --- a/erts/emulator/test/gc_SUITE.erl +++ b/erts/emulator/test/gc_SUITE.erl @@ -23,15 +23,26 @@ -module(gc_SUITE). -include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). + -export([all/0, suite/0]). --export([grow_heap/1, grow_stack/1, grow_stack_heap/1, max_heap_size/1]). +-export([ + grow_heap/1, + grow_stack/1, + grow_stack_heap/1, + max_heap_size/1, + minor_major_gc_option_async/1, + minor_major_gc_option_self/1 +]). suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [grow_heap, grow_stack, grow_stack_heap, max_heap_size]. + [grow_heap, grow_stack, grow_stack_heap, max_heap_size, + minor_major_gc_option_self, + minor_major_gc_option_async]. %% Produce a growing list of elements, @@ -190,3 +201,91 @@ long_receive() -> after 10000 -> ok end. + +minor_major_gc_option_self(_Config) -> + Endless = fun Endless() -> + receive + {gc, Type} -> erlang:garbage_collect(self(), [{type, Type}]) + after 100 -> ok end, + Endless() + end, + + %% Try as major, a test process will self-trigger GC + P1 = spawn(Endless), + erlang:garbage_collect(P1, []), + erlang:trace(P1, true, [garbage_collection]), + P1 ! {gc, major}, + expect_trace_messages(P1, [gc_major_start, gc_major_end]), + erlang:trace(P1, false, [garbage_collection]), + erlang:exit(P1, kill), + + %% Try as minor, a test process will self-trigger GC + P2 = spawn(Endless), + erlang:garbage_collect(P2, []), + erlang:trace(P2, true, [garbage_collection]), + P2 ! {gc, minor}, + expect_trace_messages(P2, [gc_minor_start, gc_minor_end]), + erlang:trace(P2, false, [garbage_collection]), + erlang:exit(P2, kill). + +minor_major_gc_option_async(_Config) -> + Endless = fun Endless() -> + receive after 100 -> ok end, + Endless() + end, + + %% Try with default option, must be major gc + P1 = spawn(Endless), + erlang:garbage_collect(P1, []), + erlang:trace(P1, true, [garbage_collection]), + erlang:garbage_collect(P1, []), + expect_trace_messages(P1, [gc_major_start, gc_major_end]), + erlang:trace(P1, false, [garbage_collection]), + erlang:exit(P1, kill), + + %% Try with the 'major' type + P2 = spawn(Endless), + erlang:garbage_collect(P2, []), + erlang:trace(P2, true, [garbage_collection]), + erlang:garbage_collect(P2, [{type, major}]), + expect_trace_messages(P2, [gc_major_start, gc_major_end]), + erlang:trace(P2, false, [garbage_collection]), + erlang:exit(P2, kill), + + %% Try with 'minor' option, once + P3 = spawn(Endless), + erlang:garbage_collect(P3, []), + erlang:trace(P3, true, [garbage_collection]), + erlang:garbage_collect(P3, [{type, minor}]), + expect_trace_messages(P3, [gc_minor_start, gc_minor_end]), + erlang:trace(P3, false, [garbage_collection]), + erlang:exit(P3, kill), + + %% Try with 'minor' option, once, async + P4 = spawn(Endless), + Ref = erlang:make_ref(), + erlang:garbage_collect(P4, []), + erlang:trace(P4, true, [garbage_collection]), + ?assertEqual(async, + erlang:garbage_collect(P4, [{type, minor}, {async, Ref}])), + expect_trace_messages(P4, [gc_minor_start, gc_minor_end]), + erlang:trace(P4, false, [garbage_collection]), + receive {garbage_collect, Ref, true} -> ok; + Other4 -> ct:pal("Unexpected message: ~p~n" + ++ "while waiting for async gc result", [Other4]) + after 2000 -> ?assert(false) + end, + erlang:exit(P4, kill). + +%% Given a list of atoms, trace tags - receives messages and checks if they are +%% trace events, and if the tag matches. Else will crash failing the test. +expect_trace_messages(_Pid, []) -> ok; +expect_trace_messages(Pid, [Tag | TraceTags]) -> + receive + {trace, Pid, Tag, _Data} -> ok; + AnythingElse -> + ct:pal("Unexpected message: ~p~nWhile expected {trace, _, ~p, _}", + [AnythingElse, Tag]), + ?assert(false) + end, + expect_trace_messages(Pid, TraceTags). diff --git a/erts/emulator/test/hipe_SUITE.erl b/erts/emulator/test/hipe_SUITE.erl index a556b4ddc0..0b44dd7fb7 100644 --- a/erts/emulator/test/hipe_SUITE.erl +++ b/erts/emulator/test/hipe_SUITE.erl @@ -19,12 +19,17 @@ %% -module(hipe_SUITE). --export([all/0, t_copy_literals/1]). +-export([all/0 + ,t_copy_literals/1 + ,t_purge/1 + ]). all() -> case erlang:system_info(hipe_architecture) of undefined -> {skip, "HiPE is disabled"}; - _ -> [t_copy_literals] + _ -> [t_copy_literals + ,t_purge + ] end. t_copy_literals(doc) -> @@ -65,3 +70,51 @@ t_copy_literals(Config) when is_list(Config) -> true = erlang:delete_module(ref_cell), true = erlang:purge_module(ref_cell), ok. + +t_purge(doc) -> "Checks that native code is properly found and purged"; +t_purge(Config) when is_list(Config) -> + Data = proplists:get_value(data_dir, Config), + Priv = proplists:get_value(priv_dir, Config), + SrcFile = filename:join(Data, "ref_cell"), + BeamFile = filename:join(Priv, "ref_cell"), + {ok,ref_cell} = c:c(SrcFile, [{outdir,Priv},native]), + true = code:is_module_native(ref_cell), + + PA = ref_cell:start_link(), + + %% Unload, PA should still be running + true = erlang:delete_module(ref_cell), + %% Can't use ref_cel:call/2, it's in old code! + call(PA, {put_res_of, fun()-> hej end}), + hej = call(PA, get), + + %% Load same module again + code:load_abs(BeamFile), + true = code:is_module_native(ref_cell), + PB = ref_cell:start_link(), + + %% Purge old code, PA should be killed, PB should survive + unlink(PA), + ARef = monitor(process, PA), + true = erlang:purge_module(ref_cell), + receive {'DOWN', ARef, process, PA, killed} -> ok + after 1 -> ct:fail("PA was not killed") + end, + + %% Unload, PB should still be running + true = erlang:delete_module(ref_cell), + call(PB, {put_res_of, fun()-> svejs end}), + svejs = call(PB, get), + + unlink(PB), + BRef = monitor(process, PB), + true = erlang:purge_module(ref_cell), + receive {'DOWN', BRef, process, PB, killed} -> ok + after 1 -> ct:fail("PB was not killed") + end, + + ok. + +call(Pid, Call) -> + Pid ! {Call, self()}, + receive {Pid, Res} -> Res end. diff --git a/erts/emulator/test/match_spec_SUITE.erl b/erts/emulator/test/match_spec_SUITE.erl index 6733237b20..34e956bc21 100644 --- a/erts/emulator/test/match_spec_SUITE.erl +++ b/erts/emulator/test/match_spec_SUITE.erl @@ -427,13 +427,13 @@ silent_no_ms(Config) when is_list(Config) -> %% [{trace,Tracee,call,{?MODULE,f1,[start]}}, {trace,Tracee,return_to, - {?MODULE,'-silent_no_ms/1-fun-2-',0}}, + {?MODULE,'-silent_no_ms/1-fun-3-',0}}, {trace,Tracee,call,{?MODULE,f2,[f,g]}}, {trace,Tracee,return_to, - {?MODULE,'-silent_no_ms/1-fun-2-',0}}, + {?MODULE,'-silent_no_ms/1-fun-3-',0}}, {trace,Tracee,call,{erlang,integer_to_list,[2]}}, {trace,Tracee,return_to, - {?MODULE,'-silent_no_ms/1-fun-2-',0}}, + {?MODULE,'-silent_no_ms/1-fun-3-',0}}, {trace,Tracee,call,{?MODULE,f2,[h,i]}}, {trace,Tracee,return_to,{?MODULE,f3,2}}] end). @@ -484,7 +484,7 @@ ms_trace2(Config) when is_list(Config) -> %% %% Expected: (no return_to for global call trace) %% - Origin = {match_spec_SUITE,'-ms_trace2/1-fun-0-',1}, + Origin = {match_spec_SUITE,'-ms_trace2/1-fun-1-',1}, [{trace_ts,Tracee,call, {?MODULE,fn, [[all],[call,return_to,{tracer,Tracer}]]}, @@ -574,7 +574,7 @@ ms_trace3(Config) when is_list(Config) -> %% %% Expected: (no return_to for global call trace) %% - Origin = {match_spec_SUITE,'-ms_trace3/1-fun-1-',2}, + Origin = {match_spec_SUITE,'-ms_trace3/1-fun-2-',2}, [{trace_ts,Controller,call, {?MODULE,fn,[TraceeName,[all], [call,return_to,send,'receive', diff --git a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c index e011aadce9..46ee8b5540 100644 --- a/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c +++ b/erts/emulator/test/mtx_SUITE_data/mtx_SUITE.c @@ -24,7 +24,7 @@ * Author: Rickard Green */ -#include "erl_nif.h" +#include <erl_nif.h> #ifdef __WIN32__ # ifndef WIN32_LEAN_AND_MEAN diff --git a/erts/emulator/test/nif_SUITE.erl b/erts/emulator/test/nif_SUITE.erl index 9c1694fa8a..3a76ee6aee 100644 --- a/erts/emulator/test/nif_SUITE.erl +++ b/erts/emulator/test/nif_SUITE.erl @@ -28,8 +28,9 @@ -export([all/0, suite/0, init_per_testcase/2, end_per_testcase/2, - basic/1, reload/1, upgrade/1, heap_frag/1, + basic/1, reload_error/1, upgrade/1, heap_frag/1, t_on_load/1, + hipe/1, types/1, many_args/1, binaries/1, get_string/1, get_atom/1, maps/1, api_macros/1, @@ -68,8 +69,9 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> - [basic, reload, upgrade, heap_frag, types, many_args, + [basic, reload_error, upgrade, heap_frag, types, many_args, t_on_load, + hipe, binaries, get_string, get_atom, maps, api_macros, from_array, iolist_as_binary, resource, resource_binary, resource_takeover, threading, send, send2, send3, @@ -88,6 +90,11 @@ all() -> init_per_testcase(t_on_load, Config) -> ets:new(nif_SUITE, [named_table]), Config; +init_per_testcase(hipe, Config) -> + case erlang:system_info(hipe_architecture) of + undefined -> {skip, "HiPE is disabled"}; + _ -> Config + end; init_per_testcase(_Case, Config) -> Config. @@ -112,8 +119,8 @@ basic(Config) when is_list(Config) -> true = lists:member(?MODULE, erlang:system_info(taints)), ok. -%% Test reload callback in nif lib -reload(Config) when is_list(Config) -> +%% Test old reload feature now always fails +reload_error(Config) when is_list(Config) -> TmpMem = tmpmem(), ensure_lib_loaded(Config), @@ -127,20 +134,20 @@ reload(Config) when is_list(Config) -> hold_nif_mod_priv_data(nif_mod:get_priv_data_ptr()), [{load,1,1,101},{get_priv_data_ptr,1,2,102}] = nif_mod_call_history(), - ok = nif_mod:load_nif_lib(Config, 2), - 2 = nif_mod:lib_version(), - [{reload,2,1,201},{lib_version,2,2,202}] = nif_mod_call_history(), + {error, {reload, _}} = nif_mod:load_nif_lib(Config, 2), + 1 = nif_mod:lib_version(), + [{lib_version,1,3,103}] = nif_mod_call_history(), - ok = nif_mod:load_nif_lib(Config, 1), + {error, {reload, _}} = nif_mod:load_nif_lib(Config, 1), 1 = nif_mod:lib_version(), - [{reload,1,1,101},{lib_version,1,2,102}] = nif_mod_call_history(), + [{lib_version,1,4,104}] = nif_mod_call_history(), true = erlang:delete_module(nif_mod), [] = nif_mod_call_history(), %%false= check_process_code(Pid, nif_mod), true = erlang:purge_module(nif_mod), - [{unload,1,3,103}] = nif_mod_call_history(), + [{unload,1,5,105}] = nif_mod_call_history(), true = lists:member(?MODULE, erlang:system_info(taints)), true = lists:member(nif_mod, erlang:system_info(taints)), @@ -411,6 +418,17 @@ t_on_load(Config) when is_list(Config) -> verify_tmpmem(TmpMem), ok. +hipe(Config) when is_list(Config) -> + Data = proplists:get_value(data_dir, Config), + Priv = proplists:get_value(priv_dir, Config), + Src = filename:join(Data, "hipe_compiled"), + {ok,hipe_compiled} = c:c(Src, [{outdir,Priv},native]), + true = code:is_module_native(hipe_compiled), + {error, {notsup,_}} = hipe_compiled:try_load_nif(), + true = code:delete(hipe_compiled), + false = code:purge(hipe_compiled), + ok. + %% Test NIF building heap fragments heap_frag(Config) when is_list(Config) -> @@ -828,7 +846,7 @@ resource_binary_do() -> -define(RT_CREATE,1). -define(RT_TAKEOVER,2). -%% Test resource takeover by module reload and upgrade +%% Test resource takeover by module upgrade resource_takeover(Config) when is_list(Config) -> TmpMem = tmpmem(), ensure_lib_loaded(Config), @@ -893,6 +911,7 @@ resource_takeover(Config) when is_list(Config) -> ok = forget_resource(NGX1), ?CHECK([], nif_mod_call_history()), % no dtor + {module,nif_mod} = erlang:load_module(nif_mod,ModBin), ok = nif_mod:load_nif_lib(Config, 2, [{resource_type, 0, ?RT_TAKEOVER, "resource_type_A",resource_dtor_A, ?RT_TAKEOVER}, @@ -911,7 +930,9 @@ resource_takeover(Config) when is_list(Config) -> {resource_type, 4, ?RT_CREATE, "resource_type_null_goneY",null, ?RT_CREATE} ]), - ?CHECK([{reload,2,1,201}], nif_mod_call_history()), + ?CHECK([{upgrade,2,1,201}], nif_mod_call_history()), + true = erlang:purge_module(nif_mod), + ?CHECK([{unload,1,1,106}], nif_mod_call_history()), BinA2 = read_resource(0,A2), ok = forget_resource(A2), @@ -1221,11 +1242,19 @@ threading_do(Config) -> ok = tester:load_nif_lib(Config, "basic"), ok = tester:run(), + erlang:load_module(tester,ModBin), + erlang:purge_module(tester), ok = tester:load_nif_lib(Config, "rwlock"), ok = tester:run(), + erlang:load_module(tester,ModBin), + erlang:purge_module(tester), ok = tester:load_nif_lib(Config, "tsd"), - ok = tester:run(). + ok = tester:run(), + + erlang:delete_module(tester), + erlang:purge_module(tester). + %% Test NIF message sending send(Config) when is_list(Config) -> @@ -1513,13 +1542,13 @@ send3_new_state(State, Blob) -> neg(Config) when is_list(Config) -> TmpMem = tmpmem(), {'EXIT',{badarg,_}} = (catch erlang:load_nif(badarg, 0)), - {error,{load_failed,_}} = erlang:load_nif("pink_unicorn", 0), Data = proplists:get_value(data_dir, Config), File = filename:join(Data, "nif_mod"), {ok,nif_mod,Bin} = compile:file(File, [binary,return_errors]), {module,nif_mod} = erlang:load_module(nif_mod,Bin), + {error,{load_failed,_}} = nif_mod:load_nif_lib(Config, 0), {error,{bad_lib,_}} = nif_mod:load_nif_lib(Config, no_init), verify_tmpmem(TmpMem), ok. @@ -1640,6 +1669,7 @@ consume_timeslice(Config) when is_list(Config) -> end. consume_timeslice_test(Config) when is_list(Config) -> + ensure_lib_loaded(Config), CONTEXT_REDS = 2000, Me = self(), Go = make_ref(), @@ -1716,7 +1746,7 @@ consume_timeslice_test(Config) when is_list(Config) -> io:format("Reductions = ~p~n", [Reductions]), ok; {RedDiff, Reductions} -> - ct:fail({unexpected_reduction_count, Reductions}) + ct:fail({unexpected_reduction_count, Reductions, ExpReds}) end, none = next_msg(P), diff --git a/erts/emulator/test/nif_SUITE_data/hipe_compiled.erl b/erts/emulator/test/nif_SUITE_data/hipe_compiled.erl new file mode 100644 index 0000000000..84ddbc8d63 --- /dev/null +++ b/erts/emulator/test/nif_SUITE_data/hipe_compiled.erl @@ -0,0 +1,6 @@ +-module(hipe_compiled). + +-export([try_load_nif/0]). + +try_load_nif() -> + erlang:load_nif("doesn't matter", 0). diff --git a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c index f2b1ef9d24..dc6d8cea76 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_SUITE.c +++ b/erts/emulator/test/nif_SUITE_data/nif_SUITE.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <string.h> @@ -184,14 +184,6 @@ static void resource_takeover(ErlNifEnv* env, PrivData* priv) msgenv_resource_type = rt; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) -{ - PrivData* priv = (PrivData*) *priv_data; - add_call(env, priv, "reload"); - resource_takeover(env,priv); - return 0; -} - static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { PrivData* priv = (PrivData*) *old_priv_data; @@ -2094,4 +2086,4 @@ static ErlNifFunc nif_funcs[] = {"format_term_nif", 2, format_term} }; -ERL_NIF_INIT(nif_SUITE,nif_funcs,load,reload,upgrade,unload) +ERL_NIF_INIT(nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/nif_SUITE_data/nif_mod.c b/erts/emulator/test/nif_SUITE_data/nif_mod.c index fd8a0d0595..4e94e7901c 100644 --- a/erts/emulator/test/nif_SUITE_data/nif_mod.c +++ b/erts/emulator/test/nif_SUITE_data/nif_mod.c @@ -17,7 +17,7 @@ * * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <string.h> #include <stdio.h> diff --git a/erts/emulator/test/nif_SUITE_data/testcase_driver.h b/erts/emulator/test/nif_SUITE_data/testcase_driver.h index e32e63069a..feb10ecaea 100644 --- a/erts/emulator/test/nif_SUITE_data/testcase_driver.h +++ b/erts/emulator/test/nif_SUITE_data/testcase_driver.h @@ -20,7 +20,7 @@ #ifndef TESTCASE_DRIVER_H__ #define TESTCASE_DRIVER_H__ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdlib.h> #include <stdio.h> diff --git a/erts/emulator/test/nif_SUITE_data/tester.c b/erts/emulator/test/nif_SUITE_data/tester.c index 257b116322..ea4afd924d 100644 --- a/erts/emulator/test/nif_SUITE_data/tester.c +++ b/erts/emulator/test/nif_SUITE_data/tester.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <stdarg.h> @@ -53,7 +53,7 @@ void testcase_free(void *ptr) void testcase_run(TestCaseState_t *tcs); -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { return 0; } @@ -70,5 +70,5 @@ static ErlNifFunc nif_funcs[] = {"run", 0, run} }; -ERL_NIF_INIT(tester,nif_funcs,NULL,reload,NULL,NULL) +ERL_NIF_INIT(tester,nif_funcs,NULL,NULL,upgrade,NULL) diff --git a/erts/emulator/test/num_bif_SUITE.erl b/erts/emulator/test/num_bif_SUITE.erl index d1c9648017..1a1ab0e5e0 100644 --- a/erts/emulator/test/num_bif_SUITE.erl +++ b/erts/emulator/test/num_bif_SUITE.erl @@ -32,6 +32,8 @@ %% list_to_integer/1 %% round/1 %% trunc/1 +%% floor/1 +%% ceil/1 %% integer_to_binary/1 %% integer_to_binary/2 %% binary_to_integer/1 @@ -41,7 +43,7 @@ t_float_to_string/1, t_integer_to_string/1, t_string_to_integer/1, t_list_to_integer_edge_cases/1, t_string_to_float_safe/1, t_string_to_float_risky/1, - t_round/1, t_trunc/1 + t_round/1, t_trunc_and_friends/1 ]). suite() -> [{ct_hooks,[ts_install_cth]}]. @@ -49,7 +51,7 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> [t_abs, t_float, t_float_to_string, t_integer_to_string, {group, t_string_to_float}, t_string_to_integer, t_round, - t_trunc, t_list_to_integer_edge_cases]. + t_trunc_and_friends, t_list_to_integer_edge_cases]. groups() -> [{t_string_to_float, [], @@ -295,30 +297,78 @@ t_round(Config) when is_list(Config) -> -4294967297 = -round(id(4294967296.9)), ok. -t_trunc(Config) when is_list(Config) -> - 0 = trunc(id(0.0)), - 5 = trunc(id(5.3333)), - -10 = trunc(id(-10.978987)), +%% Test trunc/1, floor/1, ceil/1, and round/1. +t_trunc_and_friends(_Config) -> + MinusZero = 0.0 / (-1.0), + 0 = trunc_and_friends(MinusZero), + 0 = trunc_and_friends(0.0), + 5 = trunc_and_friends(5.3333), + -10 = trunc_and_friends(-10.978987), - % The largest smallnum, converted to float (OTP-3722): + %% The largest smallnum, converted to float (OTP-3722): X = id((1 bsl 27) - 1), - F = id(X + 0.0), + F = X + 0.0, io:format("X = ~p/~w/~w, F = ~p/~w/~w, trunc(F) = ~p/~w/~w~n", [X, X, binary_to_list(term_to_binary(X)), F, F, binary_to_list(term_to_binary(F)), - trunc(F), trunc(F), binary_to_list(term_to_binary(trunc(F)))]), - X = trunc(F), - X = trunc(F+1)-1, - X = trunc(F-1)+1, - X = -trunc(-F), - X = -trunc(-F-1)-1, - X = -trunc(-F+1)+1, + trunc_and_friends(F), + trunc_and_friends(F), + binary_to_list(term_to_binary(trunc_and_friends(F)))]), + X = trunc_and_friends(F), + X = trunc_and_friends(F+1)-1, + X = trunc_and_friends(F-1)+1, + X = -trunc_and_friends(-F), + X = -trunc_and_friends(-F-1)-1, + X = -trunc_and_friends(-F+1)+1, %% Bignums. - 4294967305 = trunc(id(4294967305.7)), - -4294967305 = trunc(id(-4294967305.7)), + 4294967305 = trunc_and_friends(4294967305.7), + -4294967305 = trunc_and_friends(-4294967305.7), + 18446744073709551616 = trunc_and_friends(float(1 bsl 64)), + -18446744073709551616 = trunc_and_friends(-float(1 bsl 64)), + + %% Random. + t_trunc_and_friends_rand(100), ok. +t_trunc_and_friends_rand(0) -> + ok; +t_trunc_and_friends_rand(N) -> + F0 = rand:uniform() * math:pow(10, 50*rand:normal()), + F = case rand:uniform() of + U when U < 0.5 -> -F0; + _ -> F0 + end, + _ = trunc_and_friends(F), + t_trunc_and_friends_rand(N-1). + +trunc_and_friends(F) -> + Trunc = trunc(F), + Floor = floor(F), + Ceil = ceil(F), + Round = round(F), + + Trunc = trunc(Trunc), + Floor = floor(Floor), + Ceil = ceil(Ceil), + Round = round(Round), + + Trunc = trunc(float(Trunc)), + Floor = floor(float(Floor)), + Ceil = ceil(float(Ceil)), + Round = round(float(Round)), + + true = Floor =< Trunc andalso Trunc =< Ceil, + true = Ceil - Floor =< 1, + true = Round =:= Floor orelse Round =:= Ceil, + + if + F < 0 -> + Trunc = Ceil; + true -> + Trunc = Floor + end, + Trunc. %% Tests integer_to_binary/1. diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl index 0f999e0efe..e035fc64fe 100644 --- a/erts/emulator/test/process_SUITE.erl +++ b/erts/emulator/test/process_SUITE.erl @@ -2428,7 +2428,7 @@ no_priority_inversion2(Config) when is_list(Config) -> request_gc(Pid, Prio) -> Ref = make_ref(), - erts_internal:request_system_task(Pid, Prio, {garbage_collect, Ref}), + erts_internal:request_system_task(Pid, Prio, {garbage_collect, Ref, major}), Ref. system_task_blast(Config) when is_list(Config) -> diff --git a/erts/emulator/test/system_info_SUITE.erl b/erts/emulator/test/system_info_SUITE.erl index a4aedb31f6..3d9e74472b 100644 --- a/erts/emulator/test/system_info_SUITE.erl +++ b/erts/emulator/test/system_info_SUITE.erl @@ -36,7 +36,7 @@ -export([all/0, suite/0]). -export([process_count/1, system_version/1, misc_smoke_tests/1, - heap_size/1, wordsize/1, memory/1, ets_limit/1]). + heap_size/1, wordsize/1, memory/1, ets_limit/1, atom_limit/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -44,7 +44,7 @@ suite() -> all() -> [process_count, system_version, misc_smoke_tests, - heap_size, wordsize, memory, ets_limit]. + heap_size, wordsize, memory, ets_limit, atom_limit]. %%% %%% The test cases ------------------------------------------------------------- @@ -472,6 +472,17 @@ mapn(_Fun, 0) -> mapn(Fun, N) -> [Fun(N) | mapn(Fun, N-1)]. + +get_node_name(Config) -> + list_to_atom(atom_to_list(?MODULE) + ++ "-" + ++ atom_to_list(proplists:get_value(testcase, Config)) + ++ "-" + ++ integer_to_list(erlang:system_time(second)) + ++ "-" + ++ integer_to_list(erlang:unique_integer([positive]))). + + %% Verify system_info(ets_limit) reflects max ETS table settings. ets_limit(Config0) when is_list(Config0) -> Config = [{testcase,ets_limit}|Config0], @@ -486,7 +497,7 @@ get_ets_limit(Config, EtsMax) -> 0 -> []; _ -> [{"ERL_MAX_ETS_TABLES", integer_to_list(EtsMax)}] end, - {ok, Node} = start_node(Config, Envs), + {ok, Node} = start_node_ets(Config, Envs), Me = self(), Ref = make_ref(), spawn_link(Node, @@ -502,16 +513,40 @@ get_ets_limit(Config, EtsMax) -> stop_node(Node), Res. -start_node(Config, Envs) when is_list(Config) -> +start_node_ets(Config, Envs) when is_list(Config) -> + Pa = filename:dirname(code:which(?MODULE)), + test_server:start_node(get_node_name(Config), peer, + [{args, "-pa "++Pa}, {env, Envs}]). + +start_node_atm(Config, AtomsMax) when is_list(Config) -> Pa = filename:dirname(code:which(?MODULE)), - Name = list_to_atom(atom_to_list(?MODULE) - ++ "-" - ++ atom_to_list(proplists:get_value(testcase, Config)) - ++ "-" - ++ integer_to_list(erlang:system_time(second)) - ++ "-" - ++ integer_to_list(erlang:unique_integer([positive]))), - test_server:start_node(Name, peer, [{args, "-pa "++Pa}, {env, Envs}]). + test_server:start_node(get_node_name(Config), peer, + [{args, "-pa "++ Pa ++ AtomsMax}]). stop_node(Node) -> test_server:stop_node(Node). + + +%% Verify system_info(atom_limit) reflects max atoms settings +%% (using " +t"). +atom_limit(Config0) when is_list(Config0) -> + Config = [{testcase,atom_limit}|Config0], + 2186042 = get_atom_limit(Config, " +t 2186042 "), + ok. + +get_atom_limit(Config, AtomsMax) -> + {ok, Node} = start_node_atm(Config, AtomsMax), + Me = self(), + Ref = make_ref(), + spawn_link(Node, + fun() -> + Res = erlang:system_info(atom_limit), + unlink(Me), + Me ! {Ref, Res} + end), + receive + {Ref, Res} -> + Res + end, + stop_node(Node), + Res. diff --git a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c index 33b346aab7..786be35c9c 100644 --- a/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c +++ b/erts/emulator/test/trace_call_time_SUITE_data/trace_nif.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) @@ -6,11 +6,6 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) -{ - return 0; -} - static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { return 0; @@ -34,4 +29,4 @@ static ErlNifFunc nif_funcs[] = {"nif_dec", 1, nif_dec_1} }; -ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,reload,upgrade,unload) +ERL_NIF_INIT(trace_call_time_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/trace_nif_SUITE.erl b/erts/emulator/test/trace_nif_SUITE.erl index 8d5bff2a48..7ac6fce234 100644 --- a/erts/emulator/test/trace_nif_SUITE.erl +++ b/erts/emulator/test/trace_nif_SUITE.erl @@ -265,10 +265,16 @@ nif_process() -> nif_process(). load_nif(Config) -> - Path = proplists:get_value(data_dir, Config), - - ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0). + case is_nif_loaded() of + true -> + ok; + false -> + Path = proplists:get_value(data_dir, Config), + ok = erlang:load_nif(filename:join(Path,"trace_nif"), 0) + end. +is_nif_loaded() -> + false. nif() -> {"Stub0",[]}. %exit("nif/0 stub called"). diff --git a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c index 26f2420b8b..1afb5ee919 100644 --- a/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c +++ b/erts/emulator/test/trace_nif_SUITE_data/trace_nif.c @@ -1,4 +1,4 @@ -#include "erl_nif.h" +#include <erl_nif.h> static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) @@ -6,18 +6,18 @@ static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) return 0; } -static int reload(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) { return 0; } -static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +static void unload(ErlNifEnv* env, void* priv_data) { - return 0; } -static void unload(ErlNifEnv* env, void* priv_data) +static ERL_NIF_TERM is_nif_loaded(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) { + return enif_make_atom(env,"true"); } static ERL_NIF_TERM nif_0(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) @@ -38,9 +38,10 @@ static ERL_NIF_TERM nif_1(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) static ErlNifFunc nif_funcs[] = { + {"is_nif_loaded", 0, is_nif_loaded}, {"nif", 0, nif_0}, {"nif", 1, nif_1} }; -ERL_NIF_INIT(trace_nif_SUITE,nif_funcs,load,reload,upgrade,unload) +ERL_NIF_INIT(trace_nif_SUITE,nif_funcs,load,NULL,upgrade,unload) diff --git a/erts/emulator/test/tracer_SUITE_data/tracer_test.c b/erts/emulator/test/tracer_SUITE_data/tracer_test.c index a26bb33600..b68e480215 100644 --- a/erts/emulator/test/tracer_SUITE_data/tracer_test.c +++ b/erts/emulator/test/tracer_SUITE_data/tracer_test.c @@ -18,7 +18,7 @@ * %CopyrightEnd% */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stdio.h> #include <string.h> diff --git a/erts/emulator/utils/beam_makeops b/erts/emulator/utils/beam_makeops index 4407f7e289..9813142585 100755 --- a/erts/emulator/utils/beam_makeops +++ b/erts/emulator/utils/beam_makeops @@ -475,7 +475,7 @@ sub emulator_output { print '#include "beam_load.h"', "\n"; print "\n"; - print "char tag_to_letter[] = {\n "; + print "const char tag_to_letter[] = {\n "; for ($i = 0; $i < length($genop_types); $i++) { print "'$tag_type[$i]', "; } @@ -489,7 +489,7 @@ sub emulator_output { # Generate code for specific ops. # my($spec_opnum) = 0; - print "OpEntry opc[] = {\n"; + print "const OpEntry opc[] = {\n"; foreach $key (sort keys %specific_op) { $gen_to_spec{$key} = $spec_opnum; $num_specific{$key} = @{$specific_op{$key}}; @@ -566,13 +566,22 @@ sub emulator_output { $sep = ","; } $init .= "}"; - init_item($print_name, $init, $involves_r, $size, $pack, $sign, 0); + init_item($print_name, $init, $involves_r, $size, $pack, $sign); $op_to_name[$spec_opnum] = $instr; $spec_opnum++; } } print "};\n\n"; - print "int num_instructions = $spec_opnum;\n\n"; + print "const int num_instructions = $spec_opnum;\n\n"; + + # + # Print the array for instruction counts. + # + + print "#ifdef ERTS_OPCODE_COUNTER_SUPPORT\n"; + print "Uint erts_instr_count[$spec_opnum];\n"; + print "#endif\n"; + print "\n"; # # Generate transformations. @@ -584,7 +593,7 @@ sub emulator_output { # Print the generic instruction table. # - print "GenOpEntry gen_opc[] = {\n"; + print "const GenOpEntry gen_opc[] = {\n"; for ($i = 0; $i < @gen_opname; $i++) { if ($i == $num_file_opcodes) { print "\n/*\n * Internal generic instructions.\n */\n\n"; @@ -678,8 +687,8 @@ sub emulator_output { print "#define TE_MAX_VARS $te_max_vars\n"; print "\n"; - print "extern char tag_to_letter[];\n"; - print "extern Uint op_transform[];\n"; + print "extern const char tag_to_letter[];\n"; + print "extern const Uint op_transform[];\n"; print "\n"; for ($i = 0; $i < @op_to_name; $i++) { @@ -708,7 +717,7 @@ sub emulator_output { print "#define DEFINE_COUNTING_LABELS"; for ($i = 0; $i < @op_to_name; $i++) { my($name) = $op_to_name[$i]; - print " \\\nCountCase($name): opc[$i].count++; goto lb_$name;"; + print " \\\nCountCase($name): erts_instr_count[$i]++; goto lb_$name;"; } print "\n\n"; @@ -1417,7 +1426,7 @@ sub tr_gen { # Print the generated transformation engine. # my($offset) = 0; - print "Uint op_transform[] = {\n"; + print "const Uint op_transform[] = {\n"; foreach $key (sort keys %gen_transform) { $gen_transform_offset{$key} = $offset; my @instr = @{$gen_transform{$key}}; diff --git a/erts/emulator/utils/make_preload b/erts/emulator/utils/make_preload index f489bc2a39..8b629d9517 100755 --- a/erts/emulator/utils/make_preload +++ b/erts/emulator/utils/make_preload @@ -94,8 +94,8 @@ foreach $file (@ARGV) { close(FILE); push(@modules, " {\"$module\", " . length($_) . ", preloaded_$module},\n"); - print "unsigned preloaded_size_$module = ", length($_), ";\n"; - print "unsigned char preloaded_$module", "[] = {\n"; + print "const unsigned preloaded_size_$module = ", length($_), ";\n"; + print "const unsigned char preloaded_$module", "[] = {\n"; for ($i = 0; $i < length($_); $i++) { if ($i % 8 == 0 && $comment ne '') { $comment =~ s@/\*@..@g; # Comment start -- avoid warning. @@ -125,10 +125,10 @@ if ($gen_rc) { print @modules; print "END\n"; } elsif ($gen_old) { - print "struct {\n"; + print "const struct {\n"; print " char* name;\n"; print " int size;\n"; - print " unsigned char* code;\n"; + print " const unsigned char* code;\n"; print "} pre_loaded[] = {\n"; foreach (@modules) { print; diff --git a/erts/emulator/utils/make_tables b/erts/emulator/utils/make_tables index c158778f43..27f9dcc878 100755 --- a/erts/emulator/utils/make_tables +++ b/erts/emulator/utils/make_tables @@ -36,7 +36,6 @@ use File::Basename; # <-src>/erl_am.c # <-src>/erl_bif_table.c # <-src>/erl_bif_wrap.c -# <-src>/erl_pbifs.c # <-include>/erl_atom_table.h # <-include>/erl_bif_table.h # @@ -54,8 +53,7 @@ my %aliases; my $auto_alias_num = 0; my @bif; -my @implementation; -my @pbif; +my @bif_type; while (@ARGV && $ARGV[0] =~ /^-(\w+)/) { my $opt = shift; @@ -79,8 +77,11 @@ while (<>) { my($type, @args) = split; if ($type eq 'atom') { save_atoms(@args); - } elsif ($type eq 'bif' or $type eq 'ubif') { - my($bif,$alias,$alias2) = (@args); + } elsif ($type eq 'bif' or $type eq 'ubif' or $type eq 'gcbif') { + if (@args > 2) { + error("$type only allows two arguments"); + } + my($bif,$alias) = (@args); $bif =~ m@^([a-z_.'0-9]+):(.*)/(\d)$@ or error("invalid BIF"); my($mod,$name,$arity) = ($1,$2,$3); save_atoms($mod, $name); @@ -90,12 +91,14 @@ while (<>) { $alias .= "${name}_$arity"; } my $wrapper; - $wrapper = "wrap_$alias" if $type eq 'bif'; - $wrapper = $alias if $type eq 'ubif'; + if ($type eq 'bif') { + $wrapper = "wrap_$alias"; + } else { + $wrapper = $alias; + } push(@bif, ["am_$atom_alias{$mod}","am_$atom_alias{$name}",$arity, $alias,$wrapper]); - push(@pbif, $bif =~ m/^'/ && $alias =~ m/^ebif_/); - push(@implementation, $alias2); + push(@bif_type, $type); } else { error("invalid line"); } @@ -166,8 +169,14 @@ typedef struct bif_entry { BifFunction traced; } BifEntry; +typedef struct erts_gc_bif { + BifFunction bif; + BifFunction gc_bif; +} ErtsGcBif; + extern BifEntry bif_table[]; extern Export* bif_export[]; +extern const ErtsGcBif erts_gc_bifs[]; #define BIF_SIZE $bif_size @@ -182,8 +191,12 @@ print "\n"; for ($i = 0; $i < @bif; $i++) { my $args = join(', ', 'Process*', 'Eterm*'); - print "Eterm $bif[$i]->[3]($args);\n"; - print "Eterm wrap_$bif[$i]->[3]($args, UWord *I);\n"; + my $name = $bif[$i]->[3]; + print "Eterm $name($args);\n"; + print "Eterm wrap_$name($args, UWord *I);\n"; + print "Eterm erts_gc_$name(Process* p, Eterm* reg, Uint live);\n" + if $bif_type[$i] eq 'gcbif'; + print "\n"; } print "#endif\n"; @@ -225,27 +238,26 @@ for ($i = 0; $i < @bif; $i++) { } # -# Generate the package bif file. +# Generate erl_gc_bifs.c. # -open_file("$src/erl_pbifs.c"); +open_file("$src/erl_gc_bifs.c"); my $i; includes("export.h", "sys.h", "erl_vm.h", "global.h", "erl_process.h", "bif.h", - "erl_bif_table.h", "erl_atom_table.h"); + "erl_bif_table.h"); +print "const ErtsGcBif erts_gc_bifs[] = {\n"; for ($i = 0; $i < @bif; $i++) { + next unless $bif_type[$i] eq 'gcbif'; my $arity = $bif[$i]->[2]; my $func = $bif[$i]->[3]; - my $arg; - next unless $pbif[$i]; - next unless $func =~ m/^ebif_(.*)/; - my $orig_func = $1; - $orig_func = $implementation[$i] if $implementation[$i]; - print "Eterm\n"; - print "$func(Process* p, Eterm* BIF__ARGS)\n"; - print "{\n"; - print " return $orig_func(p, BIF__ARGS);\n"; - print "}\n\n"; + print " {$func, erts_gc_$func},\n"; } +print " {0, 0}\n"; +print "};\n"; + +# +# Utilities follow. +# sub open_file { # or die my($name) = @_; diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 1a4f641301..0e7193daf9 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -1312,23 +1312,30 @@ document etpf-msgq %--------------------------------------------------------------------------- end - +define etp-stack-preamble + set $etp_stack_p = ($arg0)->stop + set $etp_stack_end = ($arg0)->hend + printf "%% Stacktrace (%u)\n", $etp_stack_end-$etp_stack_p + etp-1 ((Eterm)($arg0)->i) 0 + printf " (I)\n" + if ($arg0)->cp != 0 + etp-1 ((Eterm)($arg0)->cp) 0 + printf " (cp)\n" + end +end define etp-stacktrace # Args: Process* # # Non-reentrant # - set $etp_stacktrace_p = ($arg0)->stop - set $etp_stacktrace_end = ($arg0)->hend - printf "%% Stacktrace (%u): ", $etp_stacktrace_end-$etp_stacktrace_p - etp ($arg0)->cp - while $etp_stacktrace_p < $etp_stacktrace_end - if ($etp_stacktrace_p[0] & 0x3) == 0x0 + etp-stack-preamble ($arg0) + while $etp_stack_p < $etp_stack_end + if ($etp_stack_p[0] & 0x3) == 0x0 # Continuation pointer - etp $etp_stacktrace_p[0] + etp $etp_stack_p[0] end - set $etp_stacktrace_p++ + set $etp_stack_p++ end end @@ -1347,13 +1354,10 @@ define etp-stackdump # # Non-reentrant # - set $etp_stackdump_p = ($arg0)->stop - set $etp_stackdump_end = ($arg0)->hend - printf "%% Stackdump (%u): ", $etp_stackdump_end-$etp_stackdump_p - etp ($arg0)->cp - while $etp_stackdump_p < $etp_stackdump_end - etp $etp_stackdump_p[0] - set $etp_stackdump_p++ + etp-stack-preamble ($arg0) + while $etp_stack_p < $etp_stack_end + etp $etp_stack_p[0] + set $etp_stack_p++ end end @@ -1835,26 +1839,35 @@ define etp-process-info printf "\n" end end + printf " Current function: " if ($etp_proc->current) - printf " Current function: " - etp-1 $etp_proc->current[0] + etp-1 $etp_proc->current->module printf ":" - etp-1 $etp_proc->current[1] - printf "/%d\n", $etp_proc->current[2] + etp-1 $etp_proc->current->function + printf "/%d\n", $etp_proc->current->arity + else + printf "unknown\n" end + printf " CP: " if ($etp_proc->cp) - printf " CP: " etp-cp-1 $etp_proc->cp printf "\n" + else + printf "unknown\n" end + printf " I: " if ($etp_proc->i) - printf " I: " etp-cp-1 $etp_proc->i printf "\n" + else + printf "unknown\n" end printf " Heap size: %ld\n", $etp_proc->heap_sz + printf " Old-heap size: " if ($etp_proc->old_heap) - printf " Old-heap size: %ld\n", $etp_proc->old_hend - $etp_proc->old_heap + printf "%ld\n", $etp_proc->old_hend - $etp_proc->old_heap + else + printf "0\n" end printf " Mbuf size: %ld\n", $etp_proc->mbuf_sz if (etp_smp_compiled) diff --git a/erts/etc/unix/run_erl.c b/erts/etc/unix/run_erl.c index 6a92e18648..a997297f65 100644 --- a/erts/etc/unix/run_erl.c +++ b/erts/etc/unix/run_erl.c @@ -863,7 +863,7 @@ static int open_log(int log_num, int flags) if (write_all(lfd, buf, strlen(buf)) < 0) status("Error in writing to log.\n"); -#if USE_FSYNC +#ifdef USE_FSYNC fsync(lfd); #endif @@ -893,7 +893,7 @@ static void write_to_log(int* lfd, int* log_num, char* buf, int len) status("Error in writing to log.\n"); } -#if USE_FSYNC +#ifdef USE_FSYNC fsync(*lfd); #endif } diff --git a/erts/example/matrix_nif.c b/erts/example/matrix_nif.c index 6452084eb7..f42204d10f 100644 --- a/erts/example/matrix_nif.c +++ b/erts/example/matrix_nif.c @@ -22,7 +22,7 @@ * for matrix calculations. */ -#include "erl_nif.h" +#include <erl_nif.h> #include <stddef.h> #include <assert.h> diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam Binary files differindex fbbfefa462..6159228bff 100644 --- a/erts/preloaded/ebin/erlang.beam +++ b/erts/preloaded/ebin/erlang.beam diff --git a/erts/preloaded/ebin/erts_code_purger.beam b/erts/preloaded/ebin/erts_code_purger.beam Binary files differindex 6956eee740..0161151785 100644 --- a/erts/preloaded/ebin/erts_code_purger.beam +++ b/erts/preloaded/ebin/erts_code_purger.beam diff --git a/erts/preloaded/ebin/erts_dirty_process_code_checker.beam b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam Binary files differindex a7ac116c05..df3bc9526b 100644 --- a/erts/preloaded/ebin/erts_dirty_process_code_checker.beam +++ b/erts/preloaded/ebin/erts_dirty_process_code_checker.beam diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam Binary files differindex 2a7bfb8b86..aae3976298 100644 --- a/erts/preloaded/ebin/erts_internal.beam +++ b/erts/preloaded/ebin/erts_internal.beam diff --git a/erts/preloaded/ebin/zlib.beam b/erts/preloaded/ebin/zlib.beam Binary files differindex 4c48742344..6a7ad9164f 100644 --- a/erts/preloaded/ebin/zlib.beam +++ b/erts/preloaded/ebin/zlib.beam diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl index 8771089b65..f502a7cd36 100644 --- a/erts/preloaded/src/erlang.erl +++ b/erts/preloaded/src/erlang.erl @@ -100,7 +100,8 @@ -export([binary_to_list/3, binary_to_term/1, binary_to_term/2]). -export([bit_size/1, bitsize/1, bitstring_to_list/1]). -export([bump_reductions/1, byte_size/1, call_on_load_function/1]). --export([cancel_timer/1, cancel_timer/2, check_old_code/1, check_process_code/2, +-export([cancel_timer/1, cancel_timer/2, ceil/1, + check_old_code/1, check_process_code/2, check_process_code/3, crc32/1]). -export([crc32/2, crc32_combine/3, date/0, decode_packet/3]). -export([delete_element/2]). @@ -109,7 +110,7 @@ -export([error/1, error/2, exit/1, exit/2, external_size/1]). -export([external_size/2, finish_after_on_load/2, finish_loading/1, float/1]). -export([float_to_binary/1, float_to_binary/2, - float_to_list/1, float_to_list/2]). + float_to_list/1, float_to_list/2, floor/1]). -export([fun_info/2, fun_info_mfa/1, fun_to_list/1, function_exported/3]). -export([garbage_collect/0, garbage_collect/1, garbage_collect/2]). -export([garbage_collect_message_area/0, get/0, get/1, get_keys/0, get_keys/1]). @@ -474,6 +475,13 @@ cancel_timer(_TimerRef) -> cancel_timer(_TimerRef, _Options) -> erlang:nif_error(undefined). +%% ceil/1 +%% Shadowed by erl_bif_types: erlang:ceil/1 +-spec ceil(Number) -> integer() when + Number :: number(). +ceil(_) -> + erlang:nif_error(undef). + %% check_old_code/1 -spec check_old_code(Module) -> boolean() when Module :: module(). @@ -837,6 +845,13 @@ float_to_list(_Float) -> float_to_list(_Float, _Options) -> erlang:nif_error(undefined). +%% floor/1 +%% Shadowed by erl_bif_types: erlang:floor/1 +-spec floor(Number) -> integer() when + Number :: number(). +floor(_) -> + erlang:nif_error(undef). + %% fun_info/2 -spec erlang:fun_info(Fun, Item) -> {Item, Info} when Fun :: function(), @@ -884,36 +899,39 @@ garbage_collect(Pid) -> error:Error -> erlang:error(Error, [Pid]) end. +-record(gcopt, { + async = sync :: sync | {async, _}, + type = major % default major, can also be minor + }). + %% garbage_collect/2 -spec garbage_collect(Pid, OptionList) -> GCResult | async when Pid :: pid(), RequestId :: term(), - Option :: {async, RequestId}, + Option :: {async, RequestId} | {type, 'major' | 'minor'}, OptionList :: [Option], GCResult :: boolean(). garbage_collect(Pid, OptionList) -> try - Async = get_gc_opts(OptionList, sync), - case Async of + GcOpts = get_gc_opts(OptionList, #gcopt{}), + case GcOpts#gcopt.async of {async, ReqId} -> {priority, Prio} = erlang:process_info(erlang:self(), priority), - erts_internal:request_system_task(Pid, - Prio, - {garbage_collect, ReqId}), + erts_internal:request_system_task( + Pid, Prio, {garbage_collect, ReqId, GcOpts#gcopt.type}), async; sync -> case Pid == erlang:self() of true -> - erlang:garbage_collect(); + erts_internal:garbage_collect(GcOpts#gcopt.type); false -> {priority, Prio} = erlang:process_info(erlang:self(), priority), ReqId = erlang:make_ref(), - erts_internal:request_system_task(Pid, - Prio, - {garbage_collect, - ReqId}), + erts_internal:request_system_task( + Pid, Prio, + {garbage_collect, ReqId, GcOpts#gcopt.type}), receive {garbage_collect, ReqId, GCResult} -> GCResult @@ -925,10 +943,12 @@ garbage_collect(Pid, OptionList) -> end. % gets async opt and verify valid option list -get_gc_opts([{async, _ReqId} = AsyncTuple | Options], _OldAsync) -> - get_gc_opts(Options, AsyncTuple); -get_gc_opts([], Async) -> - Async. +get_gc_opts([{async, _ReqId} = AsyncTuple | Options], GcOpt = #gcopt{}) -> + get_gc_opts(Options, GcOpt#gcopt{ async = AsyncTuple }); +get_gc_opts([{type, T} | Options], GcOpt = #gcopt{}) -> + get_gc_opts(Options, GcOpt#gcopt{ type = T }); +get_gc_opts([], GcOpt) -> + GcOpt. %% garbage_collect_message_area/0 -spec erlang:garbage_collect_message_area() -> boolean(). @@ -2509,6 +2529,7 @@ tuple_to_list(_Tuple) -> Alloc :: atom(); ({allocator_sizes, Alloc}) -> [_] when %% More or less anything Alloc :: atom(); + (atom_limit) -> pos_integer(); (build_type) -> opt | debug | purify | quantify | purecov | gcov | valgrind | gprof | lcnt | frmptr; (c_compiler_used) -> {atom(), term()}; diff --git a/erts/preloaded/src/erts_code_purger.erl b/erts/preloaded/src/erts_code_purger.erl index 28d71fd07e..fd214228c7 100644 --- a/erts/preloaded/src/erts_code_purger.erl +++ b/erts/preloaded/src/erts_code_purger.erl @@ -29,29 +29,34 @@ start() -> register(erts_code_purger, self()), process_flag(trap_exit, true), - loop(). - -loop() -> - _ = receive - {purge,Mod,From,Ref} when is_atom(Mod), is_pid(From) -> - Res = do_purge(Mod), - From ! {reply, purge, Res, Ref}; - - {soft_purge,Mod,From,Ref} when is_atom(Mod), is_pid(From) -> - Res = do_soft_purge(Mod), - From ! {reply, soft_purge, Res, Ref}; - - {finish_after_on_load,{Mod,Keep},From,Ref} - when is_atom(Mod), is_pid(From) -> - Res = do_finish_after_on_load(Mod, Keep), - From ! {reply, finish_after_on_load, Res, Ref}; - - {test_purge, Mod, From, Type, Ref} when is_atom(Mod), is_pid(From) -> - do_test_purge(Mod, From, Type, Ref); - - _Other -> ignore - end, - loop(). + wait_for_request(). + +wait_for_request() -> + handle_request(receive Msg -> Msg end, []). + +handle_request({purge, Mod, From, Ref}, Reqs) when is_atom(Mod), is_pid(From) -> + {Res, NewReqs} = do_purge(Mod, Reqs), + From ! {reply, purge, Res, Ref}, + check_requests(NewReqs); +handle_request({soft_purge, Mod, From, Ref}, Reqs) when is_atom(Mod), is_pid(From) -> + {Res, NewReqs} = do_soft_purge(Mod, Reqs), + From ! {reply, soft_purge, Res, Ref}, + check_requests(NewReqs); +handle_request({finish_after_on_load, {Mod,Keep}, From, Ref}, Reqs) + when is_atom(Mod), is_boolean(Keep), is_pid(From) -> + NewReqs = do_finish_after_on_load(Mod, Keep, Reqs), + From ! {reply, finish_after_on_load, ok, Ref}, + check_requests(NewReqs); +handle_request({test_purge, Mod, From, Type, Ref}, Reqs) when is_atom(Mod), is_pid(From) -> + NewReqs = do_test_purge(Mod, From, Type, Ref, Reqs), + check_requests(NewReqs); +handle_request(_Garbage, Reqs) -> + check_requests(Reqs). + +check_requests([]) -> + wait_for_request(); +check_requests([R|Rs]) -> + handle_request(R, Rs). %% %% Processes that tries to call a fun that belongs to @@ -99,14 +104,15 @@ purge(Mod) when is_atom(Mod) -> Result end. -do_purge(Mod) -> +do_purge(Mod, Reqs) -> case erts_internal:purge_module(Mod, prepare) of false -> - {false, false}; + {{false, false}, Reqs}; true -> - DidKill = check_proc_code(erlang:processes(), Mod, true), + {DidKill, NewReqs} = check_proc_code(erlang:processes(), + Mod, true, Reqs), true = erts_internal:purge_module(Mod, complete), - {true, DidKill} + {{true, DidKill}, NewReqs} end. %% soft_purge(Module) @@ -122,17 +128,14 @@ soft_purge(Mod) -> Result end. -do_soft_purge(Mod) -> +do_soft_purge(Mod, Reqs) -> case erts_internal:purge_module(Mod, prepare) of false -> - true; + {true, Reqs}; true -> - Res = check_proc_code(erlang:processes(), Mod, false), - erts_internal:purge_module(Mod, - case Res of - false -> abort; - true -> complete - end) + {PurgeOp, NewReqs} = check_proc_code(erlang:processes(), + Mod, false, Reqs), + {erts_internal:purge_module(Mod, PurgeOp), NewReqs} end. %% finish_after_on_load(Module, Keep) @@ -147,179 +150,130 @@ finish_after_on_load(Mod, Keep) -> Result end. -do_finish_after_on_load(Mod, Keep) -> +do_finish_after_on_load(Mod, Keep, Reqs) -> erlang:finish_after_on_load(Mod, Keep), case Keep of true -> - ok; + Reqs; false -> case erts_internal:purge_module(Mod, prepare_on_load) of false -> - true; + Reqs; true -> - _ = check_proc_code(erlang:processes(), Mod, true), - true = erts_internal:purge_module(Mod, complete) + {_DidKill, NewReqs} = + check_proc_code(erlang:processes(), + Mod, true, Reqs), + true = erts_internal:purge_module(Mod, complete), + NewReqs end end. - %% -%% check_proc_code(Pids, Mod, Hard) - Send asynchronous +%% check_proc_code(Pids, Mod, Hard, Preqs) - Send asynchronous %% requests to all processes to perform a check_process_code %% operation. Each process will check their own state and %% reply with the result. If 'Hard' equals %% - true, processes that refer 'Mod' will be killed. If %% any processes were killed true is returned; otherwise, %% false. -%% - false, and any processes refer 'Mod', false will -%% returned; otherwise, true. +%% - false, and any processes refer 'Mod', 'abort' will +%% be returned; otherwise, 'complete'. %% -%% Requests will be sent to all processes identified by -%% Pids at once, but without allowing GC to be performed. -%% Check process code operations that are aborted due to -%% GC need, will be restarted allowing GC. However, only -%% ?MAX_CPC_GC_PROCS outstanding operation allowing GC at -%% a time will be allowed. This in order not to blow up -%% memory wise. -%% -%% We also only allow ?MAX_CPC_NO_OUTSTANDING_KILLS +%% We only allow ?MAX_CPC_NO_OUTSTANDING_KILLS %% outstanding kills. This both in order to avoid flooding %% our message queue with 'DOWN' messages and limiting the %% amount of memory used to keep references to all %% outstanding kills. %% -%% We maybe should allow more than two outstanding -%% GC requests, but for now we play it safe... --define(MAX_CPC_GC_PROCS, 2). -define(MAX_CPC_NO_OUTSTANDING_KILLS, 10). --record(cpc_static, {hard, module, tag}). +-record(cpc_static, {hard, module, tag, purge_requests}). -record(cpc_kill, {outstanding = [], no_outstanding = 0, waiting = [], killed = false}). -check_proc_code(Pids, Mod, Hard) -> +check_proc_code(Pids, Mod, Hard, PReqs) -> Tag = erlang:make_ref(), CpcS = #cpc_static{hard = Hard, module = Mod, - tag = Tag}, - check_proc_code(CpcS, cpc_init(CpcS, Pids, 0), 0, [], #cpc_kill{}, true). - -check_proc_code(#cpc_static{hard = true}, 0, 0, [], - #cpc_kill{outstanding = [], waiting = [], killed = Killed}, - true) -> - %% No outstanding requests. We did a hard check, so result is whether or - %% not we killed any processes... - Killed; -check_proc_code(#cpc_static{hard = false}, 0, 0, [], _KillState, Success) -> - %% No outstanding requests and we did a soft check... - Success; -check_proc_code(#cpc_static{hard = false, tag = Tag} = CpcS, NoReq0, NoGcReq0, - [], _KillState, false) -> - %% Failed soft check; just cleanup the remaining replies corresponding - %% to the requests we've sent... - {NoReq1, NoGcReq1} = receive - {check_process_code, {Tag, _P, GC}, _Res} -> - case GC of - false -> {NoReq0-1, NoGcReq0}; - true -> {NoReq0, NoGcReq0-1} - end - end, - check_proc_code(CpcS, NoReq1, NoGcReq1, [], _KillState, false); -check_proc_code(#cpc_static{tag = Tag} = CpcS, NoReq0, NoGcReq0, NeedGC0, - KillState0, Success) -> - - %% Check if we should request a GC operation - {NoGcReq1, NeedGC1} = case NoGcReq0 < ?MAX_CPC_GC_PROCS of - GcOpAllowed when GcOpAllowed == false; - NeedGC0 == [] -> - {NoGcReq0, NeedGC0}; - _ -> - {NoGcReq0+1, cpc_request_gc(CpcS,NeedGC0)} - end, - - %% Wait for a cpc reply or 'DOWN' message - {NoReq1, NoGcReq2, Pid, Result, KillState1} = cpc_recv(Tag, - NoReq0, - NoGcReq1, - KillState0), - - %% Check the result of the reply - case Result of - aborted -> - %% Operation aborted due to the need to GC in order to - %% determine if the process is referring the module. - %% Schedule the operation for restart allowing GC... - check_proc_code(CpcS, NoReq1, NoGcReq2, [Pid|NeedGC1], KillState1, - Success); - false -> + tag = Tag, + purge_requests = PReqs}, + cpc_receive(CpcS, cpc_init(CpcS, Pids, 0), #cpc_kill{}, []). + +cpc_receive(#cpc_static{hard = true} = CpcS, + 0, + #cpc_kill{outstanding = [], waiting = [], killed = Killed}, + PReqs) -> + %% No outstanding cpc requests. We did a hard check, so result is + %% whether or not we killed any processes... + cpc_result(CpcS, PReqs, Killed); +cpc_receive(#cpc_static{hard = false} = CpcS, 0, _KillState, PReqs) -> + %% No outstanding cpc requests and we did a soft check that succeeded... + cpc_result(CpcS, PReqs, complete); +cpc_receive(#cpc_static{tag = Tag} = CpcS, NoReq, KillState0, PReqs) -> + receive + {check_process_code, {Tag, _Pid}, false} -> %% Process not referring the module; done with this process... - check_proc_code(CpcS, NoReq1, NoGcReq2, NeedGC1, KillState1, - Success); - true -> + cpc_receive(CpcS, NoReq-1, KillState0, PReqs); + {check_process_code, {Tag, Pid}, true} -> %% Process referring the module... case CpcS#cpc_static.hard of false -> %% ... and soft check. The whole operation failed so - %% no point continuing; clean up and fail... - check_proc_code(CpcS, NoReq1, NoGcReq2, [], KillState1, - false); + %% no point continuing; fail straight away. Garbage + %% messages from this session will be ignored + %% by following sessions... + cpc_result(CpcS, PReqs, abort); true -> %% ... and hard check; schedule kill of it... - check_proc_code(CpcS, NoReq1, NoGcReq2, NeedGC1, - cpc_sched_kill(Pid, KillState1), Success) + KillState1 = cpc_sched_kill(Pid, KillState0), + cpc_receive(CpcS, NoReq-1, KillState1, PReqs) end; - 'DOWN' -> - %% Handled 'DOWN' message - check_proc_code(CpcS, NoReq1, NoGcReq2, NeedGC1, - KillState1, Success) + {'DOWN', MonRef, process, _, _} -> + KillState1 = cpc_handle_down(MonRef, KillState0), + cpc_receive(CpcS, NoReq, KillState1, PReqs); + PReq when element(1, PReq) == purge; + element(1, PReq) == soft_purge; + element(1, PReq) == test_purge -> + %% A new purge request; save it until later... + cpc_receive(CpcS, NoReq, KillState0, [PReq | PReqs]); + _Garbage -> + %% Garbage message; ignore it... + cpc_receive(CpcS, NoReq, KillState0, PReqs) end. -cpc_recv(Tag, NoReq, NoGcReq, #cpc_kill{outstanding = []} = KillState) -> - receive - {check_process_code, {Tag, Pid, GC}, Res} -> - cpc_handle_cpc(NoReq, NoGcReq, GC, Pid, Res, KillState) - end; -cpc_recv(Tag, NoReq, NoGcReq, - #cpc_kill{outstanding = [R0, R1, R2, R3, R4 | _]} = KillState) -> - receive - {'DOWN', R, process, _, _} when R == R0; - R == R1; - R == R2; - R == R3; - R == R4 -> - cpc_handle_down(NoReq, NoGcReq, R, KillState); - {check_process_code, {Tag, Pid, GC}, Res} -> - cpc_handle_cpc(NoReq, NoGcReq, GC, Pid, Res, KillState) - end; -cpc_recv(Tag, NoReq, NoGcReq, #cpc_kill{outstanding = [R|_]} = KillState) -> - receive - {'DOWN', R, process, _, _} -> - cpc_handle_down(NoReq, NoGcReq, R, KillState); - {check_process_code, {Tag, Pid, GC}, Res} -> - cpc_handle_cpc(NoReq, NoGcReq, GC, Pid, Res, KillState) +cpc_result(#cpc_static{purge_requests = PReqs}, NewPReqs, Res) -> + {Res, PReqs ++ cpc_reverse(NewPReqs)}. + +cpc_reverse([_] = L) -> L; +cpc_reverse(Xs) -> cpc_reverse(Xs, []). + +cpc_reverse([], Ys) -> Ys; +cpc_reverse([X|Xs], Ys) -> cpc_reverse(Xs, [X|Ys]). + +cpc_handle_down(R, #cpc_kill{outstanding = Rs, + no_outstanding = N} = KillState0) -> + try + NewOutst = cpc_list_rm(R, Rs), + KillState1 = KillState0#cpc_kill{outstanding = NewOutst, + no_outstanding = N-1}, + cpc_sched_kill_waiting(KillState1) + catch + throw : undefined -> %% Triggered by garbage message... + KillState0 end. -cpc_handle_down(NoReq, NoGcReq, R, #cpc_kill{outstanding = Rs, - no_outstanding = N} = KillState) -> - {NoReq, NoGcReq, undefined, 'DOWN', - cpc_sched_kill_waiting(KillState#cpc_kill{outstanding = cpc_list_rm(R, Rs), - no_outstanding = N-1})}. - +cpc_list_rm(_R, []) -> + throw(undefined); cpc_list_rm(R, [R|Rs]) -> Rs; cpc_list_rm(R0, [R1|Rs]) -> [R1|cpc_list_rm(R0, Rs)]. -cpc_handle_cpc(NoReq, NoGcReq, false, Pid, Res, KillState) -> - {NoReq-1, NoGcReq, Pid, Res, KillState}; -cpc_handle_cpc(NoReq, NoGcReq, true, Pid, Res, KillState) -> - {NoReq, NoGcReq-1, Pid, Res, KillState}. - cpc_sched_kill_waiting(#cpc_kill{waiting = []} = KillState) -> KillState; cpc_sched_kill_waiting(#cpc_kill{outstanding = Rs, @@ -343,18 +297,13 @@ cpc_sched_kill(Pid, no_outstanding = N+1, killed = true}. -cpc_request(#cpc_static{tag = Tag, module = Mod}, Pid, AllowGc) -> - erts_internal:check_process_code(Pid, Mod, [{async, {Tag, Pid, AllowGc}}, - {allow_gc, AllowGc}]). - -cpc_request_gc(CpcS, [Pid|Pids]) -> - cpc_request(CpcS, Pid, true), - Pids. +cpc_request(#cpc_static{tag = Tag, module = Mod}, Pid) -> + erts_internal:check_process_code(Pid, Mod, [{async, {Tag, Pid}}]). cpc_init(_CpcS, [], NoReqs) -> NoReqs; cpc_init(CpcS, [Pid|Pids], NoReqs) -> - cpc_request(CpcS, Pid, false), + cpc_request(CpcS, Pid), cpc_init(CpcS, Pids, NoReqs+1). % end of check_proc_code() implementation. @@ -366,64 +315,63 @@ cpc_init(CpcS, [Pid|Pids], NoReqs) -> %% as usual, but the tester can control when to enter the %% specific phases. %% -do_test_purge(Mod, From, Type, Ref) when Type == true; Type == false -> - Mon = erlang:monitor(process, From), - Res = case Type of - true -> do_test_hard_purge(Mod, From, Ref, Mon); - false -> do_test_soft_purge(Mod, From, Ref, Mon) - end, +do_test_purge(Mod, From, true, Ref, Reqs) -> + {Res, NewReqs} = do_test_hard_purge(Mod, From, Ref, Reqs), + From ! {test_purge, Res, Ref}, + NewReqs; +do_test_purge(Mod, From, false, Ref, Reqs) -> + {Res, NewReqs} = do_test_soft_purge(Mod, From, Ref, Reqs), From ! {test_purge, Res, Ref}, - erlang:demonitor(Mon, [flush]), - ok; -do_test_purge(_, _, _, _) -> - ok. + NewReqs; +do_test_purge(_, _, _, _, Reqs) -> + Reqs. -do_test_soft_purge(Mod, From, Ref, Mon) -> +do_test_soft_purge(Mod, From, Ref, Reqs) -> PrepRes = erts_internal:purge_module(Mod, prepare), - TestRes = test_progress(started, From, Mon, Ref, ok), + TestRes = test_progress(started, From, Ref, ok), case PrepRes of false -> - _ = test_progress(continued, From, Mon, Ref, TestRes), - true; + _ = test_progress(continued, From, Ref, TestRes), + {true, Reqs}; true -> - Res = check_proc_code(erlang:processes(), Mod, false), - _ = test_progress(continued, From, Mon, Ref, TestRes), - erts_internal:purge_module(Mod, - case Res of - false -> abort; - true -> complete - end) + {PurgeOp, NewReqs} = check_proc_code(erlang:processes(), + Mod, false, Reqs), + _ = test_progress(continued, From, Ref, TestRes), + {erts_internal:purge_module(Mod, PurgeOp), NewReqs} end. -do_test_hard_purge(Mod, From, Ref, Mon) -> +do_test_hard_purge(Mod, From, Ref, Reqs) -> PrepRes = erts_internal:purge_module(Mod, prepare), - TestRes = test_progress(started, From, Mon, Ref, ok), + TestRes = test_progress(started, From, Ref, ok), case PrepRes of false -> - _ = test_progress(continued, From, Mon, Ref, TestRes), - {false, false}; + _ = test_progress(continued, From, Ref, TestRes), + {{false, false}, Reqs}; true -> - DidKill = check_proc_code(erlang:processes(), Mod, true), - _ = test_progress(continued, From, Mon, Ref, TestRes), + {DidKill, NewReqs} = check_proc_code(erlang:processes(), + Mod, true, Reqs), + _ = test_progress(continued, From, Ref, TestRes), true = erts_internal:purge_module(Mod, complete), - {true, DidKill} + {{true, DidKill}, NewReqs} end. -test_progress(_State, _From, _Mon, _Ref, died) -> +test_progress(_State, _From, _Ref, died) -> %% Test process died; continue so we wont %% leave the system in an inconsistent %% state... died; -test_progress(started, From, Mon, Ref, ok) -> +test_progress(started, From, Ref, ok) -> From ! {started, Ref}, + Mon = erlang:monitor(process, From), receive {'DOWN', Mon, process, From, _} -> died; - {continue, Ref} -> ok + {continue, Ref} -> erlang:demonitor(Mon, [flush]), ok end; -test_progress(continued, From, Mon, Ref, ok) -> +test_progress(continued, From, Ref, ok) -> From ! {continued, Ref}, + Mon = erlang:monitor(process, From), receive {'DOWN', Mon, process, From, _} -> died; - {complete, Ref} -> ok + {complete, Ref} -> erlang:demonitor(Mon, [flush]), ok end. diff --git a/erts/preloaded/src/erts_dirty_process_code_checker.erl b/erts/preloaded/src/erts_dirty_process_code_checker.erl index 911642082c..7d3fa264be 100644 --- a/erts/preloaded/src/erts_dirty_process_code_checker.erl +++ b/erts/preloaded/src/erts_dirty_process_code_checker.erl @@ -48,8 +48,7 @@ handle_request({Requester, Prio, {check_process_code, ReqId, - Module, - _Flags} = Op}) -> + Module} = Op}) -> %% %% Target may have stopped executing dirty since the %% initial request was made. Check its current state diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl index f4518c4008..a27cb2ba38 100644 --- a/erts/preloaded/src/erts_internal.erl +++ b/erts/preloaded/src/erts_internal.erl @@ -39,6 +39,7 @@ gather_system_check_result/1]). -export([request_system_task/3, request_system_task/4]). +-export([garbage_collect/1]). -export([check_process_code/3]). -export([check_dirty_process_code/2]). @@ -60,7 +61,7 @@ -export([trace/3, trace_pattern/3]). %% Auto import name clash --export([check_process_code/2]). +-export([check_process_code/1]). %% %% Await result of send to port @@ -205,8 +206,9 @@ port_info(_Result, _Item) -> -spec request_system_task(Pid, Prio, Request) -> 'ok' when Prio :: 'max' | 'high' | 'normal' | 'low', - Request :: {'garbage_collect', term()} - | {'check_process_code', term(), module(), non_neg_integer()} + Type :: 'major' | 'minor', + Request :: {'garbage_collect', term(), Type} + | {'check_process_code', term(), module()} | {'copy_literals', term(), boolean()}, Pid :: pid(). @@ -216,7 +218,7 @@ request_system_task(_Pid, _Prio, _Request) -> -spec request_system_task(RequesterPid, TargetPid, Prio, Request) -> 'ok' | 'dirty_execution' when Prio :: 'max' | 'high' | 'normal' | 'low', Request :: {'garbage_collect', term()} - | {'check_process_code', term(), module(), non_neg_integer()} + | {'check_process_code', term(), module()} | {'copy_literals', term(), boolean()}, RequesterPid :: pid(), TargetPid :: pid(). @@ -224,12 +226,14 @@ request_system_task(_Pid, _Prio, _Request) -> request_system_task(_RequesterPid, _TargetPid, _Prio, _Request) -> erlang:nif_error(undefined). --define(ERTS_CPC_ALLOW_GC, (1 bsl 0)). +-spec garbage_collect(Mode) -> 'true' when Mode :: 'major' | 'minor'. --spec check_process_code(Module, Flags) -> boolean() when - Module :: module(), - Flags :: non_neg_integer(). -check_process_code(_Module, _Flags) -> +garbage_collect(_Mode) -> + erlang:nif_error(undefined). + +-spec check_process_code(Module) -> boolean() when + Module :: module(). +check_process_code(_Module) -> erlang:nif_error(undefined). -spec check_process_code(Pid, Module, OptionList) -> CheckResult | async when @@ -240,7 +244,7 @@ check_process_code(_Module, _Flags) -> OptionList :: [Option], CheckResult :: boolean() | aborted. check_process_code(Pid, Module, OptionList) -> - {Async, Flags} = get_cpc_opts(OptionList, sync, ?ERTS_CPC_ALLOW_GC), + Async = get_cpc_opts(OptionList, sync), case Async of {async, ReqId} -> {priority, Prio} = erlang:process_info(erlang:self(), @@ -249,13 +253,12 @@ check_process_code(Pid, Module, OptionList) -> Prio, {check_process_code, ReqId, - Module, - Flags}), + Module}), async; sync -> case Pid == erlang:self() of true -> - erts_internal:check_process_code(Module, Flags); + erts_internal:check_process_code(Module); false -> {priority, Prio} = erlang:process_info(erlang:self(), priority), @@ -264,8 +267,7 @@ check_process_code(Pid, Module, OptionList) -> Prio, {check_process_code, ReqId, - Module, - Flags}), + Module}), receive {check_process_code, ReqId, CheckResult} -> CheckResult @@ -273,18 +275,14 @@ check_process_code(Pid, Module, OptionList) -> end end. -% gets async and flag opts and verify valid option list -get_cpc_opts([{async, _ReqId} = AsyncTuple | Options], _OldAsync, Flags) -> - get_cpc_opts(Options, AsyncTuple, Flags); -get_cpc_opts([{allow_gc, AllowGC} | Options], Async, Flags) -> - get_cpc_opts(Options, Async, cpc_flags(Flags, ?ERTS_CPC_ALLOW_GC, AllowGC)); -get_cpc_opts([], Async, Flags) -> - {Async, Flags}. - -cpc_flags(OldFlags, Bit, true) -> - OldFlags bor Bit; -cpc_flags(OldFlags, Bit, false) -> - OldFlags band (bnot Bit). +% gets async opt and verify valid option list +get_cpc_opts([{async, _ReqId} = AsyncTuple | Options], _OldAsync) -> + get_cpc_opts(Options, AsyncTuple); +get_cpc_opts([{allow_gc, AllowGC} | Options], Async) when AllowGC == true; + AllowGC == false -> + get_cpc_opts(Options, Async); +get_cpc_opts([], Async) -> + Async. -spec check_dirty_process_code(Pid,Module) -> 'true' | 'false' when Pid :: pid(), diff --git a/erts/preloaded/src/zlib.erl b/erts/preloaded/src/zlib.erl index fa0f28c5c3..8cd3e39fd7 100644 --- a/erts/preloaded/src/zlib.erl +++ b/erts/preloaded/src/zlib.erl @@ -23,7 +23,8 @@ -export([open/0,close/1,deflateInit/1,deflateInit/2,deflateInit/6, deflateSetDictionary/2,deflateReset/1,deflateParams/3, deflate/2,deflate/3,deflateEnd/1, - inflateInit/1,inflateInit/2,inflateSetDictionary/2, + inflateInit/1,inflateInit/2, + inflateSetDictionary/2,inflateGetDictionary/1, inflateSync/1,inflateReset/1,inflate/2,inflateEnd/1, inflateChunk/1, inflateChunk/2, setBufSize/2,getBufSize/1, @@ -98,25 +99,26 @@ -define(INFLATE_INIT, 8). -define(INFLATE_INIT2, 9). -define(INFLATE_SETDICT, 10). --define(INFLATE_SYNC, 11). --define(INFLATE_RESET, 12). --define(INFLATE_END, 13). --define(INFLATE, 14). --define(INFLATE_CHUNK, 25). +-define(INFLATE_GETDICT, 11). +-define(INFLATE_SYNC, 12). +-define(INFLATE_RESET, 13). +-define(INFLATE_END, 14). +-define(INFLATE, 15). +-define(INFLATE_CHUNK, 26). --define(CRC32_0, 15). --define(CRC32_1, 16). --define(CRC32_2, 17). +-define(CRC32_0, 16). +-define(CRC32_1, 17). +-define(CRC32_2, 18). --define(SET_BUFSZ, 18). --define(GET_BUFSZ, 19). --define(GET_QSIZE, 20). +-define(SET_BUFSZ, 19). +-define(GET_BUFSZ, 20). +-define(GET_QSIZE, 21). --define(ADLER32_1, 21). --define(ADLER32_2, 22). +-define(ADLER32_1, 22). +-define(ADLER32_2, 23). --define(CRC32_COMBINE, 23). --define(ADLER32_COMBINE, 24). +-define(CRC32_COMBINE, 24). +-define(ADLER32_COMBINE, 25). %%------------------------------------------------------------------------ @@ -242,6 +244,13 @@ inflateInit(Z, WindowBits) -> inflateSetDictionary(Z, Dictionary) -> call(Z, ?INFLATE_SETDICT, Dictionary). +-spec inflateGetDictionary(Z) -> Dictionary when + Z :: zstream(), + Dictionary :: iolist(). +inflateGetDictionary(Z) -> + _ = call(Z, ?INFLATE_GETDICT, []), + collect(Z). + -spec inflateSync(zstream()) -> 'ok'. inflateSync(Z) -> call(Z, ?INFLATE_SYNC, []). |