aboutsummaryrefslogtreecommitdiffstats
path: root/erts
diff options
context:
space:
mode:
Diffstat (limited to 'erts')
-rw-r--r--erts/aclocal.m486
-rw-r--r--erts/configure.in78
-rw-r--r--erts/doc/src/erl.xml8
-rw-r--r--erts/doc/src/erts_alloc.xml36
-rw-r--r--erts/doc/src/match_spec.xml10
-rw-r--r--erts/emulator/Makefile.in2
-rw-r--r--erts/emulator/beam/bif.c7
-rw-r--r--erts/emulator/beam/bif.tab2
-rw-r--r--erts/emulator/beam/bif_instrs.tab8
-rw-r--r--erts/emulator/beam/break.c15
-rw-r--r--erts/emulator/beam/erl_alloc.c98
-rw-r--r--erts/emulator/beam/erl_alloc_util.c1384
-rw-r--r--erts/emulator/beam/erl_alloc_util.h35
-rw-r--r--erts/emulator/beam/erl_bif_info.c105
-rw-r--r--erts/emulator/beam/erl_db.c7
-rw-r--r--erts/emulator/beam/erl_db.h1
-rw-r--r--erts/emulator/beam/erl_db_hash.c62
-rw-r--r--erts/emulator/beam/erl_init.c2
-rw-r--r--erts/emulator/beam/erl_instrument.c1257
-rw-r--r--erts/emulator/beam/erl_instrument.h42
-rw-r--r--erts/emulator/beam/erl_lock_check.c786
-rw-r--r--erts/emulator/beam/erl_lock_check.h2
-rw-r--r--erts/emulator/beam/erl_nif.c10
-rw-r--r--erts/emulator/beam/erl_process.c9
-rw-r--r--erts/emulator/beam/erl_process.h2
-rw-r--r--erts/emulator/beam/erl_trace.c2
-rw-r--r--erts/emulator/beam/global.h2
-rw-r--r--erts/emulator/beam/io.c38
-rw-r--r--erts/emulator/beam/utils.c16
-rw-r--r--erts/emulator/sys/common/erl_sys_common_misc.c13
-rw-r--r--erts/emulator/test/exception_SUITE.erl56
-rw-r--r--erts/emulator/test/process_SUITE.erl56
-rw-r--r--erts/emulator/test/tracer_SUITE.erl24
-rw-r--r--erts/emulator/test/z_SUITE.erl8
-rw-r--r--erts/etc/common/erlexec.c1
-rw-r--r--erts/lib_src/Makefile.in3
-rw-r--r--erts/preloaded/ebin/erlang.beambin102724 -> 102732 bytes
-rw-r--r--erts/preloaded/ebin/erts_internal.beambin15104 -> 15640 bytes
-rw-r--r--erts/preloaded/src/erlang.erl57
-rw-r--r--erts/preloaded/src/erts_internal.erl23
40 files changed, 2249 insertions, 2104 deletions
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4
index 887babc13f..a4d09810bd 100644
--- a/erts/aclocal.m4
+++ b/erts/aclocal.m4
@@ -2770,3 +2770,89 @@ rm -f conftest*])
#define UNSAFE_MASK 0xc0000000 /* Mask for bits that must be constant */
+dnl ----------------------------------------------------------------------
+dnl
+dnl LM_HARDWARE_ARCH
+dnl
+dnl Determine target hardware in ARCH
+dnl
+AC_DEFUN([LM_HARDWARE_ARCH], [
+ AC_MSG_CHECKING([target hardware architecture])
+ if test "x$host_alias" != "x" -a "x$host_cpu" != "x"; then
+ chk_arch_=$host_cpu
+ else
+ chk_arch_=`uname -m`
+ fi
+
+ case $chk_arch_ in
+ sun4u) ARCH=ultrasparc;;
+ sparc64) ARCH=sparc64;;
+ sun4v) ARCH=ultrasparc;;
+ i86pc) ARCH=x86;;
+ i386) ARCH=x86;;
+ i486) ARCH=x86;;
+ i586) ARCH=x86;;
+ i686) ARCH=x86;;
+ x86_64) ARCH=amd64;;
+ amd64) ARCH=amd64;;
+ macppc) ARCH=ppc;;
+ powerpc) ARCH=ppc;;
+ ppc) ARCH=ppc;;
+ ppc64) ARCH=ppc64;;
+ ppc64le) ARCH=ppc64le;;
+ "Power Macintosh") ARCH=ppc;;
+ armv5b) ARCH=arm;;
+ armv5teb) ARCH=arm;;
+ armv5tel) ARCH=arm;;
+ armv5tejl) ARCH=arm;;
+ armv6l) ARCH=arm;;
+ armv6hl) ARCH=arm;;
+ armv7l) ARCH=arm;;
+ armv7hl) ARCH=arm;;
+ tile) ARCH=tile;;
+ e2k) ARCH=e2k;;
+ *) ARCH=noarch;;
+ esac
+ AC_MSG_RESULT($ARCH)
+
+ dnl
+ dnl Convert between x86 and amd64 based on the compiler's mode.
+ dnl Ditto between ultrasparc and sparc64.
+ dnl
+ AC_MSG_CHECKING(whether compilation mode forces ARCH adjustment)
+ case "$ARCH-$ac_cv_sizeof_void_p" in
+ x86-8)
+ AC_MSG_RESULT(yes: adjusting ARCH=x86 to ARCH=amd64)
+ ARCH=amd64
+ ;;
+ amd64-4)
+ AC_MSG_RESULT(yes: adjusting ARCH=amd64 to ARCH=x86)
+ ARCH=x86
+ ;;
+ ultrasparc-8)
+ AC_MSG_RESULT(yes: adjusting ARCH=ultrasparc to ARCH=sparc64)
+ ARCH=sparc64
+ ;;
+ sparc64-4)
+ AC_MSG_RESULT(yes: adjusting ARCH=sparc64 to ARCH=ultrasparc)
+ ARCH=ultrasparc
+ ;;
+ ppc64-4)
+ AC_MSG_RESULT(yes: adjusting ARCH=ppc64 to ARCH=ppc)
+ ARCH=ppc
+ ;;
+ ppc-8)
+ AC_MSG_RESULT(yes: adjusting ARCH=ppc to ARCH=ppc64)
+ ARCH=ppc64
+ ;;
+ arm-8)
+ AC_MSG_RESULT(yes: adjusting ARCH=arm to ARCH=noarch)
+ ARCH=noarch
+ ;;
+ *)
+ AC_MSG_RESULT(no: ARCH is $ARCH)
+ ;;
+ esac
+
+ AC_SUBST(ARCH)
+])
diff --git a/erts/configure.in b/erts/configure.in
index 820247b4b8..2d0d6c6444 100644
--- a/erts/configure.in
+++ b/erts/configure.in
@@ -658,83 +658,9 @@ case $chk_opsys_ in
*) OPSYS=noopsys
esac
-if test "x$host_alias" != "x" -a "x$host_cpu" != "x"; then
- chk_arch_=$host_cpu
-else
- chk_arch_=`uname -m`
-fi
-
-case $chk_arch_ in
- sun4u) ARCH=ultrasparc;;
- sparc64) ARCH=sparc64;;
- sun4v) ARCH=ultrasparc;;
- i86pc) ARCH=x86;;
- i386) ARCH=x86;;
- i486) ARCH=x86;;
- i586) ARCH=x86;;
- i686) ARCH=x86;;
- x86_64) ARCH=amd64;;
- amd64) ARCH=amd64;;
- macppc) ARCH=ppc;;
- powerpc) ARCH=ppc;;
- ppc) ARCH=ppc;;
- ppc64) ARCH=ppc64;;
- ppc64le) ARCH=ppc64le;;
- "Power Macintosh") ARCH=ppc;;
- armv5b) ARCH=arm;;
- armv5teb) ARCH=arm;;
- armv5tel) ARCH=arm;;
- armv5tejl) ARCH=arm;;
- armv6l) ARCH=arm;;
- armv6hl) ARCH=arm;;
- armv7l) ARCH=arm;;
- armv7hl) ARCH=arm;;
- tile) ARCH=tile;;
- e2k) ARCH=e2k;;
- *) ARCH=noarch;;
-esac
-
-dnl
-dnl Convert between x86 and amd64 based on the compiler's mode.
-dnl Ditto between ultrasparc and sparc64.
-dnl
-AC_MSG_CHECKING(whether compilation mode forces ARCH adjustment)
-case "$ARCH-$ac_cv_sizeof_void_p" in
-x86-8)
- AC_MSG_RESULT(yes: adjusting ARCH=x86 to ARCH=amd64)
- ARCH=amd64
- ;;
-amd64-4)
- AC_MSG_RESULT(yes: adjusting ARCH=amd64 to ARCH=x86)
- ARCH=x86
- ;;
-ultrasparc-8)
- AC_MSG_RESULT(yes: adjusting ARCH=ultrasparc to ARCH=sparc64)
- ARCH=sparc64
- ;;
-sparc64-4)
- AC_MSG_RESULT(yes: adjusting ARCH=sparc64 to ARCH=ultrasparc)
- ARCH=ultrasparc
- ;;
-ppc64-4)
- AC_MSG_RESULT(yes: adjusting ARCH=ppc64 to ARCH=ppc)
- ARCH=ppc
- ;;
-ppc-8)
- AC_MSG_RESULT(yes: adjusting ARCH=ppc to ARCH=ppc64)
- ARCH=ppc64
- ;;
-arm-8)
- AC_MSG_RESULT(yes: adjusting ARCH=arm to ARCH=noarch)
- ARCH=noarch
- ;;
-*)
- AC_MSG_RESULT(no)
- ;;
-esac
-
AC_SUBST(OPSYS)
-AC_SUBST(ARCH)
+
+LM_HARDWARE_ARCH
dnl Check consistency of os and darwin-switches
diff --git a/erts/doc/src/erl.xml b/erts/doc/src/erl.xml
index 5089d4e0ce..74654a295d 100644
--- a/erts/doc/src/erl.xml
+++ b/erts/doc/src/erl.xml
@@ -1158,7 +1158,7 @@
<tag><marker id="+sbwtdcpu"/>
<c>+sbwtdcpu none|very_short|short|medium|long|very_long</c></tag>
<item>
- <p>As <seealso marker="+sbwt"><c>+sbwt</c></seealso> but affects
+ <p>As <seealso marker="#+sbwt"><c>+sbwt</c></seealso> but affects
dirty CPU schedulers. Defaults to <c>short</c>.</p>
<note>
<p>This flag can be removed or changed at any time
@@ -1168,7 +1168,7 @@
<tag><marker id="+sbwtdio"/>
<c>+sbwtdio none|very_short|short|medium|long|very_long</c></tag>
<item>
- <p>As <seealso marker="+sbwt"><c>+sbwt</c></seealso> but affects
+ <p>As <seealso marker="#+sbwt"><c>+sbwt</c></seealso> but affects
dirty IO schedulers. Defaults to <c>short</c>.</p>
<note>
<p>This flag can be removed or changed at any time
@@ -1443,7 +1443,7 @@
<tag><marker id="+swtdcpu"/>
<c>+swtdcpu very_low|low|medium|high|very_high</c></tag>
<item>
- <p>As <seealso marker="+swt"><c>+swt</c></seealso> but
+ <p>As <seealso marker="#+swt"><c>+swt</c></seealso> but
affects dirty CPU schedulers. Defaults to <c>medium</c>.</p>
<note>
<p>This flag can be removed or changed at any time
@@ -1453,7 +1453,7 @@
<tag><marker id="+swtdio"/>
<c>+swtdio very_low|low|medium|high|very_high</c></tag>
<item>
- <p>As <seealso marker="+swt"><c>+swt</c></seealso> but affects
+ <p>As <seealso marker="#+swt"><c>+swt</c></seealso> but affects
dirty IO schedulers. Defaults to <c>medium</c>.</p>
<note>
<p>This flag can be removed or changed at any time
diff --git a/erts/doc/src/erts_alloc.xml b/erts/doc/src/erts_alloc.xml
index 53e136d76c..0893eb291c 100644
--- a/erts/doc/src/erts_alloc.xml
+++ b/erts/doc/src/erts_alloc.xml
@@ -559,6 +559,20 @@
than this threshold, otherwise the carrier is shrunk.
See also <seealso marker="#M_rsbcst"><c>rsbcst</c></seealso>.</p>
</item>
+ <tag><marker id="M_atags"/><c><![CDATA[+M<S>atags true|false]]></c></tag>
+ <item>
+ <p>Adds a small tag to each allocated block that contains basic
+ information about what it is and who allocated it. Use the
+ <seealso marker="tools:instrument"><c>instrument</c></seealso>
+ module to inspect this information.</p>
+
+ <p>The runtime overhead is one word per allocation when enabled. This
+ may change at any time in the future.</p>
+
+ <p>The default is <c>true</c> for <c>binary_alloc</c> and
+ <c>driver_alloc</c>, and <c>false</c> for the other allocator
+ types.</p>
+ </item>
<tag><marker id="M_e"/><c><![CDATA[+M<S>e true|false]]></c></tag>
<item>
<p>Enables allocator <c><![CDATA[<S>]]></c>.</p>
@@ -724,22 +738,12 @@
<section>
<title>Instrumentation Flags</title>
<taglist>
- <tag><marker id="Mim"/><c>+Mim true|false</c></tag>
- <item>
- <p>A map over current allocations is kept by the emulator.
- The allocation map can be retrieved through module
- <seealso marker="tools:instrument">
- <c>instrument(3)</c></seealso>. <c>+Mim true</c>
- implies <c>+Mis true</c>. <c>+Mim true</c> is the same as flag
- <seealso marker="erl#instr"><c>-instr</c></seealso> in
- <c>erl(1)</c>.</p>
- </item>
- <tag><marker id="Mis"/><c>+Mis true|false</c></tag>
- <item>
- <p>Status over allocated memory is kept by the emulator.
- The allocation status can be retrieved through module
- <seealso marker="tools:instrument">
- <c>instrument(3)</c></seealso>.</p>
+ <tag><c>+M&lt;S&gt;atags</c></tag>
+ <item>
+ <p>Adds a small tag to each allocated block that contains basic
+ information about what it is and who allocated it. See
+ <seealso marker="#M_atags"><c>+M&lt;S&gt;atags</c></seealso> for a
+ more complete description.</p>
</item>
<tag><marker id="Mit"/><c>+Mit X</c></tag>
<item>
diff --git a/erts/doc/src/match_spec.xml b/erts/doc/src/match_spec.xml
index 644b989800..6cf0a0e677 100644
--- a/erts/doc/src/match_spec.xml
+++ b/erts/doc/src/match_spec.xml
@@ -169,10 +169,9 @@
<c><![CDATA[is_reference]]></c> | <c><![CDATA[is_tuple]]></c> |
<c><![CDATA[is_map]]></c> | <c><![CDATA[is_binary]]></c> |
<c><![CDATA[is_function]]></c> | <c><![CDATA[is_record]]></c> |
- <c><![CDATA[is_seq_trace]]></c> | <c><![CDATA['and']]></c> |
- <c><![CDATA['or']]></c> | <c><![CDATA['not']]></c> |
- <c><![CDATA['xor']]></c> | <c><![CDATA['andalso']]></c> |
- <c><![CDATA['orelse']]></c>
+ <c><![CDATA['and']]></c> | <c><![CDATA['or']]></c> |
+ <c><![CDATA['not']]></c> | <c><![CDATA['xor']]></c> |
+ <c><![CDATA['andalso']]></c> | <c><![CDATA['orelse']]></c>
</item>
<item>ConditionExpression ::= ExprMatchVariable | { GuardFunction } |
{ GuardFunction, ConditionExpression, ... } | TermConstruct
@@ -202,8 +201,7 @@
<c><![CDATA['>=']]></c> | <c><![CDATA['<']]></c> |
<c><![CDATA['=<']]></c> | <c><![CDATA['=:=']]></c> |
<c><![CDATA['==']]></c> | <c><![CDATA['=/=']]></c> |
- <c><![CDATA['/=']]></c> | <c><![CDATA[self]]></c> |
- <c><![CDATA[get_tcw]]></c>
+ <c><![CDATA['/=']]></c> | <c><![CDATA[self]]></c>
</item>
<item>MatchBody ::= [ ConditionExpression, ... ]
</item>
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 92cf40c1ae..5dfa60ee74 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -832,7 +832,7 @@ RUN_OBJS += \
$(OBJDIR)/erl_alloc.o $(OBJDIR)/erl_mtrace.o \
$(OBJDIR)/erl_alloc_util.o $(OBJDIR)/erl_goodfit_alloc.o \
$(OBJDIR)/erl_bestfit_alloc.o $(OBJDIR)/erl_afit_alloc.o \
- $(OBJDIR)/erl_instrument.o $(OBJDIR)/erl_init.o \
+ $(OBJDIR)/erl_init.o \
$(OBJDIR)/erl_atom_table.o $(OBJDIR)/erl_bif_table.o \
$(OBJDIR)/erl_bif_ddll.o $(OBJDIR)/erl_bif_guard.o \
$(OBJDIR)/erl_bif_info.o $(OBJDIR)/erl_bif_op.o \
diff --git a/erts/emulator/beam/bif.c b/erts/emulator/beam/bif.c
index 017faffa48..79244b8544 100644
--- a/erts/emulator/beam/bif.c
+++ b/erts/emulator/beam/bif.c
@@ -1150,6 +1150,13 @@ BIF_RETTYPE raise_3(BIF_ALIST_3)
/* Create stacktrace and store */
if (erts_backtrace_depth < depth) {
depth = erts_backtrace_depth;
+ if (depth == 0) {
+ /*
+ * For consistency with stacktraces generated
+ * automatically, always include one element.
+ */
+ depth = 1;
+ }
must_copy = 1;
}
if (must_copy) {
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 687fd39d58..bcaf9277bf 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -693,3 +693,5 @@ bif erts_internal:new_connection/1
bif erts_internal:abort_connection/2
bif erts_internal:map_next/3
bif ets:whereis/1
+bif erts_internal:gather_alloc_histograms/1
+bif erts_internal:gather_carrier_info/1
diff --git a/erts/emulator/beam/bif_instrs.tab b/erts/emulator/beam/bif_instrs.tab
index 0932b8b985..0f074280db 100644
--- a/erts/emulator/beam/bif_instrs.tab
+++ b/erts/emulator/beam/bif_instrs.tab
@@ -432,9 +432,17 @@ nif_bif.call_nif() {
live_hf_end = c_p->mbuf;
ERTS_CHK_MBUF_SZ(c_p);
erts_pre_nif(&env, c_p, (struct erl_module_nif*)I[2], NULL);
+
+ ASSERT((c_p->scheduler_data)->current_nif == NULL);
+ (c_p->scheduler_data)->current_nif = &env;
+
nif_bif_result = (*fp)(&env, bif_nif_arity, reg);
if (env.exception_thrown)
nif_bif_result = THE_NON_VALUE;
+
+ ASSERT((c_p->scheduler_data)->current_nif == &env);
+ (c_p->scheduler_data)->current_nif = NULL;
+
erts_post_nif(&env);
ERTS_CHK_MBUF_SZ(c_p);
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index ba8cc5e2ba..9ff52c92b8 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -36,7 +36,6 @@
#include "hash.h"
#include "atom.h"
#include "beam_load.h"
-#include "erl_instrument.h"
#include "erl_hl_timer.h"
#include "erl_thr_progress.h"
#include "erl_proc_sig_queue.h"
@@ -955,20 +954,6 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
erts_cbprintf(to, to_arg, "=atoms\n");
dump_atoms(to, to_arg);
- /* Keep the instrumentation data at the end of the dump */
- if (erts_instr_memory_map || erts_instr_stat) {
- erts_cbprintf(to, to_arg, "=instr_data\n");
-
- if (erts_instr_stat) {
- erts_cbprintf(to, to_arg, "=memory_status\n");
- erts_instr_dump_stat_to(to, to_arg, 0);
- }
- if (erts_instr_memory_map) {
- erts_cbprintf(to, to_arg, "=memory_map\n");
- erts_instr_dump_memory_map_to(to, to_arg);
- }
- }
-
erts_cbprintf(to, to_arg, "=end\n");
if (fp) {
fclose(fp);
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 061b9df627..d99d2ea57b 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -38,7 +38,7 @@
#include "erl_db.h"
#include "erl_binary.h"
#include "erl_bits.h"
-#include "erl_instrument.h"
+#include "erl_mtrace.h"
#include "erl_mseg.h"
#include "erl_monitor_link.h"
#include "erl_hl_timer.h"
@@ -202,8 +202,6 @@ typedef struct {
int top_pad;
AlcUInit_t alloc_util;
struct {
- int stat;
- int map;
char *mtrace;
char *nodename;
} instr;
@@ -428,6 +426,7 @@ set_default_binary_alloc_opts(struct au_init *ip)
#endif
ip->init.util.ts = ERTS_ALC_MTA_BINARY;
ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
+ ip->init.util.atags = 1;
}
static void
@@ -464,6 +463,7 @@ set_default_driver_alloc_opts(struct au_init *ip)
#endif
ip->init.util.ts = ERTS_ALC_MTA_DRIVER;
ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
+ ip->init.util.atags = 1;
}
static void
@@ -501,6 +501,7 @@ set_default_test_alloc_opts(struct au_init *ip)
ip->init.util.mmbcs = 0; /* Main carrier size */
ip->init.util.ts = ERTS_ALC_MTA_TEST;
ip->init.util.acul = ERTS_ALC_DEFAULT_ACUL;
+ ip->init.util.atags = 1;
/* Use a constant minimal MBC size */
#if ERTS_SA_MB_CARRIERS
@@ -906,7 +907,6 @@ erts_alloc_init(int *argc, char **argv, ErtsAllocInitOpts *eaiop)
&test_alloc_state);
erts_mtrace_install_wrapper_functions();
- extra_block_size += erts_instr_init(init.instr.stat, init.instr.map);
init_aireq_alloc();
@@ -1411,7 +1411,9 @@ handle_au_arg(struct au_init *auip,
}
if (!strategy_support_carrier_migration(auip))
auip->init.util.acul = 0;
- }
+ } else if (has_prefix("atags", sub_param)) {
+ auip->init.util.atags = get_bool_value(sub_param + 5, argv, ip);
+ }
else
goto bad_switch;
break;
@@ -1741,24 +1743,6 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
break;
case 'i':
switch (argv[i][3]) {
- case 's':
- arg = get_value(argv[i]+4, argv, &i);
- if (sys_strcmp("true", arg) == 0)
- init->instr.stat = 1;
- else if (sys_strcmp("false", arg) == 0)
- init->instr.stat = 0;
- else
- bad_value(param, param+3, arg);
- break;
- case 'm':
- arg = get_value(argv[i]+4, argv, &i);
- if (sys_strcmp("true", arg) == 0)
- init->instr.map = 1;
- else if (sys_strcmp("false", arg) == 0)
- init->instr.map = 0;
- else
- bad_value(param, param+3, arg);
- break;
case 't':
init->instr.mtrace = get_value(argv[i]+4, argv, &i);
break;
@@ -1817,9 +1801,7 @@ handle_args(int *argc, char **argv, erts_alc_hndl_args_init_t *init)
case '-':
if (argv[i][2] == '\0') {
/* End of system flags reached */
- if (init->instr.mtrace
- /* || init->instr.stat
- || init->instr.map */) {
+ if (init->instr.mtrace) {
while (i < *argc) {
if(sys_strcmp(argv[i], "-sname") == 0
|| sys_strcmp(argv[i], "-name") == 0) {
@@ -2097,7 +2079,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
* NOTE! When updating this function, make sure to also update
* erlang:memory/[0,1] in $ERL_TOP/erts/preloaded/src/erlang.erl
*/
-#define ERTS_MEM_NEED_ALL_ALCU (!erts_instr_stat && want_tot_or_sys)
+#define ERTS_MEM_NEED_ALL_ALCU (want_tot_or_sys)
struct {
int total;
int processes;
@@ -2108,7 +2090,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
int binary;
int code;
int ets;
- int maximum;
} want = {0};
struct {
UWord total;
@@ -2120,7 +2101,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
UWord binary;
UWord code;
UWord ets;
- UWord maximum;
} size = {0};
Eterm atoms[sizeof(size)/sizeof(UWord)];
UWord *uintps[sizeof(size)/sizeof(UWord)];
@@ -2173,12 +2153,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
want.ets = 1;
atoms[length] = am_ets;
uintps[length++] = &size.ets;
-
- want.maximum = erts_instr_stat;
- if (want.maximum) {
- atoms[length] = am_maximum;
- uintps[length++] = &size.maximum;
- }
}
else {
DeclareTmpHeapNoproc(tmp_heap,2);
@@ -2260,18 +2234,6 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
uintps[length++] = &size.ets;
}
break;
- case am_maximum:
- if (erts_instr_stat) {
- if (!want.maximum) {
- want.maximum = 1;
- atoms[length] = am_maximum;
- uintps[length++] = &size.maximum;
- }
- } else {
- UnUseTmpHeapNoproc(2);
- return am_badarg;
- }
- break;
default:
UnUseTmpHeapNoproc(2);
return am_badarg;
@@ -2437,14 +2399,7 @@ erts_memory(fmtfn_t *print_to_p, void *print_to_arg, void *proc, Eterm earg)
size.ets += erts_get_ets_misc_mem_size();
}
- if (erts_instr_stat && (want_tot_or_sys || want.maximum)) {
- if (want_tot_or_sys) {
- size.total = erts_instr_get_total();
- size.system = size.total - size.processes;
- }
- size.maximum = erts_instr_get_max_total();
- }
- else if (want_tot_or_sys) {
+ if (want_tot_or_sys) {
size.system = size.total - size.processes;
}
@@ -2522,18 +2477,6 @@ erts_allocated_areas(fmtfn_t *print_to_p, void *print_to_arg, void *proc)
i = 0;
- if (erts_instr_stat) {
- values[i].arity = 2;
- values[i].name = "total";
- values[i].ui[0] = erts_instr_get_total();
- i++;
-
- values[i].arity = 2;
- values[i].name = "maximum";
- values[i].ui[0] = erts_instr_get_max_total();
- i++;
- }
-
values[i].arity = 2;
values[i].name = "sys_misc";
values[i].ui[0] = erts_sys_misc_mem_sz();
@@ -2824,10 +2767,7 @@ erts_allocator_info(fmtfn_t to, void *arg)
erts_alcu_au_info_options(&to, arg, NULL, NULL);
erts_print(to, arg, "=allocator:instr\n");
- erts_print(to, arg, "option m: %s\n",
- erts_instr_memory_map ? "true" : "false");
- erts_print(to, arg, "option s: %s\n",
- erts_instr_stat ? "true" : "false");
+
erts_print(to, arg, "option t: %s\n",
erts_mtrace_enabled ? "true" : "false");
@@ -2933,16 +2873,12 @@ erts_allocator_options(void *proc)
NULL, hpp, szp);
#endif
{
- Eterm o[3], v[3];
- o[0] = am_atom_put("m", 1);
- v[0] = erts_instr_memory_map ? am_true : am_false;
- o[1] = am_atom_put("s", 1);
- v[1] = erts_instr_stat ? am_true : am_false;
- o[2] = am_atom_put("t", 1);
- v[2] = erts_mtrace_enabled ? am_true : am_false;
+ Eterm o[1], v[1];
+ o[0] = am_atom_put("t", 1);
+ v[0] = erts_mtrace_enabled ? am_true : am_false;
atoms[length] = am_atom_put("instr", 5);
- terms[length++] = erts_bld_2tup_list(hpp, szp, 3, o, v);
+ terms[length++] = erts_bld_2tup_list(hpp, szp, 1, o, v);
}
atoms[length] = am_atom_put("lock_physical_memory", 20);
@@ -3458,8 +3394,8 @@ badarg:
/*
* The allocator wrapper prelocking stuff below is about the locking order.
- * It only affects wrappers (erl_mtrace.c and erl_instrument.c) that keep locks
- * during alloc/realloc/free.
+ * It only affects wrappers (erl_mtrace.c) that keep locks during
+ * alloc/realloc/free.
*
* Some query functions in erl_alloc_util.c lock the allocator mutex and then
* use erts_printf that in turn may call the sys allocator through the wrappers.
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index e148be7af6..fdf355d503 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -48,6 +48,8 @@
#include "erl_mseg.h"
#include "erl_threads.h"
#include "erl_thr_progress.h"
+#include "erl_bif_unique.h"
+#include "erl_nif.h"
#ifdef ERTS_ENABLE_LOCK_COUNT
#include "erl_lock_count.h"
@@ -139,6 +141,33 @@ MBC after deallocating first block:
[Carrier_t|pad|Block_t 111| udata... ]
*/
+/* Allocation tags ...
+ *
+ * These are added to the footer of every block when enabled. Currently they
+ * consist of the allocation type and an atom identifying the allocating
+ * driver/nif (or 'system' if that can't be determined), but the format is not
+ * supposed to be set in stone.
+ *
+ * The packing scheme requires that the atom values are small enough to fit
+ * into a word with ERTS_ALC_N_BITS to spare. Users must check for overflow
+ * before MAKE_ATAG(). */
+
+typedef UWord alcu_atag_t;
+
+#define MAKE_ATAG(IdAtom, Type) \
+ (ASSERT((Type) >= ERTS_ALC_N_MIN && (Type) <= ERTS_ALC_N_MAX), \
+ ASSERT(atom_val(IdAtom) <= MAX_ATAG_ATOM_ID), \
+ (atom_val(IdAtom) << ERTS_ALC_N_BITS) | (Type))
+
+#define ATAG_ID(AT) (make_atom((AT) >> ERTS_ALC_N_BITS))
+#define ATAG_TYPE(AT) ((AT) & ERTS_ALC_N_MASK)
+
+#define MAX_ATAG_ATOM_ID (ERTS_UWORD_MAX >> ERTS_ALC_N_BITS)
+
+#define DBG_IS_VALID_ATAG(Allocator, AT) \
+ (ATAG_TYPE(AT) >= ERTS_ALC_N_MIN && \
+ ATAG_TYPE(AT) <= ERTS_ALC_N_MAX && \
+ (Allocator)->alloc_no == ERTS_ALC_T2A(ERTS_ALC_N2T(ATAG_TYPE(AT))))
/* Blocks ... */
@@ -153,10 +182,17 @@ MBC after deallocating first block:
#endif
#define FBLK_FTR_SZ (sizeof(FreeBlkFtr_t))
+#define GET_BLK_ATAG(B) \
+ (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1])
+#define SET_BLK_ATAG(B, T) \
+ (((alcu_atag_t *) (((char *) (B)) + (BLK_SZ(B))))[-1] = (T))
+
+#define BLK_ATAG_SZ(AP) ((AP)->atags ? sizeof(alcu_atag_t) : 0)
+
#define UMEMSZ2BLKSZ(AP, SZ) \
- (ABLK_HDR_SZ + (SZ) <= (AP)->min_block_size \
+ (ABLK_HDR_SZ + BLK_ATAG_SZ(AP) + (SZ) <= (AP)->min_block_size \
? (AP)->min_block_size \
- : UNIT_CEILING(ABLK_HDR_SZ + (SZ)))
+ : UNIT_CEILING(ABLK_HDR_SZ + BLK_ATAG_SZ(AP) + (SZ)))
#define UMEM2BLK(P) ((Block_t *) (((char *) (P)) - ABLK_HDR_SZ))
#define BLK2UMEM(P) ((void *) (((char *) (P)) + ABLK_HDR_SZ))
@@ -688,6 +724,62 @@ static void destroy_carrier(Allctr_t *, Block_t *, Carrier_t **);
static void mbc_free(Allctr_t *allctr, void *p, Carrier_t **busy_pcrr_pp);
static void dealloc_block(Allctr_t *, void *, ErtsAlcFixList_t *, int);
+static alcu_atag_t determine_alloc_tag(Allctr_t *allocator, ErtsAlcType_t type)
+{
+ ErtsSchedulerData *esdp;
+ Eterm id;
+
+ ERTS_CT_ASSERT(_unchecked_atom_val(am_system) <= MAX_ATAG_ATOM_ID);
+ ASSERT(allocator->atags);
+
+ esdp = erts_get_scheduler_data();
+ id = am_system;
+
+ if (esdp) {
+ if (esdp->current_nif) {
+ Module *mod = erts_nif_get_module((esdp->current_nif)->mod_nif);
+
+ /* Mod can be NULL if a resource destructor allocates memory after
+ * the module has been unloaded. */
+ if (mod) {
+ id = make_atom(mod->module);
+ }
+ } else if (esdp->current_port) {
+ Port *p = esdp->current_port;
+ id = (p->drv_ptr)->name_atom;
+ }
+
+ /* We fall back to 'system' if we can't pack the driver/NIF name into
+ * the tag. This may be a bit misleading but we've made no promises
+ * that the information is complete.
+ *
+ * This can only happen on 32-bit emulators when a new driver/NIF has
+ * been loaded *after* 16 million atoms have been used, and supporting
+ * that fringe case is not worth an extra word. 64-bit emulators are
+ * unaffected since the atom cache limits atom indexes to 32 bits. */
+ if(MAX_ATOM_TABLE_SIZE > MAX_ATAG_ATOM_ID) {
+ if (atom_val(id) > MAX_ATAG_ATOM_ID) {
+ id = am_system;
+ }
+ }
+ }
+
+ return MAKE_ATAG(id, type);
+}
+
+static void set_alloc_tag(Allctr_t *allocator, void *p, alcu_atag_t tag)
+{
+ Block_t *block;
+
+ ASSERT(DBG_IS_VALID_ATAG(allocator, tag));
+ ASSERT(allocator->atags && p);
+ (void)allocator;
+
+ block = UMEM2BLK(p);
+
+ SET_BLK_ATAG(block, tag);
+}
+
/* internal data... */
#if 0
@@ -4242,6 +4334,7 @@ static struct {
Eterm e;
Eterm t;
Eterm ramv;
+ Eterm atags;
#if HAVE_ERTS_MSEG
Eterm asbcst;
Eterm rsbcst;
@@ -4311,7 +4404,7 @@ static struct {
#endif
} am;
-static Eterm fix_type_atoms[ERTS_ALC_NO_FIXED_SIZES];
+static Eterm alloc_type_atoms[ERTS_ALC_N_MAX + 1];
static ERTS_INLINE void atom_init(Eterm *atom, char *name)
{
@@ -4342,6 +4435,7 @@ init_atoms(Allctr_t *allctr)
AM_INIT(e);
AM_INIT(t);
AM_INIT(ramv);
+ AM_INIT(atags);
#if HAVE_ERTS_MSEG
AM_INIT(asbcst);
AM_INIT(rsbcst);
@@ -4413,12 +4507,12 @@ init_atoms(Allctr_t *allctr)
}
#endif
- for (ix = 0; ix < ERTS_ALC_NO_FIXED_SIZES; ix++) {
- ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix;
- char *name = (char *) ERTS_ALC_N2TD(n);
- size_t len = sys_strlen(name);
- fix_type_atoms[ix] = am_atom_put(name, len);
- }
+ for (ix = ERTS_ALC_N_MIN; ix <= ERTS_ALC_N_MAX; ix++) {
+ const char *name = ERTS_ALC_N2TD(ix);
+ size_t len = sys_strlen(name);
+
+ alloc_type_atoms[ix] = am_atom_put(name, len);
+ }
}
if (allctr && !allctr->atoms_initialized) {
@@ -4531,6 +4625,7 @@ sz_info_fix(Allctr_t *allctr,
ErtsAlcFixList_t *fix = &allctr->fix[ix];
UWord alloced = fix->type_size * fix->u.cpool.allocated;
UWord used = fix->type_size * fix->u.cpool.used;
+ ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix;
if (print_to_p) {
fmtfn_t to = *print_to_p;
@@ -4538,15 +4633,14 @@ sz_info_fix(Allctr_t *allctr,
erts_print(to,
arg,
"fix type internal: %s %bpu %bpu\n",
- (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE
- + ix),
+ (char *) ERTS_ALC_N2TD(n),
alloced,
used);
}
if (hpp || szp) {
add_3tup(hpp, szp, &res,
- fix_type_atoms[ix],
+ alloc_type_atoms[n],
bld_unstable_uint(hpp, szp, alloced),
bld_unstable_uint(hpp, szp, used));
}
@@ -4559,6 +4653,7 @@ sz_info_fix(Allctr_t *allctr,
ErtsAlcFixList_t *fix = &allctr->fix[ix];
UWord alloced = fix->type_size * fix->u.nocpool.allocated;
UWord used = fix->type_size*fix->u.nocpool.used;
+ ErtsAlcType_t n = ERTS_ALC_N_MIN_A_FIXED_SIZE + ix;
if (print_to_p) {
fmtfn_t to = *print_to_p;
@@ -4566,15 +4661,14 @@ sz_info_fix(Allctr_t *allctr,
erts_print(to,
arg,
"fix type: %s %bpu %bpu\n",
- (char *) ERTS_ALC_N2TD(ERTS_ALC_N_MIN_A_FIXED_SIZE
- + ix),
+ (char *) ERTS_ALC_N2TD(n),
alloced,
used);
}
if (hpp || szp) {
add_3tup(hpp, szp, &res,
- fix_type_atoms[ix],
+ alloc_type_atoms[n],
bld_unstable_uint(hpp, szp, alloced),
bld_unstable_uint(hpp, szp, used));
}
@@ -5000,6 +5094,7 @@ info_options(Allctr_t *allctr,
"option e: true\n"
"option t: %s\n"
"option ramv: %s\n"
+ "option atags: %s\n"
"option sbct: %beu\n"
#if HAVE_ERTS_MSEG
"option asbcst: %bpu\n"
@@ -5018,6 +5113,7 @@ info_options(Allctr_t *allctr,
"option acul: %bpu\n",
topt,
allctr->ramv ? "true" : "false",
+ allctr->atags ? "true" : "false",
allctr->sbc_threshold,
#if HAVE_ERTS_MSEG
allctr->mseg_opt.abs_shrink_th,
@@ -5087,6 +5183,7 @@ info_options(Allctr_t *allctr,
am_sbct,
bld_uint(hpp, szp, allctr->sbc_threshold));
add_2tup(hpp, szp, &res, am.ramv, allctr->ramv ? am_true : am_false);
+ add_2tup(hpp, szp, &res, am.atags, allctr->atags ? am_true : am_false);
add_2tup(hpp, szp, &res, am.t, (allctr->t ? am_true : am_false));
add_2tup(hpp, szp, &res, am.e, am_true);
}
@@ -5408,9 +5505,8 @@ erts_alcu_current_size(Allctr_t *allctr, AllctrSize_t *size, ErtsAlcUFixInfo_t *
/* ----------------------------------------------------------------------- */
static ERTS_INLINE void *
-do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
+do_erts_alcu_alloc(ErtsAlcType_t type, Allctr_t *allctr, Uint size)
{
- Allctr_t *allctr = (Allctr_t *) extra;
void *res;
ASSERT(initialized);
@@ -5449,10 +5545,19 @@ do_erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
void *erts_alcu_alloc(ErtsAlcType_t type, void *extra, Uint size)
{
+ Allctr_t *allctr = (Allctr_t *) extra;
void *res;
+
ASSERT(!"This is not thread safe");
- res = do_erts_alcu_alloc(type, extra, size);
+
+ res = do_erts_alcu_alloc(type, allctr, size);
+
+ if (allctr->atags && res) {
+ set_alloc_tag(allctr, res, determine_alloc_tag(allctr, type));
+ }
+
DEBUG_CHECK_ALIGNMENT(res);
+
return res;
}
@@ -5462,13 +5567,25 @@ void *
erts_alcu_alloc_ts(ErtsAlcType_t type, void *extra, Uint size)
{
Allctr_t *allctr = (Allctr_t *) extra;
+ alcu_atag_t tag = 0;
void *res;
+
+ if (allctr->atags) {
+ tag = determine_alloc_tag(allctr, type);
+ }
+
erts_mtx_lock(&allctr->mutex);
- res = do_erts_alcu_alloc(type, extra, size);
- DEBUG_CHECK_ALIGNMENT(res);
+ res = do_erts_alcu_alloc(type, allctr, size);
+
+ if (allctr->atags && res) {
+ set_alloc_tag(allctr, res, tag);
+ }
erts_mtx_unlock(&allctr->mutex);
+
+ DEBUG_CHECK_ALIGNMENT(res);
+
return res;
}
@@ -5478,6 +5595,7 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size)
{
ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
int ix;
+ alcu_atag_t tag = 0;
Allctr_t *allctr;
void *res;
@@ -5487,11 +5605,19 @@ erts_alcu_alloc_thr_spec(ErtsAlcType_t type, void *extra, Uint size)
allctr = tspec->allctr[ix];
+ if (allctr->atags) {
+ tag = determine_alloc_tag(allctr, type);
+ }
+
if (allctr->thread_safe)
erts_mtx_lock(&allctr->mutex);
res = do_erts_alcu_alloc(type, allctr, size);
+ if (allctr->atags && res) {
+ set_alloc_tag(allctr, res, tag);
+ }
+
if (allctr->thread_safe)
erts_mtx_unlock(&allctr->mutex);
@@ -5504,10 +5630,15 @@ void *
erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
{
Allctr_t *pref_allctr;
+ alcu_atag_t tag = 0;
void *res;
pref_allctr = get_pref_allctr(extra);
+ if (pref_allctr->atags) {
+ tag = determine_alloc_tag(pref_allctr, type);
+ }
+
if (pref_allctr->thread_safe)
erts_mtx_lock(&pref_allctr->mutex);
@@ -5523,12 +5654,15 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
res = do_erts_alcu_alloc(type, pref_allctr, size);
}
+ if (pref_allctr->atags && res) {
+ set_alloc_tag(pref_allctr, res, tag);
+ }
+
if (pref_allctr->thread_safe)
erts_mtx_unlock(&pref_allctr->mutex);
DEBUG_CHECK_ALIGNMENT(res);
-
return res;
}
@@ -5537,10 +5671,9 @@ erts_alcu_alloc_thr_pref(ErtsAlcType_t type, void *extra, Uint size)
/* ------------------------------------------------------------------------- */
static ERTS_INLINE void
-do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p,
+do_erts_alcu_free(ErtsAlcType_t type, Allctr_t *allctr, void *p,
Carrier_t **busy_pcrr_pp)
{
- Allctr_t *allctr = (Allctr_t *) extra;
ASSERT(initialized);
ASSERT(allctr);
@@ -5572,7 +5705,8 @@ do_erts_alcu_free(ErtsAlcType_t type, void *extra, void *p,
void erts_alcu_free(ErtsAlcType_t type, void *extra, void *p)
{
- do_erts_alcu_free(type, extra, p, NULL);
+ Allctr_t *allctr = (Allctr_t *) extra;
+ do_erts_alcu_free(type, allctr, p, NULL);
}
@@ -5581,7 +5715,7 @@ erts_alcu_free_ts(ErtsAlcType_t type, void *extra, void *p)
{
Allctr_t *allctr = (Allctr_t *) extra;
erts_mtx_lock(&allctr->mutex);
- do_erts_alcu_free(type, extra, p, NULL);
+ do_erts_alcu_free(type, allctr, p, NULL);
erts_mtx_unlock(&allctr->mutex);
}
@@ -5641,13 +5775,12 @@ erts_alcu_free_thr_pref(ErtsAlcType_t type, void *extra, void *p)
static ERTS_INLINE void *
do_erts_alcu_realloc(ErtsAlcType_t type,
- void *extra,
+ Allctr_t *allctr,
void *p,
Uint size,
Uint32 alcu_flgs,
Carrier_t **busy_pcrr_pp)
{
- Allctr_t *allctr = (Allctr_t *) extra;
Block_t *blk;
void *res;
@@ -5661,7 +5794,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
ERTS_ALCU_DBG_CHK_THR_ACCESS(allctr);
if (!p) {
- res = do_erts_alcu_alloc(type, extra, size);
+ res = do_erts_alcu_alloc(type, allctr, size);
INC_CC(allctr->calls.this_realloc);
DEC_CC(allctr->calls.this_alloc);
return res;
@@ -5670,7 +5803,7 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
#if ALLOC_ZERO_EQ_NULL
if (!size) {
ASSERT(p);
- do_erts_alcu_free(type, extra, p, busy_pcrr_pp);
+ do_erts_alcu_free(type, allctr, p, busy_pcrr_pp);
INC_CC(allctr->calls.this_realloc);
DEC_CC(allctr->calls.this_free);
return NULL;
@@ -5755,19 +5888,29 @@ do_erts_alcu_realloc(ErtsAlcType_t type,
void *
erts_alcu_realloc(ErtsAlcType_t type, void *extra, void *p, Uint size)
{
+ Allctr_t *allctr = (Allctr_t *)extra;
void *res;
- res = do_erts_alcu_realloc(type, extra, p, size, 0, NULL);
+
+ res = do_erts_alcu_realloc(type, allctr, p, size, 0, NULL);
+
DEBUG_CHECK_ALIGNMENT(res);
+
+ if (allctr->atags && res) {
+ set_alloc_tag(allctr, res, determine_alloc_tag(allctr, type));
+ }
+
return res;
}
void *
erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size)
{
+ Allctr_t *allctr = (Allctr_t *)extra;
void *res;
- res = do_erts_alcu_alloc(type, extra, size);
+
+ res = do_erts_alcu_alloc(type, allctr, size);
if (!res)
- res = erts_alcu_realloc(type, extra, p, size);
+ res = do_erts_alcu_realloc(type, allctr, p, size, 0, NULL);
else {
Block_t *blk;
size_t cpy_size;
@@ -5777,23 +5920,42 @@ erts_alcu_realloc_mv(ErtsAlcType_t type, void *extra, void *p, Uint size)
if (cpy_size > size)
cpy_size = size;
sys_memcpy(res, p, cpy_size);
- do_erts_alcu_free(type, extra, p, NULL);
+ do_erts_alcu_free(type, allctr, p, NULL);
}
+
DEBUG_CHECK_ALIGNMENT(res);
+
+ if (allctr->atags && res) {
+ set_alloc_tag(allctr, res, determine_alloc_tag(allctr, type));
+ }
+
return res;
}
-
void *
erts_alcu_realloc_ts(ErtsAlcType_t type, void *extra, void *ptr, Uint size)
{
Allctr_t *allctr = (Allctr_t *) extra;
+ alcu_atag_t tag = 0;
void *res;
+
+ if (allctr->atags) {
+ tag = determine_alloc_tag(allctr, type);
+ }
+
erts_mtx_lock(&allctr->mutex);
- res = do_erts_alcu_realloc(type, extra, ptr, size, 0, NULL);
+
+ res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL);
+
+ if (allctr->atags && res) {
+ set_alloc_tag(allctr, res, tag);
+ }
+
erts_mtx_unlock(&allctr->mutex);
+
DEBUG_CHECK_ALIGNMENT(res);
+
return res;
}
@@ -5801,11 +5963,17 @@ void *
erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size)
{
Allctr_t *allctr = (Allctr_t *) extra;
+ alcu_atag_t tag = 0;
void *res;
+
+ if (allctr->atags) {
+ tag = determine_alloc_tag(allctr, type);
+ }
+
erts_mtx_lock(&allctr->mutex);
- res = do_erts_alcu_alloc(type, extra, size);
+ res = do_erts_alcu_alloc(type, allctr, size);
if (!res)
- res = erts_alcu_realloc_ts(type, extra, p, size);
+ res = do_erts_alcu_realloc(type, allctr, p, size, 0, NULL);
else {
Block_t *blk;
size_t cpy_size;
@@ -5815,10 +5983,17 @@ erts_alcu_realloc_mv_ts(ErtsAlcType_t type, void *extra, void *p, Uint size)
if (cpy_size > size)
cpy_size = size;
sys_memcpy(res, p, cpy_size);
- do_erts_alcu_free(type, extra, p, NULL);
+ do_erts_alcu_free(type, allctr, p, NULL);
}
+
+ if (allctr->atags && res) {
+ set_alloc_tag(allctr, res, tag);
+ }
+
erts_mtx_unlock(&allctr->mutex);
+
DEBUG_CHECK_ALIGNMENT(res);
+
return res;
}
@@ -5829,6 +6004,7 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra,
{
ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
int ix;
+ alcu_atag_t tag = 0;
Allctr_t *allctr;
void *res;
@@ -5838,11 +6014,19 @@ erts_alcu_realloc_thr_spec(ErtsAlcType_t type, void *extra,
allctr = tspec->allctr[ix];
+ if (allctr->atags) {
+ tag = determine_alloc_tag(allctr, type);
+ }
+
if (allctr->thread_safe)
erts_mtx_lock(&allctr->mutex);
res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL);
+ if (allctr->atags && res) {
+ set_alloc_tag(allctr, res, tag);
+ }
+
if (allctr->thread_safe)
erts_mtx_unlock(&allctr->mutex);
@@ -5857,6 +6041,7 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
{
ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t *) extra;
int ix;
+ alcu_atag_t tag = 0;
Allctr_t *allctr;
void *res;
@@ -5866,14 +6051,16 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
allctr = tspec->allctr[ix];
+ if (allctr->atags) {
+ tag = determine_alloc_tag(allctr, type);
+ }
+
if (allctr->thread_safe)
erts_mtx_lock(&allctr->mutex);
res = do_erts_alcu_alloc(type, allctr, size);
if (!res) {
- if (allctr->thread_safe)
- erts_mtx_unlock(&allctr->mutex);
- res = erts_alcu_realloc_thr_spec(type, allctr, ptr, size);
+ res = do_erts_alcu_realloc(type, allctr, ptr, size, 0, NULL);
}
else {
Block_t *blk;
@@ -5885,29 +6072,34 @@ erts_alcu_realloc_mv_thr_spec(ErtsAlcType_t type, void *extra,
cpy_size = size;
sys_memcpy(res, ptr, cpy_size);
do_erts_alcu_free(type, allctr, ptr, NULL);
- if (allctr->thread_safe)
- erts_mtx_unlock(&allctr->mutex);
}
+ if (allctr->atags && res) {
+ set_alloc_tag(allctr, res, tag);
+ }
+
+ if (allctr->thread_safe)
+ erts_mtx_unlock(&allctr->mutex);
+
DEBUG_CHECK_ALIGNMENT(res);
return res;
}
static ERTS_INLINE void *
-realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size,
+realloc_thr_pref(ErtsAlcType_t type, Allctr_t *pref_allctr, void *p, Uint size,
int force_move)
{
void *res;
- Allctr_t *pref_allctr, *used_allctr;
+ Allctr_t *used_allctr;
UWord old_user_size;
Carrier_t *busy_pcrr_p;
+ alcu_atag_t tag = 0;
int retried;
- if (!p)
- return erts_alcu_alloc_thr_pref(type, extra, size);
-
- pref_allctr = get_pref_allctr(extra);
+ if (pref_allctr->atags) {
+ tag = determine_alloc_tag(pref_allctr, type);
+ }
if (pref_allctr->thread_safe)
erts_mtx_lock(&pref_allctr->mutex);
@@ -5936,6 +6128,11 @@ restart:
retried = 1;
goto restart;
}
+
+ if (pref_allctr->atags && res) {
+ set_alloc_tag(pref_allctr, res, tag);
+ }
+
if (pref_allctr->thread_safe)
erts_mtx_unlock(&pref_allctr->mutex);
}
@@ -5944,6 +6141,9 @@ restart:
if (!res)
goto unlock_ts_return;
else {
+ if (pref_allctr->atags) {
+ set_alloc_tag(pref_allctr, res, tag);
+ }
DEBUG_CHECK_ALIGNMENT(res);
@@ -5974,20 +6174,34 @@ restart:
}
}
+ DEBUG_CHECK_ALIGNMENT(res);
+
return res;
}
void *
erts_alcu_realloc_thr_pref(ErtsAlcType_t type, void *extra, void *p, Uint size)
{
- return realloc_thr_pref(type, extra, p, size, 0);
+ if (p) {
+ Allctr_t *pref_allctr = get_pref_allctr(extra);
+
+ return realloc_thr_pref(type, pref_allctr, p, size, 0);
+ }
+
+ return erts_alcu_alloc_thr_pref(type, extra, size);
}
void *
erts_alcu_realloc_mv_thr_pref(ErtsAlcType_t type, void *extra,
void *p, Uint size)
{
- return realloc_thr_pref(type, extra, p, size, 1);
+ if (p) {
+ Allctr_t *pref_allctr = get_pref_allctr(extra);
+
+ return realloc_thr_pref(type, pref_allctr, p, size, 1);
+ }
+
+ return erts_alcu_alloc_thr_pref(type, extra, size);
}
@@ -6071,6 +6285,7 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init)
allctr->t = 0;
allctr->ramv = init->ramv;
+ allctr->atags = init->atags;
allctr->main_carrier_size = init->mmbcs;
#if HAVE_ERTS_MSEG
@@ -6320,6 +6535,1072 @@ erts_alcu_init(AlcUInit_t *init)
initialized = 1;
}
+/* ------------------------------------------------------------------------- */
+
+/* Allocation histograms and carrier information is gathered by walking through
+ * all carriers associated with each allocator instance. This is done as
+ * aux_yield_work on the scheduler that owns each instance.
+ *
+ * Yielding is implemented by temporarily inserting a "dummy carrier" at the
+ * last position. It's permanently "busy" so it won't get picked up by someone
+ * else when in the carrier pool, and we never make the employer aware of it
+ * through callbacks so we can't accidentally allocate on it.
+ *
+ * Plain malloc/free is used to guarantee we won't allocate with the allocator
+ * we're scanning. */
+
+/* Yield between carriers once this many blocks have been processed. Note that
+ * a single carrier scan may exceed this figure. */
+#ifndef DEBUG
+ #define BLOCKSCAN_REDUCTIONS (8000)
+#else
+ #define BLOCKSCAN_REDUCTIONS (400)
+#endif
+
+/* Abort a single carrier scan after this many blocks to prevent really large
+ * MBCs from blocking forever. */
+#define BLOCKSCAN_BAILOUT_THRESHOLD (16000)
+
+typedef struct alcu_blockscan {
+ /* A per-scheduler list used when multiple scans have been queued. The
+ * current scanner will always run until completion/abort before moving on
+ * to the next. */
+ struct alcu_blockscan *scanner_queue;
+
+ Allctr_t *allocator;
+ Process *process;
+
+ int (*current_op)(struct alcu_blockscan *scanner);
+ int (*next_op)(struct alcu_blockscan *scanner);
+ int reductions;
+
+ ErtsAlcCPoolData_t *cpool_cursor;
+ CarrierList_t *current_clist;
+ Carrier_t *clist_cursor;
+ Carrier_t dummy_carrier;
+
+ /* Called if the process that started this job dies before we're done. */
+ void (*abort)(void *user_data);
+
+ /* Called on each carrier. The callback must return the number of blocks
+ * scanned to yield properly between carriers.
+ *
+ * Note that it's not possible to "yield back" into a carrier. */
+ int (*scan)(Allctr_t *, void *user_data, Carrier_t *);
+
+ /* Called when all carriers have been scanned. The callback may return
+ * non-zero to yield. */
+ int (*finish)(void *user_data);
+
+ void *user_data;
+} blockscan_t;
+
+static Carrier_t *blockscan_restore_clist_cursor(blockscan_t *state)
+{
+ Carrier_t *cursor = state->clist_cursor;
+
+ ASSERT(state->clist_cursor == (state->current_clist)->first ||
+ state->clist_cursor == &state->dummy_carrier);
+
+ if (cursor == &state->dummy_carrier) {
+ cursor = cursor->next;
+
+ unlink_carrier(state->current_clist, state->clist_cursor);
+ }
+
+ return cursor;
+}
+
+static void blockscan_save_clist_cursor(blockscan_t *state, Carrier_t *after)
+{
+ ASSERT(state->clist_cursor == (state->current_clist)->first ||
+ state->clist_cursor == &state->dummy_carrier);
+
+ state->clist_cursor = &state->dummy_carrier;
+
+ (state->clist_cursor)->next = after->next;
+ (state->clist_cursor)->prev = after;
+
+ relink_carrier(state->current_clist, state->clist_cursor);
+}
+
+static int blockscan_clist_yielding(blockscan_t *state)
+{
+ Carrier_t *cursor = blockscan_restore_clist_cursor(state);
+
+ if (ERTS_PROC_IS_EXITING(state->process)) {
+ return 0;
+ }
+
+ while (cursor) {
+ /* Skip dummy carriers inserted by another (concurrent) block scan.
+ * This can happen when scanning thread-safe allocators from multiple
+ * schedulers. */
+ if (CARRIER_SZ(cursor) > 0) {
+ int blocks_scanned = state->scan(state->allocator,
+ state->user_data,
+ cursor);
+
+ state->reductions -= blocks_scanned;
+
+ if (state->reductions <= 0) {
+ blockscan_save_clist_cursor(state, cursor);
+ return 1;
+ }
+ }
+
+ cursor = cursor->next;
+ }
+
+ return 0;
+}
+
+static ErtsAlcCPoolData_t *blockscan_restore_cpool_cursor(blockscan_t *state)
+{
+ ErtsAlcCPoolData_t *cursor;
+
+ cursor = cpool_aint2cpd(cpool_read(&(state->cpool_cursor)->next));
+
+ if (state->cpool_cursor == &state->dummy_carrier.cpool) {
+ cpool_delete(state->allocator, state->allocator, &state->dummy_carrier);
+ }
+
+ return cursor;
+}
+
+static void blockscan_save_cpool_cursor(blockscan_t *state,
+ ErtsAlcCPoolData_t *after)
+{
+ ErtsAlcCPoolData_t *dummy_carrier, *prev_carrier, *next_carrier;
+
+ dummy_carrier = &state->dummy_carrier.cpool;
+
+ next_carrier = cpool_aint2cpd(cpool_mod_mark(&after->next));
+ prev_carrier = cpool_aint2cpd(cpool_mod_mark(&next_carrier->prev));
+
+ cpool_init(&dummy_carrier->next, (erts_aint_t)next_carrier);
+ cpool_init(&dummy_carrier->prev, (erts_aint_t)prev_carrier);
+
+ cpool_set_mod_marked(&prev_carrier->next,
+ (erts_aint_t)dummy_carrier,
+ (erts_aint_t)next_carrier);
+ cpool_set_mod_marked(&next_carrier->prev,
+ (erts_aint_t)dummy_carrier,
+ (erts_aint_t)prev_carrier);
+
+ state->cpool_cursor = dummy_carrier;
+}
+
+static int blockscan_cpool_yielding(blockscan_t *state)
+{
+ ErtsAlcCPoolData_t *sentinel, *cursor;
+
+ sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel;
+ cursor = blockscan_restore_cpool_cursor(state);
+
+ if (ERTS_PROC_IS_EXITING(state->process)) {
+ return 0;
+ }
+
+ while (cursor != sentinel) {
+ Carrier_t *carrier;
+ erts_aint_t exp;
+
+ /* When a deallocation happens on a pooled carrier it will be routed to
+ * its owner, so the only way to be sure that it isn't modified while
+ * scanning is to skip all carriers that aren't ours. The deallocations
+ * deferred to us will get handled when we're done. */
+ while (cursor->orig_allctr != state->allocator) {
+ cursor = cpool_aint2cpd(cpool_read(&cursor->next));
+
+ if (cursor == sentinel) {
+ return 0;
+ }
+ }
+
+ carrier = ErtsContainerStruct(cursor, Carrier_t, cpool);
+ exp = erts_atomic_read_rb(&carrier->allctr);
+
+ if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) {
+ ASSERT(state->allocator == (Allctr_t*)(exp & ~ERTS_CRR_ALCTR_FLG_MASK));
+ ASSERT(!(exp & ERTS_CRR_ALCTR_FLG_BUSY));
+
+ if (erts_atomic_cmpxchg_acqb(&carrier->allctr,
+ exp | ERTS_CRR_ALCTR_FLG_BUSY,
+ exp) == exp) {
+ /* Skip dummy carriers inserted by another (concurrent) block
+ * scan. This can happen when scanning thread-safe allocators
+ * from multiple schedulers. */
+ if (CARRIER_SZ(carrier) > 0) {
+ int blocks_scanned = state->scan(state->allocator,
+ state->user_data,
+ carrier);
+
+ state->reductions -= blocks_scanned;
+
+ if (state->reductions <= 0) {
+ blockscan_save_cpool_cursor(state, cursor);
+ erts_atomic_set_relb(&carrier->allctr, exp);
+
+ return 1;
+ }
+ }
+
+ erts_atomic_set_relb(&carrier->allctr, exp);
+ }
+ }
+
+ cursor = cpool_aint2cpd(cpool_read(&cursor->next));
+ }
+
+ return 0;
+}
+
+static int blockscan_yield_helper(blockscan_t *state,
+ int (*yielding_op)(blockscan_t*))
+{
+ /* Note that we don't check whether to abort here; only yielding_op knows
+ * whether the carrier is still in the list/pool. */
+
+ if ((state->allocator)->thread_safe) {
+ /* Locked scans have to be as short as possible. */
+ state->reductions = 1;
+
+ erts_mtx_lock(&(state->allocator)->mutex);
+ } else {
+ state->reductions = BLOCKSCAN_REDUCTIONS;
+ }
+
+ if (yielding_op(state)) {
+ state->next_op = state->current_op;
+ }
+
+ if ((state->allocator)->thread_safe) {
+ erts_mtx_unlock(&(state->allocator)->mutex);
+ }
+
+ return 1;
+}
+
+/* */
+
+static int blockscan_finish(blockscan_t *state)
+{
+ if (ERTS_PROC_IS_EXITING(state->process)) {
+ state->abort(state->user_data);
+ return 0;
+ }
+
+ state->current_op = blockscan_finish;
+
+ return state->finish(state->user_data);
+}
+
+static int blockscan_sweep_sbcs(blockscan_t *state)
+{
+ if (state->current_op != blockscan_sweep_sbcs) {
+ SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_SBC, state->allocator);
+ state->current_clist = &(state->allocator)->sbc_list;
+ state->clist_cursor = (state->current_clist)->first;
+ }
+
+ state->current_op = blockscan_sweep_sbcs;
+ state->next_op = blockscan_finish;
+
+ return blockscan_yield_helper(state, blockscan_clist_yielding);
+}
+
+static int blockscan_sweep_mbcs(blockscan_t *state)
+{
+ if (state->current_op != blockscan_sweep_mbcs) {
+ SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_MBC, state->allocator);
+ state->current_clist = &(state->allocator)->mbc_list;
+ state->clist_cursor = (state->current_clist)->first;
+ }
+
+ state->current_op = blockscan_sweep_mbcs;
+ state->next_op = blockscan_sweep_sbcs;
+
+ return blockscan_yield_helper(state, blockscan_clist_yielding);
+}
+
+static int blockscan_sweep_cpool(blockscan_t *state)
+{
+ if (state->current_op != blockscan_sweep_cpool) {
+ ErtsAlcCPoolData_t *sentinel;
+
+ SET_CARRIER_HDR(&state->dummy_carrier, 0, SCH_MBC, state->allocator);
+ sentinel = &carrier_pool[(state->allocator)->alloc_no].sentinel;
+ state->cpool_cursor = sentinel;
+ }
+
+ state->current_op = blockscan_sweep_cpool;
+ state->next_op = blockscan_sweep_mbcs;
+
+ return blockscan_yield_helper(state, blockscan_cpool_yielding);
+}
+
+static int blockscan_get_specific_allocator(int allocator_num,
+ int sched_id,
+ Allctr_t **out)
+{
+ ErtsAllocatorInfo_t *ai;
+ Allctr_t *allocator;
+
+ ASSERT(allocator_num >= ERTS_ALC_A_MIN &&
+ allocator_num <= ERTS_ALC_A_MAX);
+ ASSERT(sched_id >= 0 && sched_id <= erts_no_schedulers);
+
+ ai = &erts_allctrs_info[allocator_num];
+
+ if (!ai->enabled || !ai->alloc_util) {
+ return 0;
+ }
+
+ if (!ai->thr_spec) {
+ if (sched_id != 0) {
+ /* Only thread-specific allocators can be scanned on a specific
+ * scheduler. */
+ return 0;
+ }
+
+ allocator = (Allctr_t*)ai->extra;
+ ASSERT(allocator->thread_safe);
+ } else {
+ ErtsAllocatorThrSpec_t *tspec = (ErtsAllocatorThrSpec_t*)ai->extra;
+
+ ASSERT(sched_id < tspec->size);
+
+ allocator = tspec->allctr[sched_id];
+ }
+
+ *out = allocator;
+
+ return 1;
+}
+
+static void blockscan_sched_trampoline(void *arg)
+{
+ ErtsAlcuBlockscanYieldData *yield;
+ ErtsSchedulerData *esdp;
+ blockscan_t *scanner;
+
+ esdp = erts_get_scheduler_data();
+ scanner = (blockscan_t*)arg;
+
+ yield = ERTS_SCHED_AUX_YIELD_DATA(esdp, alcu_blockscan);
+
+ ASSERT((yield->last == NULL) == (yield->current == NULL));
+
+ if (yield->last != NULL) {
+ blockscan_t *prev_scanner = yield->last;
+
+ ASSERT(prev_scanner->scanner_queue == NULL);
+
+ prev_scanner->scanner_queue = scanner;
+ } else {
+ yield->current = scanner;
+ }
+
+ scanner->scanner_queue = NULL;
+ yield->last = scanner;
+
+ erts_notify_new_aux_yield_work(esdp);
+}
+
+static void blockscan_dispatch(blockscan_t *scanner, Process *owner,
+ Allctr_t *allocator, int sched_id)
+{
+ ASSERT(erts_get_scheduler_id() != 0);
+
+ if (sched_id == 0) {
+ /* Global instances are always handled on the current scheduler. */
+ sched_id = ERTS_ALC_GET_THR_IX();
+ ASSERT(allocator->thread_safe);
+ }
+
+ scanner->allocator = allocator;
+ scanner->process = owner;
+
+ erts_proc_inc_refc(scanner->process);
+
+ cpool_init_carrier_data(scanner->allocator, &scanner->dummy_carrier);
+ erts_atomic_init_nob(&(scanner->dummy_carrier).allctr,
+ (erts_aint_t)allocator | ERTS_CRR_ALCTR_FLG_BUSY);
+
+ if (ERTS_ALC_IS_CPOOL_ENABLED(scanner->allocator)) {
+ scanner->next_op = blockscan_sweep_cpool;
+ } else {
+ scanner->next_op = blockscan_sweep_mbcs;
+ }
+
+ /* Aux yield jobs can only be set up while running on the scheduler that
+ * services them, so we move there before continuing.
+ *
+ * We can't drive the scan itself through this since the scheduler will
+ * always finish *all* misc aux work in one go which makes it impossible to
+ * yield. */
+ erts_schedule_misc_aux_work(sched_id, blockscan_sched_trampoline, scanner);
+}
+
+int erts_handle_yielded_alcu_blockscan(ErtsSchedulerData *esdp,
+ ErtsAlcuBlockscanYieldData *yield)
+{
+ blockscan_t *scanner = yield->current;
+
+ (void)esdp;
+
+ ASSERT((yield->last == NULL) == (yield->current == NULL));
+
+ if (scanner) {
+ if (scanner->next_op(scanner)) {
+ return 1;
+ }
+
+ ASSERT(ERTS_PROC_IS_EXITING(scanner->process) ||
+ scanner->current_op == blockscan_finish);
+
+ yield->current = scanner->scanner_queue;
+
+ if (yield->current == NULL) {
+ ASSERT(scanner == yield->last);
+ yield->last = NULL;
+ }
+
+ erts_proc_dec_refc(scanner->process);
+
+ /* Plain free is intentional. */
+ free(scanner);
+
+ return yield->current != NULL;
+ }
+
+ return 0;
+}
+
+void erts_alcu_sched_spec_data_init(ErtsSchedulerData *esdp)
+{
+ ErtsAlcuBlockscanYieldData *yield;
+
+ yield = ERTS_SCHED_AUX_YIELD_DATA(esdp, alcu_blockscan);
+
+ yield->current = NULL;
+ yield->last = NULL;
+}
+
+/* ------------------------------------------------------------------------- */
+
+static ERTS_INLINE int u64_log2(Uint64 v)
+{
+ static const int log2_tab64[64] = {
+ 63, 0, 58, 1, 59, 47, 53, 2,
+ 60, 39, 48, 27, 54, 33, 42, 3,
+ 61, 51, 37, 40, 49, 18, 28, 20,
+ 55, 30, 34, 11, 43, 14, 22, 4,
+ 62, 57, 46, 52, 38, 26, 32, 41,
+ 50, 36, 17, 19, 29, 10, 13, 21,
+ 56, 45, 25, 31, 35, 16, 9, 12,
+ 44, 24, 15, 8, 23, 7, 6, 5};
+
+ v |= v >> 1;
+ v |= v >> 2;
+ v |= v >> 4;
+ v |= v >> 8;
+ v |= v >> 16;
+ v |= v >> 32;
+
+ return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58];
+}
+
+/* ------------------------------------------------------------------------- */
+
+typedef struct hist_tree__ {
+ struct hist_tree__ *parent;
+ struct hist_tree__ *left;
+ struct hist_tree__ *right;
+
+ int is_red;
+
+ alcu_atag_t tag;
+ UWord histogram[1];
+} hist_tree_t;
+
+#define ERTS_RBT_PREFIX hist_tree
+#define ERTS_RBT_T hist_tree_t
+#define ERTS_RBT_KEY_T UWord
+#define ERTS_RBT_FLAGS_T int
+#define ERTS_RBT_INIT_EMPTY_TNODE(T) ((void)0)
+#define ERTS_RBT_IS_RED(T) ((T)->is_red)
+#define ERTS_RBT_SET_RED(T) ((T)->is_red = 1)
+#define ERTS_RBT_IS_BLACK(T) (!ERTS_RBT_IS_RED(T))
+#define ERTS_RBT_SET_BLACK(T) ((T)->is_red = 0)
+#define ERTS_RBT_GET_FLAGS(T) ((T)->is_red)
+#define ERTS_RBT_SET_FLAGS(T, F) ((T)->is_red = F)
+#define ERTS_RBT_GET_PARENT(T) ((T)->parent)
+#define ERTS_RBT_SET_PARENT(T, P) ((T)->parent = P)
+#define ERTS_RBT_GET_RIGHT(T) ((T)->right)
+#define ERTS_RBT_SET_RIGHT(T, R) ((T)->right = (R))
+#define ERTS_RBT_GET_LEFT(T) ((T)->left)
+#define ERTS_RBT_SET_LEFT(T, L) ((T)->left = (L))
+#define ERTS_RBT_GET_KEY(T) ((T)->tag)
+#define ERTS_RBT_IS_LT(KX, KY) (KX < KY)
+#define ERTS_RBT_IS_EQ(KX, KY) (KX == KY)
+#define ERTS_RBT_WANT_FOREACH_DESTROY_YIELDING
+#define ERTS_RBT_WANT_FOREACH_DESTROY
+#define ERTS_RBT_WANT_INSERT
+#define ERTS_RBT_WANT_LOOKUP
+#define ERTS_RBT_UNDEF
+
+#include "erl_rbtree.h"
+
+typedef struct {
+ blockscan_t common;
+
+ ErtsIRefStorage iref;
+ Process *process;
+
+ hist_tree_rbt_yield_state_t hist_tree_yield;
+ hist_tree_t *hist_tree;
+ UWord hist_count;
+
+ UWord hist_slot_start;
+ int hist_slot_count;
+
+ UWord unscanned_size;
+
+ ErtsHeapFactory msg_factory;
+ int building_result;
+ Eterm result_list;
+} gather_ahist_t;
+
+static void gather_ahist_update(gather_ahist_t *state, UWord tag, UWord size)
+{
+ hist_tree_t *hist_node;
+ UWord size_interval;
+ int hist_slot;
+
+ hist_node = hist_tree_rbt_lookup(state->hist_tree, tag);
+
+ if (hist_node == NULL) {
+ /* Plain calloc is intentional. */
+ hist_node = (hist_tree_t*)calloc(1, sizeof(hist_tree_t) +
+ (state->hist_slot_count - 1) *
+ sizeof(hist_node->histogram[0]));
+ hist_node->tag = tag;
+
+ hist_tree_rbt_insert(&state->hist_tree, hist_node);
+ state->hist_count++;
+ }
+
+ size_interval = (size / state->hist_slot_start);
+ size_interval = u64_log2(size_interval + 1);
+
+ hist_slot = MIN(size_interval, state->hist_slot_count - 1);
+
+ hist_node->histogram[hist_slot]++;
+}
+
+static int gather_ahist_scan(Allctr_t *allocator,
+ void *user_data,
+ Carrier_t *carrier)
+{
+ gather_ahist_t *state;
+ int blocks_scanned;
+ Block_t *block;
+
+ state = (gather_ahist_t*)user_data;
+ blocks_scanned = 1;
+
+ if (IS_SB_CARRIER(carrier)) {
+ alcu_atag_t tag;
+
+ block = SBC2BLK(allocator, carrier);
+ tag = GET_BLK_ATAG(block);
+
+ ASSERT(DBG_IS_VALID_ATAG(allocator, tag));
+
+ gather_ahist_update(state, tag, SBC_BLK_SZ(block));
+ } else {
+ UWord scanned_bytes = MBC_HEADER_SIZE(allocator);
+
+ ASSERT(IS_MB_CARRIER(carrier));
+
+ block = MBC_TO_FIRST_BLK(allocator, carrier);
+
+ while (1) {
+ UWord block_size = MBC_BLK_SZ(block);
+
+ if (IS_ALLOCED_BLK(block)) {
+ alcu_atag_t tag = GET_BLK_ATAG(block);
+
+ ASSERT(DBG_IS_VALID_ATAG(allocator, tag));
+
+ gather_ahist_update(state, tag, block_size);
+ }
+
+ scanned_bytes += block_size;
+
+ if (blocks_scanned >= BLOCKSCAN_BAILOUT_THRESHOLD) {
+ state->unscanned_size += CARRIER_SZ(carrier) - scanned_bytes;
+ break;
+ } else if (IS_LAST_BLK(block)) {
+ break;
+ }
+
+ block = NXT_BLK(block);
+ blocks_scanned++;
+ }
+ }
+
+ return blocks_scanned;
+}
+
+static void gather_ahist_append_result(hist_tree_t *node, void *arg)
+{
+ gather_ahist_t *state = (gather_ahist_t*)arg;
+
+ Eterm histogram_tuple, tag_tuple;
+
+ Eterm *hp;
+ int ix;
+
+ ASSERT(state->building_result);
+
+ hp = erts_produce_heap(&state->msg_factory, 7 + state->hist_slot_count, 0);
+
+ hp[0] = make_arityval(state->hist_slot_count);
+
+ for (ix = 0; ix < state->hist_slot_count; ix++) {
+ hp[1 + ix] = make_small(node->histogram[ix]);
+ }
+
+ histogram_tuple = make_tuple(hp);
+ hp += 1 + state->hist_slot_count;
+
+ hp[0] = make_arityval(3);
+ hp[1] = ATAG_ID(node->tag);
+ hp[2] = alloc_type_atoms[ATAG_TYPE(node->tag)];
+ hp[3] = histogram_tuple;
+
+ tag_tuple = make_tuple(hp);
+ hp += 4;
+
+ state->result_list = CONS(hp, tag_tuple, state->result_list);
+
+ /* Plain free is intentional. */
+ free(node);
+}
+
+static void gather_ahist_send(gather_ahist_t *state)
+{
+ Eterm result_tuple, unscanned_size, task_ref;
+
+ Uint term_size;
+ Eterm *hp;
+
+ ASSERT((state->result_list == NIL) ^ (state->hist_count > 0));
+ ASSERT(state->building_result);
+
+ term_size = 4 + erts_iref_storage_heap_size(&state->iref);
+ term_size += IS_USMALL(0, state->unscanned_size) ? 0 : BIG_UINT_HEAP_SIZE;
+
+ hp = erts_produce_heap(&state->msg_factory, term_size, 0);
+
+ task_ref = erts_iref_storage_make_ref(&state->iref, &hp,
+ &(state->msg_factory.message)->hfrag.off_heap, 0);
+
+ unscanned_size = bld_unstable_uint(&hp, NULL, state->unscanned_size);
+
+ hp[0] = make_arityval(3);
+ hp[1] = task_ref;
+ hp[2] = unscanned_size;
+ hp[3] = state->result_list;
+
+ result_tuple = make_tuple(hp);
+
+ erts_factory_trim_and_close(&state->msg_factory, &result_tuple, 1);
+
+ erts_queue_message(state->process, 0, state->msg_factory.message,
+ result_tuple, am_system);
+}
+
+static int gather_ahist_finish(void *arg)
+{
+ gather_ahist_t *state = (gather_ahist_t*)arg;
+
+ if (!state->building_result) {
+ ErtsMessage *message;
+ Uint minimum_size;
+ Eterm *hp;
+
+ /* {Ref, unscanned size, [{Tag, {Histogram}} | Rest]} */
+ minimum_size = 4 + erts_iref_storage_heap_size(&state->iref) +
+ state->hist_count * (7 + state->hist_slot_count);
+
+ message = erts_alloc_message(minimum_size, &hp);
+ erts_factory_selfcontained_message_init(&state->msg_factory,
+ message, hp);
+
+ ERTS_RBT_YIELD_STAT_INIT(&state->hist_tree_yield);
+
+ state->result_list = NIL;
+ state->building_result = 1;
+ }
+
+ if (hist_tree_rbt_foreach_destroy_yielding(&state->hist_tree,
+ &gather_ahist_append_result,
+ state,
+ &state->hist_tree_yield,
+ BLOCKSCAN_REDUCTIONS)) {
+ return 1;
+ }
+
+ gather_ahist_send(state);
+
+ return 0;
+}
+
+static void gather_ahist_destroy_result(hist_tree_t *node, void *arg)
+{
+ (void)arg;
+ free(node);
+}
+
+static void gather_ahist_abort(void *arg)
+{
+ gather_ahist_t *state = (gather_ahist_t*)arg;
+
+ if (state->building_result) {
+ erts_factory_undo(&state->msg_factory);
+ }
+
+ hist_tree_rbt_foreach_destroy(&state->hist_tree,
+ &gather_ahist_destroy_result,
+ NULL);
+}
+
+int erts_alcu_gather_alloc_histograms(Process *p, int allocator_num,
+ int sched_id, int hist_width,
+ UWord hist_start, Eterm ref)
+{
+ gather_ahist_t *gather_state;
+ blockscan_t *scanner;
+ Allctr_t *allocator;
+
+ ASSERT(is_internal_ref(ref));
+
+ if (!blockscan_get_specific_allocator(allocator_num,
+ sched_id,
+ &allocator)) {
+ return 0;
+ } else if (!allocator->atags) {
+ return 0;
+ }
+
+ ensure_atoms_initialized(allocator);
+
+ /* Plain calloc is intentional. */
+ gather_state = (gather_ahist_t*)calloc(1, sizeof(gather_ahist_t));
+ scanner = &gather_state->common;
+
+ scanner->abort = gather_ahist_abort;
+ scanner->scan = gather_ahist_scan;
+ scanner->finish = gather_ahist_finish;
+ scanner->user_data = gather_state;
+
+ erts_iref_storage_save(&gather_state->iref, ref);
+ gather_state->hist_slot_start = hist_start;
+ gather_state->hist_slot_count = hist_width;
+ gather_state->process = p;
+
+ blockscan_dispatch(scanner, p, allocator, sched_id);
+
+ return 1;
+}
+
+/* ------------------------------------------------------------------------- */
+
+typedef struct chist_node__ {
+ struct chist_node__ *next;
+
+ UWord carrier_size;
+ UWord unscanned_size;
+ UWord allocated_size;
+
+ /* BLOCKSCAN_BAILOUT_THRESHOLD guarantees we won't overflow this or the
+ * counters in the free block histogram. */
+ int allocated_count;
+ int flags;
+
+ int histogram[1];
+} chist_node_t;
+
+typedef struct {
+ blockscan_t common;
+
+ ErtsIRefStorage iref;
+ Process *process;
+
+ Eterm allocator_desc;
+
+ chist_node_t *info_list;
+ UWord info_count;
+
+ UWord hist_slot_start;
+ int hist_slot_count;
+
+ ErtsHeapFactory msg_factory;
+ int building_result;
+ Eterm result_list;
+} gather_cinfo_t;
+
+static int gather_cinfo_scan(Allctr_t *allocator,
+ void *user_data,
+ Carrier_t *carrier)
+{
+ gather_cinfo_t *state;
+ chist_node_t *node;
+ int blocks_scanned;
+ Block_t *block;
+
+ state = (gather_cinfo_t*)user_data;
+ node = calloc(1, sizeof(chist_node_t) +
+ (state->hist_slot_count - 1) *
+ sizeof(node->histogram[0]));
+ blocks_scanned = 1;
+
+ /* ERTS_CRR_ALCTR_FLG_BUSY is ignored since we've set it ourselves and it
+ * would be misleading to include it. */
+ node->flags = erts_atomic_read_rb(&carrier->allctr) &
+ (ERTS_CRR_ALCTR_FLG_MASK & ~ERTS_CRR_ALCTR_FLG_BUSY);
+ node->carrier_size = CARRIER_SZ(carrier);
+
+ if (IS_SB_CARRIER(carrier)) {
+ UWord block_size;
+
+ block = SBC2BLK(allocator, carrier);
+ block_size = SBC_BLK_SZ(block);
+
+ node->allocated_size = block_size;
+ node->allocated_count = 1;
+ } else {
+ UWord scanned_bytes = MBC_HEADER_SIZE(allocator);
+
+ block = MBC_TO_FIRST_BLK(allocator, carrier);
+
+ while (1) {
+ UWord block_size = MBC_BLK_SZ(block);
+
+ scanned_bytes += block_size;
+
+ if (IS_ALLOCED_BLK(block)) {
+ node->allocated_size += block_size;
+ node->allocated_count++;
+ } else {
+ UWord size_interval;
+ int hist_slot;
+
+ size_interval = (block_size / state->hist_slot_start);
+ size_interval = u64_log2(size_interval + 1);
+
+ hist_slot = MIN(size_interval, state->hist_slot_count - 1);
+
+ node->histogram[hist_slot]++;
+ }
+
+ if (blocks_scanned >= BLOCKSCAN_BAILOUT_THRESHOLD) {
+ node->unscanned_size += CARRIER_SZ(carrier) - scanned_bytes;
+ break;
+ } else if (IS_LAST_BLK(block)) {
+ break;
+ }
+
+ block = NXT_BLK(block);
+ blocks_scanned++;
+ }
+ }
+
+ node->next = state->info_list;
+ state->info_list = node;
+ state->info_count++;
+
+ return blocks_scanned;
+}
+
+static void gather_cinfo_append_result(gather_cinfo_t *state,
+ chist_node_t *info)
+{
+ Eterm carrier_size, unscanned_size, allocated_size;
+ Eterm histogram_tuple, carrier_tuple;
+
+ Uint term_size;
+ Eterm *hp;
+ int ix;
+
+ ASSERT(state->building_result);
+
+ term_size = 11 + state->hist_slot_count;
+ term_size += IS_USMALL(0, info->carrier_size) ? 0 : BIG_UINT_HEAP_SIZE;
+ term_size += IS_USMALL(0, info->unscanned_size) ? 0 : BIG_UINT_HEAP_SIZE;
+ term_size += IS_USMALL(0, info->allocated_size) ? 0 : BIG_UINT_HEAP_SIZE;
+
+ hp = erts_produce_heap(&state->msg_factory, term_size, 0);
+
+ hp[0] = make_arityval(state->hist_slot_count);
+
+ for (ix = 0; ix < state->hist_slot_count; ix++) {
+ hp[1 + ix] = make_small(info->histogram[ix]);
+ }
+
+ histogram_tuple = make_tuple(hp);
+ hp += 1 + state->hist_slot_count;
+
+ carrier_size = bld_unstable_uint(&hp, NULL, info->carrier_size);
+ unscanned_size = bld_unstable_uint(&hp, NULL, info->unscanned_size);
+ allocated_size = bld_unstable_uint(&hp, NULL, info->allocated_size);
+
+ hp[0] = make_arityval(7);
+ hp[1] = state->allocator_desc;
+ hp[2] = carrier_size;
+ hp[3] = unscanned_size;
+ hp[4] = allocated_size;
+ hp[5] = make_small(info->allocated_count);
+ hp[6] = (info->flags & ERTS_CRR_ALCTR_FLG_IN_POOL) ? am_true : am_false;
+ hp[7] = histogram_tuple;
+
+ carrier_tuple = make_tuple(hp);
+ hp += 8;
+
+ state->result_list = CONS(hp, carrier_tuple, state->result_list);
+
+ free(info);
+}
+
+static void gather_cinfo_send(gather_cinfo_t *state)
+{
+ Eterm result_tuple, task_ref;
+
+ int term_size;
+ Eterm *hp;
+
+ ASSERT((state->result_list == NIL) ^ (state->info_count > 0));
+ ASSERT(state->building_result);
+
+ term_size = 3 + erts_iref_storage_heap_size(&state->iref);
+ hp = erts_produce_heap(&state->msg_factory, term_size, 0);
+
+ task_ref = erts_iref_storage_make_ref(&state->iref, &hp,
+ &(state->msg_factory.message)->hfrag.off_heap, 0);
+
+ hp[0] = make_arityval(2);
+ hp[1] = task_ref;
+ hp[2] = state->result_list;
+
+ result_tuple = make_tuple(hp);
+
+ erts_factory_trim_and_close(&state->msg_factory, &result_tuple, 1);
+
+ erts_queue_message(state->process, 0, state->msg_factory.message,
+ result_tuple, am_system);
+}
+
+static int gather_cinfo_finish(void *arg)
+{
+ gather_cinfo_t *state = (gather_cinfo_t*)arg;
+ int reductions = BLOCKSCAN_REDUCTIONS;
+
+ if (!state->building_result) {
+ ErtsMessage *message;
+ Uint minimum_size;
+ Eterm *hp;
+
+ /* {Ref, [{Carrier size, unscanned size, allocated size,
+ * allocated block count, {Free block histogram}} | Rest]} */
+ minimum_size = 3 + erts_iref_storage_heap_size(&state->iref) +
+ state->info_count * (11 + state->hist_slot_count);
+
+ message = erts_alloc_message(minimum_size, &hp);
+ erts_factory_selfcontained_message_init(&state->msg_factory,
+ message, hp);
+
+ state->result_list = NIL;
+ state->building_result = 1;
+ }
+
+ while (state->info_list) {
+ chist_node_t *current = state->info_list;
+ state->info_list = current->next;
+
+ gather_cinfo_append_result(state, current);
+
+ if (reductions-- <= 0) {
+ return 1;
+ }
+ }
+
+ gather_cinfo_send(state);
+
+ return 0;
+}
+
+static void gather_cinfo_abort(void *arg)
+{
+ gather_cinfo_t *state = (gather_cinfo_t*)arg;
+
+ if (state->building_result) {
+ erts_factory_undo(&state->msg_factory);
+ }
+
+ while (state->info_list) {
+ chist_node_t *current = state->info_list;
+ state->info_list = current->next;
+
+ free(current);
+ }
+}
+
+int erts_alcu_gather_carrier_info(struct process *p, int allocator_num,
+ int sched_id, int hist_width,
+ UWord hist_start, Eterm ref)
+{
+ gather_cinfo_t *gather_state;
+ blockscan_t *scanner;
+
+ const char *allocator_desc;
+ Allctr_t *allocator;
+
+ ASSERT(is_internal_ref(ref));
+
+ if (!blockscan_get_specific_allocator(allocator_num,
+ sched_id,
+ &allocator)) {
+ return 0;
+ }
+
+ allocator_desc = ERTS_ALC_A2AD(allocator_num);
+
+ /* Plain calloc is intentional. */
+ gather_state = (gather_cinfo_t*)calloc(1, sizeof(gather_cinfo_t));
+ scanner = &gather_state->common;
+
+ scanner->abort = gather_cinfo_abort;
+ scanner->scan = gather_cinfo_scan;
+ scanner->finish = gather_cinfo_finish;
+ scanner->user_data = gather_state;
+
+ gather_state->allocator_desc = erts_atom_put((byte *)allocator_desc,
+ sys_strlen(allocator_desc),
+ ERTS_ATOM_ENC_LATIN1, 1);
+ erts_iref_storage_save(&gather_state->iref, ref);
+ gather_state->hist_slot_start = hist_start * 2;
+ gather_state->hist_slot_count = hist_width;
+ gather_state->process = p;
+
+ blockscan_dispatch(scanner, p, allocator, sched_id);
+
+ return 1;
+}
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\
* NOTE: erts_alcu_test() is only supposed to be used for testing. *
@@ -6441,6 +7722,7 @@ erts_alcu_verify_unused_ts(Allctr_t *allctr)
erts_mtx_unlock(&allctr->mutex);
}
+
#ifdef DEBUG
int is_sbc_blk(Block_t* blk)
{
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index 05c8a0db3b..ff4d10b206 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -50,6 +50,7 @@ typedef struct {
int tspec;
int tpref;
int ramv;
+ int atags;
UWord sbct;
UWord asbcst;
UWord rsbcst;
@@ -106,6 +107,7 @@ typedef struct {
0, /* (bool) tspec: thread specific */\
0, /* (bool) tpref: thread preferred */\
0, /* (bool) ramv: realloc always moves */\
+ 0, /* (bool) atags: tagged allocations */\
512*1024, /* (bytes) sbct: sbc threshold */\
2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\
20, /* (%) rsbcst: rel sbc shrink threshold */\
@@ -142,6 +144,7 @@ typedef struct {
0, /* (bool) tspec: thread specific */\
0, /* (bool) tpref: thread preferred */\
0, /* (bool) ramv: realloc always moves */\
+ 0, /* (bool) atags: tagged allocations */\
64*1024, /* (bytes) sbct: sbc threshold */\
2*1024*2024, /* (amount) asbcst: abs sbc shrink threshold */\
20, /* (%) rsbcst: rel sbc shrink threshold */\
@@ -224,6 +227,36 @@ void erts_lcnt_update_allocator_locks(int enable);
int erts_alcu_try_set_dyn_param(Allctr_t*, Eterm param, Uint value);
+/* Gathers per-tag allocation histograms from the given allocator number
+ * (ERTS_ALC_A_*) and scheduler id. An id of 0 means the global instance will
+ * be used.
+ *
+ * The results are sent to `p`, and it returns the number of messages to wait
+ * for. */
+int erts_alcu_gather_alloc_histograms(struct process *p, int allocator_num,
+ int sched_id, int hist_width,
+ UWord hist_start, Eterm ref);
+
+/* Gathers per-carrier info from the given allocator number (ERTS_ALC_A_*) and
+ * scheduler id. An id of 0 means the global instance will be used.
+ *
+ * The results are sent to `p`, and it returns the number of messages to wait
+ * for. */
+int erts_alcu_gather_carrier_info(struct process *p, int allocator_num,
+ int sched_id, int hist_width,
+ UWord hist_start, Eterm ref);
+
+struct alcu_blockscan;
+
+typedef struct {
+ struct alcu_blockscan *current;
+ struct alcu_blockscan *last;
+} ErtsAlcuBlockscanYieldData;
+
+int erts_handle_yielded_alcu_blockscan(struct ErtsSchedulerData_ *esdp,
+ ErtsAlcuBlockscanYieldData *yield);
+void erts_alcu_sched_spec_data_init(struct ErtsSchedulerData_ *esdp);
+
#endif /* !ERL_ALLOC_UTIL__ */
#if defined(GET_ERL_ALLOC_UTIL_IMPL) && !defined(ERL_ALLOC_UTIL_IMPL__)
@@ -548,6 +581,7 @@ struct Allctr_t_ {
/* Options */
int t;
int ramv;
+ int atags;
Uint sbc_threshold;
Uint sbc_move_threshold;
Uint mbc_move_threshold;
@@ -684,6 +718,7 @@ struct Allctr_t_ {
#endif
};
+
int erts_alcu_start(Allctr_t *, AllctrInit_t *);
void erts_alcu_stop(Allctr_t *);
diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index d443e12409..8687aefd78 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -38,7 +38,7 @@
#include "erl_message.h"
#include "erl_binary.h"
#include "erl_db.h"
-#include "erl_instrument.h"
+#include "erl_mtrace.h"
#include "dist.h"
#include "erl_gc.h"
#include "erl_cpu_topology.h"
@@ -51,6 +51,7 @@
#include "erl_ptab.h"
#include "erl_time.h"
#include "erl_proc_sig_queue.h"
+#include "erl_alloc_util.h"
#ifdef HIPE
#include "hipe_arch.h"
#endif
@@ -2151,45 +2152,6 @@ info_1_tuple(Process* BIF_P, /* Pointer to current process. */
return make_small(sizeof(UWord));
}
goto badarg;
- } else if (sel == am_allocated) {
- if (arity == 2) {
- Eterm res = THE_NON_VALUE;
- char *buf;
- Sint len = is_string(*tp);
- if (len <= 0)
- return res;
- buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
- if (intlist_to_buf(*tp, buf, len) != len)
- erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
- buf[len] = '\0';
- res = erts_instr_dump_memory_map(buf) ? am_true : am_false;
- erts_free(ERTS_ALC_T_TMP, (void *) buf);
- if (is_non_value(res))
- goto badarg;
- return res;
- }
- else if (arity == 3 && tp[0] == am_status) {
- if (is_atom(tp[1]))
- return erts_instr_get_stat(BIF_P, tp[1], 1);
- else {
- Eterm res = THE_NON_VALUE;
- char *buf;
- Sint len = is_string(tp[1]);
- if (len <= 0)
- return res;
- buf = (char *) erts_alloc(ERTS_ALC_T_TMP, len+1);
- if (intlist_to_buf(tp[1], buf, len) != len)
- erts_exit(ERTS_ERROR_EXIT, "%s:%d: Internal error\n", __FILE__, __LINE__);
- buf[len] = '\0';
- res = erts_instr_dump_stat(buf, 1) ? am_true : am_false;
- erts_free(ERTS_ALC_T_TMP, (void *) buf);
- if (is_non_value(res))
- goto badarg;
- return res;
- }
- }
- else
- goto badarg;
} else if (sel == am_allocator) {
switch (arity) {
case 2:
@@ -2557,8 +2519,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
} else if (BIF_ARG_1 == am_allocated_areas) {
res = erts_allocated_areas(NULL, NULL, BIF_P);
BIF_RET(res);
- } else if (BIF_ARG_1 == am_allocated) {
- BIF_RET(erts_instr_get_memory_map(BIF_P));
} else if (BIF_ARG_1 == am_hipe_architecture) {
#if defined(HIPE)
BIF_RET(hipe_arch_name);
@@ -2699,9 +2659,6 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
sizeof(ERLANG_ARCHITECTURE)-1,
NIL));
}
- else if (BIF_ARG_1 == am_memory_types) {
- return erts_instr_get_type_info(BIF_P);
- }
else if (BIF_ARG_1 == am_os_type) {
BIF_RET(os_type_tuple);
}
@@ -4055,6 +4012,15 @@ BIF_RETTYPE erts_debug_get_internal_state_1(BIF_ALIST_1)
BIF_RET(am_false);
#endif
}
+ else if (ERTS_IS_ATOM_STR("lc_graph", BIF_ARG_1)) {
+#ifdef ERTS_ENABLE_LOCK_CHECK
+ Eterm res = erts_lc_dump_graph();
+ BIF_RET(res);
+#else
+ BIF_RET(am_notsup);
+#endif
+ }
+
}
else if (is_tuple(BIF_ARG_1)) {
Eterm* tp = tuple_val(BIF_ARG_1);
@@ -4728,6 +4694,55 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
BIF_ERROR(BIF_P, BADARG);
}
+static BIF_RETTYPE
+gather_histograms_helper(Process * c_p, Eterm arg_tuple,
+ int gather(Process *, int, int, int, UWord, Eterm))
+{
+ SWord hist_start, hist_width, sched_id;
+ int msg_count, alloc_num;
+ Eterm *args;
+
+ /* This is an internal BIF, so the error checking is mostly left to erlang
+ * code. */
+
+ ASSERT(is_tuple_arity(arg_tuple, 5));
+ args = tuple_val(arg_tuple);
+
+ for (alloc_num = ERTS_ALC_A_MIN; alloc_num <= ERTS_ALC_A_MAX; alloc_num++) {
+ if(erts_is_atom_str(ERTS_ALC_A2AD(alloc_num), args[1], 0)) {
+ break;
+ }
+ }
+
+ if (alloc_num > ERTS_ALC_A_MAX) {
+ BIF_ERROR(c_p, BADARG);
+ }
+
+ sched_id = signed_val(args[2]);
+ hist_width = signed_val(args[3]);
+ hist_start = signed_val(args[4]);
+
+ if (sched_id < 0 || sched_id > erts_no_schedulers) {
+ BIF_ERROR(c_p, BADARG);
+ }
+
+ msg_count = gather(c_p, alloc_num, sched_id, hist_width, hist_start, args[5]);
+
+ BIF_RET(make_small(msg_count));
+}
+
+BIF_RETTYPE erts_internal_gather_alloc_histograms_1(BIF_ALIST_1)
+{
+ return gather_histograms_helper(BIF_P, BIF_ARG_1,
+ erts_alcu_gather_alloc_histograms);
+}
+
+BIF_RETTYPE erts_internal_gather_carrier_info_1(BIF_ALIST_1)
+{
+ return gather_histograms_helper(BIF_P, BIF_ARG_1,
+ erts_alcu_gather_carrier_info);
+}
+
#ifdef ERTS_ENABLE_LOCK_COUNT
typedef struct {
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index a76d769283..f7ee408991 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -283,6 +283,13 @@ make_tid(Process *c_p, DbTable *tb)
return erts_mk_magic_ref(&hp, &c_p->off_heap, tb->common.btid);
}
+Eterm
+erts_db_make_tid(Process *c_p, DbTableCommon *tb)
+{
+ return make_tid(c_p, (DbTable*)tb);
+}
+
+
/*
** The meta hash table of all NAMED ets tables
diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h
index 318e90cb28..eb6da2c9fb 100644
--- a/erts/emulator/beam/erl_db.h
+++ b/erts/emulator/beam/erl_db.h
@@ -128,6 +128,7 @@ extern erts_atomic_t erts_ets_misc_mem_size;
Eterm erts_ets_colliding_names(Process*, Eterm name, Uint cnt);
Uint erts_db_get_max_tabs(void);
+Eterm erts_db_make_tid(Process *c_p, DbTableCommon *tb);
#ifdef ERTS_ENABLE_LOCK_COUNT
void erts_lcnt_enable_db_lock_count(DbTable *tb, int enable);
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 5d49b2ea14..cb5c496e90 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -340,8 +340,8 @@ typedef int (*extra_match_validator_t)(int keypos, Eterm match, Eterm guard, Ete
static struct ext_segtab* alloc_ext_segtab(DbTableHash* tb, unsigned seg_ix);
static void alloc_seg(DbTableHash *tb);
static int free_seg(DbTableHash *tb, int free_records);
-static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
- HashDbTerm *list);
+static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
+ HashDbTerm *list);
static HashDbTerm* search_list(DbTableHash* tb, Eterm key,
HashValue hval, HashDbTerm *list);
static void shrink(DbTableHash* tb, int nitems);
@@ -646,9 +646,9 @@ int db_create_hash(Process *p, DbTable *tbl)
rwmtx_opt.type = ERTS_RWMTX_TYPE_FREQUENT_READ;
if (erts_ets_rwmtx_spin_count >= 0)
rwmtx_opt.main_spincount = erts_ets_rwmtx_spin_count;
- tb->locks = (DbTableHashFineLocks*) erts_db_alloc_fnf(ERTS_ALC_T_DB_SEG, /* Other type maybe? */
- (DbTable *) tb,
- sizeof(DbTableHashFineLocks));
+ tb->locks = (DbTableHashFineLocks*) erts_db_alloc(ERTS_ALC_T_DB_SEG, /* Other type maybe? */
+ (DbTable *) tb,
+ sizeof(DbTableHashFineLocks));
for (i=0; i<DB_HASH_LOCK_CNT; ++i) {
erts_rwmtx_init_opt(&tb->locks->lck_vec[i].lck, &rwmtx_opt,
"db_hash_slot", tb->common.the_name, ERTS_LOCK_FLAGS_CATEGORY_DB);
@@ -672,19 +672,9 @@ static int db_first_hash(Process *p, DbTable *tbl, Eterm *ret)
erts_rwmtx_t* lck = RLOCK_HASH(tb,ix);
HashDbTerm* list;
- for (;;) {
- list = BUCKET(tb,ix);
- if (list != NULL) {
- if (list->hvalue == INVALID_HASH) {
- list = next(tb,&ix,&lck,list);
- }
- break;
- }
- if ((ix=next_slot(tb,ix,&lck)) == 0) {
- list = NULL;
- break;
- }
- }
+ list = BUCKET(tb,ix);
+ list = next_live(tb, &ix, &lck, list);
+
if (list != NULL) {
*ret = db_copy_key(p, tbl, &list->dbterm);
RUNLOCK_HASH(lck);
@@ -721,13 +711,13 @@ static int db_next_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret)
}
/* Key found */
- b = next(tb, &ix, &lck, b);
+ b = next_live(tb, &ix, &lck, b->next);
if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) {
while (b != 0) {
if (!has_live_key(tb, b, key, hval)) {
break;
}
- b = next(tb, &ix, &lck, b);
+ b = next_live(tb, &ix, &lck, b->next);
}
}
if (b == NULL) {
@@ -1463,20 +1453,24 @@ static ERTS_INLINE int on_mtraversal_simple_trap(Export* trap_function,
BUMP_ALL_REDS(p);
if (IS_USMALL(0, got)) {
- hp = HAlloc(p, base_halloc_sz + 5);
+ hp = HAllocX(p, base_halloc_sz + 5, ERTS_MAGIC_REF_THING_SIZE);
egot = make_small(got);
}
else {
- hp = HAlloc(p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 5);
+ hp = HAllocX(p, base_halloc_sz + BIG_UINT_HEAP_SIZE + 5,
+ ERTS_MAGIC_REF_THING_SIZE);
egot = uint_to_big(got, hp);
hp += BIG_UINT_HEAP_SIZE;
}
if (is_first_trap) {
+ if (is_atom(tid))
+ tid = erts_db_make_tid(p, &tb->common);
mpb = erts_db_make_match_prog_ref(p, *mpp, &hp);
*mpp = NULL; /* otherwise the caller will destroy it */
}
else {
+ ASSERT(!is_atom(tid));
mpb = prev_continuation_tptr[3];
}
@@ -1590,11 +1584,17 @@ static int mtraversal_select_chunk_on_loop_ended(void* context_ptr, Sint slot_ix
been in 'user space' */
}
if (rest != NIL || slot_ix >= 0) { /* Need more calls */
- sc_context_ptr->hp = HAlloc(sc_context_ptr->p, 3 + 7 + ERTS_MAGIC_REF_THING_SIZE);
+ Eterm tid = sc_context_ptr->tid;
+ sc_context_ptr->hp = HAllocX(sc_context_ptr->p,
+ 3 + 7 + ERTS_MAGIC_REF_THING_SIZE,
+ ERTS_MAGIC_REF_THING_SIZE);
mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &sc_context_ptr->hp);
+ if (is_atom(tid))
+ tid = erts_db_make_tid(sc_context_ptr->p,
+ &sc_context_ptr->tb->common);
continuation = TUPLE6(
sc_context_ptr->hp,
- sc_context_ptr->tid,
+ tid,
make_small(slot_ix),
make_small(sc_context_ptr->chunk_size),
mpb, rest,
@@ -1631,12 +1631,16 @@ static int mtraversal_select_chunk_on_trap(void* context_ptr, Sint slot_ix, Sint
BUMP_ALL_REDS(sc_context_ptr->p);
if (sc_context_ptr->prev_continuation_tptr == NULL) {
+ Eterm tid = sc_context_ptr->tid;
/* First time we're trapping */
- hp = HAlloc(sc_context_ptr->p, 7 + ERTS_MAGIC_REF_THING_SIZE);
+ hp = HAllocX(sc_context_ptr->p, 7 + ERTS_MAGIC_REF_THING_SIZE,
+ ERTS_MAGIC_REF_THING_SIZE);
+ if (is_atom(tid))
+ tid = erts_db_make_tid(sc_context_ptr->p, &sc_context_ptr->tb->common);
mpb = erts_db_make_match_prog_ref(sc_context_ptr->p, *mpp, &hp);
continuation = TUPLE6(
hp,
- sc_context_ptr->tid,
+ tid,
make_small(slot_ix),
make_small(sc_context_ptr->chunk_size),
mpb,
@@ -2905,14 +2909,14 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key,
/* It return the next live object in a table, NULL if no more */
/* In-bucket: RLOCKED */
/* Out-bucket: RLOCKED unless NULL */
-static HashDbTerm* next(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
- HashDbTerm *list)
+static HashDbTerm* next_live(DbTableHash *tb, Uint *iptr, erts_rwmtx_t** lck_ptr,
+ HashDbTerm *list)
{
int i;
ERTS_LC_ASSERT(IS_HASH_RLOCKED(tb,*iptr));
- for (list = list->next; list != NULL; list = list->next) {
+ for ( ; list != NULL; list = list->next) {
if (list->hvalue != INVALID_HASH)
return list;
}
diff --git a/erts/emulator/beam/erl_init.c b/erts/emulator/beam/erl_init.c
index 179822cc0b..57c6c10c7f 100644
--- a/erts/emulator/beam/erl_init.c
+++ b/erts/emulator/beam/erl_init.c
@@ -37,7 +37,7 @@
#include "erl_mseg.h"
#include "erl_threads.h"
#include "erl_hl_timer.h"
-#include "erl_instrument.h"
+#include "erl_mtrace.h"
#include "erl_printf_term.h"
#include "erl_misc_utils.h"
#include "packet_parser.h"
diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c
deleted file mode 100644
index 2f70e7996e..0000000000
--- a/erts/emulator/beam/erl_instrument.c
+++ /dev/null
@@ -1,1257 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-
-#ifdef HAVE_CONFIG_H
-# include "config.h"
-#endif
-
-#include "global.h"
-#include "big.h"
-#include "erl_instrument.h"
-#include "erl_threads.h"
-
-typedef union { long l; double d; } Align_t;
-
-typedef struct {
- Uint size;
-#ifdef VALGRIND
- void* valgrind_leak_suppressor;
-#endif
- Align_t mem[1];
-} StatBlock_t;
-
-#define STAT_BLOCK_HEADER_SIZE (sizeof(StatBlock_t) - sizeof(Align_t))
-
-typedef struct MapStatBlock_t_ MapStatBlock_t;
-struct MapStatBlock_t_ {
- Uint size;
- ErtsAlcType_t type_no;
- Eterm pid;
- MapStatBlock_t *prev;
- MapStatBlock_t *next;
- Align_t mem[1];
-};
-
-#define MAP_STAT_BLOCK_HEADER_SIZE (sizeof(MapStatBlock_t) - sizeof(Align_t))
-
-typedef struct {
- Uint size;
- Uint max_size;
- Uint max_size_ever;
-
- Uint blocks;
- Uint max_blocks;
- Uint max_blocks_ever;
-} Stat_t;
-
-static erts_mtx_t instr_mutex;
-static erts_mtx_t instr_x_mutex;
-
-int erts_instr_memory_map;
-int erts_instr_stat;
-
-static ErtsAllocatorFunctions_t real_allctrs[ERTS_ALC_A_MAX+1];
-
-struct stats_ {
- Stat_t tot;
- Stat_t a[ERTS_ALC_A_MAX+1];
- Stat_t *ap[ERTS_ALC_A_MAX+1];
- Stat_t c[ERTS_ALC_C_MAX+1];
- Stat_t n[ERTS_ALC_N_MAX+1];
-};
-
-static struct stats_ *stats;
-
-static MapStatBlock_t *mem_anchor;
-
-static Eterm *am_tot;
-static Eterm *am_n;
-static Eterm *am_a;
-static Eterm *am_c;
-
-static int atoms_initialized;
-
-static struct {
- Eterm total;
- Eterm allocators;
- Eterm classes;
- Eterm types;
- Eterm sizes;
- Eterm blocks;
- Eterm instr_hdr;
-#ifdef DEBUG
- Eterm end_of_atoms;
-#endif
-} am;
-
-static void ERTS_INLINE atom_init(Eterm *atom, const char *name)
-{
- *atom = am_atom_put((char *) name, sys_strlen(name));
-}
-#define AM_INIT(AM) atom_init(&am.AM, #AM)
-
-static void
-init_atoms(void)
-{
-#ifdef DEBUG
- Eterm *atom;
- for (atom = (Eterm *) &am; atom <= &am.end_of_atoms; atom++) {
- *atom = THE_NON_VALUE;
- }
-#endif
-
- AM_INIT(total);
- AM_INIT(allocators);
- AM_INIT(classes);
- AM_INIT(types);
- AM_INIT(sizes);
- AM_INIT(blocks);
- AM_INIT(instr_hdr);
-
-#ifdef DEBUG
- for (atom = (Eterm *) &am; atom < &am.end_of_atoms; atom++) {
- ASSERT(*atom != THE_NON_VALUE);
- }
-#endif
-
- atoms_initialized = 1;
-}
-
-#undef AM_INIT
-
-static void
-init_am_tot(void)
-{
- am_tot = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO,
- sizeof(Eterm));
- atom_init(am_tot, "total");
-}
-
-
-static void
-init_am_n(void)
-{
- int i;
- am_n = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO,
- (ERTS_ALC_N_MAX+1)*sizeof(Eterm));
-
- for (i = ERTS_ALC_N_MIN; i <= ERTS_ALC_N_MAX; i++) {
- atom_init(&am_n[i], ERTS_ALC_N2TD(i));
- }
-
-}
-
-static void
-init_am_c(void)
-{
- int i;
- am_c = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO,
- (ERTS_ALC_C_MAX+1)*sizeof(Eterm));
-
- for (i = ERTS_ALC_C_MIN; i <= ERTS_ALC_C_MAX; i++) {
- atom_init(&am_c[i], ERTS_ALC_C2CD(i));
- }
-
-}
-
-static void
-init_am_a(void)
-{
- int i;
- am_a = (Eterm *) erts_alloc(ERTS_ALC_T_INSTR_INFO,
- (ERTS_ALC_A_MAX+1)*sizeof(Eterm));
-
- for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
- atom_init(&am_a[i], ERTS_ALC_A2AD(i));
- }
-
-}
-
-static ERTS_INLINE void
-stat_upd_alloc(ErtsAlcType_t n, Uint size)
-{
- ErtsAlcType_t t = ERTS_ALC_N2T(n);
- ErtsAlcType_t a = ERTS_ALC_T2A(t);
- ErtsAlcType_t c = ERTS_ALC_T2C(t);
-
- stats->ap[a]->size += size;
- if (stats->ap[a]->max_size < stats->ap[a]->size)
- stats->ap[a]->max_size = stats->ap[a]->size;
-
- stats->c[c].size += size;
- if (stats->c[c].max_size < stats->c[c].size)
- stats->c[c].max_size = stats->c[c].size;
-
- stats->n[n].size += size;
- if (stats->n[n].max_size < stats->n[n].size)
- stats->n[n].max_size = stats->n[n].size;
-
- stats->tot.size += size;
- if (stats->tot.max_size < stats->tot.size)
- stats->tot.max_size = stats->tot.size;
-
- stats->ap[a]->blocks++;
- if (stats->ap[a]->max_blocks < stats->ap[a]->blocks)
- stats->ap[a]->max_blocks = stats->ap[a]->blocks;
-
- stats->c[c].blocks++;
- if (stats->c[c].max_blocks < stats->c[c].blocks)
- stats->c[c].max_blocks = stats->c[c].blocks;
-
- stats->n[n].blocks++;
- if (stats->n[n].max_blocks < stats->n[n].blocks)
- stats->n[n].max_blocks = stats->n[n].blocks;
-
- stats->tot.blocks++;
- if (stats->tot.max_blocks < stats->tot.blocks)
- stats->tot.max_blocks = stats->tot.blocks;
-
-}
-
-
-static ERTS_INLINE void
-stat_upd_free(ErtsAlcType_t n, Uint size)
-{
- ErtsAlcType_t t = ERTS_ALC_N2T(n);
- ErtsAlcType_t a = ERTS_ALC_T2A(t);
- ErtsAlcType_t c = ERTS_ALC_T2C(t);
-
- ASSERT(stats->ap[a]->size >= size);
- stats->ap[a]->size -= size;
-
- ASSERT(stats->c[c].size >= size);
- stats->c[c].size -= size;
-
- ASSERT(stats->n[n].size >= size);
- stats->n[n].size -= size;
-
- ASSERT(stats->tot.size >= size);
- stats->tot.size -= size;
-
- ASSERT(stats->ap[a]->blocks > 0);
- stats->ap[a]->blocks--;
-
- ASSERT(stats->c[c].blocks > 0);
- stats->c[c].blocks--;
-
- ASSERT(stats->n[n].blocks > 0);
- stats->n[n].blocks--;
-
- ASSERT(stats->tot.blocks > 0);
- stats->tot.blocks--;
-
-}
-
-
-static ERTS_INLINE void
-stat_upd_realloc(ErtsAlcType_t n, Uint size, Uint old_size)
-{
- if (old_size)
- stat_upd_free(n, old_size);
- stat_upd_alloc(n, size);
-}
-
-/*
- * stat instrumentation callback functions
- */
-
-static void stat_pre_lock(void)
-{
- erts_mtx_lock(&instr_mutex);
-}
-
-static void stat_pre_unlock(void)
-{
- erts_mtx_unlock(&instr_mutex);
-}
-
-static ErtsAllocatorWrapper_t instr_wrapper;
-
-static void *
-stat_alloc(ErtsAlcType_t n, void *extra, Uint size)
-{
- ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
- Uint ssize;
- void *res;
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_lock(&instr_mutex);
- }
-
- ssize = size + STAT_BLOCK_HEADER_SIZE;
- res = (*real_af->alloc)(n, real_af->extra, ssize);
- if (res) {
- stat_upd_alloc(n, size);
- ((StatBlock_t *) res)->size = size;
-#ifdef VALGRIND
- /* Suppress "possibly leaks" by storing an actual dummy pointer
- to the _start_ of the allocated block.*/
- ((StatBlock_t *) res)->valgrind_leak_suppressor = res;
-#endif
- res = (void *) ((StatBlock_t *) res)->mem;
- }
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_unlock(&instr_mutex);
- }
-
- return res;
-}
-
-static void *
-stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size)
-{
- ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
- Uint old_size;
- Uint ssize;
- void *sptr;
- void *res;
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_lock(&instr_mutex);
- }
-
- if (ptr) {
- sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE);
- old_size = ((StatBlock_t *) sptr)->size;
- }
- else {
- sptr = NULL;
- old_size = 0;
- }
-
- ssize = size + STAT_BLOCK_HEADER_SIZE;
- res = (*real_af->realloc)(n, real_af->extra, sptr, ssize);
- if (res) {
- stat_upd_realloc(n, size, old_size);
- ((StatBlock_t *) res)->size = size;
-#ifdef VALGRIND
- ((StatBlock_t *) res)->valgrind_leak_suppressor = res;
-#endif
- res = (void *) ((StatBlock_t *) res)->mem;
- }
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_unlock(&instr_mutex);
- }
-
- return res;
-}
-
-static void
-stat_free(ErtsAlcType_t n, void *extra, void *ptr)
-{
- ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
- void *sptr;
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_lock(&instr_mutex);
- }
-
- if (ptr) {
- sptr = (void *) (((char *) ptr) - STAT_BLOCK_HEADER_SIZE);
- stat_upd_free(n, ((StatBlock_t *) sptr)->size);
- }
- else {
- sptr = NULL;
- }
-
- (*real_af->free)(n, real_af->extra, sptr);
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_unlock(&instr_mutex);
- }
-
-}
-
-/*
- * map stat instrumentation callback functions
- */
-
-static void map_stat_pre_lock(void)
-{
- erts_mtx_lock(&instr_x_mutex);
- erts_mtx_lock(&instr_mutex);
-}
-
-static void map_stat_pre_unlock(void)
-{
- erts_mtx_unlock(&instr_mutex);
- erts_mtx_unlock(&instr_x_mutex);
-}
-
-static void *
-map_stat_alloc(ErtsAlcType_t n, void *extra, Uint size)
-{
- ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
- Uint msize;
- void *res;
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_lock(&instr_mutex);
- }
-
- msize = size + MAP_STAT_BLOCK_HEADER_SIZE;
- res = (*real_af->alloc)(n, real_af->extra, msize);
- if (res) {
- MapStatBlock_t *mb = (MapStatBlock_t *) res;
- stat_upd_alloc(n, size);
-
- mb->size = size;
- mb->type_no = n;
- mb->pid = erts_get_current_pid();
-
- mb->prev = NULL;
- mb->next = mem_anchor;
- if (mem_anchor)
- mem_anchor->prev = mb;
- mem_anchor = mb;
-
- res = (void *) mb->mem;
- }
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_unlock(&instr_mutex);
- }
-
- return res;
-}
-
-static void *
-map_stat_realloc(ErtsAlcType_t n, void *extra, void *ptr, Uint size)
-{
- ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
- Uint old_size;
- Uint msize;
- void *mptr;
- void *res;
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_lock(&instr_x_mutex);
- erts_mtx_lock(&instr_mutex);
- }
-
- if (ptr) {
- mptr = (void *) (((char *) ptr) - MAP_STAT_BLOCK_HEADER_SIZE);
- old_size = ((MapStatBlock_t *) mptr)->size;
- }
- else {
- mptr = NULL;
- old_size = 0;
- }
-
- msize = size + MAP_STAT_BLOCK_HEADER_SIZE;
- res = (*real_af->realloc)(n, real_af->extra, mptr, msize);
- if (res) {
- MapStatBlock_t *mb = (MapStatBlock_t *) res;
-
- mb->size = size;
- mb->type_no = n;
- mb->pid = erts_get_current_pid();
-
- stat_upd_realloc(n, size, old_size);
-
- if (mptr != res) {
-
- if (mptr) {
- if (mb->prev)
- mb->prev->next = mb;
- else {
- ASSERT(mem_anchor == (MapStatBlock_t *) mptr);
- mem_anchor = mb;
- }
- if (mb->next)
- mb->next->prev = mb;
- }
- else {
- mb->prev = NULL;
- mb->next = mem_anchor;
- if (mem_anchor)
- mem_anchor->prev = mb;
- mem_anchor = mb;
- }
-
- }
-
- res = (void *) mb->mem;
- }
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_unlock(&instr_mutex);
- erts_mtx_unlock(&instr_x_mutex);
- }
-
- return res;
-}
-
-static void
-map_stat_free(ErtsAlcType_t n, void *extra, void *ptr)
-{
- ErtsAllocatorFunctions_t *real_af = (ErtsAllocatorFunctions_t *) extra;
- void *mptr;
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_lock(&instr_x_mutex);
- erts_mtx_lock(&instr_mutex);
- }
-
- if (ptr) {
- MapStatBlock_t *mb;
-
- mptr = (void *) (((char *) ptr) - MAP_STAT_BLOCK_HEADER_SIZE);
- mb = (MapStatBlock_t *) mptr;
-
- stat_upd_free(n, mb->size);
-
- if (mb->prev)
- mb->prev->next = mb->next;
- else
- mem_anchor = mb->next;
- if (mb->next)
- mb->next->prev = mb->prev;
- }
- else {
- mptr = NULL;
- }
-
- (*real_af->free)(n, real_af->extra, mptr);
-
- if (!erts_is_allctr_wrapper_prelocked()) {
- erts_mtx_unlock(&instr_mutex);
- erts_mtx_unlock(&instr_x_mutex);
- }
-
-}
-
-static void dump_memory_map_to_stream(fmtfn_t to, void* to_arg)
-{
- ErtsAlcType_t n;
- MapStatBlock_t *bp;
- int lock = !ERTS_IS_CRASH_DUMPING;
- if (lock) {
- ASSERT(!erts_is_allctr_wrapper_prelocked());
- erts_mtx_lock(&instr_mutex);
- }
-
- /* Write header */
-
- erts_cbprintf(to, to_arg,
- "{instr_hdr,\n"
- " %lu,\n"
- " %lu,\n"
- " {",
- (unsigned long) ERTS_INSTR_VSN,
- (unsigned long) MAP_STAT_BLOCK_HEADER_SIZE);
-
-#if ERTS_ALC_N_MIN != 1
-#error ERTS_ALC_N_MIN is not 1
-#endif
-
- for (n = ERTS_ALC_N_MIN; n <= ERTS_ALC_N_MAX; n++) {
- ErtsAlcType_t t = ERTS_ALC_N2T(n);
- ErtsAlcType_t a = ERTS_ALC_T2A(t);
- ErtsAlcType_t c = ERTS_ALC_T2C(t);
- const char *astr;
-
- if (erts_allctrs_info[a].enabled)
- astr = ERTS_ALC_A2AD(a);
- else
- astr = ERTS_ALC_A2AD(ERTS_ALC_A_SYSTEM);
-
- erts_cbprintf(to, to_arg,
- "%s{%s,%s,%s}%s",
- (n == ERTS_ALC_N_MIN) ? "" : " ",
- ERTS_ALC_N2TD(n),
- astr,
- ERTS_ALC_C2CD(c),
- (n == ERTS_ALC_N_MAX) ? "" : ",\n");
- }
-
- erts_cbprintf(to, to_arg, "}}.\n");
-
- /* Write memory data */
- for (bp = mem_anchor; bp; bp = bp->next) {
- if (is_internal_pid(bp->pid))
- erts_cbprintf(to, to_arg,
- "{%lu, %lu, %lu, {%lu,%lu,%lu}}.\n",
- (UWord) bp->type_no,
- (UWord) bp->mem,
- (UWord) bp->size,
- (UWord) pid_channel_no(bp->pid),
- (UWord) pid_number(bp->pid),
- (UWord) pid_serial(bp->pid));
- else
- erts_cbprintf(to, to_arg,
- "{%lu, %lu, %lu, undefined}.\n",
- (UWord) bp->type_no,
- (UWord) bp->mem,
- (UWord) bp->size);
- }
-
- if (lock)
- erts_mtx_unlock(&instr_mutex);
-}
-
-int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg)
-{
- if (!erts_instr_memory_map)
- return 0;
-
- dump_memory_map_to_stream(to, to_arg);
- return 1;
-}
-
-int erts_instr_dump_memory_map(const char *name)
-{
- int fd;
-
- if (!erts_instr_memory_map)
- return 0;
-
- fd = open(name, O_WRONLY | O_CREAT | O_TRUNC, 0640);
- if (fd < 0)
- return 0;
-
- dump_memory_map_to_stream(erts_write_fd, (void*)&fd);
-
- close(fd);
- return 1;
-}
-
-Eterm erts_instr_get_memory_map(Process *proc)
-{
- MapStatBlock_t *org_mem_anchor;
- Eterm hdr_tuple, md_list, res;
- Eterm *hp;
- Uint hsz;
- MapStatBlock_t *bp;
-#ifdef DEBUG
- Eterm *end_hp;
-#endif
-
- if (!erts_instr_memory_map)
- return am_false;
-
- if (!atoms_initialized)
- init_atoms();
- if (!am_n)
- init_am_n();
- if (!am_c)
- init_am_c();
- if (!am_a)
- init_am_a();
-
- erts_mtx_lock(&instr_x_mutex);
- erts_mtx_lock(&instr_mutex);
-
- /* Header size */
- hsz = 5 + 1 + (ERTS_ALC_N_MAX+1-ERTS_ALC_N_MIN)*(1 + 4);
-
- /* Memory data list */
- for (bp = mem_anchor; bp; bp = bp->next) {
- if (is_internal_pid(bp->pid)) {
-#if (_PID_NUM_SIZE - 1 > MAX_SMALL)
- if (internal_pid_number(bp->pid) > MAX_SMALL)
- hsz += BIG_UINT_HEAP_SIZE;
-#endif
-#if (_PID_SER_SIZE - 1 > MAX_SMALL)
- if (internal_pid_serial(bp->pid) > MAX_SMALL)
- hsz += BIG_UINT_HEAP_SIZE;
-#endif
- hsz += 4;
- }
-
- if ((UWord) bp->mem > MAX_SMALL)
- hsz += BIG_UINT_HEAP_SIZE;
- if (bp->size > MAX_SMALL)
- hsz += BIG_UINT_HEAP_SIZE;
-
- hsz += 5 + 2;
- }
-
- hsz += 3; /* Root tuple */
-
- org_mem_anchor = mem_anchor;
- mem_anchor = NULL;
-
- erts_mtx_unlock(&instr_mutex);
-
- hp = HAlloc(proc, hsz); /* May end up calling map_stat_alloc() */
-
- erts_mtx_lock(&instr_mutex);
-
-#ifdef DEBUG
- end_hp = hp + hsz;
-#endif
-
- { /* Build header */
- ErtsAlcType_t n;
- Eterm type_map;
- Uint *hp2 = hp;
-#ifdef DEBUG
- Uint *hp2_end;
-#endif
-
- hp += (ERTS_ALC_N_MAX + 1 - ERTS_ALC_N_MIN)*4;
-
-#ifdef DEBUG
- hp2_end = hp;
-#endif
-
- type_map = make_tuple(hp);
- *(hp++) = make_arityval(ERTS_ALC_N_MAX + 1 - ERTS_ALC_N_MIN);
-
- for (n = ERTS_ALC_N_MIN; n <= ERTS_ALC_N_MAX; n++) {
- ErtsAlcType_t t = ERTS_ALC_N2T(n);
- ErtsAlcType_t a = ERTS_ALC_T2A(t);
- ErtsAlcType_t c = ERTS_ALC_T2C(t);
-
- if (!erts_allctrs_info[a].enabled)
- a = ERTS_ALC_A_SYSTEM;
-
- *(hp++) = TUPLE3(hp2, am_n[n], am_a[a], am_c[c]);
- hp2 += 4;
- }
-
- ASSERT(hp2 == hp2_end);
-
- hdr_tuple = TUPLE4(hp,
- am.instr_hdr,
- make_small(ERTS_INSTR_VSN),
- make_small(MAP_STAT_BLOCK_HEADER_SIZE),
- type_map);
-
- hp += 5;
- }
-
- /* Build memory data list */
-
- for (md_list = NIL, bp = org_mem_anchor; bp; bp = bp->next) {
- Eterm tuple;
- Eterm type;
- Eterm ptr;
- Eterm size;
- Eterm pid;
-
- if (is_not_internal_pid(bp->pid))
- pid = am_undefined;
- else {
- Eterm c;
- Eterm n;
- Eterm s;
-
-#if (ERST_INTERNAL_CHANNEL_NO > MAX_SMALL)
-#error Oversized internal channel number
-#endif
- c = make_small(ERST_INTERNAL_CHANNEL_NO);
-
-#if (_PID_NUM_SIZE - 1 > MAX_SMALL)
- if (internal_pid_number(bp->pid) > MAX_SMALL) {
- n = uint_to_big(internal_pid_number(bp->pid), hp);
- hp += BIG_UINT_HEAP_SIZE;
- }
- else
-#endif
- n = make_small(internal_pid_number(bp->pid));
-
-#if (_PID_SER_SIZE - 1 > MAX_SMALL)
- if (internal_pid_serial(bp->pid) > MAX_SMALL) {
- s = uint_to_big(internal_pid_serial(bp->pid), hp);
- hp += BIG_UINT_HEAP_SIZE;
- }
- else
-#endif
- s = make_small(internal_pid_serial(bp->pid));
- pid = TUPLE3(hp, c, n, s);
- hp += 4;
- }
-
-
-#if ERTS_ALC_N_MAX > MAX_SMALL
-#error Oversized memory type number
-#endif
- type = make_small(bp->type_no);
-
- if ((UWord) bp->mem > MAX_SMALL) {
- ptr = uint_to_big((UWord) bp->mem, hp);
- hp += BIG_UINT_HEAP_SIZE;
- }
- else
- ptr = make_small((UWord) bp->mem);
-
- if (bp->size > MAX_SMALL) {
- size = uint_to_big(bp->size, hp);
- hp += BIG_UINT_HEAP_SIZE;
- }
- else
- size = make_small(bp->size);
-
- tuple = TUPLE4(hp, type, ptr, size, pid);
- hp += 5;
-
- md_list = CONS(hp, tuple, md_list);
- hp += 2;
- }
-
- res = TUPLE2(hp, hdr_tuple, md_list);
-
- ASSERT(hp + 3 == end_hp);
-
- if (mem_anchor) {
- for (bp = mem_anchor; bp->next; bp = bp->next)
- ;
- ASSERT(org_mem_anchor);
- org_mem_anchor->prev = bp;
- bp->next = org_mem_anchor;
- }
- else {
- mem_anchor = org_mem_anchor;
- }
-
- erts_mtx_unlock(&instr_mutex);
- erts_mtx_unlock(&instr_x_mutex);
-
- return res;
-}
-
-static ERTS_INLINE void
-begin_new_max_period(Stat_t *stat, int min, int max)
-{
- int i;
- for (i = min; i <= max; i++) {
- stat[i].max_size = stat[i].size;
- stat[i].max_blocks = stat[i].blocks;
- }
-}
-
-static ERTS_INLINE void
-update_max_ever_values(Stat_t *stat, int min, int max)
-{
- int i;
- for (i = min; i <= max; i++) {
- if (stat[i].max_size_ever < stat[i].max_size)
- stat[i].max_size_ever = stat[i].max_size;
- if (stat[i].max_blocks_ever < stat[i].max_blocks)
- stat[i].max_blocks_ever = stat[i].max_blocks;
- }
-}
-
-#define bld_string erts_bld_string
-#define bld_tuple erts_bld_tuple
-#define bld_tuplev erts_bld_tuplev
-#define bld_list erts_bld_list
-#define bld_2tup_list erts_bld_2tup_list
-#define bld_uint erts_bld_uint
-
-Eterm
-erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period)
-{
- int i, len, max, min, allctr;
- Eterm *names, *values, res;
- Uint arr_size, stat_size, hsz, *hszp, *hp, **hpp;
- Stat_t *stat_src, *stat;
-
- if (!erts_instr_stat)
- return am_false;
-
- if (!atoms_initialized)
- init_atoms();
-
- if (what == am.total) {
- min = 0;
- max = 0;
- allctr = 0;
- stat_size = sizeof(Stat_t);
- stat_src = &stats->tot;
- if (!am_tot)
- init_am_tot();
- names = am_tot;
- }
- else if (what == am.allocators) {
- min = ERTS_ALC_A_MIN;
- max = ERTS_ALC_A_MAX;
- allctr = 1;
- stat_size = sizeof(Stat_t)*(ERTS_ALC_A_MAX+1);
- stat_src = stats->a;
- if (!am_a)
- init_am_a();
- names = am_a;
- }
- else if (what == am.classes) {
- min = ERTS_ALC_C_MIN;
- max = ERTS_ALC_C_MAX;
- allctr = 0;
- stat_size = sizeof(Stat_t)*(ERTS_ALC_C_MAX+1);
- stat_src = stats->c;
- if (!am_c)
- init_am_c();
- names = &am_c[ERTS_ALC_C_MIN];
- }
- else if (what == am.types) {
- min = ERTS_ALC_N_MIN;
- max = ERTS_ALC_N_MAX;
- allctr = 0;
- stat_size = sizeof(Stat_t)*(ERTS_ALC_N_MAX+1);
- stat_src = stats->n;
- if (!am_n)
- init_am_n();
- names = &am_n[ERTS_ALC_N_MIN];
- }
- else {
- return THE_NON_VALUE;
- }
-
- stat = (Stat_t *) erts_alloc(ERTS_ALC_T_TMP, stat_size);
-
- arr_size = (max - min + 1)*sizeof(Eterm);
-
- if (allctr)
- names = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, arr_size);
-
- values = (Eterm *) erts_alloc(ERTS_ALC_T_TMP, arr_size);
-
- erts_mtx_lock(&instr_mutex);
-
- update_max_ever_values(stat_src, min, max);
-
- sys_memcpy((void *) stat, (void *) stat_src, stat_size);
-
- if (begin_max_period)
- begin_new_max_period(stat_src, min, max);
-
- erts_mtx_unlock(&instr_mutex);
-
- hsz = 0;
- hszp = &hsz;
- hpp = NULL;
-
- restart_bld:
-
- len = 0;
- for (i = min; i <= max; i++) {
- if (!allctr || erts_allctrs_info[i].enabled) {
- Eterm s[2];
-
- if (allctr)
- names[len] = am_a[i];
-
- s[0] = bld_tuple(hpp, hszp, 4,
- am.sizes,
- bld_uint(hpp, hszp, stat[i].size),
- bld_uint(hpp, hszp, stat[i].max_size),
- bld_uint(hpp, hszp, stat[i].max_size_ever));
-
- s[1] = bld_tuple(hpp, hszp, 4,
- am.blocks,
- bld_uint(hpp, hszp, stat[i].blocks),
- bld_uint(hpp, hszp, stat[i].max_blocks),
- bld_uint(hpp, hszp, stat[i].max_blocks_ever));
-
- values[len] = bld_list(hpp, hszp, 2, s);
-
- len++;
- }
- }
-
- res = bld_2tup_list(hpp, hszp, len, names, values);
-
- if (!hpp) {
- hp = HAlloc(proc, hsz);
- hszp = NULL;
- hpp = &hp;
- goto restart_bld;
- }
-
- erts_free(ERTS_ALC_T_TMP, (void *) stat);
- erts_free(ERTS_ALC_T_TMP, (void *) values);
- if (allctr)
- erts_free(ERTS_ALC_T_TMP, (void *) names);
-
- return res;
-}
-
-static void
-dump_stat_to_stream(fmtfn_t to, void* to_arg, int begin_max_period)
-{
- ErtsAlcType_t i, a_max, a_min;
-
- erts_mtx_lock(&instr_mutex);
-
- erts_cbprintf(to, to_arg,
- "{instr_vsn,%lu}.\n",
- (unsigned long) ERTS_INSTR_VSN);
-
- update_max_ever_values(&stats->tot, 0, 0);
-
- erts_cbprintf(to, to_arg,
- "{total,[{total,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}]}.\n",
- (UWord) stats->tot.size,
- (UWord) stats->tot.max_size,
- (UWord) stats->tot.max_size_ever,
- (UWord) stats->tot.blocks,
- (UWord) stats->tot.max_blocks,
- (UWord) stats->tot.max_blocks_ever);
-
- a_max = 0;
- a_min = ~0;
- for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
- if (erts_allctrs_info[i].enabled) {
- if (a_min > i)
- a_min = i;
- if (a_max < i)
- a_max = i;
- }
- }
-
- ASSERT(ERTS_ALC_A_MIN <= a_min && a_min <= ERTS_ALC_A_MAX);
- ASSERT(ERTS_ALC_A_MIN <= a_max && a_max <= ERTS_ALC_A_MAX);
- ASSERT(a_min <= a_max);
-
- update_max_ever_values(stats->a, a_min, a_max);
-
- for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
- if (erts_allctrs_info[i].enabled) {
- erts_cbprintf(to, to_arg,
- "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
- i == a_min ? "{allocators,\n [" : " ",
- ERTS_ALC_A2AD(i),
- (UWord) stats->a[i].size,
- (UWord) stats->a[i].max_size,
- (UWord) stats->a[i].max_size_ever,
- (UWord) stats->a[i].blocks,
- (UWord) stats->a[i].max_blocks,
- (UWord) stats->a[i].max_blocks_ever,
- i == a_max ? "]}.\n" : ",\n");
- }
- }
-
- update_max_ever_values(stats->c, ERTS_ALC_C_MIN, ERTS_ALC_C_MAX);
-
- for (i = ERTS_ALC_C_MIN; i <= ERTS_ALC_C_MAX; i++) {
- erts_cbprintf(to, to_arg,
- "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
- i == ERTS_ALC_C_MIN ? "{classes,\n [" : " ",
- ERTS_ALC_C2CD(i),
- (UWord) stats->c[i].size,
- (UWord) stats->c[i].max_size,
- (UWord) stats->c[i].max_size_ever,
- (UWord) stats->c[i].blocks,
- (UWord) stats->c[i].max_blocks,
- (UWord) stats->c[i].max_blocks_ever,
- i == ERTS_ALC_C_MAX ? "]}.\n" : ",\n" );
- }
-
- update_max_ever_values(stats->n, ERTS_ALC_N_MIN, ERTS_ALC_N_MAX);
-
- for (i = ERTS_ALC_N_MIN; i <= ERTS_ALC_N_MAX; i++) {
- erts_cbprintf(to, to_arg,
- "%s{%s,[{sizes,%lu,%lu,%lu},{blocks,%lu,%lu,%lu}]}%s",
- i == ERTS_ALC_N_MIN ? "{types,\n [" : " ",
- ERTS_ALC_N2TD(i),
- (UWord) stats->n[i].size,
- (UWord) stats->n[i].max_size,
- (UWord) stats->n[i].max_size_ever,
- (UWord) stats->n[i].blocks,
- (UWord) stats->n[i].max_blocks,
- (UWord) stats->n[i].max_blocks_ever,
- i == ERTS_ALC_N_MAX ? "]}.\n" : ",\n" );
- }
-
- if (begin_max_period) {
- begin_new_max_period(&stats->tot, 0, 0);
- begin_new_max_period(stats->a, a_min, a_max);
- begin_new_max_period(stats->c, ERTS_ALC_C_MIN, ERTS_ALC_C_MAX);
- begin_new_max_period(stats->n, ERTS_ALC_N_MIN, ERTS_ALC_N_MAX);
- }
-
- erts_mtx_unlock(&instr_mutex);
-
-}
-
-int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period)
-{
- if (!erts_instr_stat)
- return 0;
-
- dump_stat_to_stream(to, to_arg, begin_max_period);
- return 1;
-}
-
-int erts_instr_dump_stat(const char *name, int begin_max_period)
-{
- int fd;
-
- if (!erts_instr_stat)
- return 0;
-
- fd = open(name, O_WRONLY | O_CREAT | O_TRUNC,0640);
- if (fd < 0)
- return 0;
-
- dump_stat_to_stream(erts_write_fd, (void*)&fd, begin_max_period);
-
- close(fd);
- return 1;
-}
-
-
-Uint
-erts_instr_get_total(void)
-{
- return erts_instr_stat ? stats->tot.size : 0;
-}
-
-Uint
-erts_instr_get_max_total(void)
-{
- if (erts_instr_stat) {
- update_max_ever_values(&stats->tot, 0, 0);
- return stats->tot.max_size_ever;
- }
- return 0;
-}
-
-Eterm
-erts_instr_get_type_info(Process *proc)
-{
- Eterm res, *tpls;
- Uint hsz, *hszp, *hp, **hpp;
- ErtsAlcType_t n;
-
- if (!am_n)
- init_am_n();
- if (!am_a)
- init_am_a();
- if (!am_c)
- init_am_c();
-
- tpls = (Eterm *) erts_alloc(ERTS_ALC_T_TMP,
- (ERTS_ALC_N_MAX-ERTS_ALC_N_MIN+1)
- * sizeof(Eterm));
- hsz = 0;
- hszp = &hsz;
- hpp = NULL;
-
- restart_bld:
-
-#if ERTS_ALC_N_MIN != 1
-#error ERTS_ALC_N_MIN is not 1
-#endif
-
- for (n = ERTS_ALC_N_MIN; n <= ERTS_ALC_N_MAX; n++) {
- ErtsAlcType_t t = ERTS_ALC_N2T(n);
- ErtsAlcType_t a = ERTS_ALC_T2A(t);
- ErtsAlcType_t c = ERTS_ALC_T2C(t);
-
- if (!erts_allctrs_info[a].enabled)
- a = ERTS_ALC_A_SYSTEM;
-
- tpls[n - ERTS_ALC_N_MIN]
- = bld_tuple(hpp, hszp, 3, am_n[n], am_a[a], am_c[c]);
- }
-
- res = bld_tuplev(hpp, hszp, ERTS_ALC_N_MAX-ERTS_ALC_N_MIN+1, tpls);
-
- if (!hpp) {
- hp = HAlloc(proc, hsz);
- hszp = NULL;
- hpp = &hp;
- goto restart_bld;
- }
-
- erts_free(ERTS_ALC_T_TMP, tpls);
-
- return res;
-}
-
-Uint
-erts_instr_init(int stat, int map_stat)
-{
- Uint extra_sz;
- int i;
-
- am_tot = NULL;
- am_n = NULL;
- am_c = NULL;
- am_a = NULL;
-
- erts_instr_memory_map = 0;
- erts_instr_stat = 0;
- atoms_initialized = 0;
-
- if (!stat && !map_stat)
- return 0;
-
- stats = erts_alloc(ERTS_ALC_T_INSTR_INFO, sizeof(struct stats_));
-
- erts_mtx_init(&instr_mutex, "instr", NIL,
- ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
-
- mem_anchor = NULL;
-
- /* Install instrumentation functions */
- ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs));
-
- sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs));
-
- sys_memzero((void *) &stats->tot, sizeof(Stat_t));
- sys_memzero((void *) stats->a, sizeof(Stat_t)*(ERTS_ALC_A_MAX+1));
- sys_memzero((void *) stats->c, sizeof(Stat_t)*(ERTS_ALC_C_MAX+1));
- sys_memzero((void *) stats->n, sizeof(Stat_t)*(ERTS_ALC_N_MAX+1));
-
- for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
- if (erts_allctrs_info[i].enabled)
- stats->ap[i] = &stats->a[i];
- else
- stats->ap[i] = &stats->a[ERTS_ALC_A_SYSTEM];
- }
-
- if (map_stat) {
-
- erts_mtx_init(&instr_x_mutex, "instr_x", NIL,
- ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
-
- erts_instr_memory_map = 1;
- erts_instr_stat = 1;
- for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
- erts_allctrs[i].alloc = map_stat_alloc;
- erts_allctrs[i].realloc = map_stat_realloc;
- erts_allctrs[i].free = map_stat_free;
- erts_allctrs[i].extra = (void *) &real_allctrs[i];
- }
- instr_wrapper.lock = map_stat_pre_lock;
- instr_wrapper.unlock = map_stat_pre_unlock;
- extra_sz = MAP_STAT_BLOCK_HEADER_SIZE;
- }
- else {
- erts_instr_stat = 1;
- for (i = ERTS_ALC_A_MIN; i <= ERTS_ALC_A_MAX; i++) {
- erts_allctrs[i].alloc = stat_alloc;
- erts_allctrs[i].realloc = stat_realloc;
- erts_allctrs[i].free = stat_free;
- erts_allctrs[i].extra = (void *) &real_allctrs[i];
- }
- instr_wrapper.lock = stat_pre_lock;
- instr_wrapper.unlock = stat_pre_unlock;
- extra_sz = STAT_BLOCK_HEADER_SIZE;
- }
- erts_allctr_wrapper_prelock_init(&instr_wrapper);
- return extra_sz;
-}
-
diff --git a/erts/emulator/beam/erl_instrument.h b/erts/emulator/beam/erl_instrument.h
deleted file mode 100644
index 351172b2fa..0000000000
--- a/erts/emulator/beam/erl_instrument.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * %CopyrightBegin%
- *
- * Copyright Ericsson AB 2003-2016. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- * %CopyrightEnd%
- */
-
-#ifndef ERL_INSTRUMENT_H__
-#define ERL_INSTRUMENT_H__
-
-#include "erl_mtrace.h"
-
-#define ERTS_INSTR_VSN 2
-
-extern int erts_instr_memory_map;
-extern int erts_instr_stat;
-
-Uint erts_instr_init(int stat, int map_stat);
-int erts_instr_dump_memory_map_to(fmtfn_t to, void* to_arg);
-int erts_instr_dump_memory_map(const char *name);
-Eterm erts_instr_get_memory_map(Process *process);
-int erts_instr_dump_stat_to(fmtfn_t to, void* to_arg, int begin_max_period);
-int erts_instr_dump_stat(const char *name, int begin_max_period);
-Eterm erts_instr_get_stat(Process *proc, Eterm what, int begin_max_period);
-Eterm erts_instr_get_type_info(Process *proc);
-Uint erts_instr_get_total(void);
-Uint erts_instr_get_max_total(void);
-
-#endif
diff --git a/erts/emulator/beam/erl_lock_check.c b/erts/emulator/beam/erl_lock_check.c
index 0ced5ec310..d66410367b 100644
--- a/erts/emulator/beam/erl_lock_check.c
+++ b/erts/emulator/beam/erl_lock_check.c
@@ -41,6 +41,7 @@
#include "erl_lock_check.h"
#include "erl_term.h"
#include "erl_threads.h"
+#include "erl_atom_table.h"
typedef struct {
char *name;
@@ -75,10 +76,10 @@ static erts_lc_lock_order_t erts_lock_order[] = {
* if only one lock use
* the lock name)"
*/
+ { "NO LOCK", NULL },
{ "driver_lock", "driver_name" },
{ "port_lock", "port_id" },
{ "port_data_lock", "address" },
- { "bif_timers", NULL },
{ "reg_tab", NULL },
{ "proc_main", "pid" },
{ "old_code", "address" },
@@ -103,7 +104,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "node_table", NULL },
{ "dist_table", NULL },
{ "sys_tracers", NULL },
- { "module_tab", NULL },
{ "export_tab", NULL },
{ "fun_tab", NULL },
{ "environ", NULL },
@@ -111,7 +111,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "drv_ev_state_grow", NULL, },
{ "drv_ev_state", "address" },
{ "safe_hash", "address" },
- { "removed_fd_pre_alloc_lock", "address" },
{ "state_prealloc", NULL },
{ "schdlr_sspnd", NULL },
{ "migration_info_update", NULL },
@@ -134,10 +133,6 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "msacc_list_mutex", NULL },
{ "msacc_unmanaged_mutex", NULL },
{ "atom_tab", NULL },
- { "misc_op_list_pre_alloc_lock", "address" },
- { "message_pre_alloc_lock", "address" },
- { "ptimer_pre_alloc_lock", "address", },
- { "btm_pre_alloc_lock", NULL, },
{ "dist_entry_out_queue", "address" },
{ "port_sched_lock", "port_id" },
{ "sys_msg_q", NULL },
@@ -147,20 +142,12 @@ static erts_lc_lock_order_t erts_lock_order[] = {
{ "mtrace_op", NULL },
{ "instr_x", NULL },
{ "instr", NULL },
- { "pollsets_lock", NULL },
{ "alcu_allocator", "index" },
{ "mseg", NULL },
- { "port_task_pre_alloc_lock", "address" },
- { "proclist_pre_alloc_lock", "address" },
- { "xports_list_pre_alloc_lock", "address" },
- { "inet_buffer_stack_lock", NULL },
- { "system_block", NULL },
{ "get_time", NULL },
{ "get_corrected_time", NULL },
{ "runtime", NULL },
- { "breakpoints", NULL },
{ "pix_lock", "address" },
- { "run_queues_lists", NULL },
{ "sched_stat", NULL },
{ "async_init_mtx", NULL },
#ifdef __WIN32__
@@ -194,10 +181,10 @@ static const char *rw_op_str(erts_lock_options_t options)
return erts_lock_options_get_short_desc(options);
}
-typedef struct erts_lc_locked_lock_t_ erts_lc_locked_lock_t;
-struct erts_lc_locked_lock_t_ {
- erts_lc_locked_lock_t *next;
- erts_lc_locked_lock_t *prev;
+typedef struct lc_locked_lock_t_ lc_locked_lock_t;
+struct lc_locked_lock_t_ {
+ lc_locked_lock_t *next;
+ lc_locked_lock_t *prev;
UWord extra;
Sint16 id;
char *file;
@@ -207,32 +194,47 @@ struct erts_lc_locked_lock_t_ {
};
typedef struct {
- erts_lc_locked_lock_t *first;
- erts_lc_locked_lock_t *last;
-} erts_lc_locked_lock_list_t;
+ lc_locked_lock_t *first;
+ lc_locked_lock_t *last;
+} lc_locked_lock_list_t;
+
+typedef union lc_free_block_t_ lc_free_block_t;
+union lc_free_block_t_ {
+ lc_free_block_t *next;
+ lc_locked_lock_t lock;
+};
+
+typedef struct {
+ /*
+ * m[X][Y] & 1 if we locked X directly after Y was locked.
+ * m[X][Y] & 2 if we locked X indirectly after Y was locked.
+ * m[X][0] = 1 if we locked X when nothing else was locked.
+ * m[0][] is unused as it would represent locking "NO LOCK"
+ */
+ char m[ERTS_LOCK_ORDER_SIZE][ERTS_LOCK_ORDER_SIZE];
+
+} lc_matrix_t;
-typedef struct erts_lc_locked_locks_t_ erts_lc_locked_locks_t;
-struct erts_lc_locked_locks_t_ {
+static lc_matrix_t tot_lc_matrix;
+
+typedef struct lc_thread_t_ lc_thread_t;
+struct lc_thread_t_ {
char *thread_name;
int emu_thread;
erts_tid_t tid;
- erts_lc_locked_locks_t *next;
- erts_lc_locked_locks_t *prev;
- erts_lc_locked_lock_list_t locked;
- erts_lc_locked_lock_list_t required;
-};
-
-typedef union erts_lc_free_block_t_ erts_lc_free_block_t;
-union erts_lc_free_block_t_ {
- erts_lc_free_block_t *next;
- erts_lc_locked_lock_t lock;
+ lc_thread_t *next;
+ lc_thread_t *prev;
+ lc_locked_lock_list_t locked;
+ lc_locked_lock_list_t required;
+ lc_free_block_t *free_blocks;
+ lc_matrix_t matrix;
};
static ethr_tsd_key locks_key;
-static erts_lc_locked_locks_t *erts_locked_locks = NULL;
+static lc_thread_t *lc_threads = NULL;
+static ethr_spinlock_t lc_threads_lock;
-static erts_lc_free_block_t *free_blocks = NULL;
#ifdef ERTS_LC_STATIC_ALLOC
#define ERTS_LC_FB_CHUNK_SIZE 10000
@@ -240,176 +242,165 @@ static erts_lc_free_block_t *free_blocks = NULL;
#define ERTS_LC_FB_CHUNK_SIZE 10
#endif
-static ethr_spinlock_t free_blocks_lock;
static ERTS_INLINE void
-lc_lock(void)
+lc_lock_threads(void)
{
- ethr_spin_lock(&free_blocks_lock);
+ ethr_spin_lock(&lc_threads_lock);
}
static ERTS_INLINE void
-lc_unlock(void)
+lc_unlock_threads(void)
{
- ethr_spin_unlock(&free_blocks_lock);
+ ethr_spin_unlock(&lc_threads_lock);
}
-static ERTS_INLINE void lc_free(void *p)
+static ERTS_INLINE void lc_free(lc_thread_t* thr, lc_locked_lock_t *p)
{
- erts_lc_free_block_t *fb = (erts_lc_free_block_t *) p;
+ lc_free_block_t *fb = (lc_free_block_t *) p;
#ifdef DEBUG
- sys_memset((void *) p, 0xdf, sizeof(erts_lc_free_block_t));
+ sys_memset((void *) p, 0xdf, sizeof(lc_free_block_t));
#endif
- lc_lock();
- fb->next = free_blocks;
- free_blocks = fb;
- lc_unlock();
+ fb->next = thr->free_blocks;
+ thr->free_blocks = fb;
}
-#ifdef ERTS_LC_STATIC_ALLOC
-
-static void *lc_core_alloc(void)
-{
- lc_unlock();
- ERTS_INTERNAL_ERROR("Lock checker out of memory!\n");
-}
-
-#else
-
-static void *lc_core_alloc(void)
+static lc_locked_lock_t *lc_core_alloc(lc_thread_t* thr)
{
int i;
- erts_lc_free_block_t *fbs;
- lc_unlock();
- fbs = (erts_lc_free_block_t *) malloc(sizeof(erts_lc_free_block_t)
+ lc_free_block_t *fbs;
+ fbs = (lc_free_block_t *) malloc(sizeof(lc_free_block_t)
* ERTS_LC_FB_CHUNK_SIZE);
if (!fbs) {
ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
}
for (i = 1; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
#ifdef DEBUG
- sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t));
+ sys_memset((void *) &fbs[i], 0xdf, sizeof(lc_free_block_t));
#endif
fbs[i].next = &fbs[i+1];
}
#ifdef DEBUG
sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
- 0xdf, sizeof(erts_lc_free_block_t));
+ 0xdf, sizeof(lc_free_block_t));
#endif
- lc_lock();
- fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = free_blocks;
- free_blocks = &fbs[1];
- return (void *) &fbs[0];
+ fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = thr->free_blocks;
+ thr->free_blocks = &fbs[1];
+ return &fbs[0].lock;
}
-#endif
-
-static ERTS_INLINE void *lc_alloc(void)
+static ERTS_INLINE lc_locked_lock_t *lc_alloc(lc_thread_t* thr)
{
- void *res;
- lc_lock();
- if (!free_blocks)
- res = lc_core_alloc();
+ lc_locked_lock_t *res;
+ if (!thr->free_blocks)
+ res = lc_core_alloc(thr);
else {
- res = (void *) free_blocks;
- free_blocks = free_blocks->next;
+ res = &thr->free_blocks->lock;
+ thr->free_blocks = thr->free_blocks->next;
}
- lc_unlock();
return res;
}
-static erts_lc_locked_locks_t *
-create_locked_locks(char *thread_name)
+static lc_thread_t *
+create_thread_data(char *thread_name)
{
- erts_lc_locked_locks_t *l_lcks = malloc(sizeof(erts_lc_locked_locks_t));
- if (!l_lcks)
+ lc_thread_t *thr = malloc(sizeof(lc_thread_t));
+ if (!thr)
ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
- l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
- if (!l_lcks->thread_name)
+ thr->thread_name = strdup(thread_name ? thread_name : "unknown");
+ if (!thr->thread_name)
ERTS_INTERNAL_ERROR("Lock checker failed to allocate memory!");
- l_lcks->emu_thread = 0;
- l_lcks->tid = erts_thr_self();
- l_lcks->required.first = NULL;
- l_lcks->required.last = NULL;
- l_lcks->locked.first = NULL;
- l_lcks->locked.last = NULL;
- l_lcks->prev = NULL;
- lc_lock();
- l_lcks->next = erts_locked_locks;
- if (erts_locked_locks)
- erts_locked_locks->prev = l_lcks;
- erts_locked_locks = l_lcks;
- lc_unlock();
- erts_tsd_set(locks_key, (void *) l_lcks);
- return l_lcks;
+ thr->emu_thread = 0;
+ thr->tid = erts_thr_self();
+ thr->required.first = NULL;
+ thr->required.last = NULL;
+ thr->locked.first = NULL;
+ thr->locked.last = NULL;
+ thr->prev = NULL;
+ thr->free_blocks = NULL;
+ sys_memzero(&thr->matrix, sizeof(thr->matrix));
+
+ lc_lock_threads();
+ thr->next = lc_threads;
+ if (lc_threads)
+ lc_threads->prev = thr;
+ lc_threads = thr;
+ lc_unlock_threads();
+ erts_tsd_set(locks_key, (void *) thr);
+ return thr;
}
+static void collect_matrix(lc_matrix_t*);
+
static void
-destroy_locked_locks(erts_lc_locked_locks_t *l_lcks)
-{
- ASSERT(l_lcks->thread_name);
- free((void *) l_lcks->thread_name);
- ASSERT(l_lcks->required.first == NULL);
- ASSERT(l_lcks->required.last == NULL);
- ASSERT(l_lcks->locked.first == NULL);
- ASSERT(l_lcks->locked.last == NULL);
-
- lc_lock();
- if (l_lcks->prev)
- l_lcks->prev->next = l_lcks->next;
+destroy_locked_locks(lc_thread_t *thr)
+{
+ ASSERT(thr->thread_name);
+ free((void *) thr->thread_name);
+ ASSERT(thr->required.first == NULL);
+ ASSERT(thr->required.last == NULL);
+ ASSERT(thr->locked.first == NULL);
+ ASSERT(thr->locked.last == NULL);
+
+ lc_lock_threads();
+ if (thr->prev)
+ thr->prev->next = thr->next;
else {
- ASSERT(erts_locked_locks == l_lcks);
- erts_locked_locks = l_lcks->next;
+ ASSERT(lc_threads == thr);
+ lc_threads = thr->next;
}
+ if (thr->next)
+ thr->next->prev = thr->prev;
+
+ collect_matrix(&thr->matrix);
- if (l_lcks->next)
- l_lcks->next->prev = l_lcks->prev;
- lc_unlock();
+ lc_unlock_threads();
- free((void *) l_lcks);
+ free((void *) thr);
}
-static ERTS_INLINE erts_lc_locked_locks_t *
+static ERTS_INLINE lc_thread_t *
get_my_locked_locks(void)
{
return erts_tsd_get(locks_key);
}
-static ERTS_INLINE erts_lc_locked_locks_t *
+static ERTS_INLINE lc_thread_t *
make_my_locked_locks(void)
{
- erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
- if (l_lcks)
- return l_lcks;
+ lc_thread_t *thr = get_my_locked_locks();
+ if (thr)
+ return thr;
else
- return create_locked_locks(NULL);
+ return create_thread_data(NULL);
}
-static ERTS_INLINE erts_lc_locked_lock_t *
-new_locked_lock(erts_lc_lock_t *lck, erts_lock_options_t options,
+static ERTS_INLINE lc_locked_lock_t *
+new_locked_lock(lc_thread_t* thr,
+ erts_lc_lock_t *lck, erts_lock_options_t options,
char *file, unsigned int line)
{
- erts_lc_locked_lock_t *l_lck = (erts_lc_locked_lock_t *) lc_alloc();
- l_lck->next = NULL;
- l_lck->prev = NULL;
- l_lck->id = lck->id;
- l_lck->extra = lck->extra;
- l_lck->file = file;
- l_lck->line = line;
- l_lck->flags = lck->flags;
- l_lck->taken_options = options;
- return l_lck;
+ lc_locked_lock_t *ll = lc_alloc(thr);
+ ll->next = NULL;
+ ll->prev = NULL;
+ ll->id = lck->id;
+ ll->extra = lck->extra;
+ ll->file = file;
+ ll->line = line;
+ ll->flags = lck->flags;
+ ll->taken_options = options;
+ return ll;
}
static void
raw_print_lock(char *prefix, Sint16 id, Wterm extra, erts_lock_flags_t flags,
char* file, unsigned int line, char *suffix)
{
- char *lname = (0 <= id && id < ERTS_LOCK_ORDER_SIZE
+ char *lname = (1 <= id && id < ERTS_LOCK_ORDER_SIZE
? erts_lock_order[id].name
: "unknown");
erts_fprintf(stderr,"%s'%s:",prefix,lname);
@@ -439,20 +430,20 @@ print_lock(char *prefix, erts_lc_lock_t *lck, char *suffix)
}
static void
-print_curr_locks(erts_lc_locked_locks_t *l_lcks)
+print_curr_locks(lc_thread_t *thr)
{
- erts_lc_locked_lock_t *l_lck;
- if (!l_lcks || !l_lcks->locked.first)
+ lc_locked_lock_t *ll;
+ if (!thr || !thr->locked.first)
erts_fprintf(stderr,
"Currently no locks are locked by the %s thread.\n",
- l_lcks->thread_name);
+ thr->thread_name);
else {
erts_fprintf(stderr,
"Currently these locks are locked by the %s thread:\n",
- l_lcks->thread_name);
- for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next)
- raw_print_lock(" ", l_lck->id, l_lck->extra, l_lck->flags,
- l_lck->file, l_lck->line, "\n");
+ thr->thread_name);
+ for (ll = thr->locked.first; ll; ll = ll->next)
+ raw_print_lock(" ", ll->id, ll->extra, ll->flags,
+ ll->file, ll->line, "\n");
}
}
@@ -481,55 +472,55 @@ uninitialized_lock(void)
}
static void
-lock_twice(char *prefix, erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
+lock_twice(char *prefix, lc_thread_t *thr, erts_lc_lock_t *lck,
erts_lock_options_t options)
{
erts_fprintf(stderr, "%s (%s)", prefix, rw_op_str(options));
print_lock(" ", lck, " lock which is already locked by thread!\n");
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
lc_abort();
}
static void
-unlock_op_mismatch(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck,
+unlock_op_mismatch(lc_thread_t *thr, erts_lc_lock_t *lck,
erts_lock_options_t options)
{
erts_fprintf(stderr, "Unlocking (%s) ", rw_op_str(options));
print_lock("", lck, " lock which mismatch previous lock operation!\n");
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
lc_abort();
}
static void
-unlock_of_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+unlock_of_not_locked(lc_thread_t *thr, erts_lc_lock_t *lck)
{
print_lock("Unlocking ", lck, " lock which is not locked by thread!\n");
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
lc_abort();
}
static void
-lock_order_violation(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+lock_order_violation(lc_thread_t *thr, erts_lc_lock_t *lck)
{
print_lock("Lock order violation occured when locking ", lck, "!\n");
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
print_lock_order();
lc_abort();
}
static void
-type_order_violation(char *op, erts_lc_locked_locks_t *l_lcks,
+type_order_violation(char *op, lc_thread_t *thr,
erts_lc_lock_t *lck)
{
erts_fprintf(stderr, "Lock type order violation occured when ");
print_lock(op, lck, "!\n");
- ASSERT(l_lcks);
- print_curr_locks(l_lcks);
+ ASSERT(thr);
+ print_curr_locks(thr);
lc_abort();
}
static void
-lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact,
+lock_mismatch(lc_thread_t *thr, int exact,
int failed_have, erts_lc_lock_t *have, int have_len,
int failed_have_not, erts_lc_lock_t *have_not, int have_not_len)
{
@@ -576,39 +567,39 @@ lock_mismatch(erts_lc_locked_locks_t *l_lcks, int exact,
print_lock2(" ", have_not[i].id, have_not[i].extra, 0, "\n");
}
}
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
lc_abort();
}
static void
-unlock_of_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+unlock_of_required_lock(lc_thread_t *thr, erts_lc_lock_t *lck)
{
print_lock("Unlocking required ", lck, " lock!\n");
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
lc_abort();
}
static void
-unrequire_of_not_required_lock(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+unrequire_of_not_required_lock(lc_thread_t *thr, erts_lc_lock_t *lck)
{
print_lock("Unrequire on ", lck, " lock not required!\n");
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
lc_abort();
}
static void
-require_twice(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+require_twice(lc_thread_t *thr, erts_lc_lock_t *lck)
{
print_lock("Require on ", lck, " lock already required!\n");
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
lc_abort();
}
static void
-required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
+required_not_locked(lc_thread_t *thr, erts_lc_lock_t *lck)
{
print_lock("Required ", lck, " lock not locked!\n");
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
lc_abort();
}
@@ -616,15 +607,15 @@ required_not_locked(erts_lc_locked_locks_t *l_lcks, erts_lc_lock_t *lck)
static void
thread_exit_handler(void)
{
- erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
- if (l_lcks) {
- if (l_lcks->locked.first) {
+ lc_thread_t *thr = get_my_locked_locks();
+ if (thr) {
+ if (thr->locked.first) {
erts_fprintf(stderr,
"Thread exiting while having locked locks!\n");
- print_curr_locks(l_lcks);
+ print_curr_locks(thr);
lc_abort();
}
- destroy_locked_locks(l_lcks);
+ destroy_locked_locks(thr);
/* erts_tsd_set(locks_key, NULL); */
}
}
@@ -642,24 +633,24 @@ lc_abort(void)
void
erts_lc_set_thread_name(char *thread_name)
{
- erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
- if (!l_lcks)
- l_lcks = create_locked_locks(thread_name);
+ lc_thread_t *thr = get_my_locked_locks();
+ if (!thr)
+ thr = create_thread_data(thread_name);
else {
- ASSERT(l_lcks->thread_name);
- free((void *) l_lcks->thread_name);
- l_lcks->thread_name = strdup(thread_name ? thread_name : "unknown");
- if (!l_lcks->thread_name)
+ ASSERT(thr->thread_name);
+ free((void *) thr->thread_name);
+ thr->thread_name = strdup(thread_name ? thread_name : "unknown");
+ if (!thr->thread_name)
ERTS_INTERNAL_ERROR("strdup failed");
}
- l_lcks->emu_thread = 1;
+ thr->emu_thread = 1;
}
int
erts_lc_is_emu_thr(void)
{
- erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
- return l_lcks->emu_thread;
+ lc_thread_t *thr = get_my_locked_locks();
+ return thr->emu_thread;
}
int
@@ -705,7 +696,7 @@ erts_lc_get_lock_order_id(char *name)
return (Sint16) -1;
}
-static int compare_locked_by_id(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
+static int compare_locked_by_id(lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
{
if(locked_lock->id < comparand->id) {
return -1;
@@ -716,7 +707,7 @@ static int compare_locked_by_id(erts_lc_locked_lock_t *locked_lock, erts_lc_lock
return 0;
}
-static int compare_locked_by_id_extra(erts_lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
+static int compare_locked_by_id_extra(lc_locked_lock_t *locked_lock, erts_lc_lock_t *comparand)
{
int order = compare_locked_by_id(locked_lock, comparand);
@@ -731,18 +722,18 @@ static int compare_locked_by_id_extra(erts_lc_locked_lock_t *locked_lock, erts_l
return 0;
}
-typedef int (*locked_compare_func)(erts_lc_locked_lock_t *, erts_lc_lock_t *);
+typedef int (*locked_compare_func)(lc_locked_lock_t *, erts_lc_lock_t *);
/* Searches through a list of taken locks, bailing when it hits an entry whose
* order relative to the search template is the opposite of the one at the
* start of the search. (*closest_neighbor) is either set to the exact match,
* or the one closest to it in the sort order. */
static int search_locked_list(locked_compare_func compare,
- erts_lc_locked_lock_t *locked_locks,
+ lc_locked_lock_t *locked_locks,
erts_lc_lock_t *search_template,
- erts_lc_locked_lock_t **closest_neighbor)
+ lc_locked_lock_t **closest_neighbor)
{
- erts_lc_locked_lock_t *iterator = locked_locks;
+ lc_locked_lock_t *iterator = locked_locks;
(*closest_neighbor) = iterator;
@@ -778,9 +769,9 @@ static int search_locked_list(locked_compare_func compare,
/* Searches for a lock in the given list that matches search_template, and sets
* (*locked_locks) to the closest lock in the sort order. */
static int
-find_lock(erts_lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template)
+find_lock(lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template)
{
- erts_lc_locked_lock_t *closest_neighbor;
+ lc_locked_lock_t *closest_neighbor;
int found_lock;
found_lock = search_locked_list(compare_locked_by_id_extra,
@@ -809,9 +800,9 @@ find_lock(erts_lc_locked_lock_t **locked_locks, erts_lc_lock_t *search_template)
/* Searches for a lock in the given list by id, and sets (*locked_locks) to the
* closest lock in the sort order. */
static int
-find_id(erts_lc_locked_lock_t **locked_locks, Sint16 id)
+find_id(lc_locked_lock_t **locked_locks, Sint16 id)
{
- erts_lc_locked_lock_t *closest_neighbor;
+ lc_locked_lock_t *closest_neighbor;
erts_lc_lock_t search_template;
int found_lock;
@@ -830,34 +821,34 @@ find_id(erts_lc_locked_lock_t **locked_locks, Sint16 id)
void
erts_lc_have_locks(int *resv, erts_lc_lock_t *locks, int len)
{
- erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
+ lc_thread_t *thr = get_my_locked_locks();
int i;
- if (!l_lcks) {
+ if (!thr) {
for (i = 0; i < len; i++)
resv[i] = 0;
}
else {
- erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
+ lc_locked_lock_t *ll = thr->locked.first;
for (i = 0; i < len; i++)
- resv[i] = find_lock(&l_lck, &locks[i]);
+ resv[i] = find_lock(&ll, &locks[i]);
}
}
void
erts_lc_have_lock_ids(int *resv, int *ids, int len)
{
- erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
+ lc_thread_t *thr = get_my_locked_locks();
int i;
- if (!l_lcks) {
+ if (!thr) {
for (i = 0; i < len; i++)
resv[i] = 0;
}
else {
- erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
+ lc_locked_lock_t *ll = thr->locked.first;
for (i = 0; i < len; i++)
- resv[i] = find_id(&l_lck, ids[i]);
+ resv[i] = find_id(&ll, ids[i]);
}
}
@@ -866,27 +857,27 @@ erts_lc_check(erts_lc_lock_t *have, int have_len,
erts_lc_lock_t *have_not, int have_not_len)
{
int i;
- erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
- erts_lc_locked_lock_t *l_lck;
+ lc_thread_t *thr = get_my_locked_locks();
+ lc_locked_lock_t *ll;
if (have && have_len > 0) {
- if (!l_lcks)
+ if (!thr)
lock_mismatch(NULL, 0,
-1, have, have_len,
-1, have_not, have_not_len);
- l_lck = l_lcks->locked.first;
+ ll = thr->locked.first;
for (i = 0; i < have_len; i++) {
- if (!find_lock(&l_lck, &have[i]))
- lock_mismatch(l_lcks, 0,
+ if (!find_lock(&ll, &have[i]))
+ lock_mismatch(thr, 0,
i, have, have_len,
-1, have_not, have_not_len);
}
}
- if (have_not && have_not_len > 0 && l_lcks) {
- l_lck = l_lcks->locked.first;
+ if (have_not && have_not_len > 0 && thr) {
+ ll = thr->locked.first;
for (i = 0; i < have_not_len; i++) {
- if (find_lock(&l_lck, &have_not[i]))
- lock_mismatch(l_lcks, 0,
+ if (find_lock(&ll, &have_not[i]))
+ lock_mismatch(thr, 0,
-1, have, have_len,
i, have_not, have_not_len);
}
@@ -896,8 +887,8 @@ erts_lc_check(erts_lc_lock_t *have, int have_len,
void
erts_lc_check_exact(erts_lc_lock_t *have, int have_len)
{
- erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
- if (!l_lcks) {
+ lc_thread_t *thr = get_my_locked_locks();
+ if (!thr) {
if (have && have_len > 0)
lock_mismatch(NULL, 1,
-1, have, have_len,
@@ -905,17 +896,17 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len)
}
else {
int i;
- erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
+ lc_locked_lock_t *ll = thr->locked.first;
for (i = 0; i < have_len; i++) {
- if (!find_lock(&l_lck, &have[i]))
- lock_mismatch(l_lcks, 1,
+ if (!find_lock(&ll, &have[i]))
+ lock_mismatch(thr, 1,
i, have, have_len,
-1, NULL, 0);
}
- for (i = 0, l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next)
+ for (i = 0, ll = thr->locked.first; ll; ll = ll->next)
i++;
if (i != have_len)
- lock_mismatch(l_lcks, 1,
+ lock_mismatch(thr, 1,
-1, have, have_len,
-1, NULL, 0);
}
@@ -924,16 +915,16 @@ erts_lc_check_exact(erts_lc_lock_t *have, int have_len)
void
erts_lc_check_no_locked_of_type(erts_lock_flags_t type)
{
- erts_lc_locked_locks_t *l_lcks = get_my_locked_locks();
- if (l_lcks) {
- erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
- for (l_lck = l_lcks->locked.first; l_lck; l_lck = l_lck->next) {
- if ((l_lck->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == type) {
+ lc_thread_t *thr = get_my_locked_locks();
+ if (thr) {
+ lc_locked_lock_t *ll = thr->locked.first;
+ for (ll = thr->locked.first; ll; ll = ll->next) {
+ if ((ll->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == type) {
erts_fprintf(stderr,
"Locked lock of type %s found which isn't "
"allowed here!\n",
- erts_lock_flags_get_type_name(l_lck->flags));
- print_curr_locks(l_lcks);
+ erts_lock_flags_get_type_name(ll->flags));
+ print_curr_locks(thr);
lc_abort();
}
}
@@ -951,7 +942,7 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
* This in order to make sure that caller can handle
* the situation without causing a lock order violation.
*/
- erts_lc_locked_locks_t *l_lcks;
+ lc_thread_t *thr;
if (lck->inited != ERTS_LC_INITITALIZED)
uninitialized_lock();
@@ -959,25 +950,25 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
if (lck->id < 0)
return 0;
- l_lcks = get_my_locked_locks();
+ thr = get_my_locked_locks();
- if (!l_lcks || !l_lcks->locked.first) {
- ASSERT(!l_lcks || !l_lcks->locked.last);
+ if (!thr || !thr->locked.first) {
+ ASSERT(!thr || !thr->locked.last);
return 0;
}
else {
- erts_lc_locked_lock_t *tl_lck;
+ lc_locked_lock_t *tl_lck;
- ASSERT(l_lcks->locked.last);
+ ASSERT(thr->locked.last);
#if 0 /* Ok when trylocking I guess... */
- if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, l_lcks->locked.last->flags))
- type_order_violation("trylocking ", l_lcks, lck);
+ if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags))
+ type_order_violation("trylocking ", thr, lck);
#endif
- if (l_lcks->locked.last->id < lck->id
- || (l_lcks->locked.last->id == lck->id
- && l_lcks->locked.last->extra < lck->extra))
+ if (thr->locked.last->id < lck->id
+ || (thr->locked.last->id == lck->id
+ && thr->locked.last->extra < lck->extra))
return 0;
/*
@@ -986,11 +977,11 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
/* Check that we are not trying to lock this lock twice */
- for (tl_lck = l_lcks->locked.last; tl_lck; tl_lck = tl_lck->prev) {
+ for (tl_lck = thr->locked.last; tl_lck; tl_lck = tl_lck->prev) {
if (tl_lck->id < lck->id
|| (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) {
if (tl_lck->id == lck->id && tl_lck->extra == lck->extra)
- lock_twice("Trylocking", l_lcks, lck, options);
+ lock_twice("Trylocking", thr, lck, options);
break;
}
}
@@ -1015,8 +1006,8 @@ erts_lc_trylock_force_busy_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t options,
char *file, unsigned int line)
{
- erts_lc_locked_locks_t *l_lcks;
- erts_lc_locked_lock_t *l_lck;
+ lc_thread_t *thr;
+ lc_locked_lock_t *ll;
if (lck->inited != ERTS_LC_INITITALIZED)
uninitialized_lock();
@@ -1024,43 +1015,43 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t
if (lck->id < 0)
return;
- l_lcks = make_my_locked_locks();
- l_lck = locked ? new_locked_lock(lck, options, file, line) : NULL;
+ thr = make_my_locked_locks();
+ ll = locked ? new_locked_lock(thr, lck, options, file, line) : NULL;
- if (!l_lcks->locked.last) {
- ASSERT(!l_lcks->locked.first);
+ if (!thr->locked.last) {
+ ASSERT(!thr->locked.first);
if (locked)
- l_lcks->locked.first = l_lcks->locked.last = l_lck;
+ thr->locked.first = thr->locked.last = ll;
}
else {
- erts_lc_locked_lock_t *tl_lck;
+ lc_locked_lock_t *tl_lck;
#if 0 /* Ok when trylocking I guess... */
- if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, l_lcks->locked.last->flags))
- type_order_violation("trylocking ", l_lcks, lck);
+ if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags))
+ type_order_violation("trylocking ", thr, lck);
#endif
- for (tl_lck = l_lcks->locked.last; tl_lck; tl_lck = tl_lck->prev) {
+ for (tl_lck = thr->locked.last; tl_lck; tl_lck = tl_lck->prev) {
if (tl_lck->id < lck->id
|| (tl_lck->id == lck->id && tl_lck->extra <= lck->extra)) {
if (tl_lck->id == lck->id && tl_lck->extra == lck->extra)
- lock_twice("Trylocking", l_lcks, lck, options);
+ lock_twice("Trylocking", thr, lck, options);
if (locked) {
- l_lck->next = tl_lck->next;
- l_lck->prev = tl_lck;
+ ll->next = tl_lck->next;
+ ll->prev = tl_lck;
if (tl_lck->next)
- tl_lck->next->prev = l_lck;
+ tl_lck->next->prev = ll;
else
- l_lcks->locked.last = l_lck;
- tl_lck->next = l_lck;
+ thr->locked.last = ll;
+ tl_lck->next = ll;
}
return;
}
}
if (locked) {
- l_lck->next = l_lcks->locked.first;
- l_lcks->locked.first->prev = l_lck;
- l_lcks->locked.first = l_lck;
+ ll->next = thr->locked.first;
+ thr->locked.first->prev = ll;
+ thr->locked.first = ll;
}
}
@@ -1069,83 +1060,83 @@ void erts_lc_trylock_flg_x(int locked, erts_lc_lock_t *lck, erts_lock_options_t
void erts_lc_require_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options,
char *file, unsigned int line)
{
- erts_lc_locked_locks_t *l_lcks = make_my_locked_locks();
- erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
- if (!find_lock(&l_lck, lck))
- required_not_locked(l_lcks, lck);
- l_lck = new_locked_lock(lck, options, file, line);
- if (!l_lcks->required.last) {
- ASSERT(!l_lcks->required.first);
- l_lck->next = l_lck->prev = NULL;
- l_lcks->required.first = l_lcks->required.last = l_lck;
+ lc_thread_t *thr = make_my_locked_locks();
+ lc_locked_lock_t *ll = thr->locked.first;
+ if (!find_lock(&ll, lck))
+ required_not_locked(thr, lck);
+ ll = new_locked_lock(thr, lck, options, file, line);
+ if (!thr->required.last) {
+ ASSERT(!thr->required.first);
+ ll->next = ll->prev = NULL;
+ thr->required.first = thr->required.last = ll;
}
else {
- erts_lc_locked_lock_t *l_lck2;
- ASSERT(l_lcks->required.first);
- for (l_lck2 = l_lcks->required.last;
+ lc_locked_lock_t *l_lck2;
+ ASSERT(thr->required.first);
+ for (l_lck2 = thr->required.last;
l_lck2;
l_lck2 = l_lck2->prev) {
if (l_lck2->id < lck->id
|| (l_lck2->id == lck->id && l_lck2->extra < lck->extra))
break;
else if (l_lck2->id == lck->id && l_lck2->extra == lck->extra)
- require_twice(l_lcks, lck);
+ require_twice(thr, lck);
}
if (!l_lck2) {
- l_lck->next = l_lcks->required.first;
- l_lck->prev = NULL;
- l_lcks->required.first->prev = l_lck;
- l_lcks->required.first = l_lck;
+ ll->next = thr->required.first;
+ ll->prev = NULL;
+ thr->required.first->prev = ll;
+ thr->required.first = ll;
}
else {
- l_lck->next = l_lck2->next;
- if (l_lck->next) {
- ASSERT(l_lcks->required.last != l_lck2);
- l_lck->next->prev = l_lck;
+ ll->next = l_lck2->next;
+ if (ll->next) {
+ ASSERT(thr->required.last != l_lck2);
+ ll->next->prev = ll;
}
else {
- ASSERT(l_lcks->required.last == l_lck2);
- l_lcks->required.last = l_lck;
+ ASSERT(thr->required.last == l_lck2);
+ thr->required.last = ll;
}
- l_lck->prev = l_lck2;
- l_lck2->next = l_lck;
+ ll->prev = l_lck2;
+ l_lck2->next = ll;
}
}
}
void erts_lc_unrequire_lock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
{
- erts_lc_locked_locks_t *l_lcks = make_my_locked_locks();
- erts_lc_locked_lock_t *l_lck = l_lcks->locked.first;
- if (!find_lock(&l_lck, lck))
- required_not_locked(l_lcks, lck);
- l_lck = l_lcks->required.first;
- if (!find_lock(&l_lck, lck))
- unrequire_of_not_required_lock(l_lcks, lck);
- if (l_lck->prev) {
- ASSERT(l_lcks->required.first != l_lck);
- l_lck->prev->next = l_lck->next;
+ lc_thread_t *thr = make_my_locked_locks();
+ lc_locked_lock_t *ll = thr->locked.first;
+ if (!find_lock(&ll, lck))
+ required_not_locked(thr, lck);
+ ll = thr->required.first;
+ if (!find_lock(&ll, lck))
+ unrequire_of_not_required_lock(thr, lck);
+ if (ll->prev) {
+ ASSERT(thr->required.first != ll);
+ ll->prev->next = ll->next;
}
else {
- ASSERT(l_lcks->required.first == l_lck);
- l_lcks->required.first = l_lck->next;
+ ASSERT(thr->required.first == ll);
+ thr->required.first = ll->next;
}
- if (l_lck->next) {
- ASSERT(l_lcks->required.last != l_lck);
- l_lck->next->prev = l_lck->prev;
+ if (ll->next) {
+ ASSERT(thr->required.last != ll);
+ ll->next->prev = ll->prev;
}
else {
- ASSERT(l_lcks->required.last == l_lck);
- l_lcks->required.last = l_lck->prev;
+ ASSERT(thr->required.last == ll);
+ thr->required.last = ll->prev;
}
- lc_free((void *) l_lck);
+ lc_free(thr, ll);
}
void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options,
char *file, unsigned int line)
{
- erts_lc_locked_locks_t *l_lcks;
- erts_lc_locked_lock_t *l_lck;
+ lc_thread_t *thr;
+ lc_locked_lock_t *new_ll;
if (lck->inited != ERTS_LC_INITITALIZED)
uninitialized_lock();
@@ -1153,32 +1144,45 @@ void erts_lc_lock_flg_x(erts_lc_lock_t *lck, erts_lock_options_t options,
if (lck->id < 0)
return;
- l_lcks = make_my_locked_locks();
- l_lck = new_locked_lock(lck, options, file, line);
+ thr = make_my_locked_locks();
+ new_ll = new_locked_lock(thr, lck, options, file, line);
- if (!l_lcks->locked.last) {
- ASSERT(!l_lcks->locked.first);
- l_lcks->locked.last = l_lcks->locked.first = l_lck;
+ if (!thr->locked.last) {
+ ASSERT(!thr->locked.first);
+ thr->locked.last = thr->locked.first = new_ll;
+ ASSERT(0 < lck->id && lck->id < ERTS_LOCK_ORDER_SIZE);
+ thr->matrix.m[lck->id][0] = 1;
}
- else if (l_lcks->locked.last->id < lck->id
- || (l_lcks->locked.last->id == lck->id
- && l_lcks->locked.last->extra < lck->extra)) {
- if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, l_lcks->locked.last->flags))
- type_order_violation("locking ", l_lcks, lck);
- l_lck->prev = l_lcks->locked.last;
- l_lcks->locked.last->next = l_lck;
- l_lcks->locked.last = l_lck;
+ else if (thr->locked.last->id < lck->id
+ || (thr->locked.last->id == lck->id
+ && thr->locked.last->extra < lck->extra)) {
+ lc_locked_lock_t* ll;
+ if (LOCK_IS_TYPE_ORDER_VIOLATION(lck->flags, thr->locked.last->flags)) {
+ type_order_violation("locking ", thr, lck);
+ }
+
+ ASSERT(0 < lck->id && lck->id < ERTS_LOCK_ORDER_SIZE);
+ ll = thr->locked.last;
+ thr->matrix.m[lck->id][ll->id] |= 1;
+ for (ll = ll->prev; ll; ll = ll->prev) {
+ ASSERT(0 < ll->id && ll->id < ERTS_LOCK_ORDER_SIZE);
+ thr->matrix.m[lck->id][ll->id] |= 2;
+ }
+
+ new_ll->prev = thr->locked.last;
+ thr->locked.last->next = new_ll;
+ thr->locked.last = new_ll;
}
- else if (l_lcks->locked.last->id == lck->id && l_lcks->locked.last->extra == lck->extra)
- lock_twice("Locking", l_lcks, lck, options);
+ else if (thr->locked.last->id == lck->id && thr->locked.last->extra == lck->extra)
+ lock_twice("Locking", thr, lck, options);
else
- lock_order_violation(l_lcks, lck);
+ lock_order_violation(thr, lck);
}
void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
{
- erts_lc_locked_locks_t *l_lcks;
- erts_lc_locked_lock_t *l_lck;
+ lc_thread_t *thr;
+ lc_locked_lock_t *ll;
if (lck->inited != ERTS_LC_INITITALIZED)
uninitialized_lock();
@@ -1186,38 +1190,38 @@ void erts_lc_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
if (lck->id < 0)
return;
- l_lcks = get_my_locked_locks();
+ thr = get_my_locked_locks();
- if (l_lcks) {
- l_lck = l_lcks->required.first;
- if (find_lock(&l_lck, lck))
- unlock_of_required_lock(l_lcks, lck);
+ if (thr) {
+ ll = thr->required.first;
+ if (find_lock(&ll, lck))
+ unlock_of_required_lock(thr, lck);
}
- for (l_lck = l_lcks ? l_lcks->locked.last : NULL; l_lck; l_lck = l_lck->prev) {
- if (l_lck->id == lck->id && l_lck->extra == lck->extra) {
- if ((l_lck->taken_options & ERTS_LOCK_OPTIONS_RDWR) != options)
- unlock_op_mismatch(l_lcks, lck, options);
- if (l_lck->prev)
- l_lck->prev->next = l_lck->next;
+ for (ll = thr ? thr->locked.last : NULL; ll; ll = ll->prev) {
+ if (ll->id == lck->id && ll->extra == lck->extra) {
+ if ((ll->taken_options & ERTS_LOCK_OPTIONS_RDWR) != options)
+ unlock_op_mismatch(thr, lck, options);
+ if (ll->prev)
+ ll->prev->next = ll->next;
else
- l_lcks->locked.first = l_lck->next;
- if (l_lck->next)
- l_lck->next->prev = l_lck->prev;
+ thr->locked.first = ll->next;
+ if (ll->next)
+ ll->next->prev = ll->prev;
else
- l_lcks->locked.last = l_lck->prev;
- lc_free((void *) l_lck);
+ thr->locked.last = ll->prev;
+ lc_free(thr, ll);
return;
}
}
- unlock_of_not_locked(l_lcks, lck);
+ unlock_of_not_locked(thr, lck);
}
void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
{
- erts_lc_locked_locks_t *l_lcks;
- erts_lc_locked_lock_t *l_lck;
+ lc_thread_t *thr;
+ lc_locked_lock_t *ll;
if (lck->inited != ERTS_LC_INITITALIZED)
uninitialized_lock();
@@ -1225,17 +1229,17 @@ void erts_lc_might_unlock_flg(erts_lc_lock_t *lck, erts_lock_options_t options)
if (lck->id < 0)
return;
- l_lcks = get_my_locked_locks();
+ thr = get_my_locked_locks();
- if (l_lcks) {
- l_lck = l_lcks->required.first;
- if (find_lock(&l_lck, lck))
- unlock_of_required_lock(l_lcks, lck);
+ if (thr) {
+ ll = thr->required.first;
+ if (find_lock(&ll, lck))
+ unlock_of_required_lock(thr, lck);
}
- l_lck = l_lcks->locked.first;
- if (!find_lock(&l_lck, lck))
- unlock_of_not_locked(l_lcks, lck);
+ ll = thr->locked.first;
+ if (!find_lock(&ll, lck))
+ unlock_of_not_locked(thr, lck);
}
int
@@ -1316,26 +1320,7 @@ erts_lc_destroy_lock(erts_lc_lock_t *lck)
void
erts_lc_init(void)
{
-#ifdef ERTS_LC_STATIC_ALLOC
- int i;
- static erts_lc_free_block_t fbs[ERTS_LC_FB_CHUNK_SIZE];
- for (i = 0; i < ERTS_LC_FB_CHUNK_SIZE - 1; i++) {
-#ifdef DEBUG
- sys_memset((void *) &fbs[i], 0xdf, sizeof(erts_lc_free_block_t));
-#endif
- fbs[i].next = &fbs[i+1];
- }
-#ifdef DEBUG
- sys_memset((void *) &fbs[ERTS_LC_FB_CHUNK_SIZE-1],
- 0xdf, sizeof(erts_lc_free_block_t));
-#endif
- fbs[ERTS_LC_FB_CHUNK_SIZE-1].next = NULL;
- free_blocks = &fbs[0];
-#else /* #ifdef ERTS_LC_STATIC_ALLOC */
- free_blocks = NULL;
-#endif /* #ifdef ERTS_LC_STATIC_ALLOC */
-
- if (ethr_spinlock_init(&free_blocks_lock) != 0)
+ if (ethr_spinlock_init(&lc_threads_lock) != 0)
ERTS_INTERNAL_ERROR("spinlock_init failed");
erts_tsd_key_create(&locks_key,"erts_lock_check_key");
@@ -1357,5 +1342,76 @@ erts_lc_pll(void)
print_curr_locks(get_my_locked_locks());
}
+static void collect_matrix(lc_matrix_t* matrix)
+{
+ int i, j;
+ for (i = 1; i < ERTS_LOCK_ORDER_SIZE; i++) {
+ for (j = 0; j <= i; j++) {
+ tot_lc_matrix.m[i][j] |= matrix->m[i][j];
+ }
+#ifdef DEBUG
+ for ( ; j < ERTS_LOCK_ORDER_SIZE; j++) {
+ ASSERT(matrix->m[i][j] == 0);
+ }
+#endif
+ }
+}
+
+Eterm
+erts_lc_dump_graph(void)
+{
+ const char* basename = "lc_graph.";
+ char filename[40];
+ lc_matrix_t* tot = &tot_lc_matrix;
+ lc_thread_t* thr;
+ int i, j, name_max = 0;
+ FILE* ff;
+
+ lc_lock_threads();
+ for (thr = lc_threads; thr; thr = thr->next) {
+ collect_matrix(&thr->matrix);
+ }
+ lc_unlock_threads();
+
+ sys_strcpy(filename, basename);
+ sys_get_pid(filename + strlen(basename),
+ sizeof(filename) - strlen(basename));
+ ff = fopen(filename, "w");
+ if (!ff)
+ return am_error;
+
+ for (i = 1; i < ERTS_LOCK_ORDER_SIZE; i++) {
+ int len = strlen(erts_lock_order[i].name);
+ if (name_max < len)
+ name_max = len;
+ }
+ fputs("%This file was generated by erts_debug:lc_graph()\n\n", ff);
+ fputs("%{ThisLockName, ThisLockId, LockedDirectlyBeforeThis, LockedIndirectlyBeforeThis}\n", ff);
+ fprintf(ff, "[{%*s, %2d}", name_max, "\"NO LOCK\"", 0);
+ for (i = 1; i < ERTS_LOCK_ORDER_SIZE; i++) {
+ char* delim = "";
+ fprintf(ff, ",\n {%*s, %2d, [", name_max, erts_lock_order[i].name, i);
+ for (j = 0; j < ERTS_LOCK_ORDER_SIZE; j++) {
+ if (tot->m[i][j] & 1) {
+ fprintf(ff, "%s%d", delim, j);
+ delim = ",";
+ }
+ }
+ fprintf(ff, "], [");
+ delim = "";
+ for (j = 0; j < ERTS_LOCK_ORDER_SIZE; j++) {
+ if (tot->m[i][j] == 2) {
+ fprintf(ff, "%s%d", delim, j);
+ delim = ",";
+ }
+ }
+ fputs("]}", ff);
+ }
+ fputs("].", ff);
+ fclose(ff);
+ erts_fprintf(stderr, "Created file '%s' in current working directory\n",
+ filename);
+ return am_ok;
+}
#endif /* #ifdef ERTS_ENABLE_LOCK_CHECK */
diff --git a/erts/emulator/beam/erl_lock_check.h b/erts/emulator/beam/erl_lock_check.h
index 5c2c38e8f2..138bc810bd 100644
--- a/erts/emulator/beam/erl_lock_check.h
+++ b/erts/emulator/beam/erl_lock_check.h
@@ -94,6 +94,8 @@ void erts_lc_unrequire_lock(erts_lc_lock_t *lck);
int erts_lc_is_emu_thr(void);
+Eterm erts_lc_dump_graph(void);
+
#define ERTS_LC_ASSERT(A) \
((void) (((A) || ERTS_SOMEONE_IS_CRASH_DUMPING) ? 1 : erts_lc_assert_failed(__FILE__, __LINE__, #A)))
#else /* #ifdef ERTS_ENABLE_LOCK_CHECK */
diff --git a/erts/emulator/beam/erl_nif.c b/erts/emulator/beam/erl_nif.c
index 0e0013e8a4..260656e2d3 100644
--- a/erts/emulator/beam/erl_nif.c
+++ b/erts/emulator/beam/erl_nif.c
@@ -388,12 +388,18 @@ erts_call_dirty_nif(ErtsSchedulerData *esdp, Process *c_p, BeamInstr *I, Eterm *
erts_atomic32_read_band_mb(&c_p->state, ~(ERTS_PSFLG_DIRTY_CPU_PROC
| ERTS_PSFLG_DIRTY_IO_PROC));
+ ASSERT(esdp->current_nif == NULL);
+ esdp->current_nif = &env;
+
erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
result = (*dirty_nif)(&env, codemfa->arity, argv); /* Call dirty NIF */
erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
+ ASSERT(esdp->current_nif == &env);
+ esdp->current_nif = NULL;
+
ASSERT(env.proc->static_flags & ERTS_STC_FLG_SHADOW_PROC);
ASSERT(env.proc->next == c_p);
@@ -4258,6 +4264,10 @@ int erts_nif_get_funcs(struct erl_module_nif* mod,
return mod->entry.num_of_funcs;
}
+Module *erts_nif_get_module(struct erl_module_nif *nif_mod) {
+ return nif_mod->mod;
+}
+
Eterm erts_nif_call_function(Process *p, Process *tracee,
struct erl_module_nif* mod,
ErlNifFunc *fun, int argc, Eterm *argv)
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index 71541786d0..650ec0958c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -36,7 +36,6 @@
#include "erl_db.h"
#include "dist.h"
#include "beam_catches.h"
-#include "erl_instrument.h"
#include "erl_threads.h"
#include "erl_binary.h"
#include "beam_bp.h"
@@ -2508,6 +2507,8 @@ handle_yield(ErtsAuxWorkData *awdp, erts_aint32_t aux_work, int waiting)
yield |= erts_handle_yielded_ets_all_request(awdp->esdp,
&awdp->yield.ets_all);
+ yield |= erts_handle_yielded_alcu_blockscan(awdp->esdp,
+ &awdp->yield.alcu_blockscan);
/*
* Other yielding operations...
@@ -5698,6 +5699,7 @@ init_scheduler_data(ErtsSchedulerData* esdp, int num,
esdp->ssi = ssi;
esdp->current_process = NULL;
esdp->current_port = NULL;
+ esdp->current_nif = NULL;
esdp->virtual_reds = 0;
esdp->cpu_id = -1;
@@ -6093,7 +6095,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
proxy = prev_proxy;
ASSERT(erts_atomic32_read_nob(&proxy->state) & ERTS_PSFLG_PROXY);
erts_atomic32_set_nob(&proxy->state, state);
- (void) erts_set_runq_proc(proc, rq, &bound);
+ (void) erts_set_runq_proc(proxy, rq, &bound);
}
else {
proxy = erts_alloc(ERTS_ALC_T_PROC, sizeof(Process));
@@ -6106,7 +6108,7 @@ make_proxy_proc(Process *prev_proxy, Process *proc, erts_aint32_t prio)
}
#endif
erts_atomic32_init_nob(&proxy->state, state);
- erts_init_runq_proc(proc, rq, bound);
+ erts_init_runq_proc(proxy, rq, bound);
}
proxy->common.id = proc->common.id;
@@ -8333,6 +8335,7 @@ sched_thread_func(void *vesdp)
ERTS_VERIFY_UNUSED_TEMP_ALLOC(NULL);
#endif
+ erts_alcu_sched_spec_data_init(esdp);
erts_ets_sched_spec_data_init(esdp);
process_main(esdp->x_reg_array, esdp->f_reg_array);
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index e232776016..e2aa1d9f84 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -593,6 +593,7 @@ typedef struct {
ErtsDelayedAuxWorkWakeupJob *job;
} delayed_wakeup;
struct {
+ ErtsAlcuBlockscanYieldData alcu_blockscan;
ErtsEtsAllYieldData ets_all;
/* Other yielding operations... */
} yield;
@@ -637,6 +638,7 @@ struct ErtsSchedulerData_ {
ErtsSchedType type;
Uint no; /* Scheduler number for normal schedulers */
Uint dirty_no; /* Scheduler number for dirty schedulers */
+ struct enif_environment_t *current_nif;
Process *dirty_shadow_process;
Port *current_port;
ErtsRunQueue *run_queue;
diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c
index 018894f685..1e833539b3 100644
--- a/erts/emulator/beam/erl_trace.c
+++ b/erts/emulator/beam/erl_trace.c
@@ -2593,7 +2593,7 @@ erts_term_to_tracer(Eterm prefix, Eterm t)
state = tp[3];
}
} else {
- if (arityval(tp[0]) == 2 && is_atom(tp[2])) {
+ if (arityval(tp[0]) == 2 && is_atom(tp[1])) {
module = tp[1];
state = tp[2];
}
diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h
index 256670ff22..2cf268162d 100644
--- a/erts/emulator/beam/global.h
+++ b/erts/emulator/beam/global.h
@@ -123,6 +123,7 @@ void erts_unload_nif(struct erl_module_nif* nif);
extern void erl_nif_init(void);
extern int erts_nif_get_funcs(struct erl_module_nif*,
struct enif_func_t **funcs);
+extern Module *erts_nif_get_module(struct erl_module_nif*);
extern Eterm erts_nif_call_function(Process *p, Process *tracee,
struct erl_module_nif*,
struct enif_func_t *,
@@ -200,6 +201,7 @@ typedef struct {
struct erts_driver_t_ {
erts_driver_t *next;
erts_driver_t *prev;
+ Eterm name_atom;
char *name;
struct {
int major;
diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c
index 66b8005975..2446b3c074 100644
--- a/erts/emulator/beam/io.c
+++ b/erts/emulator/beam/io.c
@@ -956,6 +956,9 @@ try_imm_drv_call(ErtsTryImmDrvCallState *sp)
reds_left_in = ERTS_BIF_REDS_LEFT(c_p);
erts_proc_unlock(c_p, ERTS_PROC_LOCK_MAIN);
+
+ ASSERT((c_p->scheduler_data)->current_port == NULL);
+ (c_p->scheduler_data)->current_port = prt;
}
ASSERT(0 <= reds_left_in && reds_left_in <= CONTEXT_REDS);
@@ -1017,6 +1020,9 @@ finalize_imm_drv_call(ErtsTryImmDrvCallState *sp)
erts_port_release(prt);
if (c_p) {
+ ASSERT((c_p->scheduler_data)->current_port == prt);
+ (c_p->scheduler_data)->current_port = NULL;
+
erts_proc_lock(c_p, ERTS_PROC_LOCK_MAIN);
if (reds != (CONTEXT_REDS - sp->reds_left_in)) {
@@ -2906,11 +2912,8 @@ static void lcnt_enable_driver_lock_count(erts_driver_t *dp, int enable)
{
if (dp->lock) {
if (enable) {
- Eterm name_as_atom = erts_atom_put((byte*)dp->name, sys_strlen(dp->name),
- ERTS_ATOM_ENC_LATIN1, 1);
-
- erts_lcnt_install_new_lock_info(&dp->lock->lcnt, "driver_lock", name_as_atom,
- ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
+ erts_lcnt_install_new_lock_info(&dp->lock->lcnt, "driver_lock",
+ dp->name_atom, ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
} else {
erts_lcnt_uninstall(&dp->lock->lcnt);
}
@@ -7357,7 +7360,11 @@ no_stop_select_callback(ErlDrvEvent event, void* private)
static int
init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
{
+ drv->name_atom = erts_atom_put((byte*)de->driver_name,
+ sys_strlen(de->driver_name),
+ ERTS_ATOM_ENC_LATIN1, 1);
drv->name = de->driver_name;
+
ASSERT(de->extended_marker == ERL_DRV_EXTENDED_MARKER);
ASSERT(de->major_version >= 2);
drv->version.major = de->major_version;
@@ -7367,13 +7374,10 @@ init_driver(erts_driver_t *drv, ErlDrvEntry *de, DE_Handle *handle)
if (drv->flags & ERL_DRV_FLAG_USE_PORT_LOCKING) {
drv->lock = NULL;
} else {
- Eterm driver_id = erts_atom_put((byte *) drv->name,
- sys_strlen(drv->name),
- ERTS_ATOM_ENC_LATIN1, 1);
-
drv->lock = erts_alloc(ERTS_ALC_T_DRIVER_LOCK, sizeof(erts_mtx_t));
- erts_mtx_init(drv->lock, "driver_lock", driver_id, ERTS_LOCK_FLAGS_CATEGORY_IO);
+ erts_mtx_init(drv->lock, "driver_lock", drv->name_atom,
+ ERTS_LOCK_FLAGS_CATEGORY_IO);
}
drv->entry = de;
@@ -7459,18 +7463,12 @@ int erts_add_driver_entry(ErlDrvEntry *de, DE_Handle *handle,
erts_tsd_set(driver_list_lock_status_key, (void *) 1);
}
- if (taint) {
- Eterm name_atom = erts_atom_put((byte*)de->driver_name,
- sys_strlen(de->driver_name),
- ERTS_ATOM_ENC_LATIN1, 0);
- if (is_atom(name_atom))
- erts_add_taint(name_atom);
- else
- err = 1;
- }
-
if (!err) {
err = init_driver(dp, de, handle);
+
+ if (taint) {
+ erts_add_taint(dp->name_atom);
+ }
}
if (err) {
diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c
index 188e02eff8..2e22130524 100644
--- a/erts/emulator/beam/utils.c
+++ b/erts/emulator/beam/utils.c
@@ -140,7 +140,7 @@ Eterm*
erts_set_hole_marker(Eterm* ptr, Uint sz)
{
Eterm* p = ptr;
- int i;
+ Uint i;
for (i = 0; i < sz; i++) {
*p++ = ERTS_HOLE_MARKER;
@@ -1961,7 +1961,7 @@ static void do_send_logger_message(Eterm *hp, ErlOffHeap *ohp, ErlHeapFragment *
{notify,{info_msg,gleader,{emulator,format,[args]}}} |
{notify,{error,gleader,{emulator,format,[args]}}} |
{notify,{warning_msg,gleader,{emulator,format,[args}]}} */
-static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
+static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, size_t len)
{
Uint sz;
Eterm gl;
@@ -1974,7 +1974,7 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
ASSERT(is_atom(tag));
- if (len <= 0) {
+ if (len == 0) {
return -1;
}
@@ -2007,7 +2007,7 @@ static int do_send_to_logger(Eterm tag, Eterm gleader, char *buf, int len)
}
static int do_send_term_to_logger(Eterm tag, Eterm gleader,
- char *buf, int len, Eterm args)
+ char *buf, size_t len, Eterm args)
{
Uint sz;
Eterm gl;
@@ -2048,13 +2048,13 @@ static int do_send_term_to_logger(Eterm tag, Eterm gleader,
}
static ERTS_INLINE int
-send_info_to_logger(Eterm gleader, char *buf, int len)
+send_info_to_logger(Eterm gleader, char *buf, size_t len)
{
return do_send_to_logger(am_info_msg, gleader, buf, len);
}
static ERTS_INLINE int
-send_warning_to_logger(Eterm gleader, char *buf, int len)
+send_warning_to_logger(Eterm gleader, char *buf, size_t len)
{
Eterm tag;
switch (erts_error_logger_warnings) {
@@ -2066,13 +2066,13 @@ send_warning_to_logger(Eterm gleader, char *buf, int len)
}
static ERTS_INLINE int
-send_error_to_logger(Eterm gleader, char *buf, int len)
+send_error_to_logger(Eterm gleader, char *buf, size_t len)
{
return do_send_to_logger(am_error, gleader, buf, len);
}
static ERTS_INLINE int
-send_error_term_to_logger(Eterm gleader, char *buf, int len, Eterm args)
+send_error_term_to_logger(Eterm gleader, char *buf, size_t len, Eterm args)
{
return do_send_term_to_logger(am_error, gleader, buf, len, args);
}
diff --git a/erts/emulator/sys/common/erl_sys_common_misc.c b/erts/emulator/sys/common/erl_sys_common_misc.c
index 826307c077..41a6fcb7e1 100644
--- a/erts/emulator/sys/common/erl_sys_common_misc.c
+++ b/erts/emulator/sys/common/erl_sys_common_misc.c
@@ -217,14 +217,16 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
}
do {
+ Uint64 n;
if (!frac_part) {
do {
*p++ = '0';
} while (--decimals);
break;
}
- *p++ = (char)((frac_part % 10) + '0');
- frac_part /= 10;
+ n = frac_part / 10;
+ *p++ = (char)((frac_part - n*10) + '0');
+ frac_part = n;
} while (--decimals);
*p++ = '.';
@@ -236,9 +238,10 @@ sys_double_to_chars_fast(double f, char *buffer, int buffer_size, int decimals,
*p++ = '0';
} else {
do {
- *p++ = (char)((int_part % 10) + '0');
- int_part /= 10;
- }while (int_part);
+ Uint64 n = int_part / 10;
+ *p++ = (char)((int_part - n*10) + '0');
+ int_part = n;
+ } while (int_part);
}
if (neg)
*p++ = '-';
diff --git a/erts/emulator/test/exception_SUITE.erl b/erts/emulator/test/exception_SUITE.erl
index da0292f385..60d14ce841 100644
--- a/erts/emulator/test/exception_SUITE.erl
+++ b/erts/emulator/test/exception_SUITE.erl
@@ -23,7 +23,8 @@
-export([all/0, suite/0,
badmatch/1, pending_errors/1, nil_arith/1, top_of_stacktrace/1,
stacktrace/1, nested_stacktrace/1, raise/1, gunilla/1, per/1,
- exception_with_heap_frag/1, line_numbers/1]).
+ exception_with_heap_frag/1, backtrace_depth/1,
+ line_numbers/1]).
-export([bad_guy/2]).
-export([crash/1]).
@@ -42,7 +43,7 @@ suite() ->
all() ->
[badmatch, pending_errors, nil_arith, top_of_stacktrace,
stacktrace, nested_stacktrace, raise, gunilla, per,
- exception_with_heap_frag, line_numbers].
+ exception_with_heap_frag, backtrace_depth, line_numbers].
-define(try_match(E),
catch ?MODULE:bar(),
@@ -572,6 +573,57 @@ do_exception_with_heap_frag(Bin, [Sz|Sizes]) ->
do_exception_with_heap_frag(Bin, Sizes);
do_exception_with_heap_frag(_, []) -> ok.
+backtrace_depth(Config) when is_list(Config) ->
+ _ = [do_backtrace_depth(D) || D <- lists:seq(0, 8)],
+ ok.
+
+do_backtrace_depth(D) ->
+ Old = erlang:system_flag(backtrace_depth, D),
+ try
+ Expected = max(1, D),
+ do_backtrace_depth_1(Expected)
+ after
+ _ = erlang:system_flag(backtrace_depth, Old)
+ end.
+
+do_backtrace_depth_1(D) ->
+ Exit = fun() ->
+ error(reason)
+ end,
+ HandCrafted = fun() ->
+ {'EXIT',{_,Stk0}} = (catch error(get_stacktrace)),
+ %% Fool the compiler to force a hand-crafted
+ %% stacktrace.
+ Stk = [hd(Stk0)|tl(Stk0)],
+ erlang:raise(error, reason, Stk)
+ end,
+ PassedOn = fun() ->
+ try error(get_stacktrace)
+ catch error:_:Stk ->
+ %% Just pass on the given stacktrace.
+ erlang:raise(error, reason, Stk)
+ end
+ end,
+ do_backtrace_depth_2(D, Exit),
+ do_backtrace_depth_2(D, HandCrafted),
+ do_backtrace_depth_2(D, PassedOn),
+ ok.
+
+do_backtrace_depth_2(D, Exc) ->
+ try
+ Exc()
+ catch
+ error:reason:Stk ->
+ if
+ length(Stk) =/= D ->
+ io:format("Expected depth: ~p\n", [D]),
+ io:format("~p\n", [Stk]),
+ error(bad_depth);
+ true ->
+ ok
+ end
+ end.
+
line_numbers(Config) when is_list(Config) ->
{'EXIT',{{case_clause,bad_tag},
[{?MODULE,line1,2,
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index 46ece531a8..585c5a1871 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -2217,36 +2217,44 @@ handle_event(Event, Pid) ->
processes_term_proc_list(Config) when is_list(Config) ->
Tester = self(),
- as_expected = processes_term_proc_list_test(false),
- {ok, Node} = start_node(Config, "+Mis true"),
- RT = spawn_link(Node, fun () ->
- receive after 1000 -> ok end,
- processes_term_proc_list_test(false),
- Tester ! {it_worked, self()}
- end),
- receive {it_worked, RT} -> ok end,
- stop_node(Node),
+
+ Run = fun(Args) ->
+ {ok, Node} = start_node(Config, Args),
+ RT = spawn_link(Node, fun () ->
+ receive after 1000 -> ok end,
+ as_expected = processes_term_proc_list_test(false),
+ Tester ! {it_worked, self()}
+ end),
+ receive {it_worked, RT} -> ok end,
+ stop_node(Node)
+ end,
+
+ %% We have to run this test case with +S1 since instrument:allocations()
+ %% will report a free()'d block as present until it's actually deallocated
+ %% by its employer.
+ Run("+MSe true +MSatags false +S1"),
+ Run("+MSe true +MSatags true +S1"),
+
ok.
-
+
-define(CHK_TERM_PROC_LIST(MC, XB),
chk_term_proc_list(?LINE, MC, XB)).
chk_term_proc_list(Line, MustChk, ExpectBlks) ->
- case {MustChk, instrument:memory_status(types)} of
- {false, false} ->
+ Allocs = instrument:allocations(#{ allocator_types => [sl_alloc] }),
+ case {MustChk, Allocs} of
+ {false, {error, not_enabled}} ->
not_enabled;
- {_, MS} ->
- {value,
- {ptab_list_deleted_el,
- DL}} = lists:keysearch(ptab_list_deleted_el, 1, MS),
- case lists:keysearch(blocks, 1, DL) of
- {value, {blocks, ExpectBlks, _, _}} ->
- ok;
- {value, {blocks, Blks, _, _}} ->
- exit({line, Line,
- mismatch, expected, ExpectBlks, actual, Blks});
- Unexpected ->
- exit(Unexpected)
+ {_, {ok, {_Shift, _Unscanned, ByOrigin}}} ->
+ ByType = maps:get(system, ByOrigin, #{}),
+ Hist = maps:get(ptab_list_deleted_el, ByType, {}),
+ case lists:sum(tuple_to_list(Hist)) of
+ ExpectBlks ->
+ ok;
+ Blks ->
+ exit({line, Line, mismatch,
+ expected, ExpectBlks,
+ actual, Blks})
end
end,
ok.
diff --git a/erts/emulator/test/tracer_SUITE.erl b/erts/emulator/test/tracer_SUITE.erl
index ab7d047bc3..e1362ef07a 100644
--- a/erts/emulator/test/tracer_SUITE.erl
+++ b/erts/emulator/test/tracer_SUITE.erl
@@ -30,7 +30,8 @@
-export([load/1, unload/1, reload/1, invalid_tracers/1]).
-export([send/1, recv/1, call/1, call_return/1, spawn/1, exit/1,
link/1, unlink/1, getting_linked/1, getting_unlinked/1,
- register/1, unregister/1, in/1, out/1, gc_start/1, gc_end/1]).
+ register/1, unregister/1, in/1, out/1, gc_start/1, gc_end/1,
+ seq_trace/1]).
suite() -> [{ct_hooks,[ts_install_cth]},
{timetrap, {minutes, 1}}].
@@ -41,7 +42,8 @@ all() ->
groups() ->
[{ basic, [], [send, recv, call, call_return, spawn, exit,
link, unlink, getting_linked, getting_unlinked,
- register, unregister, in, out, gc_start, gc_end]}].
+ register, unregister, in, out, gc_start, gc_end,
+ seq_trace]}].
init_per_suite(Config) ->
erlang:trace_pattern({'_','_','_'}, false, [local]),
@@ -583,6 +585,24 @@ gc_end(_Config) ->
test(gc_major_end, garbage_collection, Tc, Expect, false).
+seq_trace(_Config) ->
+
+ seq_trace:set_system_tracer({tracer_test,
+ {#{ seq_trace => trace }, self(), []}}),
+ erlang:spawn(fun() ->
+ seq_trace:set_token(label,17),
+ seq_trace:set_token(print,true),
+ seq_trace:print(17,"**** Trace Started ****")
+ end),
+ receive
+ {seq_trace, _, 17, {print, _, _, _, _}, _} ->
+ ok;
+ M ->
+ ct:fail("~p~n",[M])
+ after 100 ->
+ ct:fail(timeout)
+ end.
+
test(Event, Tc, Expect) ->
test(Event, Tc, Expect, false).
test(Event, Tc, Expect, Removes) ->
diff --git a/erts/emulator/test/z_SUITE.erl b/erts/emulator/test/z_SUITE.erl
index ac3df8bfbf..103f9f1550 100644
--- a/erts/emulator/test/z_SUITE.erl
+++ b/erts/emulator/test/z_SUITE.erl
@@ -37,6 +37,7 @@
-export([schedulers_alive/1, node_container_refc_check/1,
long_timers/1, pollset_size/1,
check_io_debug/1, get_check_io_info/0,
+ lc_graph/1,
leaked_processes/1]).
suite() ->
@@ -46,6 +47,7 @@ suite() ->
all() ->
[schedulers_alive, node_container_refc_check,
long_timers, pollset_size, check_io_debug,
+ lc_graph,
%% Make sure that the leaked_processes/1 is always
%% run last.
leaked_processes].
@@ -289,6 +291,12 @@ has_gethost([P|T]) ->
has_gethost([]) ->
false.
+lc_graph(Config) when is_list(Config) ->
+ %% Create "lc_graph" file in current working dir
+ %% if lock checker is enabled
+ erts_debug:lc_graph(),
+ ok.
+
leaked_processes(Config) when is_list(Config) ->
%% Replace the defualt timetrap with a timetrap with
%% known pid.
diff --git a/erts/etc/common/erlexec.c b/erts/etc/common/erlexec.c
index cf6d1bacca..21a3f40c97 100644
--- a/erts/etc/common/erlexec.c
+++ b/erts/etc/common/erlexec.c
@@ -79,6 +79,7 @@ static const char plusM_au_allocs[]= {
static char *plusM_au_alloc_switches[] = {
"as",
"asbcst",
+ "atags",
"acul",
"acnl",
"acfml",
diff --git a/erts/lib_src/Makefile.in b/erts/lib_src/Makefile.in
index 601f3917a8..48660f7c71 100644
--- a/erts/lib_src/Makefile.in
+++ b/erts/lib_src/Makefile.in
@@ -334,7 +334,10 @@ ETHREAD_LIB=
endif
+ifneq ($(CREATE_DIRS),)
_create_dirs := $(shell mkdir -p $(CREATE_DIRS))
+endif
+
#
# Everything to build
#
diff --git a/erts/preloaded/ebin/erlang.beam b/erts/preloaded/ebin/erlang.beam
index e93f053e01..3ef5ad0646 100644
--- a/erts/preloaded/ebin/erlang.beam
+++ b/erts/preloaded/ebin/erlang.beam
Binary files differ
diff --git a/erts/preloaded/ebin/erts_internal.beam b/erts/preloaded/ebin/erts_internal.beam
index f5967780ad..d1bc4ecd2c 100644
--- a/erts/preloaded/ebin/erts_internal.beam
+++ b/erts/preloaded/ebin/erts_internal.beam
Binary files differ
diff --git a/erts/preloaded/src/erlang.erl b/erts/preloaded/src/erlang.erl
index bffa59338e..6fad5adde8 100644
--- a/erts/preloaded/src/erlang.erl
+++ b/erts/preloaded/src/erlang.erl
@@ -50,7 +50,7 @@
dist_ctrl_get_data_notification/1,
dist_get_stat/1]).
--deprecated([now/0]).
+-deprecated([get_stacktrace/0,now/0]).
%% Get rid of autoimports of spawn to avoid clashes with ourselves.
-compile({no_auto_import,[spawn_link/1]}).
@@ -3552,11 +3552,9 @@ max(A, _) -> A.
%%
-type memory_type() :: 'total' | 'processes' | 'processes_used' | 'system'
- | 'atom' | 'atom_used' | 'binary' | 'code' | 'ets'
- | 'low' | 'maximum'.
+ | 'atom' | 'atom_used' | 'binary' | 'code' | 'ets'.
-define(CARRIER_ALLOCS, [mseg_alloc]).
--define(LOW_ALLOCS, [ll_low_alloc, std_low_alloc]).
-define(ALL_NEEDED_ALLOCS, (erlang:system_info(alloc_util_allocators)
-- ?CARRIER_ALLOCS)).
@@ -3568,9 +3566,7 @@ max(A, _) -> A.
atom_used = 0,
binary = 0,
code = 0,
- ets = 0,
- low = 0,
- maximum = 0}).
+ ets = 0}).
-spec erlang:memory() -> [{Type, Size}] when
Type :: memory_type(),
@@ -3580,14 +3576,6 @@ memory() ->
notsup ->
erlang:error(notsup);
Mem ->
- InstrTail = case Mem#memory.maximum of
- 0 -> [];
- _ -> [{maximum, Mem#memory.maximum}]
- end,
- Tail = case Mem#memory.low of
- 0 -> InstrTail;
- _ -> [{low, Mem#memory.low} | InstrTail]
- end,
[{total, Mem#memory.total},
{processes, Mem#memory.processes},
{processes_used, Mem#memory.processes_used},
@@ -3596,7 +3584,7 @@ memory() ->
{atom_used, Mem#memory.atom_used},
{binary, Mem#memory.binary},
{code, Mem#memory.code},
- {ets, Mem#memory.ets} | Tail]
+ {ets, Mem#memory.ets}]
end.
-spec erlang:memory(Type :: memory_type()) -> non_neg_integer();
@@ -3684,16 +3672,6 @@ need_mem_info(binary) ->
{false, [binary_alloc], true, false};
need_mem_info(ets) ->
{true, [ets_alloc], true, false};
-need_mem_info(low) ->
- LowAllocs = ?LOW_ALLOCS -- ?CARRIER_ALLOCS,
- {_, _, FeatureList, _} = erlang:system_info(allocator),
- AlcUAllocs = case LowAllocs -- FeatureList of
- [] -> LowAllocs;
- _ -> []
- end,
- {false, AlcUAllocs, true, true};
-need_mem_info(maximum) ->
- {true, [], true, true};
need_mem_info(_) ->
{false, [], false, true}.
@@ -3706,8 +3684,6 @@ get_memval(atom_used, #memory{atom_used = V}) -> V;
get_memval(binary, #memory{binary = V}) -> V;
get_memval(code, #memory{code = V}) -> V;
get_memval(ets, #memory{ets = V}) -> V;
-get_memval(low, #memory{low = V}) -> V;
-get_memval(maximum, #memory{maximum = V}) -> V;
get_memval(_, #memory{}) -> 0.
memory_is_supported() ->
@@ -3762,16 +3738,6 @@ fix_proc([_ | Rest], Acc) ->
fix_proc([], Acc) ->
Acc.
-is_low_alloc(_A, []) ->
- false;
-is_low_alloc(A, [A|_As]) ->
- true;
-is_low_alloc(A, [_A|As]) ->
- is_low_alloc(A, As).
-
-is_low_alloc(A) ->
- is_low_alloc(A, ?LOW_ALLOCS).
-
au_mem_data(notsup, _) ->
notsup;
au_mem_data(_, [{_, false} | _]) ->
@@ -3824,16 +3790,11 @@ au_mem_data(#memory{total = Tot,
Rest)
end;
au_mem_data(#memory{total = Tot,
- system = Sys,
- low = Low} = Mem,
- [{A, _, Data} | Rest]) ->
+ system = Sys} = Mem,
+ [{_, _, Data} | Rest]) ->
Sz = blocks_size(Data, 0),
au_mem_data(Mem#memory{total = Tot+Sz,
- system = Sys+Sz,
- low = case is_low_alloc(A) of
- true -> Low+Sz;
- false -> Low
- end},
+ system = Sys+Sz},
Rest);
au_mem_data(EMD, []) ->
EMD.
@@ -3855,10 +3816,6 @@ receive_emd(Ref) ->
receive_emd(Ref, #memory{}, erlang:system_info(schedulers)).
aa_mem_data(#memory{} = Mem,
- [{maximum, Max} | Rest]) ->
- aa_mem_data(Mem#memory{maximum = Max},
- Rest);
-aa_mem_data(#memory{} = Mem,
[{total, Tot} | Rest]) ->
aa_mem_data(Mem#memory{total = Tot,
system = 0}, % system will be adjusted later
diff --git a/erts/preloaded/src/erts_internal.erl b/erts/preloaded/src/erts_internal.erl
index da5c9c68ed..79169b7d23 100644
--- a/erts/preloaded/src/erts_internal.erl
+++ b/erts/preloaded/src/erts_internal.erl
@@ -80,6 +80,8 @@
-export([is_process_alive/1, is_process_alive/2]).
+-export([gather_alloc_histograms/1, gather_carrier_info/1]).
+
%%
%% Await result of send to port
%%
@@ -621,3 +623,24 @@ is_process_alive(Pid) ->
Res
end.
+-spec gather_alloc_histograms({Type, SchedId, HistWidth, HistStart, Ref}) -> MsgCount when
+ Type :: atom(),
+ SchedId :: non_neg_integer(),
+ HistWidth :: non_neg_integer(),
+ HistStart :: non_neg_integer(),
+ Ref :: reference(),
+ MsgCount :: non_neg_integer().
+
+gather_alloc_histograms(_) ->
+ erlang:nif_error(undef).
+
+-spec gather_carrier_info({Type, SchedId, HistWidth, HistStart, Ref}) -> MsgCount when
+ Type :: atom(),
+ SchedId :: non_neg_integer(),
+ HistWidth :: non_neg_integer(),
+ HistStart :: non_neg_integer(),
+ Ref :: reference(),
+ MsgCount :: non_neg_integer().
+
+gather_carrier_info(_) ->
+ erlang:nif_error(undef).