diff options
author | Björn Gustavsson <[email protected]> | 2017-10-20 14:07:29 +0200 |
---|---|---|
committer | Björn Gustavsson <[email protected]> | 2017-10-20 14:07:29 +0200 |
commit | 806c2de5ca3806ad8110531c99749063e5603ac0 (patch) | |
tree | e340c1edd12135e1413a2eb890eb2773a17ffe02 | |
parent | 202d62c473e00fd066a70c85ba1d5c26ef2607a1 (diff) | |
parent | 7f8e605b242913b6f58f008aaa27a2edd1ab94b2 (diff) | |
download | otp-806c2de5ca3806ad8110531c99749063e5603ac0.tar.gz otp-806c2de5ca3806ad8110531c99749063e5603ac0.tar.bz2 otp-806c2de5ca3806ad8110531c99749063e5603ac0.zip |
Merge branch 'bjorn/improve-crash-dumps/OTP-14685' into maint
* bjorn/improve-crash-dumps/OTP-14685:
Bump version of crash dumps to 0.4
Verify that binaries of different sizes are dumped correctly
Don't dump literal areas that are not referenced at all
Dump literals separately to avoid incomplete heap data
Implement dumping of maps in crash dumps
Buffer writing of crash dumps
-rw-r--r-- | erts/emulator/beam/beam_ranges.c | 22 | ||||
-rw-r--r-- | erts/emulator/beam/break.c | 51 | ||||
-rw-r--r-- | erts/emulator/beam/erl_alloc.types | 1 | ||||
-rw-r--r-- | erts/emulator/beam/erl_process_dump.c | 363 | ||||
-rw-r--r-- | erts/emulator/beam/global.h | 2 | ||||
-rw-r--r-- | erts/emulator/beam/module.c | 1 | ||||
-rw-r--r-- | erts/include/internal/erl_printf.h | 1 | ||||
-rw-r--r-- | erts/lib_src/common/erl_printf.c | 12 | ||||
-rw-r--r-- | lib/observer/src/crashdump_viewer.erl | 93 | ||||
-rw-r--r-- | lib/observer/test/crashdump_helper.erl | 40 | ||||
-rw-r--r-- | lib/observer/test/crashdump_viewer_SUITE.erl | 47 |
11 files changed, 602 insertions, 31 deletions
diff --git a/erts/emulator/beam/beam_ranges.c b/erts/emulator/beam/beam_ranges.c index 9b0335e83d..fac4289271 100644 --- a/erts/emulator/beam/beam_ranges.c +++ b/erts/emulator/beam/beam_ranges.c @@ -32,6 +32,15 @@ typedef struct { erts_smp_atomic_t end; /* (BeamInstr*) Points one word beyond last function in module. */ } Range; +/* + * Used for crash dumping of literals. The size of erts_dump_lit_areas is + * always twice the number of active ranges (to allow for literals in both + * current and old code). + */ + +ErtsLiteralArea** erts_dump_lit_areas; +Uint erts_dump_num_lit_areas; + /* Range 'end' needs to be atomic as we purge module by setting end=start in active code_ix */ #define RANGE_END(R) ((BeamInstr*)erts_smp_atomic_read_nob(&(R)->end)) @@ -97,6 +106,11 @@ erts_init_ranges(void) r[i].allocated = 0; erts_smp_atomic_init_nob(&r[i].mid, 0); } + + erts_dump_num_lit_areas = 8; + erts_dump_lit_areas = (ErtsLiteralArea **) + erts_alloc(ERTS_ALC_T_CRASH_DUMP, + erts_dump_num_lit_areas * sizeof(ErtsLiteralArea*)); } void @@ -164,6 +178,14 @@ erts_end_staging_ranges(int commit) erts_smp_atomic_set_nob(&r[dst].mid, (erts_aint_t) (r[dst].modules + r[dst].n / 2)); + + if (r[dst].allocated * 2 > erts_dump_num_lit_areas) { + erts_dump_num_lit_areas *= 2; + erts_dump_lit_areas = (ErtsLiteralArea **) + erts_realloc(ERTS_ALC_T_CRASH_DUMP, + (void *) erts_dump_lit_areas, + erts_dump_num_lit_areas * sizeof(ErtsLiteralArea*)); + } } } diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 76a0c5c716..e57be5a595 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -58,6 +58,8 @@ static void dump_attributes(fmtfn_t to, void *to_arg, byte* ptr, int size); extern char* erts_system_version[]; +#define WRITE_BUFFER_SIZE (64*1024) + static void port_info(fmtfn_t to, void *to_arg) { @@ -677,18 +679,28 @@ bin_check(void) static Sint64 crash_dump_limit = ERTS_SINT64_MAX; static Sint64 crash_dump_written = 0; -static int crash_dump_limited_writer(void* vfdp, char* buf, size_t len) +typedef struct LimitedWriterInfo_ { + fmtfn_t to; + void* to_arg; +} LimitedWriterInfo; + +static int +crash_dump_limited_writer(void* vfdp, char* buf, size_t len) { const char stop_msg[] = "\n=abort:CRASH DUMP SIZE LIMIT REACHED\n"; + LimitedWriterInfo* lwi = (LimitedWriterInfo *) vfdp; crash_dump_written += len; if (crash_dump_written <= crash_dump_limit) { - return erts_write_fd(vfdp, buf, len); + return lwi->to(lwi->to_arg, buf, len); } len -= (crash_dump_written - crash_dump_limit); - erts_write_fd(vfdp, buf, len); - erts_write_fd(vfdp, (char*)stop_msg, sizeof(stop_msg)-1); + lwi->to(lwi->to_arg, buf, len); + lwi->to(lwi->to_arg, (char*)stop_msg, sizeof(stop_msg)-1); + if (lwi->to == &erts_write_fp) { + fclose((FILE *) lwi->to_arg); + } /* We assume that crash dump was called from erts_exit_vv() */ erts_exit_epilogue(); @@ -713,6 +725,9 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) int i; fmtfn_t to = &erts_write_fd; void* to_arg; + FILE* fp = 0; + LimitedWriterInfo lwi; + static char* write_buffer; /* 'static' to avoid a leak warning in valgrind */ if (ERTS_SOMEONE_IS_CRASH_DUMPING) return; @@ -820,9 +835,30 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) fd = open(dumpname,O_WRONLY | O_CREAT | O_TRUNC,0640); if (fd < 0) return; /* Can't create the crash dump, skip it */ - to_arg = (void*)&fd; + + /* + * Wrap into a FILE* so that we can use buffered output. Set an + * explicit buffer to make sure the first write does not fail because + * of a failure to allocate a buffer. + */ + write_buffer = (char *) erts_alloc_fnf(ERTS_ALC_T_TMP, WRITE_BUFFER_SIZE); + if (write_buffer && (fp = fdopen(fd, "w")) != NULL) { + setvbuf(fp, write_buffer, _IOFBF, WRITE_BUFFER_SIZE); + lwi.to = &erts_write_fp; + lwi.to_arg = (void*)fp; + } else { + lwi.to = &erts_write_fd; + lwi.to_arg = (void*)&fd; + } + if (to == &crash_dump_limited_writer) { + to_arg = (void *) &lwi; + } else { + to = lwi.to; + to_arg = lwi.to_arg; + } + time(&now); - erts_cbprintf(to, to_arg, "=erl_crash_dump:0.3\n%s", ctime(&now)); + erts_cbprintf(to, to_arg, "=erl_crash_dump:0.4\n%s", ctime(&now)); if (file != NULL) erts_cbprintf(to, to_arg, "The error occurred in file %s, line %d\n", file, line); @@ -932,6 +968,9 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) } erts_cbprintf(to, to_arg, "=end\n"); + if (fp) { + fclose(fp); + } close(fd); erts_fprintf(stderr,"done\n"); } diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 50a1d97dd5..252bf1cc7e 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -285,6 +285,7 @@ type MREF_ENT STANDARD SYSTEM magic_ref_entry type MREF_TAB_BKTS STANDARD SYSTEM magic_ref_table_buckets type MREF_TAB LONG_LIVED SYSTEM magic_ref_table type MINDIRECTION FIXED_SIZE SYSTEM magic_indirection +type CRASH_DUMP STANDARD SYSTEM crash_dump +if threads_no_smp # Need thread safe allocs, but std_alloc and fix_alloc are not; diff --git a/erts/emulator/beam/erl_process_dump.c b/erts/emulator/beam/erl_process_dump.c index b826e6c5d3..9fd024cae8 100644 --- a/erts/emulator/beam/erl_process_dump.c +++ b/erts/emulator/beam/erl_process_dump.c @@ -32,6 +32,7 @@ #include "dist.h" #include "beam_catches.h" #include "erl_binary.h" +#include "erl_map.h" #define ERTS_WANT_EXTERNAL_TAGS #include "external.h" @@ -51,6 +52,11 @@ static void print_function_from_pc(fmtfn_t to, void *to_arg, BeamInstr* x); static void heap_dump(fmtfn_t to, void *to_arg, Eterm x); static void dump_binaries(fmtfn_t to, void *to_arg, Binary* root); static void dump_externally(fmtfn_t to, void *to_arg, Eterm term); +static void mark_literal(Eterm* ptr); +static void init_literal_areas(void); +static void dump_literals(fmtfn_t to, void *to_arg); +static void dump_module_literals(fmtfn_t to, void *to_arg, + ErtsLiteralArea* lit_area); static Binary* all_binaries; @@ -58,14 +64,14 @@ extern BeamInstr beam_apply[]; extern BeamInstr beam_exit[]; extern BeamInstr beam_continue_exit[]; - void erts_deep_process_dump(fmtfn_t to, void *to_arg) { int i, max = erts_ptab_max(&erts_proc); all_binaries = NULL; - + init_literal_areas(); + for (i = 0; i < max; i++) { Process *p = erts_pix2proc(i); if (p && p->i != ENULL) { @@ -75,6 +81,7 @@ erts_deep_process_dump(fmtfn_t to, void *to_arg) } } + dump_literals(to, to_arg); dump_binaries(to, to_arg, all_binaries); } @@ -373,7 +380,9 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x) next = (Eterm *) x; } else if (is_list(x)) { ptr = list_val(x); - if (ptr[0] != OUR_NIL) { + if (erts_is_literal(x, ptr)) { + mark_literal(ptr); + } else if (ptr[0] != OUR_NIL) { erts_print(to, to_arg, PTR_FMT ":l", ptr); dump_element(to, to_arg, ptr[0]); erts_putc(to, to_arg, '|'); @@ -392,7 +401,9 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x) ptr = boxed_val(x); hdr = *ptr; - if (hdr != OUR_NIL) { /* If not visited */ + if (erts_is_literal(x, ptr)) { + mark_literal(ptr); + } else if (hdr != OUR_NIL) { erts_print(to, to_arg, PTR_FMT ":", ptr); if (is_arity_value(hdr)) { Uint i; @@ -498,11 +509,77 @@ heap_dump(fmtfn_t to, void *to_arg, Eterm x) erts_print(to, to_arg, "p<%beu.%beu>\n", port_channel_no(x), port_number(x)); *ptr = OUR_NIL; + } else if (is_map_header(hdr)) { + if (is_flatmap_header(hdr)) { + flatmap_t* fmp = (flatmap_t *) flatmap_val(x); + Eterm* values = ptr + sizeof(flatmap_t) / sizeof(Eterm); + Uint map_size = fmp->size; + int i; + + erts_print(to, to_arg, "Mf" ETERM_FMT ":", map_size); + dump_element(to, to_arg, fmp->keys); + erts_putc(to, to_arg, ':'); + for (i = 0; i < map_size; i++) { + dump_element(to, to_arg, values[i]); + if (is_immed(values[i])) { + values[i] = make_small(0); + } + if (i < map_size-1) { + erts_putc(to, to_arg, ','); + } + } + erts_putc(to, to_arg, '\n'); + *ptr = OUR_NIL; + x = fmp->keys; + if (map_size) { + fmp->keys = (Eterm) next; + next = &values[map_size-1]; + } + continue; + } else { + Uint i; + Uint sz = 0; + Eterm* nodes = ptr + 1; + + switch (MAP_HEADER_TYPE(hdr)) { + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY: + nodes++; + sz = 16; + erts_print(to, to_arg, "Mh" ETERM_FMT ":" ETERM_FMT ":", + hashmap_size(x), sz); + break; + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP: + nodes++; + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + erts_print(to, to_arg, "Mh" ETERM_FMT ":" ETERM_FMT ":", + hashmap_size(x), sz); + break; + case MAP_HEADER_TAG_HAMT_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(hdr)); + erts_print(to, to_arg, "Mn" ETERM_FMT ":", sz); + break; + } + *ptr = OUR_NIL; + for (i = 0; i < sz; i++) { + dump_element(to, to_arg, nodes[i]); + if (is_immed(nodes[i])) { + nodes[i] = make_small(0); + } + if (i < sz-1) { + erts_putc(to, to_arg, ','); + } + } + erts_putc(to, to_arg, '\n'); + x = nodes[0]; + nodes[0] = (Eterm) next; + next = &nodes[sz-1]; + continue; + } } else { /* * All other we dump in the external term format. */ - dump_externally(to, to_arg, x); + dump_externally(to, to_arg, x); erts_putc(to, to_arg, '\n'); *ptr = OUR_NIL; } @@ -564,11 +641,6 @@ dump_externally(fmtfn_t to, void *to_arg, Eterm term) } } - /* Do not handle maps */ - if (is_map(term)) { - term = am_undefined; - } - s = p = sbuf; erts_encode_ext(term, &p); erts_print(to, to_arg, "E%X:", p-s); @@ -577,6 +649,277 @@ dump_externally(fmtfn_t to, void *to_arg, Eterm term) } } +/* + * Handle dumping of literal areas. + */ + +static ErtsLiteralArea** lit_areas; +static Uint num_lit_areas; + +static int compare_areas(const void * a, const void * b) +{ + ErtsLiteralArea** a_p = (ErtsLiteralArea **) a; + ErtsLiteralArea** b_p = (ErtsLiteralArea **) b; + + if (*a_p < *b_p) { + return -1; + } else if (*b_p < *a_p) { + return 1; + } else { + return 0; + } +} + + +static void +init_literal_areas(void) +{ + int i; + Module* modp; + ErtsCodeIndex code_ix; + ErtsLiteralArea** area_p; + + code_ix = erts_active_code_ix(); + erts_rlock_old_code(code_ix); + + lit_areas = area_p = erts_dump_lit_areas; + num_lit_areas = 0; + for (i = 0; i < module_code_size(code_ix); i++) { + modp = module_code(i, code_ix); + if (modp == NULL) { + continue; + } + if (modp->curr.code_length > 0 && + modp->curr.code_hdr->literal_area) { + *area_p++ = modp->curr.code_hdr->literal_area; + } + if (modp->old.code_length > 0 && modp->old.code_hdr->literal_area) { + *area_p++ = modp->old.code_hdr->literal_area; + } + } + + num_lit_areas = area_p - lit_areas; + ASSERT(num_lit_areas <= erts_dump_num_lit_areas); + for (i = 0; i < num_lit_areas; i++) { + lit_areas[i]->off_heap = 0; + } + + qsort(lit_areas, num_lit_areas, sizeof(ErtsLiteralArea *), + compare_areas); + + erts_runlock_old_code(code_ix); +} + +static int search_areas(const void * a, const void * b) { + Eterm* key = (Eterm *) a; + ErtsLiteralArea** b_p = (ErtsLiteralArea **) b; + if (key < b_p[0]->start) { + return -1; + } else if (b_p[0]->end <= key) { + return 1; + } else { + return 0; + } +} + +static void mark_literal(Eterm* ptr) +{ + ErtsLiteralArea** ap; + + ap = bsearch(ptr, lit_areas, num_lit_areas, sizeof(ErtsLiteralArea*), + search_areas); + ASSERT(ap); + ap[0]->off_heap = (struct erl_off_heap_header *) 1; +} + + +static void +dump_literals(fmtfn_t to, void *to_arg) +{ + ErtsCodeIndex code_ix; + int i; + + code_ix = erts_active_code_ix(); + erts_rlock_old_code(code_ix); + + erts_print(to, to_arg, "=literals\n"); + for (i = 0; i < num_lit_areas; i++) { + if (lit_areas[i]->off_heap) { + dump_module_literals(to, to_arg, lit_areas[i]); + } + } + + erts_runlock_old_code(code_ix); +} + +static void +dump_module_literals(fmtfn_t to, void *to_arg, ErtsLiteralArea* lit_area) +{ + Eterm* htop; + Eterm* hend; + + htop = lit_area->start; + hend = lit_area->end; + while (htop < hend) { + Eterm w = *htop; + Eterm term; + Uint size; + + switch (primary_tag(w)) { + case TAG_PRIMARY_HEADER: + term = make_boxed(htop); + erts_print(to, to_arg, PTR_FMT ":", htop); + if (is_arity_value(w)) { + Uint i; + Uint arity = arityval(w); + + erts_print(to, to_arg, "t" ETERM_FMT ":", arity); + for (i = 1; i <= arity; i++) { + dump_element(to, to_arg, htop[i]); + if (i < arity) { + erts_putc(to, to_arg, ','); + } + } + erts_putc(to, to_arg, '\n'); + } else if (w == HEADER_FLONUM) { + FloatDef f; + char sbuf[31]; + int i; + + GET_DOUBLE_DATA((htop+1), f); + i = sys_double_to_chars(f.fd, sbuf, sizeof(sbuf)); + sys_memset(sbuf+i, 0, 31-i); + erts_print(to, to_arg, "F%X:%s\n", i, sbuf); + } else if (_is_bignum_header(w)) { + erts_print(to, to_arg, "B%T\n", term); + } else if (is_binary_header(w)) { + Uint tag = thing_subtag(w); + Uint size = binary_size(term); + Uint i; + + if (tag == HEAP_BINARY_SUBTAG) { + byte* p; + + erts_print(to, to_arg, "Yh%X:", size); + p = binary_bytes(term); + for (i = 0; i < size; i++) { + erts_print(to, to_arg, "%02X", p[i]); + } + } else if (tag == REFC_BINARY_SUBTAG) { + ProcBin* pb = (ProcBin *) binary_val(term); + Binary* val = pb->val; + + if (erts_atomic_xchg_nob(&val->intern.refc, 0) != 0) { + val->intern.flags = (UWord) all_binaries; + all_binaries = val; + } + erts_print(to, to_arg, + "Yc" PTR_FMT ":" PTR_FMT ":" PTR_FMT, + val, + pb->bytes - (byte *)val->orig_bytes, + size); + } else if (tag == SUB_BINARY_SUBTAG) { + ErlSubBin* Sb = (ErlSubBin *) binary_val(term); + Eterm* real_bin; + void* val; + + real_bin = boxed_val(Sb->orig); + if (thing_subtag(*real_bin) == REFC_BINARY_SUBTAG) { + /* + * Unvisited REFC_BINARY: Point directly to + * the binary. + */ + ProcBin* pb = (ProcBin *) real_bin; + val = pb->val; + } else { + /* + * Heap binary or visited REFC binary: Point + * to heap binary or ProcBin on the heap. + */ + val = real_bin; + } + erts_print(to, to_arg, + "Ys" PTR_FMT ":" PTR_FMT ":" PTR_FMT, + val, Sb->offs, size); + } + erts_putc(to, to_arg, '\n'); + } else if (is_map_header(w)) { + if (is_flatmap_header(w)) { + flatmap_t* fmp = (flatmap_t *) flatmap_val(term); + Eterm* values = htop + sizeof(flatmap_t) / sizeof(Eterm); + Uint map_size = fmp->size; + int i; + + erts_print(to, to_arg, "Mf" ETERM_FMT ":", map_size); + dump_element(to, to_arg, fmp->keys); + erts_putc(to, to_arg, ':'); + for (i = 0; i < map_size; i++) { + dump_element(to, to_arg, values[i]); + if (i < map_size-1) { + erts_putc(to, to_arg, ','); + } + } + erts_putc(to, to_arg, '\n'); + } else { + Uint i; + Uint sz = 0; + Eterm* nodes = htop + 1; + + switch (MAP_HEADER_TYPE(w)) { + case MAP_HEADER_TAG_HAMT_HEAD_ARRAY: + nodes++; + sz = 16; + erts_print(to, to_arg, "Mh" ETERM_FMT ":" ETERM_FMT ":", + hashmap_size(term), sz); + break; + case MAP_HEADER_TAG_HAMT_HEAD_BITMAP: + nodes++; + sz = hashmap_bitcount(MAP_HEADER_VAL(w)); + erts_print(to, to_arg, "Mh" ETERM_FMT ":" ETERM_FMT ":", + hashmap_size(term), sz); + break; + case MAP_HEADER_TAG_HAMT_NODE_BITMAP: + sz = hashmap_bitcount(MAP_HEADER_VAL(w)); + erts_print(to, to_arg, "Mn" ETERM_FMT ":", sz); + break; + } + for (i = 0; i < sz; i++) { + dump_element(to, to_arg, nodes[i]); + if (i < sz-1) { + erts_putc(to, to_arg, ','); + } + } + erts_putc(to, to_arg, '\n'); + } + } + size = 1 + header_arity(w); + switch (w & _HEADER_SUBTAG_MASK) { + case MAP_SUBTAG: + if (is_flatmap_header(w)) { + size += 1 + flatmap_get_size(htop); + } else { + size += hashmap_bitcount(MAP_HEADER_VAL(w)); + } + break; + case SUB_BINARY_SUBTAG: + size += 1; + break; + } + break; + default: + ASSERT(!is_header(htop[1])); + erts_print(to, to_arg, PTR_FMT ":l", htop); + dump_element(to, to_arg, htop[0]); + erts_putc(to, to_arg, '|'); + dump_element(to, to_arg, htop[1]); + erts_putc(to, to_arg, '\n'); + size = 2; + break; + } + htop += size; + } +} + void erts_dump_process_state(fmtfn_t to, void *to_arg, erts_aint32_t psflg) { char *s; diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 182d3aa44e..440723aea6 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -955,6 +955,8 @@ void erts_update_ranges(BeamInstr* code, Uint size); void erts_remove_from_ranges(BeamInstr* code); UWord erts_ranges_sz(void); void erts_lookup_function_info(FunctionInfo* fi, BeamInstr* pc, int full_info); +ErtsLiteralArea** erts_dump_lit_areas; +Uint erts_dump_num_lit_areas; /* break.c */ void init_break_handler(void); diff --git a/erts/emulator/beam/module.c b/erts/emulator/beam/module.c index 7987cb2eb5..a386407ec2 100644 --- a/erts/emulator/beam/module.c +++ b/erts/emulator/beam/module.c @@ -256,4 +256,3 @@ void module_end_staging(int commit) IF_DEBUG(dbg_load_code_ix = -1); } - diff --git a/erts/include/internal/erl_printf.h b/erts/include/internal/erl_printf.h index f180a53f18..7e9807f6a8 100644 --- a/erts/include/internal/erl_printf.h +++ b/erts/include/internal/erl_printf.h @@ -44,6 +44,7 @@ struct erts_dsprintf_buf_t_ { typedef int (*fmtfn_t)(void*, char*, size_t); int erts_write_fd(void *vfdp, char* buf, size_t len); +int erts_write_fp(void *vfdp, char* buf, size_t len); int erts_write_ds(void *vdsbufp, char* buf, size_t len); int erts_printf(const char *, ...); diff --git a/erts/lib_src/common/erl_printf.c b/erts/lib_src/common/erl_printf.c index 7781fc2196..3b073bcd1b 100644 --- a/erts/lib_src/common/erl_printf.c +++ b/erts/lib_src/common/erl_printf.c @@ -147,8 +147,8 @@ write_f_add_cr(void *vfp, char* buf, size_t len) return len; } -static int -write_f(void *vfp, char* buf, size_t len) +int +erts_write_fp(void *vfp, char* buf, size_t len) { ASSERT(vfp); #ifdef PUTC_ON_SMALL_WRITES @@ -257,7 +257,7 @@ erts_printf(const char *format, ...) FLOCKFILE(stdout); res = erts_printf_format(erts_printf_add_cr_to_stdout ? write_f_add_cr - : write_f, + : erts_write_fp, (void *) stdout, (char *) format, arglist); @@ -285,7 +285,7 @@ erts_fprintf(FILE *filep, const char *format, ...) else if (erts_printf_add_cr_to_stderr && filep == stderr) fmt_f = write_f_add_cr; else - fmt_f = write_f; + fmt_f = erts_write_fp; FLOCKFILE(filep); res = erts_printf_format(fmt_f,(void *)filep,(char *)format,arglist); FUNLOCKFILE(filep); @@ -390,7 +390,7 @@ erts_vprintf(const char *format, va_list arglist) errno = 0; res = erts_printf_format(erts_printf_add_cr_to_stdout ? write_f_add_cr - : write_f, + : erts_write_fp, (void *) stdout, (char *) format, arglist); @@ -414,7 +414,7 @@ erts_vfprintf(FILE *filep, const char *format, va_list arglist) else if (erts_printf_add_cr_to_stderr && filep == stderr) fmt_f = write_f_add_cr; else - fmt_f = write_f; + fmt_f = erts_write_fp; res = erts_printf_format(fmt_f,(void *)filep,(char *)format,arglist); } return res; diff --git a/lib/observer/src/crashdump_viewer.erl b/lib/observer/src/crashdump_viewer.erl index 95e12887cd..ddf2879a97 100644 --- a/lib/observer/src/crashdump_viewer.erl +++ b/lib/observer/src/crashdump_viewer.erl @@ -104,6 +104,7 @@ -define(index_table,index_table). -define(instr_data,instr_data). -define(internal_ets,internal_ets). +-define(literals,literals). -define(loaded_modules,loaded_modules). -define(memory,memory). -define(memory_map,memory_map). @@ -785,6 +786,9 @@ parse_vsn_str(Str,WS) -> %%% Progress is reported during the time and MUST be checked with %%% crashdump_viewer:get_progress/0 until it returns {ok,done}. do_read_file(File) -> + erase(?literals), %Clear literal cache. + put(truncated,false), %Not truncated (yet). + erase(truncated_reason), %Not truncated (yet). case file:read_file_info(File) of {ok,#file_info{type=regular, access=FileA, @@ -856,6 +860,19 @@ indexify(Fd,AddrAdj,Bin,N) -> {?proc_heap,LastId} -> [{_,LastPos}] = lookup_index(?proc_heap,LastId), ets:insert(cdv_heap_file_chars,{LastId,N+Start+1-LastPos}); + {?literals,[]} -> + case get(truncated_reason) of + undefined -> + [{_,LastPos}] = lookup_index(?literals,[]), + ets:insert(cdv_heap_file_chars, + {literals,N+Start+1-LastPos}); + _ -> + %% Literals are truncated. Make sure we never + %% attempt to read in the literals. (Heaps that + %% references literals will show markers for + %% incomplete heaps, but will otherwise work.) + delete_index(?literals, []) + end; _ -> ok end, indexify(Fd,AddrAdj,Rest,N1); @@ -908,6 +925,7 @@ check_if_truncated() -> find_truncated_proc({Tag,_Id}) when Tag==?atoms; Tag==?binary; Tag==?instr_data; + Tag==?literals; Tag==?memory_status; Tag==?memory_map -> put(truncated_proc,false); @@ -1386,13 +1404,33 @@ maybe_other_node2(Channel) -> expand_memory(Fd,Pid,DumpVsn) -> BinAddrAdj = get_bin_addr_adj(DumpVsn), put(fd,Fd), - Dict = read_heap(Fd,Pid,BinAddrAdj,gb_trees:empty()), + Dict0 = case get(?literals) of + undefined -> + Literals = read_literals(Fd), + put(?literals,Literals), + put(fd,Fd), + Literals; + Literals -> + Literals + end, + Dict = read_heap(Fd,Pid,BinAddrAdj,Dict0), Expanded = {read_stack_dump(Fd,Pid,BinAddrAdj,Dict), read_messages(Fd,Pid,BinAddrAdj,Dict), read_dictionary(Fd,Pid,BinAddrAdj,Dict)}, erase(fd), Expanded. +read_literals(Fd) -> + case lookup_index(?literals,[]) of + [{_,Start}] -> + [{_,Chars}] = ets:lookup(cdv_heap_file_chars,literals), + init_progress("Reading literals",Chars), + pos_bof(Fd,Start), + read_heap(0,gb_trees:empty()); + [] -> + gb_trees:empty() + end. + %%%----------------------------------------------------------------- %%% This is a workaround for a bug in dump versions prior to 0.3: %%% Addresses were truncated to 32 bits. This could cause binaries to @@ -2589,8 +2627,26 @@ parse_heap_term("Ys"++Line0, Addr, BinAddrAdj, D0) -> %Sub binary. end end, D = gb_trees:insert(Addr, Term, D0), - {Term,Line,D}. - + {Term,Line,D}; +parse_heap_term("Mf"++Line0, Addr, BinAddrAdj, D0) -> %Flatmap. + {Size,":"++Line1} = get_hex(Line0), + {Keys,":"++Line2,D1} = parse_term(Line1, BinAddrAdj, D0), + {Values,Line,D2} = parse_tuple(Size, Line2, Addr,BinAddrAdj, D1, []), + Pairs = zip_tuples(tuple_size(Keys), Keys, Values, []), + Map = maps:from_list(Pairs), + D = gb_trees:update(Addr, Map, D2), + {Map,Line,D}; +parse_heap_term("Mh"++Line0, Addr, BinAddrAdj, D0) -> %Head node in a hashmap. + {MapSize,":"++Line1} = get_hex(Line0), + {N,":"++Line2} = get_hex(Line1), + {Nodes,Line,D1} = parse_tuple(N, Line2, Addr, BinAddrAdj, D0, []), + Map = maps:from_list(flatten_hashmap_nodes(Nodes)), + MapSize = maps:size(Map), %Assertion. + D = gb_trees:update(Addr, Map, D1), + {Map,Line,D}; +parse_heap_term("Mn"++Line0, Addr, BinAddrAdj, D) -> %Interior node in a hashmap. + {N,":"++Line} = get_hex(Line0), + parse_tuple(N, Line, Addr, BinAddrAdj, D, []). parse_tuple(0, Line, Addr, _, D0, Acc) -> Tuple = list_to_tuple(lists:reverse(Acc)), @@ -2604,6 +2660,25 @@ parse_tuple(N, Line0, Addr, BinAddrAdj, D0, Acc) -> parse_tuple(N-1, Line, Addr, BinAddrAdj, D, [Term|Acc]) end. +zip_tuples(0, _T1, _T2, Acc) -> + Acc; +zip_tuples(N, T1, T2, Acc) when N =< tuple_size(T1) -> + zip_tuples(N-1, T1, T2, [{element(N, T1),element(N, T2)}|Acc]). + +flatten_hashmap_nodes(Tuple) -> + flatten_hashmap_nodes_1(tuple_size(Tuple), Tuple, []). + +flatten_hashmap_nodes_1(0, _Tuple, Acc) -> + Acc; +flatten_hashmap_nodes_1(N, Tuple0, Acc0) -> + case element(N, Tuple0) of + [K|V] -> + flatten_hashmap_nodes_1(N-1, Tuple0, [{K,V}|Acc0]); + Tuple when is_tuple(Tuple) -> + Acc = flatten_hashmap_nodes_1(N-1, Tuple0, Acc0), + flatten_hashmap_nodes_1(tuple_size(Tuple), Tuple, Acc) + end. + parse_term([$H|Line0], BinAddrAdj, D) -> %Pointer to heap term. {Ptr,Line} = get_hex(Line0), deref_ptr(Ptr, Line, BinAddrAdj, D); @@ -2792,6 +2867,10 @@ reset_tables() -> insert_index(Tag,Id,Pos) -> ets:insert(cdv_dump_index_table,{{Tag,Pos},Id}). +delete_index(Tag,Id) -> + Ms = [{{{Tag,'$1'},Id},[],[true]}], + ets:select_delete(cdv_dump_index_table, Ms). + lookup_index({Tag,Id}) -> lookup_index(Tag,Id); lookup_index(Tag) -> @@ -2808,6 +2887,7 @@ insert_binary_index(Addr,Pos) -> lookup_binary_index(Addr) -> ets:lookup(cdv_binary_index_table,Addr). + %%----------------------------------------------------------------- %% Convert tags read from crashdump to atoms used as first part of key %% in cdv_dump_index_table @@ -2825,6 +2905,7 @@ tag_to_atom("hidden_node") -> ?hidden_node; tag_to_atom("index_table") -> ?index_table; tag_to_atom("instr_data") -> ?instr_data; tag_to_atom("internal_ets") -> ?internal_ets; +tag_to_atom("literals") -> ?literals; tag_to_atom("loaded_modules") -> ?loaded_modules; tag_to_atom("memory") -> ?memory; tag_to_atom("mod") -> ?mod; @@ -2848,8 +2929,10 @@ tag_to_atom(UnknownTag) -> %%%----------------------------------------------------------------- %%% Store last tag for use when truncated, and reason if aborted put_last_tag(?abort,Reason) -> - %% Don't overwrite the real last tag - put(truncated_reason,Reason); + %% Don't overwrite the real last tag, and make sure to return + %% the previous last tag. + put(truncated_reason,Reason), + get(last_tag); put_last_tag(Tag,Id) -> put(last_tag,{Tag,Id}). diff --git a/lib/observer/test/crashdump_helper.erl b/lib/observer/test/crashdump_helper.erl index f37d9057cb..41041682c2 100644 --- a/lib/observer/test/crashdump_helper.erl +++ b/lib/observer/test/crashdump_helper.erl @@ -19,7 +19,9 @@ %% -module(crashdump_helper). --export([n1_proc/2,remote_proc/2]). +-export([n1_proc/2,remote_proc/2, + dump_maps/0,create_maps/0, + create_binaries/0]). -compile(r18). -include_lib("common_test/include/ct.hrl"). @@ -60,6 +62,7 @@ n1_proc(Creator,_N2,Pid2,Port2,_L) -> put(ref,Ref), put(pid,Pid), put(bin,Bin), + put(bins,create_binaries()), put(sub_bin,SubBin), put(bignum,83974938738373873), put(neg_bignum,-38748762783736367), @@ -92,3 +95,38 @@ remote_proc(P1,Creator) -> Creator ! {self(),done}, receive after infinity -> ok end end). + +create_binaries() -> + Sizes = lists:seq(60, 70) ++ lists:seq(120, 140), + [begin + <<H:16/unit:8>> = erlang:md5(<<Size:32>>), + Data = ((H bsl (8*150)) div (H+7919)), + <<Data:Size/unit:8>> + end || Size <- Sizes]. + +%%% +%%% Test dumping of maps. Dumping of maps only from OTP 20.2. +%%% + +dump_maps() -> + Parent = self(), + F = fun() -> + register(aaaaaaaa_maps, self()), + put(maps, create_maps()), + Parent ! {self(),done}, + receive _ -> ok end + end, + Pid = spawn_link(F), + receive + {Pid,done} -> + {ok,Pid} + end. + +create_maps() -> + Map0 = maps:from_list([{I,[I,I+1]} || I <- lists:seq(1, 40)]), + Map1 = maps:from_list([{I,{a,[I,I*I],{}}} || I <- lists:seq(1, 100)]), + Map2 = maps:from_list([{{I},(I*I) bsl 24} || I <- lists:seq(1, 10000)]), + Map3 = lists:foldl(fun(I, A) -> + A#{I=>I*I} + end, Map2, lists:seq(-10, 0)), + #{a=>Map0,b=>Map1,c=>Map2,d=>Map3,e=>#{}}. diff --git a/lib/observer/test/crashdump_viewer_SUITE.erl b/lib/observer/test/crashdump_viewer_SUITE.erl index f9ac884743..86a60e15f4 100644 --- a/lib/observer/test/crashdump_viewer_SUITE.erl +++ b/lib/observer/test/crashdump_viewer_SUITE.erl @@ -364,6 +364,10 @@ special(File,Procs) -> crashdump_viewer:expand_binary({SOffset,SSize,SPos}), io:format(" expand binary ok",[]), + Binaries = crashdump_helper:create_binaries(), + verify_binaries(Binaries, proplists:get_value(bins,Dict)), + io:format(" binaries ok",[]), + #proc{last_calls=LastCalls} = ProcDetails, true = length(LastCalls) =< 4, @@ -513,11 +517,36 @@ special(File,Procs) -> io:format(" unicode table name ok",[]), ok; + ".maps" -> + %% I registered a process as aaaaaaaa_maps in the map dump + %% to make sure it will be the first in the list when sorted + %% on names. + [#proc{pid=Pid0,name=Name}|_Rest] = lists:keysort(#proc.name,Procs), + "aaaaaaaa_maps" = Name, + Pid = pid_to_list(Pid0), + {ok,ProcDetails=#proc{},[]} = crashdump_viewer:proc_details(Pid), + io:format(" process details ok",[]), + + #proc{dict=Dict} = ProcDetails, + io:format("~p\n", [Dict]), + Maps = crashdump_helper:create_maps(), + Maps = proplists:get_value(maps,Dict), + io:format(" maps ok",[]), + ok; _ -> ok end, ok. +verify_binaries([H|T1], [H|T2]) -> + %% Heap binary. + verify_binaries(T1, T2); +verify_binaries([Bin|T1], [['#CDVBin',Offset,Size,Pos]|T2]) -> + %% Refc binary. + {ok,<<Bin:Size/binary>>} = crashdump_viewer:expand_binary({Offset,Size,Pos}), + verify_binaries(T1, T2); +verify_binaries([], []) -> + ok. lookat_all_pids([]) -> ok; @@ -582,8 +611,9 @@ do_create_dumps(DataDir,Rel) -> "-env ERL_CRASH_DUMP_BYTES " ++ integer_to_list(Bytes)), CD6 = dump_with_unicode_atoms(DataDir,Rel,"unicode"), + CD7 = dump_with_maps(DataDir,Rel,"maps"), TruncatedDumps = truncate_dump(CD1), - {[CD1,CD2,CD3,CD4,CD5,CD6|TruncatedDumps], DosDump}; + {[CD1,CD2,CD3,CD4,CD5,CD6,CD7|TruncatedDumps], DosDump}; _ -> {[CD1,CD2], DosDump} end. @@ -596,7 +626,10 @@ truncate_dump(File) -> {win32,_} -> <<"\r\n">>; _ -> <<"\n">> end, - [StartBin,AfterTag] = binary:split(Bin,BinTag), + %% Split after "our binary" created by crashdump_helper + %% (it may not be the first binary). + RE = <<"\n=binary:(?=[0-9A-Z]+",NewLine/binary,"FF:010203)">>, + [StartBin,AfterTag] = re:split(Bin,RE,[{parts,2}]), [AddrAndSize,BinaryAndRest] = binary:split(AfterTag,Colon), [Binary,_Rest] = binary:split(BinaryAndRest,NewLine), TruncSize = byte_size(Binary) - 2, @@ -699,6 +732,16 @@ dump_with_unicode_atoms(DataDir,Rel,DumpName) -> ?t:stop_node(n1), CD. +dump_with_maps(DataDir,Rel,DumpName) -> + Opt = rel_opt(Rel), + Pz = "-pz \"" ++ filename:dirname(code:which(?MODULE)) ++ "\"", + PzOpt = [{args,Pz}], + {ok,N1} = ?t:start_node(n1,peer,Opt ++ PzOpt), + {ok,_Pid} = rpc:call(N1,crashdump_helper,dump_maps,[]), + CD = dump(N1,DataDir,Rel,DumpName), + ?t:stop_node(n1), + CD. + dump(Node,DataDir,Rel,DumpName) -> Crashdump = filename:join(DataDir, dump_prefix(Rel)++DumpName), rpc:call(Node,os,putenv,["ERL_CRASH_DUMP",Crashdump]), |