diff options
Diffstat (limited to 'erts')
27 files changed, 841 insertions, 177 deletions
diff --git a/erts/emulator/beam/beam_bif_load.c b/erts/emulator/beam/beam_bif_load.c index bb1b2e5b27..4c8ee5178a 100644 --- a/erts/emulator/beam/beam_bif_load.c +++ b/erts/emulator/beam/beam_bif_load.c @@ -1799,6 +1799,78 @@ erts_queue_release_literals(Process* c_p, ErtsLiteralArea* literals) } } +struct debug_la_oh { + void (*func)(ErlOffHeap *, void *); + void *arg; +}; + +static void debug_later_cleanup_literal_area_off_heap(void *vfap, + ErtsThrPrgrVal val, + void *vlrlap) +{ + struct debug_la_oh *fap = vfap; + ErtsLaterReleasLiteralArea *lrlap = vlrlap; + ErtsLiteralArea *lap = lrlap->la; + if (!erts_debug_have_accessed_literal_area(lap)) { + ErlOffHeap oh; + ERTS_INIT_OFF_HEAP(&oh); + oh.first = lap->off_heap; + (*fap->func)(&oh, fap->arg); + erts_debug_save_accessed_literal_area(lap); + } +} + +static void debug_later_complete_literal_area_switch_off_heap(void *vfap, + ErtsThrPrgrVal val, + void *vlap) +{ + struct debug_la_oh *fap = vfap; + ErtsLiteralArea *lap = vlap; + if (lap && !erts_debug_have_accessed_literal_area(lap)) { + ErlOffHeap oh; + ERTS_INIT_OFF_HEAP(&oh); + oh.first = lap->off_heap; + (*fap->func)(&oh, fap->arg); + erts_debug_save_accessed_literal_area(lap); + } +} + + +void +erts_debug_foreach_release_literal_area_off_heap(void (*func)(ErlOffHeap *, void *), void *arg) +{ + ErtsLiteralArea *lap; + ErlOffHeap oh; + ErtsLiteralAreaRef *ref; + struct debug_la_oh fa; + erts_mtx_lock(&release_literal_areas.mtx); + for (ref = release_literal_areas.first; ref; ref = ref->next) { + lap = ref->literal_area; + if (!erts_debug_have_accessed_literal_area(lap)) { + ERTS_INIT_OFF_HEAP(&oh); + oh.first = lap->off_heap; + (*func)(&oh, arg); + erts_debug_save_accessed_literal_area(lap); + } + } + erts_mtx_unlock(&release_literal_areas.mtx); + lap = ERTS_COPY_LITERAL_AREA(); + if (lap && !erts_debug_have_accessed_literal_area(lap)) { + ERTS_INIT_OFF_HEAP(&oh); + oh.first = lap->off_heap; + (*func)(&oh, arg); + erts_debug_save_accessed_literal_area(lap); + } + fa.func = func; + fa.arg = arg; + erts_debug_later_op_foreach(later_release_literal_area, + debug_later_cleanup_literal_area_off_heap, + (void *) &fa); + erts_debug_later_op_foreach(complete_literal_area_switch, + debug_later_complete_literal_area_switch_off_heap, + (void *) &fa); +} + /* * Move code from current to old and null all export entries for the module */ diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index db74b06cc5..9bbcba6f3b 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -604,7 +604,12 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, erts_literal_area_t *litopt) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, + Uint *bsz, erts_literal_area_t *litopt +#ifdef ERTS_COPY_REGISTER_LOCATION + , char *file, int line +#endif + ) { char* hstart; Uint hsize; @@ -854,7 +859,11 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint case EXTERNAL_REF_SUBTAG: { ExternalThing *etp = (ExternalThing *) objp; +#if defined(ERTS_COPY_REGISTER_LOCATION) && defined(ERL_NODE_BOOKKEEP) + erts_ref_node_entry__(etp->node, 2, make_boxed(htop), file, line); +#else erts_ref_node_entry(etp->node, 2, make_boxed(htop)); +#endif } L_off_heap_node_container_common: { @@ -1326,8 +1335,13 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) * Copy object "obj" preserving sharing. * Second half: copy and restore the object. */ -Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, - Eterm** hpp, ErlOffHeap* off_heap) { +Uint copy_shared_perform_x(Eterm obj, Uint size, erts_shcopy_t *info, + Eterm** hpp, ErlOffHeap* off_heap +#ifdef ERTS_COPY_REGISTER_LOCATION + , char *file, int line +#endif + ) +{ Uint e; unsigned sz; Eterm* ptr; @@ -1393,7 +1407,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL +#ifdef ERTS_COPY_REGISTER_LOCATION + , file, line +#endif + ); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1461,7 +1479,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL +#ifdef ERTS_COPY_REGISTER_LOCATION + , file, line +#endif + ); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1660,7 +1682,12 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, case EXTERNAL_REF_SUBTAG: { ExternalThing *etp = (ExternalThing *) ptr; + +#if defined(ERTS_COPY_REGISTER_LOCATION) && defined(ERL_NODE_BOOKKEEP) + erts_ref_node_entry__(etp->node, 2, make_boxed(hp), file, line); +#else erts_ref_node_entry(etp->node, 2, make_boxed(hp)); +#endif } off_heap_node_container_common: { @@ -1823,8 +1850,12 @@ all_clean: * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, - ErlOffHeap* off_heap) +Eterm +copy_shallow_x(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap +#ifdef ERTS_COPY_REGISTER_LOCATION + , char *file, int line +#endif + ) { Eterm* tp = ptr; Eterm* hp = *hpp; @@ -1866,7 +1897,11 @@ Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, case EXTERNAL_REF_SUBTAG: { ExternalThing* etp = (ExternalThing *) (tp-1); +#if defined(ERTS_COPY_REGISTER_LOCATION) && defined(ERL_NODE_BOOKKEEP) + erts_ref_node_entry__(etp->node, 2, make_boxed(hp-1), file, line); +#else erts_ref_node_entry(etp->node, 2, make_boxed(hp-1)); +#endif } off_heap_common: { diff --git a/erts/emulator/beam/dist.c b/erts/emulator/beam/dist.c index 4537e3e569..eb9e749a08 100644 --- a/erts/emulator/beam/dist.c +++ b/erts/emulator/beam/dist.c @@ -1266,18 +1266,6 @@ erts_dsig_send_group_leader(ErtsDSigSendContext *ctx, Eterm leader, Eterm remote return dsig_send_ctl(ctx, ctl); } -struct dist_sequences { - ErlHeapFragment hfrag; - struct dist_sequences *parent; - struct dist_sequences *left; - struct dist_sequences *right; - char is_red; - - Uint64 seq_id; - int cnt; - Sint ctl_len; -}; - #define ERTS_RBT_PREFIX dist_seq #define ERTS_RBT_T DistSeqNode #define ERTS_RBT_KEY_T Uint @@ -1312,25 +1300,25 @@ struct dist_sequences { #include "erl_rbtree.h" -struct erts_dist_seq_tree_foreach_iter_arg { - int (*func)(ErtsDistExternal *, void *, Sint); +struct erts_debug_dist_seq_tree_foreach_iter_arg { + int (*func)(DistSeqNode *, void *, Sint); void *arg; }; static int -erts_dist_seq_tree_foreach_iter(DistSeqNode *seq, void *arg, Sint reds) +erts_debug_dist_seq_tree_foreach_iter(DistSeqNode *seq, void *arg, Sint reds) { - struct erts_dist_seq_tree_foreach_iter_arg *state = arg; - return state->func(erts_get_dist_ext(&seq->hfrag), state->arg, reds); + struct erts_debug_dist_seq_tree_foreach_iter_arg *state = arg; + return state->func(seq, state->arg, reds); } void -erts_dist_seq_tree_foreach(DistEntry *dep, int (*func)(ErtsDistExternal *, void *, Sint), void *arg) +erts_debug_dist_seq_tree_foreach(DistEntry *dep, int (*func)(DistSeqNode *, void *, Sint), void *arg) { - struct erts_dist_seq_tree_foreach_iter_arg state; + struct erts_debug_dist_seq_tree_foreach_iter_arg state; state.func = func; state.arg = arg; - dist_seq_rbt_foreach(dep->sequences, erts_dist_seq_tree_foreach_iter, &state); + dist_seq_rbt_foreach(dep->sequences, erts_debug_dist_seq_tree_foreach_iter, &state); } static int dist_seq_cleanup(DistSeqNode *seq, void *unused, Sint reds) diff --git a/erts/emulator/beam/dist.h b/erts/emulator/beam/dist.h index 067028634b..a33fb7efcf 100644 --- a/erts/emulator/beam/dist.h +++ b/erts/emulator/beam/dist.h @@ -274,6 +274,18 @@ typedef struct erts_dsig_send_context { typedef struct dist_sequences DistSeqNode; +struct dist_sequences { + ErlHeapFragment hfrag; + struct dist_sequences *parent; + struct dist_sequences *left; + struct dist_sequences *right; + char is_red; + + Uint64 seq_id; + int cnt; + Sint ctl_len; +}; + /* * erts_dsig_send_* return values. */ @@ -306,9 +318,9 @@ extern Uint erts_dist_cache_size(void); extern Sint erts_abort_connection_rwunlock(DistEntry *dep); -extern void erts_dist_seq_tree_foreach( +extern void erts_debug_dist_seq_tree_foreach( DistEntry *dep, - int (*func)(ErtsDistExternal *, void*, Sint), void *args); + int (*func)(DistSeqNode *, void*, Sint), void *args); extern int erts_dsig_prepare(ErtsDSigSendContext *, DistEntry*, diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 92e5069c71..21941ba96e 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -149,6 +149,7 @@ type MODULE_TABLE LONG_LIVED CODE module_tab type TAINT LONG_LIVED CODE taint_list type MODULE_REFS STANDARD CODE module_refs type NC_TMP TEMPORARY SYSTEM nc_tmp +type NC_STD STANDARD SYSTEM nc_std type TMP TEMPORARY SYSTEM tmp type UNDEF SYSTEM SYSTEM undefined type DCACHE STANDARD SYSTEM dcache diff --git a/erts/emulator/beam/erl_bif_persistent.c b/erts/emulator/beam/erl_bif_persistent.c index f38e0cc5cb..8662a9e33a 100644 --- a/erts/emulator/beam/erl_bif_persistent.c +++ b/erts/emulator/beam/erl_bif_persistent.c @@ -1234,3 +1234,102 @@ next_to_delete(void) erts_mtx_unlock(&delete_queue_mtx); return table; } + +/* + * test/debug functionality follow... + */ + +static Uint accessed_literal_areas_size; +static Uint accessed_no_literal_areas; +static ErtsLiteralArea **accessed_literal_areas; + +int +erts_debug_have_accessed_literal_area(ErtsLiteralArea *lap) +{ + Uint i; + for (i = 0; i < accessed_no_literal_areas; i++) { + if (accessed_literal_areas[i] == lap) + return !0; + } + return 0; +} + +void +erts_debug_save_accessed_literal_area(ErtsLiteralArea *lap) +{ + if (accessed_no_literal_areas == accessed_literal_areas_size) { + accessed_literal_areas_size += 10; + accessed_literal_areas = erts_realloc(ERTS_ALC_T_TMP, + accessed_literal_areas, + (sizeof(ErtsLiteralArea *) + *accessed_literal_areas_size)); + } + accessed_literal_areas[accessed_no_literal_areas++] = lap; +} + +static void debug_foreach_off_heap(HashTable *tbl, void (*func)(ErlOffHeap *, void *), void *arg) +{ + int i; + + for (i = 0; i < tbl->allocated; i++) { + Eterm term = tbl->term[i]; + if (is_tuple_arity(term, 2)) { + ErtsLiteralArea *lap = term_to_area(term); + ErlOffHeap oh; + if (!erts_debug_have_accessed_literal_area(lap)) { + ERTS_INIT_OFF_HEAP(&oh); + oh.first = lap->off_heap; + (*func)(&oh, arg); + erts_debug_save_accessed_literal_area(lap); + } + } + } +} + +struct debug_la_oh { + void (*func)(ErlOffHeap *, void *); + void *arg; +}; + +static void debug_handle_table(void *vfap, + ErtsThrPrgrVal val, + void *vtbl) +{ + struct debug_la_oh *fap = vfap; + HashTable *tbl = vtbl; + debug_foreach_off_heap(tbl, fap->func, fap->arg); +} + +void +erts_debug_foreach_persistent_term_off_heap(void (*func)(ErlOffHeap *, void *), void *arg) +{ + HashTable *tbl; + struct debug_la_oh fa; + accessed_no_literal_areas = 0; + accessed_literal_areas_size = 10; + accessed_literal_areas = erts_alloc(ERTS_ALC_T_TMP, + (sizeof(ErtsLiteralArea *) + * accessed_literal_areas_size)); + + tbl = (HashTable *) erts_atomic_read_nob(&the_hash_table); + debug_foreach_off_heap(tbl, func, arg); + erts_mtx_lock(&delete_queue_mtx); + for (tbl = delete_queue_head; tbl; tbl = tbl->delete_next) + debug_foreach_off_heap(tbl, func, arg); + erts_mtx_unlock(&delete_queue_mtx); + fa.func = func; + fa.arg = arg; + erts_debug_later_op_foreach(table_updater, + debug_handle_table, + (void *) &fa); + erts_debug_later_op_foreach(table_deleter, + debug_handle_table, + (void *) &fa); + erts_debug_foreach_release_literal_area_off_heap(func, arg); + + erts_free(ERTS_ALC_T_TMP, accessed_literal_areas); + accessed_no_literal_areas = 0; + accessed_literal_areas_size = 0; + accessed_literal_areas = NULL; +} + diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index d24f30f126..3c74ef493b 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -4415,7 +4415,7 @@ void db_info(fmtfn_t to, void *to_arg, int show) /* Called by break handler * pdbi.to_arg = to_arg; pdbi.show = show; - erts_db_foreach_table(db_info_print, &pdbi); + erts_db_foreach_table(db_info_print, &pdbi, !0); } Uint @@ -4428,7 +4428,7 @@ erts_get_ets_misc_mem_size(void) /* SMP Note: May only be used when system is locked */ void -erts_db_foreach_table(void (*func)(DbTable *, void *), void *arg) +erts_db_foreach_table(void (*func)(DbTable *, void *), void *arg, int alive_only) { int ix; @@ -4440,7 +4440,7 @@ erts_db_foreach_table(void (*func)(DbTable *, void *), void *arg) if (first) { DbTable *tb = first; do { - if (is_table_alive(tb)) + if (!alive_only || is_table_alive(tb)) (*func)(tb, arg); tb = tb->common.all.next; } while (tb != first); @@ -4457,6 +4457,15 @@ erts_db_foreach_offheap(DbTable *tb, tb->common.meth->db_foreach_offheap(tb, func, arg); } +void +erts_db_foreach_thr_prgr_offheap(void (*func)(ErlOffHeap *, void *), + void *arg) +{ + erts_db_foreach_thr_prgr_offheap_hash(func, arg); + erts_db_foreach_thr_prgr_offheap_tree(func, arg); + erts_db_foreach_thr_prgr_offheap_catree(func, arg); +} + /* retrieve max number of ets tables */ Uint erts_db_get_max_tabs() diff --git a/erts/emulator/beam/erl_db.h b/erts/emulator/beam/erl_db.h index b3dc1b9ba3..c604744687 100644 --- a/erts/emulator/beam/erl_db.h +++ b/erts/emulator/beam/erl_db.h @@ -114,10 +114,12 @@ void init_db(ErtsDbSpinCount); int erts_db_process_exiting(Process *, ErtsProcLocks, void **); int erts_db_execute_free_fixation(Process*, DbFixation*); void db_info(fmtfn_t, void *, int); -void erts_db_foreach_table(void (*)(DbTable *, void *), void *); +void erts_db_foreach_table(void (*)(DbTable *, void *), void *, int); void erts_db_foreach_offheap(DbTable *, void (*func)(ErlOffHeap *, void *), void *); +void erts_db_foreach_thr_prgr_offheap(void (*func)(ErlOffHeap *, void *), + void *); extern int erts_ets_rwmtx_spin_count; extern int user_requested_db_max_tabs; /* set in erl_init */ diff --git a/erts/emulator/beam/erl_db_catree.c b/erts/emulator/beam/erl_db_catree.c index fed4b44a9b..8a89eb72df 100644 --- a/erts/emulator/beam/erl_db_catree.c +++ b/erts/emulator/beam/erl_db_catree.c @@ -2395,6 +2395,34 @@ void db_calc_stats_catree(DbTableCATree* tb, DbCATreeStats* stats) } while (depth > 0); } +struct debug_catree_fa { + void (*func)(ErlOffHeap *, void *); + void *arg; +}; + +static void debug_free_route_node(void *vfap, ErtsThrPrgrVal val, void *vnp) +{ + DbTableCATreeNode *np = vnp; + if (np->u.route.key.oh) { + struct debug_catree_fa *fap = vfap; + ErlOffHeap oh; + ERTS_INIT_OFF_HEAP(&oh); + oh.first = np->u.route.key.oh; + (*fap->func)(&oh, fap->arg); + } +} + +void +erts_db_foreach_thr_prgr_offheap_catree(void (*func)(ErlOffHeap *, void *), + void *arg) +{ + struct debug_catree_fa fa; + fa.func = func; + fa.arg = arg; + erts_debug_later_op_foreach(do_free_route_node, debug_free_route_node, &fa); +} + + #ifdef HARDDEBUG /* diff --git a/erts/emulator/beam/erl_db_catree.h b/erts/emulator/beam/erl_db_catree.h index c2c884eee3..2ede85e04e 100644 --- a/erts/emulator/beam/erl_db_catree.h +++ b/erts/emulator/beam/erl_db_catree.h @@ -132,6 +132,9 @@ typedef struct { Uint max_depth; } DbCATreeStats; void db_calc_stats_catree(DbTableCATree*, DbCATreeStats*); +void +erts_db_foreach_thr_prgr_offheap_catree(void (*func)(ErlOffHeap *, void *), + void *arg); #endif /* _DB_CATREE_H */ diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index ceaccf7e44..4904d3fa42 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -3262,6 +3262,12 @@ Eterm erts_ets_hash_sizeof_ext_segtab(void) return make_small(((SIZEOF_EXT_SEGTAB(0)-1) / sizeof(UWord)) + 1); } +void +erts_db_foreach_thr_prgr_offheap_hash(void (*func)(ErlOffHeap *, void *), + void *arg) +{ +} + #ifdef ERTS_ENABLE_LOCK_COUNT void erts_lcnt_enable_db_hash_lock_count(DbTableHash *tb, int enable) { int i; diff --git a/erts/emulator/beam/erl_db_hash.h b/erts/emulator/beam/erl_db_hash.h index eae5537ba4..9759d8b466 100644 --- a/erts/emulator/beam/erl_db_hash.h +++ b/erts/emulator/beam/erl_db_hash.h @@ -110,6 +110,9 @@ typedef struct { void db_calc_stats_hash(DbTableHash* tb, DbHashStats*); Eterm erts_ets_hash_sizeof_ext_segtab(void); +void +erts_db_foreach_thr_prgr_offheap_hash(void (*func)(ErlOffHeap *, void *), + void *arg); #ifdef ERTS_ENABLE_LOCK_COUNT void erts_lcnt_enable_db_hash_lock_count(DbTableHash *tb, int enable); diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index 492ea81b63..19b94b0634 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -3936,6 +3936,12 @@ static int doit_select_replace(DbTableCommon *tb, TreeDbTerm **this, return 1; } +void +erts_db_foreach_thr_prgr_offheap_tree(void (*func)(ErlOffHeap *, void *), + void *arg) +{ +} + #ifdef TREE_DEBUG static void do_dump_tree2(DbTableCommon* tb, int to, void *to_arg, int show, TreeDbTerm *t, int offset) diff --git a/erts/emulator/beam/erl_db_tree.h b/erts/emulator/beam/erl_db_tree.h index 54da2a6bc1..06e0005801 100644 --- a/erts/emulator/beam/erl_db_tree.h +++ b/erts/emulator/beam/erl_db_tree.h @@ -53,4 +53,8 @@ void db_initialize_tree(void); int db_create_tree(Process *p, DbTable *tbl); +void +erts_db_foreach_thr_prgr_offheap_tree(void (*func)(ErlOffHeap *, void *), + void *arg); + #endif /* _DB_TREE_H */ diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 7ab8034606..13b1f8ab4d 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2887,7 +2887,7 @@ sweep_off_heap(Process *p, int fullsweep) if (is_external_header(((struct erl_off_heap_header*) boxed_val(ptr->thing_word))->thing_word)) erts_node_bookkeep(((ExternalThing*)ptr)->node, make_boxed(&ptr->thing_word), - ERL_NODE_DEC); + ERL_NODE_DEC, __FILE__, __LINE__); *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); switch (ptr->thing_word) { @@ -2918,7 +2918,7 @@ sweep_off_heap(Process *p, int fullsweep) if (is_external_header(ptr->thing_word)) { erts_node_bookkeep(((ExternalThing*)ptr)->node, make_boxed(&ptr->thing_word), - ERL_NODE_INC); + ERL_NODE_INC, __FILE__, __LINE__); } prev = &ptr->next; ptr = ptr->next; @@ -3089,9 +3089,11 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) if (is_external_header(oh->thing_word)) { erts_node_bookkeep(((ExternalThing*)oh)->node, - make_boxed(((Eterm*)oh)-offs), ERL_NODE_DEC); + make_boxed(((Eterm*)oh)-offs), + ERL_NODE_DEC, __FILE__, __LINE__); erts_node_bookkeep(((ExternalThing*)oh)->node, - make_boxed((Eterm*)oh), ERL_NODE_INC); + make_boxed((Eterm*)oh), ERL_NODE_INC, + __FILE__, __LINE__); } if (ErtsInArea(oh->next, area, area_size)) { diff --git a/erts/emulator/beam/erl_monitor_link.c b/erts/emulator/beam/erl_monitor_link.c index 1c6b4afaa3..43028be39d 100644 --- a/erts/emulator/beam/erl_monitor_link.c +++ b/erts/emulator/beam/erl_monitor_link.c @@ -671,6 +671,24 @@ erts_monitor_tree_foreach(ErtsMonitor *root, arg); } +void +erts_debug_monitor_tree_destroying_foreach(ErtsMonitor *root, + ErtsMonitorFunc func, + void *arg, + void *vysp) +{ + void *tmp_vysp = erts_alloc(ERTS_ALC_T_ML_YIELD_STATE, + sizeof(ErtsMonLnkYieldState)); + Sint reds; + sys_memcpy(tmp_vysp, tmp_vysp, sizeof(ErtsMonLnkYieldState)); + do { + reds = ml_rbt_foreach_yielding((ErtsMonLnkNode *) root, + (ErtsMonLnkNodeFunc) func, + arg, &tmp_vysp, (Sint) INT_MAX); + } while (reds <= 0); + ERTS_ML_ASSERT(!tmp_vysp); +} + int erts_monitor_tree_foreach_yielding(ErtsMonitor *root, ErtsMonitorFunc func, @@ -716,6 +734,19 @@ erts_monitor_list_foreach(ErtsMonitor *list, arg, &ystate, (Sint) INT_MAX)); } +void +erts_debug_monitor_list_destroying_foreach(ErtsMonitor *list, + ErtsMonitorFunc func, + void *arg, + void *vysp) +{ + void *tmp_vysp = vysp; + while (!ml_dl_list_foreach_yielding((ErtsMonLnkNode *) list, + (int (*)(ErtsMonLnkNode *, void *, Sint)) func, + arg, &tmp_vysp, (Sint) INT_MAX)); + ERTS_ML_ASSERT(!tmp_vysp); +} + int erts_monitor_list_foreach_yielding(ErtsMonitor *list, ErtsMonitorFunc func, @@ -1080,6 +1111,24 @@ erts_link_tree_foreach(ErtsLink *root, } +void +erts_debug_link_tree_destroying_foreach(ErtsLink *root, + ErtsLinkFunc func, + void *arg, + void *vysp) +{ + void *tmp_vysp = erts_alloc(ERTS_ALC_T_ML_YIELD_STATE, + sizeof(ErtsMonLnkYieldState)); + Sint reds; + sys_memcpy(tmp_vysp, vysp, sizeof(ErtsMonLnkYieldState)); + do { + reds = ml_rbt_foreach_yielding((ErtsMonLnkNode *) root, + (ErtsMonLnkNodeFunc) func, + arg, &tmp_vysp, (Sint) INT_MAX); + } while (reds <= 0); + ERTS_ML_ASSERT(!tmp_vysp); +} + int erts_link_tree_foreach_yielding(ErtsLink *root, ErtsLinkFunc func, diff --git a/erts/emulator/beam/erl_monitor_link.h b/erts/emulator/beam/erl_monitor_link.h index eff861fce8..86be400c09 100644 --- a/erts/emulator/beam/erl_monitor_link.h +++ b/erts/emulator/beam/erl_monitor_link.h @@ -1509,6 +1509,17 @@ ERTS_GLB_INLINE ErtsMonitorSuspend *erts_monitor_suspend(ErtsMonitor *mon) #endif +void +erts_debug_monitor_tree_destroying_foreach(ErtsMonitor *root, + ErtsMonitorFunc func, + void *arg, + void *vysp); +void +erts_debug_monitor_list_destroying_foreach(ErtsMonitor *list, + ErtsMonitorFunc func, + void *arg, + void *vysp); + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *\ * Link Operations * \* */ @@ -2365,4 +2376,10 @@ erts_link_dist_delete(ErtsLink *lnk) #endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ +void +erts_debug_link_tree_destroying_foreach(ErtsLink *root, + ErtsLinkFunc func, + void *arg, + void *vysp); + #endif /* ERL_MONITOR_LINK_H__ */ diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 4eb6c3e214..50e9812534 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -932,7 +932,8 @@ static void try_delete_node(void *venp) * * If refc > 0, the entry is in use. Keep the entry. */ - erts_node_bookkeep(enp, THE_NON_VALUE, ERL_NODE_DEC); + erts_node_bookkeep(enp, THE_NON_VALUE, ERL_NODE_DEC, + __FILE__, __LINE__); refc = erts_refc_dectest(&enp->refc, -1); if (refc == -1) (void) hash_erase(&erts_node_table, (void *) enp); @@ -1164,6 +1165,12 @@ void erts_lcnt_update_distribution_locks(int enable) { * can damage the real-time properties of the system. * \* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#ifdef ERL_NODE_BOOKKEEP +#define ERTS_DBG_NC_ALLOC_TYPE ERTS_ALC_T_NC_STD +#else +#define ERTS_DBG_NC_ALLOC_TYPE ERTS_ALC_T_NC_TMP +#endif + #include "erl_db.h" #undef INIT_AM @@ -1188,10 +1195,12 @@ static Eterm AM_delayed_delete_timer; static Eterm AM_thread_progress_delete_timer; static Eterm AM_sequence; static Eterm AM_signal; +static Eterm AM_persistent_term; static void setup_reference_table(void); static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp); static void delete_reference_table(void); +static void clear_system(void); #undef ERTS_MAX__ #define ERTS_MAX__(A, B) ((A) > (B) ? (A) : (B)) @@ -1244,11 +1253,11 @@ typedef struct inserted_bin_ { Binary *bin_val; } InsertedBin; -static ReferredNode *referred_nodes; +static ReferredNode *referred_nodes = NULL; static int no_referred_nodes; -static ReferredDist *referred_dists; +static ReferredDist *referred_dists = NULL; static int no_referred_dists; -static InsertedBin *inserted_bins; +static InsertedBin *inserted_bins = NULL; Eterm erts_get_node_and_dist_references(struct process *proc) @@ -1284,9 +1293,15 @@ erts_get_node_and_dist_references(struct process *proc) INIT_AM(thread_progress_delete_timer); INIT_AM(signal); INIT_AM(sequence); + INIT_AM(persistent_term); references_atoms_need_init = 0; } +#ifdef ERL_NODE_BOOKKEEP + if (referred_nodes || referred_dists || inserted_bins) + delete_reference_table(); +#endif + setup_reference_table(); /* Get term size */ @@ -1304,7 +1319,11 @@ erts_get_node_and_dist_references(struct process *proc) ASSERT(endp == hp); +#ifndef ERL_NODE_BOOKKEEP delete_reference_table(); +#endif + + clear_system(); erts_thr_progress_unblock(); erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN); @@ -1339,7 +1358,7 @@ insert_dist_referrer(ReferredDist *referred_dist, break; if(!drp) { - drp = (DistReferrer *) erts_alloc(ERTS_ALC_T_NC_TMP, + drp = (DistReferrer *) erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, sizeof(DistReferrer)); drp->next = referred_dist->referrers; referred_dist->referrers = drp; @@ -1402,7 +1421,7 @@ insert_node_referrer(ReferredNode *referred_node, int type, Eterm id) break; if(!nrp) { - nrp = (NodeReferrer *) erts_alloc(ERTS_ALC_T_NC_TMP, + nrp = (NodeReferrer *) erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, sizeof(NodeReferrer)); nrp->next = referred_node->referrers; ERTS_INIT_OFF_HEAP(&nrp->off_heap); @@ -1516,7 +1535,8 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) erts_match_prog_foreach_offheap((Binary *) u.mref->mb, insert_offheap2, (void *) &a); - nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin)); + nib = erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, + sizeof(InsertedBin)); nib->bin_val = (Binary *) u.mref->mb; nib->next = inserted_bins; inserted_bins = nib; @@ -1574,18 +1594,6 @@ static int clear_visited_monitor(ErtsMonitor *mon, void *p, Sint reds) } static void -insert_p_monitors(ErtsPTabElementCommon *p) -{ - Eterm id = p->id; - erts_monitor_tree_foreach(p->u.alive.monitors, - insert_monitor, - (void *) &id); - erts_monitor_list_foreach(p->u.alive.lt_monitors, - insert_monitor, - (void *) &id); -} - -static void insert_dist_monitors(DistEntry *dep) { if (dep->mld) { @@ -1600,8 +1608,10 @@ insert_dist_monitors(DistEntry *dep) static int -insert_sequence(ErtsDistExternal *edep, void *arg, Sint reds) +insert_sequence(DistSeqNode *seq, void *arg, Sint reds) { + ErtsDistExternal *edep = erts_get_dist_ext(&seq->hfrag); + insert_offheap(&seq->hfrag.off_heap, SEQUENCE_REF, *(Eterm*)arg); insert_dist_entry(edep->dep, SEQUENCE_REF, *(Eterm*)arg, 0); return 1; } @@ -1609,18 +1619,7 @@ insert_sequence(ErtsDistExternal *edep, void *arg, Sint reds) static void insert_dist_sequences(DistEntry *dep) { - erts_dist_seq_tree_foreach(dep, insert_sequence, (void *) &dep->sysname); -} - -static void -clear_visited_p_monitors(ErtsPTabElementCommon *p) -{ - erts_monitor_tree_foreach(p->u.alive.monitors, - clear_visited_monitor, - NULL); - erts_monitor_list_foreach(p->u.alive.lt_monitors, - clear_visited_monitor, - NULL); + erts_debug_dist_seq_tree_foreach(dep, insert_sequence, (void *) &dep->sysname); } static void @@ -1667,13 +1666,6 @@ static int clear_visited_link(ErtsLink *lnk, void *p, Sint reds) } static void -insert_p_links(ErtsPTabElementCommon *p) -{ - Eterm id = p->id; - erts_link_tree_foreach(p->u.alive.links, insert_link, (void *) &id); -} - -static void insert_dist_links(DistEntry *dep) { if (dep->mld) @@ -1683,14 +1675,6 @@ insert_dist_links(DistEntry *dep) } static void -clear_visited_p_links(ErtsPTabElementCommon *p) -{ - erts_link_tree_foreach(p->u.alive.links, - clear_visited_link, - NULL); -} - -static void clear_visited_dist_links(DistEntry *dep) { if (dep->mld) @@ -1885,15 +1869,11 @@ insert_process(Process *proc) insert_sig_ext, (void *) proc); - /* If the process is FREE, the proc->common field has been - re-used by the ptab delete, so we cannot trust it. */ - if (!(erts_atomic32_read_nob(&proc->state) & ERTS_PSFLG_FREE)) { - /* Insert links */ - insert_p_links(&proc->common); - - /* Insert monitors */ - insert_p_monitors(&proc->common); - } + /* Insert monitors and links... */ + erts_debug_proc_monitor_link_foreach(proc, + insert_monitor, + insert_link, + (void *) &proc->common.id); { DistEntry *dep = ERTS_PROC_GET_DIST_ENTRY(proc); @@ -1906,6 +1886,12 @@ insert_process(Process *proc) } static void +insert_process2(Process *proc, void *arg) +{ + insert_process(proc); +} + +static void insert_dist_suspended_procs(DistEntry *dep) { ErtsProcList *plist = erts_proclist_peek_first(dep->suspended); @@ -1916,16 +1902,47 @@ insert_dist_suspended_procs(DistEntry *dep) } } +static void clear_process(Process *proc); + +static void +clear_dist_suspended_procs(DistEntry *dep) +{ + ErtsProcList *plist = erts_proclist_peek_first(dep->suspended); + while (plist) { + if (is_not_immed(plist->u.pid)) + clear_process(plist->u.p); + plist = erts_proclist_peek_next(dep->suspended, plist); + } +} + +static void +insert_persistent_term(ErlOffHeap *ohp, void *arg) +{ + Eterm heap[3]; + insert_offheap(ohp, SYSTEM_REF, + TUPLE2(&heap[0], AM_system, AM_persistent_term)); +} + +static void +insert_ets_offheap_thr_prgr(ErlOffHeap *ohp, void *arg) +{ + Eterm heap[3]; + insert_offheap(ohp, ETS_REF, + TUPLE2(&heap[0], AM_system, AM_ets)); +} + #ifdef ERL_NODE_BOOKKEEP void -erts_node_bookkeep(ErlNode *np, Eterm term, int what) +erts_node_bookkeep(ErlNode *np, Eterm term, int what, char *f, int l) { - erts_aint_t slot = (erts_atomic_inc_read_nob(&np->slot) - 1) % 1024; + erts_aint_t slot = (erts_atomic_inc_read_nob(&np->slot) - 1) % ERTS_BOOKKEEP_SIZE; ErtsSchedulerData *esdp = erts_get_scheduler_data(); Eterm who = THE_NON_VALUE; ASSERT(np); np->books[slot].what = what; np->books[slot].term = term; + np->books[slot].file = f; + np->books[slot].line = l; if (esdp->current_process) { who = esdp->current_process->common.id; } else if (esdp->current_port) { @@ -1946,14 +1963,14 @@ setup_reference_table(void) inserted_bins = NULL; hash_get_info(&hi, &erts_node_table); - referred_nodes = erts_alloc(ERTS_ALC_T_NC_TMP, + referred_nodes = erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, hi.objs*sizeof(ReferredNode)); no_referred_nodes = 0; hash_foreach(&erts_node_table, init_referred_node, NULL); ASSERT(no_referred_nodes == hi.objs); hash_get_info(&hi, &erts_dist_table); - referred_dists = erts_alloc(ERTS_ALC_T_NC_TMP, + referred_dists = erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, hi.objs*sizeof(ReferredDist)); no_referred_dists = 0; hash_foreach(&erts_dist_table, init_referred_dist, NULL); @@ -1990,8 +2007,9 @@ setup_reference_table(void) if (proc) insert_process(proc); } + erts_debug_free_process_foreach(insert_process2, NULL); - erts_foreach_sys_msg_in_q(insert_sys_msg); + erts_debug_foreach_sys_msg_in_q(insert_sys_msg); /* Insert all ports */ max = erts_ptab_max(&erts_port); @@ -2008,10 +2026,18 @@ setup_reference_table(void) if (state & ERTS_PORT_SFLGS_DEAD) continue; - /* Insert links */ - insert_p_links(&prt->common); - /* Insert monitors */ - insert_p_monitors(&prt->common); + /* Insert links */ + erts_link_tree_foreach(ERTS_P_LINKS(prt), + insert_link, + (void *) &prt->common.id); + /* Insert monitors */ + erts_monitor_tree_foreach(ERTS_P_MONITORS(prt), + insert_monitor, + (void *) &prt->common.id); + /* Insert local target monitors */ + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(prt), + insert_monitor, + (void *) &prt->common.id); /* Insert port data */ ohp = erts_port_data_offheap(prt); if (ohp) @@ -2091,11 +2117,16 @@ setup_reference_table(void) } /* Insert all ets tables */ - erts_db_foreach_table(insert_ets_table, NULL); + erts_db_foreach_table(insert_ets_table, NULL, 0); + erts_db_foreach_thr_prgr_offheap(insert_ets_offheap_thr_prgr, NULL); /* Insert all bif timers */ erts_debug_bif_timer_foreach(insert_bif_timer, NULL); + /* Insert persistent term storage */ + erts_debug_foreach_persistent_term_off_heap(insert_persistent_term, + NULL); + /* Insert node table (references to dist) */ hash_foreach(&erts_node_table, insert_erl_node, NULL); } @@ -2348,8 +2379,7 @@ static void noop_sig_ext(ErtsDistExternal *ext, void *arg) static void delete_reference_table(void) { - DistEntry *dep; - int i, max; + int i; for(i = 0; i < no_referred_nodes; i++) { NodeReferrer *nrp; NodeReferrer *tnrp; @@ -2358,11 +2388,13 @@ delete_reference_table(void) erts_cleanup_offheap(&nrp->off_heap); tnrp = nrp; nrp = nrp->next; - erts_free(ERTS_ALC_T_NC_TMP, (void *) tnrp); + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *) tnrp); } } - if (referred_nodes) - erts_free(ERTS_ALC_T_NC_TMP, (void *) referred_nodes); + if (referred_nodes) { + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *) referred_nodes); + referred_nodes = NULL; + } for(i = 0; i < no_referred_dists; i++) { DistReferrer *drp; @@ -2371,34 +2403,57 @@ delete_reference_table(void) while(drp) { tdrp = drp; drp = drp->next; - erts_free(ERTS_ALC_T_NC_TMP, (void *) tdrp); + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *) tdrp); } } - if (referred_dists) - erts_free(ERTS_ALC_T_NC_TMP, (void *) referred_dists); + if (referred_dists) { + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *) referred_dists); + referred_dists = NULL; + } while(inserted_bins) { InsertedBin *ib = inserted_bins; inserted_bins = inserted_bins->next; - erts_free(ERTS_ALC_T_NC_TMP, (void *)ib); + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *)ib); } +} + +static void clear_process(Process *proc) +{ + erts_proc_sig_debug_foreach_sig(proc, + noop_sig_msg, + noop_sig_offheap, + clear_visited_monitor, + clear_visited_link, + noop_sig_ext, + (void *) proc); + + + /* Clear monitors and links... */ + erts_debug_proc_monitor_link_foreach(proc, + clear_visited_monitor, + clear_visited_link, + (void *) &proc->common.id); +} + +static void clear_process2(Process *proc, void *arg) +{ + clear_process(proc); +} - /* Cleanup... */ +static void +clear_system(void) +{ + DistEntry *dep; + int i, max; + /* Clear... */ max = erts_ptab_max(&erts_proc); for (i = 0; i < max; i++) { Process *proc = erts_pix2proc(i); - if (proc) { - clear_visited_p_links(&proc->common); - clear_visited_p_monitors(&proc->common); - erts_proc_sig_debug_foreach_sig(proc, - noop_sig_msg, - noop_sig_offheap, - clear_visited_monitor, - clear_visited_link, - noop_sig_ext, - (void *) proc); - } + if (proc) + clear_process(proc); } + erts_debug_free_process_foreach(clear_process2, NULL); max = erts_ptab_max(&erts_port); for (i = 0; i < max; i++) { @@ -2413,28 +2468,42 @@ delete_reference_table(void) if (state & ERTS_PORT_SFLGS_DEAD) continue; - clear_visited_p_links(&prt->common); - clear_visited_p_monitors(&prt->common); + /* Clear links */ + erts_link_tree_foreach(ERTS_P_LINKS(prt), + clear_visited_link, + (void *) &prt->common.id); + /* Clear monitors */ + erts_monitor_tree_foreach(ERTS_P_MONITORS(prt), + clear_visited_monitor, + (void *) &prt->common.id); + /* Clear local target monitors */ + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(prt), + clear_visited_monitor, + (void *) &prt->common.id); } for(dep = erts_visible_dist_entries; dep; dep = dep->next) { clear_visited_dist_links(dep); clear_visited_dist_monitors(dep); + clear_dist_suspended_procs(dep); } for(dep = erts_hidden_dist_entries; dep; dep = dep->next) { clear_visited_dist_links(dep); clear_visited_dist_monitors(dep); + clear_dist_suspended_procs(dep); } for(dep = erts_pending_dist_entries; dep; dep = dep->next) { clear_visited_dist_links(dep); clear_visited_dist_monitors(dep); + clear_dist_suspended_procs(dep); } for(dep = erts_not_connected_dist_entries; dep; dep = dep->next) { clear_visited_dist_links(dep); clear_visited_dist_monitors(dep); + clear_dist_suspended_procs(dep); } } diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index aa8af12555..ffaafbbbea 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -187,6 +187,7 @@ set pagination off set $i = 0 set $node = referred_nodes[$node_ix].node while $i < $node->slot.counter + printf "%s:%d ", $node->books[$i].file, $node->books[$i].line printf "%p: ", $node->books[$i].term etp-1 $node->books[$i].who printf " " @@ -211,8 +212,12 @@ lists:usort(lists:filter(fun({V,N}) -> N /= 0 end, maps:to_list(Accs))). struct erl_node_bookkeeping { Eterm who; Eterm term; + char *file; + int line; enum { ERL_NODE_INC, ERL_NODE_DEC } what; }; + +#define ERTS_BOOKKEEP_SIZE (1024) #endif typedef struct erl_node_ { @@ -222,7 +227,7 @@ typedef struct erl_node_ { Uint32 creation; /* Creation */ DistEntry *dist_entry; /* Corresponding dist entry */ #ifdef ERL_NODE_BOOKKEEP - struct erl_node_bookkeeping books[1024]; + struct erl_node_bookkeeping books[ERTS_BOOKKEEP_SIZE]; erts_atomic_t slot; #endif } ErlNode; @@ -276,14 +281,21 @@ Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*, Uint32 conn_id); Eterm erts_make_dhandle(Process *c_p, DistEntry*, Uint32 conn_id); ERTS_GLB_INLINE void erts_init_node_entry(ErlNode *np, erts_aint_t val); +#ifdef ERL_NODE_BOOKKEEP +#define erts_ref_node_entry(NP, MIN, T) erts_ref_node_entry__((NP), (MIN), (T), __FILE__, __LINE__) +#define erts_deref_node_entry(NP, T) erts_deref_node_entry__((NP), (T), __FILE__, __LINE__) +ERTS_GLB_INLINE erts_aint_t erts_ref_node_entry__(ErlNode *np, int min_val, Eterm term, char *file, int line); +ERTS_GLB_INLINE void erts_deref_node_entry__(ErlNode *np, Eterm term, char *file, int line); +#else ERTS_GLB_INLINE erts_aint_t erts_ref_node_entry(ErlNode *np, int min_val, Eterm term); ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np, Eterm term); +#endif ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_runlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_rwlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_rwunlock(DistEntry *dep); #ifdef ERL_NODE_BOOKKEEP -void erts_node_bookkeep(ErlNode *, Eterm , int); +void erts_node_bookkeep(ErlNode *, Eterm , int, char *file, int line); #else #define erts_node_bookkeep(...) #endif @@ -296,21 +308,40 @@ erts_init_node_entry(ErlNode *np, erts_aint_t val) erts_refc_init(&np->refc, val); } +#ifdef ERL_NODE_BOOKKEEP + +ERTS_GLB_INLINE erts_aint_t +erts_ref_node_entry__(ErlNode *np, int min_val, Eterm term, char *file, int line) +{ + erts_node_bookkeep(np, term, ERL_NODE_INC, file, line); + return erts_refc_inctest(&np->refc, min_val); +} + +ERTS_GLB_INLINE void +erts_deref_node_entry__(ErlNode *np, Eterm term, char *file, int line) +{ + erts_node_bookkeep(np, term, ERL_NODE_DEC, file, line); + if (erts_refc_dectest(&np->refc, 0) == 0) + erts_schedule_delete_node(np); +} + +#else + ERTS_GLB_INLINE erts_aint_t erts_ref_node_entry(ErlNode *np, int min_val, Eterm term) { - erts_node_bookkeep(np, term, ERL_NODE_INC); return erts_refc_inctest(&np->refc, min_val); } ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np, Eterm term) { - erts_node_bookkeep(np, term, ERL_NODE_DEC); if (erts_refc_dectest(&np->refc, 0) == 0) erts_schedule_delete_node(np); } +#endif + ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep) { diff --git a/erts/emulator/beam/erl_proc_sig_queue.c b/erts/emulator/beam/erl_proc_sig_queue.c index fb900ca7ba..d5e0e3b218 100644 --- a/erts/emulator/beam/erl_proc_sig_queue.c +++ b/erts/emulator/beam/erl_proc_sig_queue.c @@ -4689,10 +4689,12 @@ erts_proc_sig_debug_foreach_sig(Process *c_p, case ERTS_SIG_Q_OP_MONITOR_DOWN: switch (type) { case ERTS_SIG_Q_TYPE_GEN_EXIT: - if (ERTS_SIG_IS_GEN_EXIT_EXTERNAL(sig)) - debug_foreach_sig_external(sig, ext_func, arg); - else + if (!ERTS_SIG_IS_GEN_EXIT_EXTERNAL(sig)) debug_foreach_sig_heap_frags(&sig->hfrag, oh_func, arg); + else { + oh_func(&sig->hfrag.off_heap, arg); + debug_foreach_sig_external(sig, ext_func, arg); + } break; case ERTS_LNK_TYPE_PORT: case ERTS_LNK_TYPE_PROC: diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index de0564292d..1c1ef1db84 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -13468,3 +13468,79 @@ erts_debug_later_op_foreach(void (*callback)(void*), } } } + +void +erts_debug_free_process_foreach(void (*func)(Process *, void *), void *arg) +{ + ErtsRunQueue *rq; + int ix, prio; + for (ix = 0; ix < erts_no_run_queues; ix++) { + rq = ERTS_RUNQ_IX(ix); + for (prio = PRIORITY_MAX; prio < PRIORITY_LOW; prio++) { + Process *p = rq->procs.prio[prio].first; + for (; p; p = p->next) { + if (ERTS_PSFLG_FREE & erts_atomic32_read_nob(&p->state)) + (*func)(p, arg); + } + } + } +} + +void +erts_debug_proc_monitor_link_foreach(Process *proc, + int (*monitor_func)(ErtsMonitor *, void *, Sint ), + int (*link_func)(ErtsLink *, void *, Sint ), + void *arg) +{ + if (!(erts_atomic32_read_nob(&proc->state) & ERTS_PSFLG_FREE)) { + /* For all links */ + erts_link_tree_foreach(ERTS_P_LINKS(proc), + link_func, + arg); + /* For all monitors */ + erts_monitor_tree_foreach(ERTS_P_MONITORS(proc), + monitor_func, + arg); + /* For all local target monitors */ + erts_monitor_list_foreach(ERTS_P_LT_MONITORS(proc), + monitor_func, + arg); + } + else { + struct continue_exit_state *ce_state = proc->u.terminate; + + /* For all links */ + if (ce_state->phase == ERTS_CONTINUE_EXIT_LINKS) + erts_debug_link_tree_destroying_foreach(ce_state->links, + link_func, + arg, + ce_state->yield_state); + else + erts_link_tree_foreach(ce_state->links, + link_func, + arg); + + /* For all monitors */ + if (ce_state->phase == ERTS_CONTINUE_EXIT_MONITORS) + erts_debug_monitor_tree_destroying_foreach(ce_state->monitors, + monitor_func, + arg, + ce_state->yield_state); + else + erts_monitor_tree_foreach(ce_state->monitors, + monitor_func, + arg); + + /* For all local target monitors */ + if (ce_state->phase == ERTS_CONTINUE_EXIT_LT_MONITORS) + erts_debug_monitor_list_destroying_foreach(ce_state->lt_monitors, + monitor_func, + arg, + ce_state->yield_state); + else + erts_monitor_list_foreach(ce_state->lt_monitors, + monitor_func, + arg); + + } +} diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index 745a2a482c..c0d7cfd13d 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -701,6 +701,13 @@ void erts_debug_later_op_foreach(void (*callback)(void*), void (*func)(void *, ErtsThrPrgrVal, void *), void *arg); +void +erts_debug_free_process_foreach(void (*func)(Process *, void *), void *arg); +void +erts_debug_proc_monitor_link_foreach(Process *proc, + int (*monitor_func)(ErtsMonitor *, void *, Sint ), + int (*link_func)(ErtsLink *, void *, Sint ), + void *arg); #ifdef ERTS_INCLUDE_SCHEDULER_INTERNALS diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 9c835ac357..f6f177887c 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2194,11 +2194,12 @@ sys_msg_dispatcher_wait(void *vwait_p) erts_mtx_unlock(&smq_mtx); } +static ErtsSysMsgQ *local_sys_message_queue = NULL; + static void * sys_msg_dispatcher_func(void *unused) { ErtsThrPrgrCallbacks callbacks; - ErtsSysMsgQ *local_sys_message_queue = NULL; ErtsThrPrgrData *tpd; int wait = 0; @@ -2206,6 +2207,8 @@ sys_msg_dispatcher_func(void *unused) erts_lc_set_thread_name("system message dispatcher"); #endif + local_sys_message_queue = NULL; + callbacks.arg = (void *) &wait; callbacks.wakeup = sys_msg_dispatcher_wakeup; callbacks.prepare_wait = sys_msg_dispatcher_prep_wait; @@ -2262,6 +2265,8 @@ sys_msg_dispatcher_func(void *unused) Process *proc = NULL; Port *port = NULL; + ASSERT(is_value(smqp->msg)); + if (erts_thr_progress_update(tpd)) erts_thr_progress_leader_update(tpd); @@ -2374,6 +2379,7 @@ sys_msg_dispatcher_func(void *unused) erts_fprintf(stderr, "dropped\n"); #endif } + smqp->msg = THE_NON_VALUE; } } @@ -2381,32 +2387,38 @@ sys_msg_dispatcher_func(void *unused) } void -erts_foreach_sys_msg_in_q(void (*func)(Eterm, - Eterm, - Eterm, - ErlHeapFragment *)) +erts_debug_foreach_sys_msg_in_q(void (*func)(Eterm, + Eterm, + Eterm, + ErlHeapFragment *)) { - ErtsSysMsgQ *sm; - erts_mtx_lock(&smq_mtx); - for (sm = sys_message_queue; sm; sm = sm->next) { - Eterm to; - switch (sm->type) { - case SYS_MSG_TYPE_SYSMON: - to = erts_get_system_monitor(); - break; - case SYS_MSG_TYPE_SYSPROF: - to = erts_get_system_profile(); - break; - case SYS_MSG_TYPE_ERRLGR: - to = erts_get_system_logger(); - break; - default: - to = NIL; - break; - } - (*func)(sm->from, to, sm->msg, sm->bp); + ErtsSysMsgQ *smq[] = {sys_message_queue, local_sys_message_queue}; + int i; + + ERTS_LC_ASSERT(erts_thr_progress_is_blocking()); + + for (i = 0; i < sizeof(smq)/sizeof(smq[0]); i++) { + ErtsSysMsgQ *sm; + for (sm = smq[i]; sm; sm = sm->next) { + Eterm to; + switch (sm->type) { + case SYS_MSG_TYPE_SYSMON: + to = erts_get_system_monitor(); + break; + case SYS_MSG_TYPE_SYSPROF: + to = erts_get_system_profile(); + break; + case SYS_MSG_TYPE_ERRLGR: + to = erts_get_system_logger(); + break; + default: + to = NIL; + break; + } + if (is_value(sm->msg)) + (*func)(sm->from, to, sm->msg, sm->bp); + } } - erts_mtx_unlock(&smq_mtx); } diff --git a/erts/emulator/beam/erl_trace.h b/erts/emulator/beam/erl_trace.h index b7844d1cb0..af38ef52db 100644 --- a/erts/emulator/beam/erl_trace.h +++ b/erts/emulator/beam/erl_trace.h @@ -90,10 +90,10 @@ int erts_is_tracer_valid(Process* p); void erts_check_my_tracer_proc(Process *); void erts_block_sys_msg_dispatcher(void); void erts_release_sys_msg_dispatcher(void); -void erts_foreach_sys_msg_in_q(void (*func)(Eterm, - Eterm, - Eterm, - ErlHeapFragment *)); +void erts_debug_foreach_sys_msg_in_q(void (*func)(Eterm, + Eterm, + Eterm, + ErlHeapFragment *)); Eterm erts_set_system_logger(Eterm); Eterm erts_get_system_logger(void); void erts_queue_error_logger_message(Eterm, Eterm, ErlHeapFragment *); diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 4c8d3d3dbe..0c2cf98033 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -907,7 +907,8 @@ Eterm erl_is_function(Process* p, Eterm arg1, Eterm arg2); /* beam_bif_load.c */ Eterm erts_check_process_code(Process *c_p, Eterm module, int *redsp, int fcalls); Eterm erts_proc_copy_literal_area(Process *c_p, int *redsp, int fcalls, int gc_allowed); - +void erts_debug_foreach_release_literal_area_off_heap(void (*func)(ErlOffHeap *, void *), + void *arg); typedef struct ErtsLiteralArea_ { struct erl_off_heap_header *off_heap; Eterm *end; @@ -1072,17 +1073,46 @@ Uint size_object_x(Eterm, erts_literal_area_t*); #define size_object_litopt(Term,LitArea) size_object_x(Term,LitArea) Uint copy_shared_calculate(Eterm, erts_shcopy_t*); -Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); - Uint size_shared(Eterm); +/* #define ERTS_COPY_REGISTER_LOCATION */ + +#ifdef ERTS_COPY_REGISTER_LOCATION + +#define copy_shared_perform(U, V, X, Y, Z) \ + copy_shared_perform_x((U), (V), (X), (Y), (Z), __FILE__, __LINE__) +Eterm copy_shared_perform_x(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*, + char *file, int line); + +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_t*, + char *file, int line); +#define copy_struct(Obj,Sz,HPP,OH) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL,__FILE__,__LINE__) +#define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea,__FILE__,__LINE__) + +#define copy_shallow(R, SZ, HPP, OH) \ + copy_shallow_x((R), (SZ), (HPP), (OH), __FILE__, __LINE__) +Eterm copy_shallow_x(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*, + char *file, int line); + +#else + +#define copy_shared_perform(U, V, X, Y, Z) \ + copy_shared_perform_x((U), (V), (X), (Y), (Z)) +Eterm copy_shared_perform_x(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); + Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_t*); #define copy_struct(Obj,Sz,HPP,OH) \ copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL) #define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) -Eterm copy_shallow(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*); +#define copy_shallow(R, SZ, HPP, OH) \ + copy_shallow_x((R), (SZ), (HPP), (OH)) +Eterm copy_shallow_x(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*); + +#endif void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, Eterm* refs, unsigned nrefs, int literals); @@ -1257,6 +1287,10 @@ Uint erts_persistent_term_count(void); void erts_init_persistent_dumping(void); extern ErtsLiteralArea** erts_persistent_areas; extern Uint erts_num_persistent_areas; +void erts_debug_foreach_persistent_term_off_heap(void (*func)(ErlOffHeap *, void *), + void *arg); +int erts_debug_have_accessed_literal_area(ErtsLiteralArea *lap); +void erts_debug_save_accessed_literal_area(ErtsLiteralArea *lap); /* external.c */ void erts_init_external(void); diff --git a/erts/emulator/test/node_container_SUITE.erl b/erts/emulator/test/node_container_SUITE.erl index 7d2db5257c..c44693f8d9 100644 --- a/erts/emulator/test/node_container_SUITE.erl +++ b/erts/emulator/test/node_container_SUITE.erl @@ -51,7 +51,8 @@ unique_pid/1, iter_max_procs/1, magic_ref/1, - dist_entry_gc/1]). + dist_entry_gc/1, + persistent_term/1]). suite() -> [{ct_hooks,[ts_install_cth]}, @@ -63,7 +64,8 @@ all() -> node_table_gc, dist_link_refc, dist_monitor_refc, node_controller_refc, ets_refc, match_spec_refc, timer_refc, pid_wrap, port_wrap, bad_nc, - unique_pid, iter_max_procs, magic_ref]. + unique_pid, iter_max_procs, + magic_ref, persistent_term]. init_per_suite(Config) -> Config. @@ -906,6 +908,44 @@ magic_ref(Config) when is_list(Config) -> true = erts_debug:get_internal_state({magic_ref,MRef2}), ok. +persistent_term(Config) when is_list(Config) -> + {ok, Node} = start_node(get_nodefirstname()), + Self = self(), + NcData = make_ref(), + RPid = spawn_link(Node, + fun () -> + Self ! {NcData, self(), hd(erlang:ports()), erlang:make_ref()} + end), + Data = receive + {NcData, RPid, RPort, RRef} -> + {RPid, RPort, RRef} + end, + unlink(RPid), + stop_node(Node), + Stuff = lists:foldl(fun (N, Acc) -> + persistent_term:put({?MODULE, N}, Data), + persistent_term:erase({?MODULE, N-1}), + node_container_refc_check(node()), + Data = persistent_term:get({?MODULE, N}), + try + persistent_term:get({?MODULE, N-1}) + catch + error:badarg -> + ok + end, + case N rem 4 of + 0 -> [persistent_term:get({?MODULE, N})|Acc]; + _ -> Acc + end + end, + [], + lists:seq(1, 100)), + persistent_term:erase({?MODULE, 100}), + receive after 2000 -> ok end, %% give literal gc some time to run... + node_container_refc_check(node()), + id(Stuff), + ok. + lost_pending_connection(Node) -> _ = (catch erts_internal:new_connection(Node)), diff --git a/erts/etc/unix/etp-commands.in b/erts/etc/unix/etp-commands.in index 8b6abb5336..20809d61e8 100644 --- a/erts/etc/unix/etp-commands.in +++ b/erts/etc/unix/etp-commands.in @@ -3993,22 +3993,79 @@ end # ETS table debug # +define etp-ets-tab-status-int +# Args: +# +# Non-reentrant + if ($arg0 & 0x1) + printf "priv" + end + if ($arg0 & 0x2) + printf "prot" + end + if ($arg0 & 0x4) + printf "pub" + end + if ($arg0 & 0x8) + printf "|del" + end + if ($arg0 & 0x10) + printf "|set" + end + if ($arg0 & 0x20) + printf "|bag" + end + if ($arg0 & 0x40) + printf "|dbag" + end + if ($arg0 & 0x80) + printf "|oset" + end + if ($arg0 & 0x100) + printf "|caoset" + end + if ($arg0 & 0x200) + printf "|flocked" + end + if ($arg0 & 0x400) + printf "|fread" + end + if ($arg0 & 0x800) + printf "|named" + end + if ($arg0 & 0x1000) + printf "|busy" + end +end + define etp-ets-tables # Args: # # Non-reentrant - printf "%% Dumping < %lu ETS tables\n", (unsigned long)db_max_tabs - while $etp_ets_tables_i < db_max_tabs - if (meta_main_tab[$etp_ets_tables_i].u.next_free & 3) == 0 - printf "%% %d:", $etp_ets_tables_i - etp-1 ((Eterm)(meta_main_tab[$etp_ets_tables_i].u.tb->common.id)) 0 + set $sched_ix = 0 + while $sched_ix < erts_no_schedulers + set $ets_tabs = &erts_aligned_scheduler_data[$sched_ix].esd.ets_tables + set $no_ets_tabs = *(unsigned long *)&($ets_tabs->count) + printf "\n%% %lu ETS tables created on scheduler %lu...\n\n", $no_ets_tabs, (unsigned long)$sched_ix+1 + set $ets_tab = $ets_tabs->clist + set $first_ets_tab = $ets_tab + while $ets_tab + set $refn = &((ErtsMagicBinary *)$ets_tab->common->btid)->refn + printf "%% " + etp-1 $ets_tab->common.the_name 0 + printf " #Ref<0.%u.%u.%u> ", (unsigned)$refn[2], (unsigned)$refn[1], (unsigned)$refn[0] + etp-1 $ets_tab->common.owner 0 printf " " - etp-1 ((Eterm)(meta_main_tab[$etp_ets_tables_i].u.tb->common.owner)) 0 - printf "\n" + etp-ets-tab-status-int $ets_tab->common.status + printf " (DbTable*)%p\n", $ets_tab + if $ets_tab->common.all.next == $first_ets_tab + set $ets_tab = (DbTable *) 0 + else + set $ets_tab = $ets_tab->common.all.next + end end - set $etp_ets_tables_i++ + set $sched_ix++ end - set $etp_ets_tables_i = 0 end document etp-ets-tables |