diff options
-rw-r--r-- | erts/emulator/beam/copy.c | 49 | ||||
-rw-r--r-- | erts/emulator/beam/erl_alloc.types | 1 | ||||
-rw-r--r-- | erts/emulator/beam/erl_gc.c | 10 | ||||
-rw-r--r-- | erts/emulator/beam/erl_node_tables.c | 69 | ||||
-rw-r--r-- | erts/emulator/beam/erl_node_tables.h | 39 | ||||
-rw-r--r-- | erts/emulator/beam/global.h | 35 |
6 files changed, 165 insertions, 38 deletions
diff --git a/erts/emulator/beam/copy.c b/erts/emulator/beam/copy.c index db74b06cc5..9bbcba6f3b 100644 --- a/erts/emulator/beam/copy.c +++ b/erts/emulator/beam/copy.c @@ -604,7 +604,12 @@ cleanup: /* * Copy a structure to a heap. */ -Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint *bsz, erts_literal_area_t *litopt) +Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, + Uint *bsz, erts_literal_area_t *litopt +#ifdef ERTS_COPY_REGISTER_LOCATION + , char *file, int line +#endif + ) { char* hstart; Uint hsize; @@ -854,7 +859,11 @@ Eterm copy_struct_x(Eterm obj, Uint sz, Eterm** hpp, ErlOffHeap* off_heap, Uint case EXTERNAL_REF_SUBTAG: { ExternalThing *etp = (ExternalThing *) objp; +#if defined(ERTS_COPY_REGISTER_LOCATION) && defined(ERL_NODE_BOOKKEEP) + erts_ref_node_entry__(etp->node, 2, make_boxed(htop), file, line); +#else erts_ref_node_entry(etp->node, 2, make_boxed(htop)); +#endif } L_off_heap_node_container_common: { @@ -1326,8 +1335,13 @@ Uint copy_shared_calculate(Eterm obj, erts_shcopy_t *info) * Copy object "obj" preserving sharing. * Second half: copy and restore the object. */ -Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, - Eterm** hpp, ErlOffHeap* off_heap) { +Uint copy_shared_perform_x(Eterm obj, Uint size, erts_shcopy_t *info, + Eterm** hpp, ErlOffHeap* off_heap +#ifdef ERTS_COPY_REGISTER_LOCATION + , char *file, int line +#endif + ) +{ Uint e; unsigned sz; Eterm* ptr; @@ -1393,7 +1407,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL +#ifdef ERTS_COPY_REGISTER_LOCATION + , file, line +#endif + ); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1461,7 +1479,11 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, *resp = obj; } else { Uint bsz = 0; - *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL); /* copy literal */ + *resp = copy_struct_x(obj, hbot - hp, &hp, off_heap, &bsz, NULL +#ifdef ERTS_COPY_REGISTER_LOCATION + , file, line +#endif + ); /* copy literal */ hbot -= bsz; } goto cleanup_next; @@ -1660,7 +1682,12 @@ Uint copy_shared_perform(Eterm obj, Uint size, erts_shcopy_t *info, case EXTERNAL_REF_SUBTAG: { ExternalThing *etp = (ExternalThing *) ptr; + +#if defined(ERTS_COPY_REGISTER_LOCATION) && defined(ERL_NODE_BOOKKEEP) + erts_ref_node_entry__(etp->node, 2, make_boxed(hp), file, line); +#else erts_ref_node_entry(etp->node, 2, make_boxed(hp)); +#endif } off_heap_node_container_common: { @@ -1823,8 +1850,12 @@ all_clean: * * NOTE: Assumes that term is a tuple (ptr is an untagged tuple ptr). */ -Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, - ErlOffHeap* off_heap) +Eterm +copy_shallow_x(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, ErlOffHeap* off_heap +#ifdef ERTS_COPY_REGISTER_LOCATION + , char *file, int line +#endif + ) { Eterm* tp = ptr; Eterm* hp = *hpp; @@ -1866,7 +1897,11 @@ Eterm copy_shallow(Eterm* ERTS_RESTRICT ptr, Uint sz, Eterm** hpp, case EXTERNAL_REF_SUBTAG: { ExternalThing* etp = (ExternalThing *) (tp-1); +#if defined(ERTS_COPY_REGISTER_LOCATION) && defined(ERL_NODE_BOOKKEEP) + erts_ref_node_entry__(etp->node, 2, make_boxed(hp-1), file, line); +#else erts_ref_node_entry(etp->node, 2, make_boxed(hp-1)); +#endif } off_heap_common: { diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 92e5069c71..21941ba96e 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -149,6 +149,7 @@ type MODULE_TABLE LONG_LIVED CODE module_tab type TAINT LONG_LIVED CODE taint_list type MODULE_REFS STANDARD CODE module_refs type NC_TMP TEMPORARY SYSTEM nc_tmp +type NC_STD STANDARD SYSTEM nc_std type TMP TEMPORARY SYSTEM tmp type UNDEF SYSTEM SYSTEM undefined type DCACHE STANDARD SYSTEM dcache diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 67a73e4d57..17b7f6b46a 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -2848,7 +2848,7 @@ sweep_off_heap(Process *p, int fullsweep) if (is_external_header(((struct erl_off_heap_header*) boxed_val(ptr->thing_word))->thing_word)) erts_node_bookkeep(((ExternalThing*)ptr)->node, make_boxed(&ptr->thing_word), - ERL_NODE_DEC); + ERL_NODE_DEC, __FILE__, __LINE__); *prev = ptr = (struct erl_off_heap_header*) boxed_val(ptr->thing_word); ASSERT(!IS_MOVED_BOXED(ptr->thing_word)); switch (ptr->thing_word) { @@ -2879,7 +2879,7 @@ sweep_off_heap(Process *p, int fullsweep) if (is_external_header(ptr->thing_word)) { erts_node_bookkeep(((ExternalThing*)ptr)->node, make_boxed(&ptr->thing_word), - ERL_NODE_INC); + ERL_NODE_INC, __FILE__, __LINE__); } prev = &ptr->next; ptr = ptr->next; @@ -3050,9 +3050,11 @@ offset_heap(Eterm* hp, Uint sz, Sint offs, char* area, Uint area_size) if (is_external_header(oh->thing_word)) { erts_node_bookkeep(((ExternalThing*)oh)->node, - make_boxed(((Eterm*)oh)-offs), ERL_NODE_DEC); + make_boxed(((Eterm*)oh)-offs), + ERL_NODE_DEC, __FILE__, __LINE__); erts_node_bookkeep(((ExternalThing*)oh)->node, - make_boxed((Eterm*)oh), ERL_NODE_INC); + make_boxed((Eterm*)oh), ERL_NODE_INC, + __FILE__, __LINE__); } if (ErtsInArea(oh->next, area, area_size)) { diff --git a/erts/emulator/beam/erl_node_tables.c b/erts/emulator/beam/erl_node_tables.c index 850176e1cf..bdf30ef496 100644 --- a/erts/emulator/beam/erl_node_tables.c +++ b/erts/emulator/beam/erl_node_tables.c @@ -932,7 +932,8 @@ static void try_delete_node(void *venp) * * If refc > 0, the entry is in use. Keep the entry. */ - erts_node_bookkeep(enp, THE_NON_VALUE, ERL_NODE_DEC); + erts_node_bookkeep(enp, THE_NON_VALUE, ERL_NODE_DEC, + __FILE__, __LINE__); refc = erts_refc_dectest(&enp->refc, -1); if (refc == -1) (void) hash_erase(&erts_node_table, (void *) enp); @@ -1164,6 +1165,12 @@ void erts_lcnt_update_distribution_locks(int enable) { * can damage the real-time properties of the system. * \* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#ifdef ERL_NODE_BOOKKEEP +#define ERTS_DBG_NC_ALLOC_TYPE ERTS_ALC_T_NC_STD +#else +#define ERTS_DBG_NC_ALLOC_TYPE ERTS_ALC_T_NC_TMP +#endif + #include "erl_db.h" #undef INIT_AM @@ -1193,6 +1200,7 @@ static Eterm AM_persistent_term; static void setup_reference_table(void); static Eterm reference_table_term(Uint **hpp, ErlOffHeap *ohp, Uint *szp); static void delete_reference_table(void); +static void cleanup_system(void); #undef ERTS_MAX__ #define ERTS_MAX__(A, B) ((A) > (B) ? (A) : (B)) @@ -1245,11 +1253,11 @@ typedef struct inserted_bin_ { Binary *bin_val; } InsertedBin; -static ReferredNode *referred_nodes; +static ReferredNode *referred_nodes = NULL; static int no_referred_nodes; -static ReferredDist *referred_dists; +static ReferredDist *referred_dists = NULL; static int no_referred_dists; -static InsertedBin *inserted_bins; +static InsertedBin *inserted_bins = NULL; Eterm erts_get_node_and_dist_references(struct process *proc) @@ -1289,6 +1297,11 @@ erts_get_node_and_dist_references(struct process *proc) references_atoms_need_init = 0; } +#ifdef ERL_NODE_BOOKKEEP + if (referred_nodes || referred_dists || inserted_bins) + delete_reference_table(); +#endif + setup_reference_table(); /* Get term size */ @@ -1306,7 +1319,11 @@ erts_get_node_and_dist_references(struct process *proc) ASSERT(endp == hp); +#ifndef ERL_NODE_BOOKKEEP delete_reference_table(); +#endif + + cleanup_system(); erts_thr_progress_unblock(); erts_proc_lock(proc, ERTS_PROC_LOCK_MAIN); @@ -1341,7 +1358,7 @@ insert_dist_referrer(ReferredDist *referred_dist, break; if(!drp) { - drp = (DistReferrer *) erts_alloc(ERTS_ALC_T_NC_TMP, + drp = (DistReferrer *) erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, sizeof(DistReferrer)); drp->next = referred_dist->referrers; referred_dist->referrers = drp; @@ -1404,7 +1421,7 @@ insert_node_referrer(ReferredNode *referred_node, int type, Eterm id) break; if(!nrp) { - nrp = (NodeReferrer *) erts_alloc(ERTS_ALC_T_NC_TMP, + nrp = (NodeReferrer *) erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, sizeof(NodeReferrer)); nrp->next = referred_node->referrers; ERTS_INIT_OFF_HEAP(&nrp->off_heap); @@ -1518,7 +1535,8 @@ insert_offheap(ErlOffHeap *oh, int type, Eterm id) erts_match_prog_foreach_offheap((Binary *) u.mref->mb, insert_offheap2, (void *) &a); - nib = erts_alloc(ERTS_ALC_T_NC_TMP, sizeof(InsertedBin)); + nib = erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, + sizeof(InsertedBin)); nib->bin_val = (Binary *) u.mref->mb; nib->next = inserted_bins; inserted_bins = nib; @@ -1938,14 +1956,16 @@ insert_ets_offheap_thr_prgr(ErlOffHeap *ohp, void *arg) #ifdef ERL_NODE_BOOKKEEP void -erts_node_bookkeep(ErlNode *np, Eterm term, int what) +erts_node_bookkeep(ErlNode *np, Eterm term, int what, char *f, int l) { - erts_aint_t slot = (erts_atomic_inc_read_nob(&np->slot) - 1) % 1024; + erts_aint_t slot = (erts_atomic_inc_read_nob(&np->slot) - 1) % ERTS_BOOKKEEP_SIZE; ErtsSchedulerData *esdp = erts_get_scheduler_data(); Eterm who = THE_NON_VALUE; ASSERT(np); np->books[slot].what = what; np->books[slot].term = term; + np->books[slot].file = f; + np->books[slot].line = l; if (esdp->current_process) { who = esdp->current_process->common.id; } else if (esdp->current_port) { @@ -1966,14 +1986,14 @@ setup_reference_table(void) inserted_bins = NULL; hash_get_info(&hi, &erts_node_table); - referred_nodes = erts_alloc(ERTS_ALC_T_NC_TMP, + referred_nodes = erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, hi.objs*sizeof(ReferredNode)); no_referred_nodes = 0; hash_foreach(&erts_node_table, init_referred_node, NULL); ASSERT(no_referred_nodes == hi.objs); hash_get_info(&hi, &erts_dist_table); - referred_dists = erts_alloc(ERTS_ALC_T_NC_TMP, + referred_dists = erts_alloc(ERTS_DBG_NC_ALLOC_TYPE, hi.objs*sizeof(ReferredDist)); no_referred_dists = 0; hash_foreach(&erts_dist_table, init_referred_dist, NULL); @@ -2373,8 +2393,7 @@ static void noop_sig_ext(ErtsDistExternal *ext, void *arg) static void delete_reference_table(void) { - DistEntry *dep; - int i, max; + int i; for(i = 0; i < no_referred_nodes; i++) { NodeReferrer *nrp; NodeReferrer *tnrp; @@ -2383,11 +2402,13 @@ delete_reference_table(void) erts_cleanup_offheap(&nrp->off_heap); tnrp = nrp; nrp = nrp->next; - erts_free(ERTS_ALC_T_NC_TMP, (void *) tnrp); + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *) tnrp); } } - if (referred_nodes) - erts_free(ERTS_ALC_T_NC_TMP, (void *) referred_nodes); + if (referred_nodes) { + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *) referred_nodes); + referred_nodes = NULL; + } for(i = 0; i < no_referred_dists; i++) { DistReferrer *drp; @@ -2396,17 +2417,25 @@ delete_reference_table(void) while(drp) { tdrp = drp; drp = drp->next; - erts_free(ERTS_ALC_T_NC_TMP, (void *) tdrp); + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *) tdrp); } } - if (referred_dists) - erts_free(ERTS_ALC_T_NC_TMP, (void *) referred_dists); + if (referred_dists) { + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *) referred_dists); + referred_dists = NULL; + } while(inserted_bins) { InsertedBin *ib = inserted_bins; inserted_bins = inserted_bins->next; - erts_free(ERTS_ALC_T_NC_TMP, (void *)ib); + erts_free(ERTS_DBG_NC_ALLOC_TYPE, (void *)ib); } +} +static void +cleanup_system(void) +{ + DistEntry *dep; + int i, max; /* Cleanup... */ max = erts_ptab_max(&erts_proc); diff --git a/erts/emulator/beam/erl_node_tables.h b/erts/emulator/beam/erl_node_tables.h index aa8af12555..ffaafbbbea 100644 --- a/erts/emulator/beam/erl_node_tables.h +++ b/erts/emulator/beam/erl_node_tables.h @@ -187,6 +187,7 @@ set pagination off set $i = 0 set $node = referred_nodes[$node_ix].node while $i < $node->slot.counter + printf "%s:%d ", $node->books[$i].file, $node->books[$i].line printf "%p: ", $node->books[$i].term etp-1 $node->books[$i].who printf " " @@ -211,8 +212,12 @@ lists:usort(lists:filter(fun({V,N}) -> N /= 0 end, maps:to_list(Accs))). struct erl_node_bookkeeping { Eterm who; Eterm term; + char *file; + int line; enum { ERL_NODE_INC, ERL_NODE_DEC } what; }; + +#define ERTS_BOOKKEEP_SIZE (1024) #endif typedef struct erl_node_ { @@ -222,7 +227,7 @@ typedef struct erl_node_ { Uint32 creation; /* Creation */ DistEntry *dist_entry; /* Corresponding dist entry */ #ifdef ERL_NODE_BOOKKEEP - struct erl_node_bookkeeping books[1024]; + struct erl_node_bookkeeping books[ERTS_BOOKKEEP_SIZE]; erts_atomic_t slot; #endif } ErlNode; @@ -276,14 +281,21 @@ Eterm erts_build_dhandle(Eterm **hpp, ErlOffHeap*, DistEntry*, Uint32 conn_id); Eterm erts_make_dhandle(Process *c_p, DistEntry*, Uint32 conn_id); ERTS_GLB_INLINE void erts_init_node_entry(ErlNode *np, erts_aint_t val); +#ifdef ERL_NODE_BOOKKEEP +#define erts_ref_node_entry(NP, MIN, T) erts_ref_node_entry__((NP), (MIN), (T), __FILE__, __LINE__) +#define erts_deref_node_entry(NP, T) erts_deref_node_entry__((NP), (T), __FILE__, __LINE__) +ERTS_GLB_INLINE erts_aint_t erts_ref_node_entry__(ErlNode *np, int min_val, Eterm term, char *file, int line); +ERTS_GLB_INLINE void erts_deref_node_entry__(ErlNode *np, Eterm term, char *file, int line); +#else ERTS_GLB_INLINE erts_aint_t erts_ref_node_entry(ErlNode *np, int min_val, Eterm term); ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np, Eterm term); +#endif ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_runlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_rwlock(DistEntry *dep); ERTS_GLB_INLINE void erts_de_rwunlock(DistEntry *dep); #ifdef ERL_NODE_BOOKKEEP -void erts_node_bookkeep(ErlNode *, Eterm , int); +void erts_node_bookkeep(ErlNode *, Eterm , int, char *file, int line); #else #define erts_node_bookkeep(...) #endif @@ -296,21 +308,40 @@ erts_init_node_entry(ErlNode *np, erts_aint_t val) erts_refc_init(&np->refc, val); } +#ifdef ERL_NODE_BOOKKEEP + +ERTS_GLB_INLINE erts_aint_t +erts_ref_node_entry__(ErlNode *np, int min_val, Eterm term, char *file, int line) +{ + erts_node_bookkeep(np, term, ERL_NODE_INC, file, line); + return erts_refc_inctest(&np->refc, min_val); +} + +ERTS_GLB_INLINE void +erts_deref_node_entry__(ErlNode *np, Eterm term, char *file, int line) +{ + erts_node_bookkeep(np, term, ERL_NODE_DEC, file, line); + if (erts_refc_dectest(&np->refc, 0) == 0) + erts_schedule_delete_node(np); +} + +#else + ERTS_GLB_INLINE erts_aint_t erts_ref_node_entry(ErlNode *np, int min_val, Eterm term) { - erts_node_bookkeep(np, term, ERL_NODE_INC); return erts_refc_inctest(&np->refc, min_val); } ERTS_GLB_INLINE void erts_deref_node_entry(ErlNode *np, Eterm term) { - erts_node_bookkeep(np, term, ERL_NODE_DEC); if (erts_refc_dectest(&np->refc, 0) == 0) erts_schedule_delete_node(np); } +#endif + ERTS_GLB_INLINE void erts_de_rlock(DistEntry *dep) { diff --git a/erts/emulator/beam/global.h b/erts/emulator/beam/global.h index 64fb080250..0c2cf98033 100644 --- a/erts/emulator/beam/global.h +++ b/erts/emulator/beam/global.h @@ -1073,17 +1073,46 @@ Uint size_object_x(Eterm, erts_literal_area_t*); #define size_object_litopt(Term,LitArea) size_object_x(Term,LitArea) Uint copy_shared_calculate(Eterm, erts_shcopy_t*); -Eterm copy_shared_perform(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); - Uint size_shared(Eterm); +/* #define ERTS_COPY_REGISTER_LOCATION */ + +#ifdef ERTS_COPY_REGISTER_LOCATION + +#define copy_shared_perform(U, V, X, Y, Z) \ + copy_shared_perform_x((U), (V), (X), (Y), (Z), __FILE__, __LINE__) +Eterm copy_shared_perform_x(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*, + char *file, int line); + +Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_t*, + char *file, int line); +#define copy_struct(Obj,Sz,HPP,OH) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL,__FILE__,__LINE__) +#define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ + copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea,__FILE__,__LINE__) + +#define copy_shallow(R, SZ, HPP, OH) \ + copy_shallow_x((R), (SZ), (HPP), (OH), __FILE__, __LINE__) +Eterm copy_shallow_x(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*, + char *file, int line); + +#else + +#define copy_shared_perform(U, V, X, Y, Z) \ + copy_shared_perform_x((U), (V), (X), (Y), (Z)) +Eterm copy_shared_perform_x(Eterm, Uint, erts_shcopy_t*, Eterm**, ErlOffHeap*); + Eterm copy_struct_x(Eterm, Uint, Eterm**, ErlOffHeap*, Uint*, erts_literal_area_t*); #define copy_struct(Obj,Sz,HPP,OH) \ copy_struct_x(Obj,Sz,HPP,OH,NULL,NULL) #define copy_struct_litopt(Obj,Sz,HPP,OH,LitArea) \ copy_struct_x(Obj,Sz,HPP,OH,NULL,LitArea) -Eterm copy_shallow(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*); +#define copy_shallow(R, SZ, HPP, OH) \ + copy_shallow_x((R), (SZ), (HPP), (OH)) +Eterm copy_shallow_x(Eterm* ERTS_RESTRICT, Uint, Eterm**, ErlOffHeap*); + +#endif void erts_move_multi_frags(Eterm** hpp, ErlOffHeap*, ErlHeapFragment* first, Eterm* refs, unsigned nrefs, int literals); |