Merge branch 'john/erts/runtime-lcnt' into maint

* john/erts/runtime-lcnt:

  Document rt_mask and add warnings about copy_save
  Add an emulator test suite for lock counting
  Break erts_debug:lock_counters/1 into separate BIFs
  Allow toggling lock counting at runtime
  Move lock flags to a common header
  Enable register_SUITE for lcnt builds
  Enable lcnt smoke test on all builds that have lcnt enabled
  Make lock counter info independent of the locks being counted

OTP-14412
OTP-13170
OTP-14413

1 files changed, 308 insertions, 155 deletions

diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c
index 96f9b284b3..e5d7efcc72 100644
--- a/erts/emulator/beam/erl_bif_info.c
+++ b/erts/emulator/beam/erl_bif_info.c
@@ -4452,48 +4452,120 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2)
 }
 
 #ifdef ERTS_ENABLE_LOCK_COUNT
+
+typedef struct {
+    /* info->location_count may increase between size calculation and term
+     * building, so we cap it at the value sampled in lcnt_build_result_vector.
+     *
+     * Shrinking is safe though. */
+    int max_location_count;
+    erts_lcnt_lock_info_t *info;
+} lcnt_sample_t;
+
+typedef struct lcnt_sample_vector_ {
+    lcnt_sample_t *elements;
+    size_t size;
+} lcnt_sample_vector_t;
+
+static lcnt_sample_vector_t lcnt_build_sample_vector(erts_lcnt_lock_info_list_t *list) {
+    erts_lcnt_lock_info_t *iterator;
+    lcnt_sample_vector_t result;
+    size_t allocated_entries;
+
+    allocated_entries = 64;
+    result.size = 0;
+
+    result.elements = erts_alloc(ERTS_ALC_T_LCNT_VECTOR,
+        allocated_entries * sizeof(lcnt_sample_t));
+
+    iterator = NULL;
+    while(erts_lcnt_iterate_list(list, &iterator)) {
+        erts_lcnt_retain_lock_info(iterator);
+
+        result.elements[result.size].max_location_count = iterator->location_count;
+        result.elements[result.size].info = iterator;
+
+        result.size++;
+
+        if(result.size >= allocated_entries) {
+            allocated_entries *= 2;
+
+            result.elements = erts_realloc(ERTS_ALC_T_LCNT_VECTOR, result.elements,
+                allocated_entries * sizeof(lcnt_sample_t));
+        }
+    }
+
+    return result;
+}
+
+static void lcnt_destroy_sample_vector(lcnt_sample_vector_t *vector) {
+    size_t i;
+
+    for(i = 0; i < vector->size; i++) {
+        erts_lcnt_release_lock_info(vector->elements[i].info);
+    }
+
+    erts_free(ERTS_ALC_T_LCNT_VECTOR, vector->elements);
+}
+
+/* The size of an integer is not guaranteed to be constant since we're walking
+ * over live data, and may cross over into bignum territory between size calc
+ * and the actual build. This takes care of that through always assuming the
+ * worst, but needs to be fixed up with HRelease once the final term has been
+ * built. */
+static ERTS_INLINE Eterm bld_unstable_uint64(Uint **hpp, Uint *szp, Uint64 ui) {
+    Eterm res = THE_NON_VALUE;
+
+    if(szp) {
+        *szp += ERTS_UINT64_HEAP_SIZE(~((Uint64) 0));
+    }
+
+    if(hpp) {
+        if (IS_USMALL(0, ui)) {
+            res = make_small(ui);
+        } else {
+            res = erts_uint64_to_big(ui, hpp);
+        }
+    }
+
+    return res;
+}
+
 static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_stats_t *stats, Eterm res) {
-    Uint tries = 0, colls = 0;
-    unsigned long timer_s = 0, timer_ns = 0, timer_n = 0;
-    unsigned int  line = 0;
     unsigned int  i;
-    
+    const char *file;
+
     Eterm af, uil;
     Eterm uit, uic;
     Eterm uits, uitns, uitn;
     Eterm tt, tstat, tloc, t;
     Eterm thist, vhist[ERTS_LCNT_HISTOGRAM_SLOT_SIZE];
-	
+
     /* term:
-     * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}},
-     *   { .. histogram .. }]
-     */
+     *  [{{file, line},
+         {tries, colls, {seconds, nanoseconds, n_blocks}},
+     *   { .. histogram .. }] */
 
-    tries = (Uint) ethr_atomic_read(&stats->tries);
-    colls = (Uint) ethr_atomic_read(&stats->colls);
-   
-    line     = stats->line; 
-    timer_s  = stats->timer.s;
-    timer_ns = stats->timer.ns;
-    timer_n  = stats->timer_n;
-   
-    af    = erts_atom_put((byte *)stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1);
-    uil   = erts_bld_uint( hpp, szp, line);
+    file = stats->file ? stats->file : "undefined";
+
+    af    = erts_atom_put((byte *)file, strlen(file), ERTS_ATOM_ENC_LATIN1, 1);
+    uil   = erts_bld_uint( hpp, szp, stats->line);
     tloc  = erts_bld_tuple(hpp, szp, 2, af, uil);
-    
-    uit   = erts_bld_uint( hpp, szp, tries);
-    uic   = erts_bld_uint( hpp, szp, colls);
 
-    uits  = erts_bld_uint( hpp, szp, timer_s);
-    uitns = erts_bld_uint( hpp, szp, timer_ns);
-    uitn  = erts_bld_uint( hpp, szp, timer_n);
+    uit   = bld_unstable_uint64(hpp, szp, (Uint)ethr_atomic_read(&stats->attempts));
+    uic   = bld_unstable_uint64(hpp, szp, (Uint)ethr_atomic_read(&stats->collisions));
+
+    uits  = bld_unstable_uint64(hpp, szp, stats->total_time_waited.s);
+    uitns = bld_unstable_uint64(hpp, szp, stats->total_time_waited.ns);
+    uitn  = bld_unstable_uint64(hpp, szp, stats->times_waited);
     tt    = erts_bld_tuple(hpp, szp, 3, uits, uitns, uitn);
 
     tstat = erts_bld_tuple(hpp, szp, 3, uit, uic, tt);
 
     for(i = 0; i < ERTS_LCNT_HISTOGRAM_SLOT_SIZE; i++) {
-	vhist[i] = erts_bld_uint(hpp, szp, stats->hist.ns[i]);
+        vhist[i] = bld_unstable_uint64(hpp, szp, stats->wait_time_histogram.ns[i]);
     }
+
     thist  = erts_bld_tuplev(hpp, szp, ERTS_LCNT_HISTOGRAM_SLOT_SIZE, vhist);
 
     t   = erts_bld_tuple(hpp, szp, 3, tloc, tstat, thist);
@@ -4502,185 +4574,266 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s
     return res;
 }
 
-static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock, Eterm res) {
+static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) {
+    Eterm id = info->id;
+
+    if((info->flags & ERTS_LOCK_FLAGS_MASK_TYPE) == ERTS_LOCK_TYPE_PROCLOCK) {
+        /* Use registered names as id's for process locks if available. Thread
+         * progress is delayed since we may be running on a dirty scheduler. */
+        ErtsThrPrgrDelayHandle delay_handle;
+        Process *process;
+
+        delay_handle = erts_thr_progress_unmanaged_delay();
+
+        process = erts_proc_lookup(info->id);
+        if (process && process->common.u.alive.reg) {
+            id = process->common.u.alive.reg->name;
+        }
+
+        erts_thr_progress_unmanaged_continue(delay_handle);
+    } else if(info->flags & ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR) {
+        if(is_small(id) && !sys_strcmp(info->name, "alcu_allocator")) {
+            const char *name = (const char*)ERTS_ALC_A2AD(signed_val(id));
+            id = erts_atom_put((byte*)name, strlen(name), ERTS_ATOM_ENC_LATIN1, 1);
+        }
+    }
+
+    return id;
+}
+
+static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, lcnt_sample_t *sample, Eterm res) {
+    erts_lcnt_lock_info_t *info = sample->info;
+
     Eterm name, type, id, stats = NIL, t;
-    Process *proc = NULL;
-    char *ltype;
+    const char *lock_desc;
     int i;
+
+    /* term: [{name, id, type, stats()}] */
+
+    ASSERT(info->name);
     
-    /* term:
-     * [{name, id, type, stats()}] 
-     */
-	
-    ASSERT(lock->name);
-    
-    ltype = erts_lcnt_lock_type(lock->flag);
-    
-    ASSERT(ltype);
-    
-    type  = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
-    name  = erts_atom_put((byte *)lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1);
-
-    if (lock->flag & ERTS_LCNT_LT_ALLOC) {
-	/* use allocator types names as id's for allocator locks */
-	ltype = (char *) ERTS_ALC_A2AD(signed_val(lock->id));
-	id    = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1);
-    } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) {
-	/* use registered names as id's for process locks if available */
-	proc  = erts_proc_lookup(lock->id);
-	if (proc && proc->common.u.alive.reg) {
-	    id = proc->common.u.alive.reg->name;
-	} else {
-	    /* otherwise use process id */
-	    id = lock->id;
-	}
+    lock_desc = erts_lock_flags_get_type_name(info->flags);
+
+    type  = erts_atom_put((byte*)lock_desc, strlen(lock_desc), ERTS_ATOM_ENC_LATIN1, 1);
+    name  = erts_atom_put((byte*)info->name, strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1);
+
+    /* Only attempt to resolve ids when actually emitting the term. This ought
+     * to be safe since all immediates are the same size. */
+    if(hpp != NULL) {
+        id = lcnt_pretty_print_lock_id(info);
     } else {
-	id = lock->id;
+        id = NIL;
     }
 
-    for (i = 0; i < lock->n_stats; i++) {
-	stats = lcnt_build_lock_stats_term(hpp, szp, &(lock->stats[i]), stats);
+    for(i = 0; i < MIN(info->location_count, sample->max_location_count); i++) {
+        stats = lcnt_build_lock_stats_term(hpp, szp, &(info->location_stats[i]), stats);
     }
 
     t   = erts_bld_tuple(hpp, szp, 4, name, id, type, stats);
-    res = erts_bld_cons( hpp, szp, t, res);
+    res = erts_bld_cons(hpp, szp, t, res);
 
     return res;
 }
 
-static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_data_t *data, Eterm res) {
+static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *duration,
+                                    lcnt_sample_vector_t *current_locks,
+                                    lcnt_sample_vector_t *deleted_locks, Eterm res) {
+    const char *str_duration = "duration";
+    const char *str_locks = "locks";
+
     Eterm dts, dtns, tdt, adur, tdur, aloc, lloc = NIL, tloc;
-    erts_lcnt_lock_t *lock = NULL;
-    char *str_duration = "duration";
-    char *str_locks    = "locks";
-    
-    /* term:
-     * [{'duration', {seconds, nanoseconds}}, {'locks', locks()}]
-     */
-   
+    size_t i;
+
+    /* term: [{'duration', {seconds, nanoseconds}}, {'locks', locks()}] */
+
     /* duration tuple */ 
-    dts  = erts_bld_uint( hpp, szp, data->duration.s);
-    dtns = erts_bld_uint( hpp, szp, data->duration.ns);
+    dts  = bld_unstable_uint64(hpp, szp, duration->s);
+    dtns = bld_unstable_uint64(hpp, szp, duration->ns);
     tdt  = erts_bld_tuple(hpp, szp, 2, dts, dtns);
-    
+
     adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1);
     tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt);
    
     /* lock tuple */
-    
     aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1);
-    	
-    for (lock = data->current_locks->head; lock != NULL ; lock = lock->next ) {
-	lloc = lcnt_build_lock_term(hpp, szp, lock, lloc);
+
+    for(i = 0; i < current_locks->size; i++) {
+        lloc = lcnt_build_lock_term(hpp, szp, &current_locks->elements[i], lloc);
     }
-    
-    for (lock = data->deleted_locks->head; lock != NULL ; lock = lock->next ) {
-	lloc = lcnt_build_lock_term(hpp, szp, lock, lloc);
+
+    for(i = 0; i < deleted_locks->size; i++) {
+        lloc = lcnt_build_lock_term(hpp, szp, &deleted_locks->elements[i], lloc);
     }
-    
+
     tloc = erts_bld_tuple(hpp, szp, 2, aloc, lloc);
-    
-    res  = erts_bld_cons( hpp, szp, tloc, res);          
-    res  = erts_bld_cons( hpp, szp, tdur, res);          
+
+    res  = erts_bld_cons(hpp, szp, tloc, res);
+    res  = erts_bld_cons(hpp, szp, tdur, res);
 
     return res;
-}    
+}
+
+static struct {
+    const char *name;
+    erts_lock_flags_t flag;
+} lcnt_category_map[] = {
+        {"allocator", ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR},
+        {"db", ERTS_LOCK_FLAGS_CATEGORY_DB},
+        {"debug", ERTS_LOCK_FLAGS_CATEGORY_DEBUG},
+        {"distribution", ERTS_LOCK_FLAGS_CATEGORY_DISTRIBUTION},
+        {"generic", ERTS_LOCK_FLAGS_CATEGORY_GENERIC},
+        {"io", ERTS_LOCK_FLAGS_CATEGORY_IO},
+        {"process", ERTS_LOCK_FLAGS_CATEGORY_PROCESS},
+        {"scheduler", ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER},
+        {NULL, 0}
+    };
+
+static erts_lock_flags_t lcnt_atom_to_lock_category(Eterm atom) {
+    int i = 0;
+
+    for(i = 0; lcnt_category_map[i].name != NULL; i++) {
+        if(erts_is_atom_str(lcnt_category_map[i].name, atom, 0)) {
+            return lcnt_category_map[i].flag;
+        }
+    }
+
+    return 0;
+}
+
+static Eterm lcnt_build_category_list(Eterm **hpp, Uint *szp, erts_lock_flags_t mask) {
+    Eterm res;
+    int i;
+
+    res = NIL;
+
+    for(i = 0; lcnt_category_map[i].name != NULL; i++) {
+        if(mask & lcnt_category_map[i].flag) {
+            Eterm category = erts_atom_put((byte*)lcnt_category_map[i].name,
+                                           strlen(lcnt_category_map[i].name),
+                                           ERTS_ATOM_ENC_UTF8, 0);
+
+            res = erts_bld_cons(hpp, szp, category, res);
+        }
+    }
+
+    return res;
+}
+
 #endif
 
-BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1)
+BIF_RETTYPE erts_debug_lcnt_clear_0(BIF_ALIST_0)
 {
-#ifdef ERTS_ENABLE_LOCK_COUNT
-    Eterm res = NIL;
-#endif
+#ifndef ERTS_ENABLE_LOCK_COUNT
+    BIF_RET(am_error);
+#else
+    erts_lcnt_clear_counters();
 
+    BIF_RET(am_ok);
+#endif
+}
 
-    if (BIF_ARG_1 == am_enabled) {
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	BIF_RET(am_true);
+BIF_RETTYPE erts_debug_lcnt_collect_0(BIF_ALIST_0)
+{
+#ifndef ERTS_ENABLE_LOCK_COUNT
+    BIF_RET(am_error);
 #else
-	BIF_RET(am_false);
-#endif
-    }
-#ifdef ERTS_ENABLE_LOCK_COUNT
+    lcnt_sample_vector_t current_locks, deleted_locks;
+    erts_lcnt_data_t data;
 
-    else if (BIF_ARG_1 == am_info) {
-	erts_lcnt_data_t *data; 
-	Uint hsize = 0;
-	Uint *szp;
-    	Eterm* hp;
+    Eterm *term_heap_start, *term_heap_end;
+    Uint term_heap_size = 0;
+    Eterm result;
 
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+    data = erts_lcnt_get_data();
 
-	erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_SUSPEND);
-	data = erts_lcnt_get_data();
+    current_locks = lcnt_build_sample_vector(data.current_locks);
+    deleted_locks = lcnt_build_sample_vector(data.deleted_locks);
 
-	/* calculate size */
+    lcnt_build_result_term(NULL, &term_heap_size, &data.duration,
+        &current_locks, &deleted_locks, NIL);
 
-	szp = &hsize;
-	lcnt_build_result_term(NULL, szp, data, NIL);
+    term_heap_start = HAlloc(BIF_P, term_heap_size);
+    term_heap_end = term_heap_start;
 
-	/* alloc and build */
+    result = lcnt_build_result_term(&term_heap_end, NULL,
+        &data.duration, &current_locks, &deleted_locks, NIL);
 
-	hp = HAlloc(BIF_P, hsize);
+    HRelease(BIF_P, term_heap_start + term_heap_size, term_heap_end);
 
-	res = lcnt_build_result_term(&hp, NULL, data, res);
-	
-	erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_SUSPEND);
+    lcnt_destroy_sample_vector(&current_locks);
+    lcnt_destroy_sample_vector(&deleted_locks);
 
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	
-	BIF_RET(res);
-    } else if (BIF_ARG_1 == am_clear) {
-	erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-	erts_smp_thr_progress_block();
+    BIF_RET(result);
+#endif
+}
+
+BIF_RETTYPE erts_debug_lcnt_control_1(BIF_ALIST_1)
+{
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    if(ERTS_IS_ATOM_STR("mask", BIF_ARG_1)) {
+        erts_lock_flags_t mask;
+        Eterm *term_heap_block;
+        Uint term_heap_size;
 
-	erts_lcnt_clear_counters();
+        mask = erts_lcnt_get_category_mask();
+        term_heap_size = 0;
 
-	erts_smp_thr_progress_unblock();
-	erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
+        lcnt_build_category_list(NULL, &term_heap_size, mask);
 
-	BIF_RET(am_ok);
-    } else if (is_tuple(BIF_ARG_1)) {
-        Eterm* ptr = tuple_val(BIF_ARG_1);
-
-        if ((arityval(ptr[0]) == 2) && (ptr[2] == am_false || ptr[2] == am_true)) {
-            int lock_opt = 0, enable = (ptr[2] == am_true) ? 1 : 0;
-            if (ERTS_IS_ATOM_STR("copy_save", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_COPYSAVE;
-            } else if (ERTS_IS_ATOM_STR("process_locks", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_PROCLOCK;
-            } else if (ERTS_IS_ATOM_STR("port_locks", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_PORTLOCK;
-            } else if (ERTS_IS_ATOM_STR("suspend", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_SUSPEND;
-            } else if (ERTS_IS_ATOM_STR("location", ptr[1])) {
-                lock_opt = ERTS_LCNT_OPT_LOCATION;
-            } else {
-                BIF_ERROR(BIF_P, BADARG);
-            }
+        term_heap_block = HAlloc(BIF_P, term_heap_size);
 
-            erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN);
-            erts_smp_thr_progress_block();
+        BIF_RET(lcnt_build_category_list(&term_heap_block, NULL, mask));
+    } else if(ERTS_IS_ATOM_STR("copy_save", BIF_ARG_1)) {
+        if(erts_lcnt_get_preserve_info()) {
+            BIF_RET(am_true);
+        }
 
-            if (enable) res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false;
-            else res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false;
-            
-#ifdef ERTS_SMP
-            if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PORTLOCK) {
-                erts_lcnt_enable_io_lock_count(enable);
-            } else if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PROCLOCK) {
-                erts_lcnt_enable_proc_lock_count(enable);
-            }
+        BIF_RET(am_false);
+    }
 #endif
-            erts_smp_thr_progress_unblock();
-            erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN);
-            BIF_RET(res);
+    BIF_ERROR(BIF_P, BADARG);
+}
+
+BIF_RETTYPE erts_debug_lcnt_control_2(BIF_ALIST_2)
+{
+#ifdef ERTS_ENABLE_LOCK_COUNT
+    if(ERTS_IS_ATOM_STR("mask", BIF_ARG_1)) {
+        erts_lock_flags_t category_mask = 0;
+        Eterm categories = BIF_ARG_2;
+
+        if(!(is_list(categories) || is_nil(categories))) {
+            BIF_ERROR(BIF_P, BADARG);
         }
-    } 
 
-#endif 
+        while(is_list(categories)) {
+            Eterm *cell = list_val(categories);
+            erts_lock_flags_t category;
+
+            category = lcnt_atom_to_lock_category(CAR(cell));
+
+            if(!category) {
+                Eterm *hp = HAlloc(BIF_P, 4);
+
+                BIF_RET(TUPLE3(hp, am_error, am_badarg, CAR(cell)));
+            }
+
+            category_mask |= category;
+            categories = CDR(cell);
+        }
+
+        erts_lcnt_set_category_mask(category_mask);
+
+        BIF_RET(am_ok);
+    } else if(BIF_ARG_2 == am_true || BIF_ARG_2 == am_false) {
+        int enabled = (BIF_ARG_2 == am_true);
+
+        if(ERTS_IS_ATOM_STR("copy_save", BIF_ARG_1)) {
+            erts_lcnt_set_preserve_info(enabled);
+
+            BIF_RET(am_ok);
+        }
+    }
+#endif
     BIF_ERROR(BIF_P, BADARG);
 }