Make lock counter info independent of the locks being counted

This allows us to enable/disable lock counting at will, and greatly improves
the performance of erts_debug:lock_counters/1 since we no longer have to
worry about the lock counters "dying" while we're enumerating them.

OTP-14412

1 files changed, 352 insertions, 480 deletions

diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c
index aee9796171..2cf59aa367 100644
--- a/erts/emulator/beam/erl_lock_count.c
+++ b/erts/emulator/beam/erl_lock_count.c
@@ -1,7 +1,7 @@
 /*
  * %CopyrightBegin%
  *
- * Copyright Ericsson AB 2008-2016. All Rights Reserved.
+ * Copyright Ericsson AB 2008-2017. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,51 +18,30 @@
  * %CopyrightEnd%
  */
 
-/*
- * Description:	Statistics for locks.
- *
- * Author:	Björn-Egil Dahlberg
- * Date:	2008-07-03
- */
-
 #ifdef HAVE_CONFIG_H
 #  include "config.h"
 #endif
 
-/* Needed for VxWorks va_arg */
-#include "sys.h"
-
 #ifdef ERTS_ENABLE_LOCK_COUNT
 
+#include "sys.h"
+
 #include "erl_lock_count.h"
-#include "ethread.h"
-#include "erl_term.h"
-#include "atom.h"
-#include <stdio.h>
+#include "erl_thr_progress.h"
 
-/* globals, dont access these without locks or blocks */
+#define LCNT_MAX_CARRIER_ENTRIES 255
 
-ethr_mutex lcnt_data_lock;
-erts_lcnt_data_t *erts_lcnt_data;
-Uint16 erts_lcnt_rt_options;
-erts_lcnt_time_t timer_start;
-const char *str_undefined = "undefined";
+/* - Exported global - */
 
-static ethr_tsd_key lcnt_thr_data_key;
-static int lcnt_n_thr;
-static erts_lcnt_thread_data_t *lcnt_thread_data[2048];
+Uint16 erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK | ERTS_LCNT_OPT_LOCATION;
 
-/* local functions */
+/* - Locals that are shared with the header implementation - */
 
-static ERTS_INLINE void lcnt_lock(void) {
-    ethr_mutex_lock(&lcnt_data_lock);
-}
+int lcnt_initialization_completed__ = 0;
 
-static ERTS_INLINE void lcnt_unlock(void) {
-    ethr_mutex_unlock(&lcnt_data_lock);
-}
+ethr_tsd_key lcnt_thr_data_key__;
 
-const int log2_tab64[64] = {
+const int lcnt_log2_tab64__[64] = {
     63,  0, 58,  1, 59, 47, 53,  2,
     60, 39, 48, 27, 54, 33, 42,  3,
     61, 51, 37, 40, 49, 18, 28, 20,
@@ -72,99 +51,52 @@ const int log2_tab64[64] = {
     56, 45, 25, 31, 35, 16,  9, 12,
     44, 24, 15,  8, 23,  7,  6,  5};
 
-static ERTS_INLINE int lcnt_log2(Uint64 v) {
-    v |= v >> 1;
-    v |= v >> 2;
-    v |= v >> 4;
-    v |= v >> 8;
-    v |= v >> 16;
-    v |= v >> 32;
-    return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58];
-}
-
-static char* lcnt_lock_type(Uint16 flag) {
-    switch(flag & ERTS_LCNT_LT_ALL) {
-        case ERTS_LCNT_LT_SPINLOCK:   return "spinlock";
-        case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock";
-        case ERTS_LCNT_LT_MUTEX:      return "mutex";
-        case ERTS_LCNT_LT_RWMUTEX:    return "rw_mutex";
-        case ERTS_LCNT_LT_PROCLOCK:   return "proclock";
-        default:                      return "";
-    }
-}
+/* - Local variables - */
 
-static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) {
-    ethr_atomic_set(&stats->tries, 0);
-    ethr_atomic_set(&stats->colls, 0);
-    stats->timer.s  = 0;
-    stats->timer.ns = 0;
-    stats->timer_n  = 0;
-    stats->file     = (char *)str_undefined;
-    stats->line     = 0;
-    sys_memzero(stats->hist.ns, sizeof(stats->hist.ns));
-}
+static erts_lcnt_lock_info_list_t lcnt_current_lock_list;
+static erts_lcnt_lock_info_list_t lcnt_deleted_lock_list;
 
-static void lcnt_time(erts_lcnt_time_t *time) {
-    /*
-     * erts_sys_hrtime() is the highest resolution
-     * we could find, it may or may not be monotonic...
-     */
-    ErtsMonotonicTime mtime = erts_sys_hrtime();
-    time->s  = (unsigned long) (mtime / 1000000000LL);
-    time->ns = (unsigned long) (mtime - 1000000000LL*time->s);
-}
+static erts_lcnt_time_t lcnt_timer_start;
 
-static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) {
-    long ds;
-    long dns;
+/* local functions */
 
-    ds  = t1->s  - t0->s;
-    dns = t1->ns - t0->ns;
+static void lcnt_clear_stats(erts_lcnt_lock_info_t *info) {
+    size_t i;
 
-    /* the difference should not be able to get bigger than 1 sec in ns*/
+    for(i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
+        erts_lcnt_lock_stats_t *stats = &info->location_stats[i];
 
-    if (dns < 0) {
-        ds  -= 1;
-        dns += 1000000000LL;
-    }
+        sys_memzero(&stats->wait_time_histogram, sizeof(stats->wait_time_histogram));
 
-    ASSERT(ds >= 0);
+        stats->total_time_waited.s = 0;
+        stats->total_time_waited.ns = 0;
 
-    d->s  = ds;
-    d->ns = dns;
-}
+        stats->times_waited = 0;
 
-/* difference d must be non-negative */
+        stats->file = NULL;
+        stats->line = 0;
 
-static void lcnt_time_add(erts_lcnt_time_t *t, erts_lcnt_time_t *d) {
-    t->s  += d->s;
-    t->ns += d->ns;
+        ethr_atomic_set(&stats->attempts, 0);
+        ethr_atomic_set(&stats->collisions, 0);
+    }
 
-    t->s  += t->ns / 1000000000LL;
-    t->ns  = t->ns % 1000000000LL;
+    info->location_count = 1;
 }
 
-static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) {
-    erts_lcnt_thread_data_t *eltd;
+static lcnt_thread_data_t__ *lcnt_thread_data_alloc(void) {
+    lcnt_thread_data_t__ *eltd =
+        (lcnt_thread_data_t__*)malloc(sizeof(lcnt_thread_data_t__));
 
-    eltd = (erts_lcnt_thread_data_t*)malloc(sizeof(erts_lcnt_thread_data_t));
-    if (!eltd) {
-        ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
+    if(!eltd) {
+        ERTS_INTERNAL_ERROR("Failed to allocate lcnt thread data.");
     }
+
     eltd->timer_set = 0;
     eltd->lock_in_conflict = 0;
 
-    eltd->id = lcnt_n_thr++;
-    /* set thread data to array */
-    lcnt_thread_data[eltd->id] = eltd;
-
     return eltd;
 }
 
-static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) {
-    return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key);
-}
-
 /* debug */
 
 #if 0
@@ -175,472 +107,391 @@ static char* lock_opt(Uint16 flag) {
     return "--";
 }
 
-static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action) {
-    erts_aint_t w_state, r_state;
+static void print_lock_x(erts_lcnt_lock_info_t *info, Uint16 flag, char *action) {
+    ethr_sint_t w_state, r_state;
     char *type;
 
-    if (strcmp(lock->name, "run_queue") != 0) return;
-    type = lcnt_lock_type(lock->flag);
-    r_state = ethr_atomic_read(&lock->r_state);
-    w_state = ethr_atomic_read(&lock->w_state);
+    if (strcmp(info->name, "run_queue") != 0) return;
+    type = erts_lcnt_lock_type(info->flag);
+    r_state = ethr_atomic_read(&info->r_state);
+    w_state = ethr_atomic_read(&info->w_state);
 
-    if (lock->flag & flag) {
+    if (info->flag & flag) {
         erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n",
                 action,
-                lock->name,
+                info->name,
                 r_state,
                 w_state,
                 type,
-                lock->id);
+                info->id);
     }
 }
 #endif
 
-static erts_lcnt_lock_stats_t *lcnt_get_lock_stats(erts_lcnt_lock_t *lock, char *file, unsigned int line) {
-    unsigned int i;
-    erts_lcnt_lock_stats_t *stats = NULL;
-
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) {
-        for (i = 0; i < lock->n_stats; i++) {
-            if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) {
-                return &(lock->stats[i]);
-            }
-        }
-        if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) {
-            stats = &lock->stats[lock->n_stats];
-            lock->n_stats++;
-            stats->file = file;
-            stats->line = line;
-            return stats;
-        }
-    }
-    return &lock->stats[0];
+/* - List operations -
+ *
+ * Info entries are kept in a doubly linked list where each entry is locked
+ * with its neighbors rather than a global lock. Deletion is rather quick, but
+ * insertion is still serial since the head becomes a de facto global lock.
+ *
+ * We rely on ad-hoc spinlocks to avoid "recursing" into this module. */
+
+#define LCNT_SPINLOCK_YIELD_ITERATIONS 50
+
+#define LCNT_SPINLOCK_HELPER_INIT \
+    Uint failed_spin_count = 0;
+
+#define LCNT_SPINLOCK_HELPER_YIELD \
+    do { \
+        failed_spin_count++; \
+        if(!(failed_spin_count % LCNT_SPINLOCK_YIELD_ITERATIONS)) { \
+            erts_thr_yield(); \
+        } else { \
+            ERTS_SPIN_BODY; \
+        } \
+    } while(0)
+
+static void lcnt_unlock_list_entry(erts_lcnt_lock_info_t *info) {
+    ethr_atomic32_set_relb(&info->lock, 0);
 }
 
-static void lcnt_update_stats_hist(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_wait) {
-    int idx;
-    unsigned long r;
+static int lcnt_try_lock_list_entry(erts_lcnt_lock_info_t *info) {
+    return ethr_atomic32_cmpxchg_acqb(&info->lock, 1, 0) == 0;
+}
 
-    if (time_wait->s > 0 || time_wait->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) {
-        idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1;
-    } else {
-        r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT;
-        if (r) idx = lcnt_log2(r);
-        else idx = 0;
+static void lcnt_lock_list_entry(erts_lcnt_lock_info_t *info) {
+    LCNT_SPINLOCK_HELPER_INIT;
+
+    while(!lcnt_try_lock_list_entry(info)) {
+        LCNT_SPINLOCK_HELPER_YIELD;
     }
-    hist->ns[idx]++;
 }
 
-static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict,
-        erts_lcnt_time_t *time_wait) {
+static void lcnt_lock_list_entry_with_neighbors(erts_lcnt_lock_info_t *info) {
+    LCNT_SPINLOCK_HELPER_INIT;
 
-    ethr_atomic_inc(&stats->tries);
+    for(;;) {
+        if(!lcnt_try_lock_list_entry(info))
+            goto retry_after_entry_failed;
+        if(!lcnt_try_lock_list_entry(info->next))
+            goto retry_after_next_failed;
+        if(!lcnt_try_lock_list_entry(info->prev))
+            goto retry_after_prev_failed;
 
-    if (lock_in_conflict)
-        ethr_atomic_inc(&stats->colls);
+        return;
 
-    if (time_wait) {
-        lcnt_time_add(&(stats->timer), time_wait);
-        stats->timer_n++;
-        lcnt_update_stats_hist(&stats->hist,time_wait);
+    retry_after_prev_failed:
+        lcnt_unlock_list_entry(info->next);
+    retry_after_next_failed:
+        lcnt_unlock_list_entry(info);
+    retry_after_entry_failed:
+        LCNT_SPINLOCK_HELPER_YIELD;
     }
 }
 
-/* interface */
+static void lcnt_unlock_list_entry_with_neighbors(erts_lcnt_lock_info_t *info) {
+    lcnt_unlock_list_entry(info->prev);
+    lcnt_unlock_list_entry(info->next);
+    lcnt_unlock_list_entry(info);
+}
 
-void erts_lcnt_init() {
-    erts_lcnt_thread_data_t *eltd = NULL;
+static void lcnt_insert_list_entry(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t *info) {
+    erts_lcnt_lock_info_t *next, *prev;
 
-    /* init lock */
-    if (ethr_mutex_init(&lcnt_data_lock) != 0) abort();
+    prev = &list->head;
 
-    /* init tsd */
-    lcnt_n_thr = 0;
-    ethr_tsd_key_create(&lcnt_thr_data_key, "lcnt_data");
+    lcnt_lock_list_entry(prev);
 
-    lcnt_lock();
+    next = prev->next;
 
-    erts_lcnt_rt_options = ERTS_LCNT_OPT_LOCATION | ERTS_LCNT_OPT_PROCLOCK;
-    eltd = lcnt_thread_data_alloc();
-    ethr_tsd_set(lcnt_thr_data_key, eltd);
+    lcnt_lock_list_entry(next);
 
-    /* init lcnt structure */
-    erts_lcnt_data = (erts_lcnt_data_t*)malloc(sizeof(erts_lcnt_data_t));
-    if (!erts_lcnt_data) {
-        ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
-    }
-    erts_lcnt_data->current_locks = erts_lcnt_list_init();
-    erts_lcnt_data->deleted_locks = erts_lcnt_list_init();
+    info->next = next;
+    info->prev = prev;
 
-    lcnt_unlock();
+    prev->next = info;
+    next->prev = info;
 
+    lcnt_unlock_list_entry(next);
+    lcnt_unlock_list_entry(prev);
 }
 
-void erts_lcnt_late_init() {
-    /* set start timer and zero statistics */
-    erts_lcnt_clear_counters();
-    erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler);
-}
+static void lcnt_insert_list_carrier(erts_lcnt_lock_info_list_t *list,
+                                     erts_lcnt_lock_info_carrier_t *carrier) {
+    erts_lcnt_lock_info_t *next, *prev;
+    size_t i;
 
-/* list operations */
+    for(i = 0; i < carrier->entry_count; i++) {
+        erts_lcnt_lock_info_t *info = &carrier->entries[i];
 
-/* BEGIN ASSUMPTION: lcnt_data_lock taken */
+        info->prev = &carrier->entries[i - 1];
+        info->next = &carrier->entries[i + 1];
+    }
 
-erts_lcnt_lock_list_t *erts_lcnt_list_init(void) {
-    erts_lcnt_lock_list_t *list;
+    prev = &list->head;
 
-    list = (erts_lcnt_lock_list_t*)malloc(sizeof(erts_lcnt_lock_list_t));
-    if (!list) {
-        ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
-    }
-    list->head = NULL;
-    list->tail = NULL;
-    list->n    = 0;
-    return list;
-}
+    lcnt_lock_list_entry(prev);
 
-static void lcnt_list_free(erts_lcnt_lock_t *head) {
-    erts_lcnt_lock_t *lock, *next;
+    next = prev->next;
 
-    lock = head;
+    lcnt_lock_list_entry(next);
 
-    while(lock != NULL) {
-        next = lock->next;
-        free(lock);
-        lock = next;
-    }
+    next->prev = &carrier->entries[carrier->entry_count - 1];
+    carrier->entries[carrier->entry_count - 1].next = next;
+
+    prev->next = &carrier->entries[0];
+    carrier->entries[0].prev = prev;
+
+    lcnt_unlock_list_entry(next);
+    lcnt_unlock_list_entry(prev);
 }
 
-void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) {
-    erts_lcnt_lock_t *tail = NULL;
+static void lcnt_init_list(erts_lcnt_lock_info_list_t *list) {
+    /* Ensure that ref_count operations explode when touching the sentinels in
+     * DEBUG mode. */
+    ethr_atomic_init(&(list->head.ref_count), -1);
+    ethr_atomic_init(&(list->tail.ref_count), -1);
 
-    tail = list->tail;
-    if (tail) {
-        tail->next = lock;
-        lock->prev = tail;
-    } else {
-        list->head = lock;
-        lock->prev = NULL;
-        ASSERT(!lock->next);
-    }
-    lock->next = NULL;
-    list->tail = lock;
+    ethr_atomic32_init(&(list->head.lock), 0);
+    (list->head).next = &list->tail;
+    (list->head).prev = &list->tail;
 
-    list->n++;
+    ethr_atomic32_init(&(list->tail.lock), 0);
+    (list->tail).next = &list->head;
+    (list->tail).prev = &list->head;
 }
 
-void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) {
-    if (lock->next) lock->next->prev = lock->prev;
-    if (lock->prev) lock->prev->next = lock->next;
-    if (list->head == lock) list->head = lock->next;
-    if (list->tail == lock) list->tail = lock->prev;
+/* - Carrier operations - */
 
-    lock->prev = NULL;
-    lock->next = NULL;
-    list->n--;
+int lcnt_thr_progress_unmanaged_delay__(void) {
+    return erts_thr_progress_unmanaged_delay();
 }
-/* END ASSUMPTION: lcnt_data_lock taken */
 
+void lcnt_thr_progress_unmanaged_continue__(int handle) {
+    return erts_thr_progress_unmanaged_continue(handle);
+}
 
-/* lock operations */
+static void lcnt_deallocate_carrier_malloc(erts_lcnt_lock_info_carrier_t *carrier) {
+    ASSERT(ethr_atomic_read(&carrier->ref_count) == 0);
+    free(carrier);
+}
 
-/* interface to erl_threads.h */
-/* only lock on init and destroy, all others should use atomics */
-void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) {
-    erts_lcnt_init_lock_x(lock, name, flag, NIL);
+static void lcnt_deallocate_carrier_erts(erts_lcnt_lock_info_carrier_t *carrier) {
+    ASSERT(ethr_atomic_read(&carrier->ref_count) == 0);
+    erts_free(ERTS_ALC_T_LCNT_CARRIER, (void*)carrier);
 }
 
-void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) {
-    int i;
+static void lcnt_thr_prg_cleanup_carrier(void *data) {
+    erts_lcnt_lock_info_carrier_t *carrier = data;
+    size_t entry_count, i;
 
-    if (flag & ERTS_LCNT_LT_DISABLE) {
-        ERTS_LCNT_CLEAR_FLAG(lock);
-        return;
+    /* carrier->entry_count will be replaced with garbage if it's deallocated
+     * on the final iteration, so we'll tuck it away to get a clean exit. */
+    entry_count = carrier->entry_count;
+
+    for(i = 0; i < entry_count; i++) {
+        ASSERT(ethr_atomic_read(&carrier->ref_count) >= (entry_count - i));
+
+        erts_lcnt_release_lock_info(&carrier->entries[i]);
     }
+}
 
-    lock->next = NULL;
-    lock->prev = NULL;
-    lock->flag = flag;
-    lock->name = name;
-    lock->id   = id;
+static void lcnt_schedule_carrier_cleanup(void *data) {
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
 
-    ethr_atomic_init(&lock->r_state, 0);
-    ethr_atomic_init(&lock->w_state, 0);
-#ifdef DEBUG
-    ethr_atomic_init(&lock->flowstate, 0);
-#endif
+    /* We can't issue cleanup jobs on anything other than normal schedulers, so
+     * we move to the first scheduler if required. */
+
+    if(!esdp || esdp->type != ERTS_SCHED_NORMAL) {
+        erts_schedule_misc_aux_work(1, &lcnt_schedule_carrier_cleanup, data);
+    } else {
+        erts_lcnt_lock_info_carrier_t *carrier = data;
+        size_t carrier_size;
 
-    lock->n_stats = 1;
+        carrier_size = sizeof(erts_lcnt_lock_info_carrier_t) +
+                       sizeof(erts_lcnt_lock_info_t) * carrier->entry_count;
 
-    for (i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
-        lcnt_clear_stats(&lock->stats[i]);
+        erts_schedule_thr_prgr_later_cleanup_op(&lcnt_thr_prg_cleanup_carrier,
+            data, (ErtsThrPrgrLaterOp*)&carrier->release_entries, carrier_size);
     }
+}
 
-    lcnt_lock();
-    erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock);
-    lcnt_unlock();
+static void lcnt_info_deallocate(erts_lcnt_lock_info_t *info) {
+    lcnt_release_carrier__(info->carrier);
 }
 
-/* init empty, instead of zero struct
- * used by process locks probes
- */
-void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) {
-    lock->next = NULL;
-    lock->prev = NULL;
-    lock->flag = 0;
-    lock->name = NULL;
-    lock->id   = NIL;
-    ethr_atomic_init(&lock->r_state, 0);
-    ethr_atomic_init(&lock->w_state, 0);
-#ifdef DEBUG
-    ethr_atomic_init(&lock->flowstate, 0);
-#endif
-    lock->n_stats = 0;
-    sys_memzero(lock->stats, sizeof(lock->stats));
-}
-/* destroy lock */
-void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) {
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-    lcnt_lock();
-
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) {
-        erts_lcnt_lock_t *deleted_lock;
-        /* copy structure and insert the copy */
-        deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t));
-        if (!deleted_lock) {
-            ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!");
-        }
-        memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t));
-        deleted_lock->next = NULL;
-        deleted_lock->prev = NULL;
-        erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock);
-    }
-    /* delete original */
-    erts_lcnt_list_delete(erts_lcnt_data->current_locks, lock);
-    ERTS_LCNT_CLEAR_FLAG(lock);
+static void lcnt_info_dispose(erts_lcnt_lock_info_t *info) {
+    ASSERT(ethr_atomic_read(&info->ref_count) == 0);
 
-    lcnt_unlock();
+    if(erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) {
+        ethr_atomic_set(&info->ref_count, 1);
+
+        /* Move straight to deallocation the next time around. */
+        info->dispose = &lcnt_info_deallocate;
+
+        lcnt_insert_list_entry(&lcnt_deleted_lock_list, info);
+    } else {
+        lcnt_info_deallocate(info);
+    }
 }
 
-/* lock */
+static void lcnt_lock_info_init_helper(erts_lcnt_lock_info_t *info) {
+#ifdef DEBUG
+    ethr_atomic_init(&info->flowstate, 0);
+#endif
 
-void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
-    erts_aint_t r_state = 0, w_state = 0;
-    erts_lcnt_thread_data_t *eltd;
+    ethr_atomic_init(&info->ref_count, 1);
+    ethr_atomic32_init(&info->lock, 0);
 
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
+    ethr_atomic_init(&info->r_state, 0);
+    ethr_atomic_init(&info->w_state, 0);
 
-    eltd = lcnt_get_thread_data();
-    ASSERT(eltd);
+    info->dispose = &lcnt_info_dispose;
 
-    w_state = ethr_atomic_read(&lock->w_state);
+    lcnt_clear_stats(info);
+}
 
-    if (option & ERTS_LCNT_LO_WRITE) {
-        r_state = ethr_atomic_read(&lock->r_state);
-        ethr_atomic_inc( &lock->w_state);
-    }
-    if (option & ERTS_LCNT_LO_READ) {
-        ethr_atomic_inc( &lock->r_state);
-    }
+erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int entry_count) {
+    erts_lcnt_lock_info_carrier_t *result;
+    size_t carrier_size, i;
+
+    ASSERT(entry_count > 0 && entry_count <= LCNT_MAX_CARRIER_ENTRIES);
 
-    /* we cannot acquire w_lock if either w or r are taken */
-    /* we cannot acquire r_lock if w_lock is taken */
+    carrier_size = sizeof(erts_lcnt_lock_info_carrier_t) +
+                   sizeof(erts_lcnt_lock_info_t) * entry_count;
 
-    if ((w_state > 0) || (r_state > 0)) {
-        eltd->lock_in_conflict = 1;
-        if (eltd->timer_set == 0) {
-            lcnt_time(&eltd->timer);
-        }
-        eltd->timer_set++;
+    if(lcnt_initialization_completed__) {
+        result = (erts_lcnt_lock_info_carrier_t*)erts_alloc(ERTS_ALC_T_LCNT_CARRIER, carrier_size);
+        result->deallocate = &lcnt_deallocate_carrier_erts;
     } else {
-        eltd->lock_in_conflict = 0;
+        result = (erts_lcnt_lock_info_carrier_t*)malloc(carrier_size);
+        result->deallocate = &lcnt_deallocate_carrier_malloc;
     }
-}
 
-void erts_lcnt_lock(erts_lcnt_lock_t *lock) {
-    erts_aint_t w_state;
-    erts_lcnt_thread_data_t *eltd;
+    ethr_atomic_init(&result->ref_count, entry_count);
 
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
+    result->entry_count = entry_count;
 
-    w_state = ethr_atomic_read(&lock->w_state);
-    ethr_atomic_inc(&lock->w_state);
-    eltd = lcnt_get_thread_data();
+    for(i = 0; i < entry_count; i++) {
+        erts_lcnt_lock_info_t *info = &result->entries[i];
 
-    ASSERT(eltd);
+        lcnt_lock_info_init_helper(info);
 
-    if (w_state > 0) {
-        eltd->lock_in_conflict = 1;
-        /* only set the timer if nobody else has it
-         * This should only happen when proc_locks aquires several locks
-         * 'atomicly'. All other locks will block the thread if w_state > 0
-         * i.e. locked.
-         */
-        if (eltd->timer_set == 0) {
-            lcnt_time(&eltd->timer);
-        }
-        eltd->timer_set++;
-    } else {
-        eltd->lock_in_conflict = 0;
+        info->carrier = result;
     }
-}
-
-/* if a lock wasn't really a lock operation, bad bad process locks */
-
-void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) {
-    /* should check if this thread was "waiting" */
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
 
-    ethr_atomic_dec(&lock->w_state);
+    return result;
 }
 
-/*
- * erts_lcnt_lock_post
- *
- * Used when we get a lock (i.e. directly after a lock operation)
- * if the timer was set then we had to wait for the lock
- * lock_post will calculate the wait time.
- */
+void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carrier) {
+    ethr_sint_t swapped_carrier;
 
-void erts_lcnt_lock_post(erts_lcnt_lock_t *lock) {
-    erts_lcnt_lock_post_x(lock, (char*)str_undefined, 0);
-}
+    swapped_carrier = ethr_atomic_cmpxchg_mb(ref, (ethr_sint_t)carrier, (ethr_sint_t)NULL);
 
-void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line) {
-    erts_lcnt_thread_data_t *eltd;
-    erts_lcnt_time_t timer;
-    erts_lcnt_time_t time_wait;
-    erts_lcnt_lock_stats_t *stats;
+    if(swapped_carrier != (ethr_sint_t)NULL) {
 #ifdef DEBUG
-    erts_aint_t flowstate;
+        ASSERT(ethr_atomic_read(&carrier->ref_count) == carrier->entry_count);
+        ethr_atomic_set(&carrier->ref_count, 0);
 #endif
 
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
+        carrier->deallocate(carrier);
+    } else {
+        lcnt_insert_list_carrier(&lcnt_current_lock_list, carrier);
+    }
+}
+
+void erts_lcnt_uninstall(erts_lcnt_ref_t *ref) {
+    ethr_sint_t previous_carrier, swapped_carrier;
 
-#ifdef DEBUG
-    if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) {
-        flowstate = ethr_atomic_read(&lock->flowstate);
-        ASSERT(flowstate == 0);
-        ethr_atomic_inc(&lock->flowstate);
+    previous_carrier = ethr_atomic_read(ref);
+    swapped_carrier = ethr_atomic_cmpxchg_mb(ref, (ethr_sint_t)NULL, previous_carrier);
+
+    if(previous_carrier && previous_carrier == swapped_carrier) {
+        lcnt_schedule_carrier_cleanup((void*)previous_carrier);
     }
-#endif
+}
 
-    eltd = lcnt_get_thread_data();
+/* - Initialization - */
 
-    ASSERT(eltd);
+void erts_lcnt_pre_thr_init() {
+    /* Ensure that the dependency hack mentioned in the header doesn't
+     * explode at runtime. */
+    ERTS_CT_ASSERT(sizeof(LcntThrPrgrLaterOp) >= sizeof(ErtsThrPrgrLaterOp));
+    ERTS_CT_ASSERT(ERTS_THR_PRGR_DHANDLE_MANAGED == 
+        (ErtsThrPrgrDelayHandle)LCNT_THR_PRGR_DHANDLE_MANAGED);
 
-    /* if lock was in conflict, time it */
-    stats = lcnt_get_lock_stats(lock, file, line);
-    if (eltd->timer_set) {
-        lcnt_time(&timer);
+    lcnt_init_list(&lcnt_current_lock_list);
+    lcnt_init_list(&lcnt_deleted_lock_list);
+}
 
-        lcnt_time_diff(&time_wait, &timer, &(eltd->timer));
-        lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait);
-        eltd->timer_set--;
-        ASSERT(eltd->timer_set >= 0);
-    } else {
-        lcnt_update_stats(stats, eltd->lock_in_conflict, NULL);
-    }
+void erts_lcnt_post_thr_init() {
+    /* ASSUMPTION: this is safe since it runs prior to the creation of other
+     * threads (Directly after ethread init). */
 
+    ethr_tsd_key_create(&lcnt_thr_data_key__, "lcnt_data");
+
+    erts_lcnt_thread_setup();
 }
 
-/* unlock */
+void erts_lcnt_late_init() {
+    /* Set start timer and zero all statistics */
+    erts_lcnt_clear_counters();
+    erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler);
 
-void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-    if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_dec(&lock->w_state);
-    if (option & ERTS_LCNT_LO_READ ) ethr_atomic_dec(&lock->r_state);
+    /* It's safe to use erts_alloc and thread progress past this point. */
+    lcnt_initialization_completed__ = 1;
 }
 
-void erts_lcnt_unlock(erts_lcnt_lock_t *lock) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-#ifdef DEBUG
-    {
-        erts_aint_t w_state;
-        erts_aint_t flowstate;
-
-        /* flowstate */
-        flowstate = ethr_atomic_read(&lock->flowstate);
-        ASSERT(flowstate == 1);
-        ethr_atomic_dec(&lock->flowstate);
-
-        /* write state */
-        w_state = ethr_atomic_read(&lock->w_state);
-        ASSERT(w_state > 0);
-    }
-#endif
-    ethr_atomic_dec(&lock->w_state);
+void erts_lcnt_thread_setup() {
+    lcnt_thread_data_t__ *eltd = lcnt_thread_data_alloc();
+
+    ASSERT(eltd);
+
+    ethr_tsd_set(lcnt_thr_data_key__, eltd);
 }
 
-/* trylock */
+void erts_lcnt_thread_exit_handler() {
+    lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__();
 
-void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) {
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-    /* Determine lock_state via res instead of state */
-    if (res != EBUSY) {
-        if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state);
-        if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state);
-        lcnt_update_stats(&(lock->stats[0]), 0, NULL);
-    } else {
-        ethr_atomic_inc(&lock->stats[0].tries);
-        ethr_atomic_inc(&lock->stats[0].colls);
+    if (eltd) {
+        free(eltd);
     }
 }
 
+/* - BIF interface - */
 
-void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) {
-    /* Determine lock_state via res instead of state */
-    if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return;
-    if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return;
-    if (res != EBUSY) {
+void erts_lcnt_retain_lock_info(erts_lcnt_lock_info_t *info) {
 #ifdef DEBUG
-        {
-            erts_aint_t flowstate;
-            flowstate = ethr_atomic_read(&lock->flowstate);
-            ASSERT(flowstate == 0);
-            ethr_atomic_inc( &lock->flowstate);
-        }
+    ASSERT(ethr_atomic_inc_read_acqb(&info->ref_count) >= 2);
+#else
+    ethr_atomic_inc_acqb(&info->ref_count);
 #endif
-        ethr_atomic_inc(&lock->w_state);
-        lcnt_update_stats(&(lock->stats[0]), 0, NULL);
-    } else {
-        ethr_atomic_inc(&lock->stats[0].tries);
-        ethr_atomic_inc(&lock->stats[0].colls);
-    }
 }
 
-/* thread operations */
+void erts_lcnt_release_lock_info(erts_lcnt_lock_info_t *info) {
+    ethr_sint_t count;
 
-void erts_lcnt_thread_setup(void) {
-    erts_lcnt_thread_data_t *eltd;
+    /* We need to acquire the lock before decrementing ref_count to avoid
+     * racing with list iteration; there's a short window between reading the
+     * reference to info and increasing its ref_count. */
+    lcnt_lock_list_entry_with_neighbors(info);
 
-    lcnt_lock();
-    /* lock for thread id global update */
-    eltd = lcnt_thread_data_alloc();
-    lcnt_unlock();
-    ASSERT(eltd);
-    ethr_tsd_set(lcnt_thr_data_key, eltd);
-}
+    count = ethr_atomic_dec_read(&info->ref_count);
 
-void erts_lcnt_thread_exit_handler() {
-    erts_lcnt_thread_data_t *eltd;
+    ASSERT(count >= 0);
 
-    eltd = ethr_tsd_get(lcnt_thr_data_key);
+    if(count > 0) {
+        lcnt_unlock_list_entry_with_neighbors(info);
+    } else {
+        (info->next)->prev = info->prev;
+        (info->prev)->next = info->next;
 
-    if (eltd) {
-        free(eltd);
+        lcnt_unlock_list_entry_with_neighbors(info);
+
+        info->dispose(info);
     }
 }
 
-/* bindings for bifs */
-
 Uint16 erts_lcnt_set_rt_opt(Uint16 opt) {
     Uint16 prev;
     prev = (erts_lcnt_rt_options & opt);
@@ -656,51 +507,72 @@ Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) {
 }
 
 void erts_lcnt_clear_counters(void) {
-    erts_lcnt_lock_t *lock;
-    erts_lcnt_lock_list_t *list;
-    erts_lcnt_lock_stats_t *stats;
-    int i;
+    erts_lcnt_lock_info_t *iterator;
 
-    lcnt_lock();
+    lcnt_time__(&lcnt_timer_start);
 
-    list = erts_lcnt_data->current_locks;
+    iterator = NULL;
+    while(erts_lcnt_iterate_list(&lcnt_current_lock_list, &iterator)) {
+        lcnt_clear_stats(iterator);
+    }
 
-    for (lock = list->head; lock != NULL; lock = lock->next) {
-        for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) {
-            stats = &lock->stats[i];
-            lcnt_clear_stats(stats);
-        }
-        lock->n_stats = 1;
+    iterator = NULL;
+    while(erts_lcnt_iterate_list(&lcnt_deleted_lock_list, &iterator)) {
+        erts_lcnt_release_lock_info(iterator);
     }
+}
+
+erts_lcnt_data_t erts_lcnt_get_data(void) {
+    erts_lcnt_time_t timer_stop;
+    erts_lcnt_data_t result;
 
-    lock = erts_lcnt_data->deleted_locks->head;
-    erts_lcnt_data->deleted_locks->head = NULL;
-    erts_lcnt_data->deleted_locks->tail = NULL;
-    erts_lcnt_data->deleted_locks->n = 0;
+    lcnt_time__(&timer_stop);
 
-    lcnt_time(&timer_start);
+    result.timer_start = lcnt_timer_start;
 
-    lcnt_unlock();
+    result.current_locks = &lcnt_current_lock_list;
+    result.deleted_locks = &lcnt_deleted_lock_list;
 
-    /* free deleted locks */
-    lcnt_list_free(lock);
+    lcnt_time_diff__(&result.duration, &timer_stop, &result.timer_start);
+
+    return result;
 }
 
-erts_lcnt_data_t *erts_lcnt_get_data(void) {
-    erts_lcnt_time_t timer_stop;
+const char *erts_lcnt_lock_type(Uint16 type) {
+    switch(type & ERTS_LCNT_LT_ALL) {
+        case ERTS_LCNT_LT_SPINLOCK:   return "spinlock";
+        case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock";
+        case ERTS_LCNT_LT_MUTEX:      return "mutex";
+        case ERTS_LCNT_LT_RWMUTEX:    return "rw_mutex";
+        case ERTS_LCNT_LT_PROCLOCK:   return "proclock";
+        default:                      return "";
+    }
+}
 
-    lcnt_lock();
+int erts_lcnt_iterate_list(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t **iterator) {
+    erts_lcnt_lock_info_t *current, *next;
 
-    lcnt_time(&timer_stop);
-    lcnt_time_diff(&(erts_lcnt_data->duration), &timer_stop, &timer_start);
+    current = *iterator ? *iterator : &list->head;
 
-    lcnt_unlock();
+    ASSERT(current != &list->tail);
 
-    return erts_lcnt_data;
-}
+    lcnt_lock_list_entry(current);
+
+    next = current->next;
+
+    if(next != &list->tail) {
+        erts_lcnt_retain_lock_info(next);
+    }
+
+    lcnt_unlock_list_entry(current);
+
+    if(current != &list->head) {
+        erts_lcnt_release_lock_info(current);
+    }
+
+    *iterator = next;
 
-char *erts_lcnt_lock_type(Uint16 type) {
-    return lcnt_lock_type(type);
+    return next != &list->tail;
 }
 
-#endif /* ifdef ERTS_ENABLE_LOCK_COUNT */
+#endif /* #ifdef ERTS_ENABLE_LOCK_COUNT */