Allow toggling lock counting at runtime

The implementation is still hidden behind ERTS_ENABLE_LOCK_COUNT, and
all categories are still enabled by default, but the actual counting can be
toggled at will.

OTP-13170

11 files changed, 90 insertions, 36 deletions

diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c
index ad580e7d52..799f67fc45 100644
--- a/erts/emulator/sys/common/erl_check_io.c
+++ b/erts/emulator/sys/common/erl_check_io.c
@@ -2564,24 +2564,22 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 #ifdef ERTS_SMP
     init_removed_fd_alloc();
     pollset.removed_list = NULL;
-    erts_smp_spinlock_init(&pollset.removed_list_lock,
-			   "pollset_rm_list");
+    erts_smp_spinlock_init(&pollset.removed_list_lock, "pollset_rm_list", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     {
-	int i;
-	for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
-#ifdef ERTS_ENABLE_LOCK_COUNT
-	    erts_smp_mtx_init_x(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i));
-#else
-	    erts_smp_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state");
-#endif
-	}
+        int i;
+        for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) {
+            erts_smp_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i),
+                ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
+        }
     }
 #endif
 #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS
     max_fds = ERTS_CIO_POLL_MAX_FDS();
     erts_smp_atomic_init_nob(&drv_ev_state_len, 0);
     drv_ev_state = NULL;
-    erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow");
+    erts_smp_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 #else
     {
 	SafeHashFunctions hf;
@@ -2591,10 +2589,11 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void)
 	hf.free = &drv_ev_state_free;
 	num_state_prealloc = 0;
 	state_prealloc_first = NULL;
-	erts_smp_spinlock_init(&state_prealloc_lock,"state_prealloc");
+	erts_smp_spinlock_init(&state_prealloc_lock,"state_prealloc", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 
-	safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab", 
-		       DRV_EV_STATE_HTAB_SIZE, hf);
+	safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab",
+            ERTS_LOCK_FLAGS_CATEGORY_IO, DRV_EV_STATE_HTAB_SIZE, hf);
     }
 #endif
 }
@@ -3130,3 +3129,12 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip)
     return counters.num_errors;
 }
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) {
+#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS
+    erts_lcnt_enable_hash_lock_count(&drv_ev_state_tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable);
+#else
+    (void)enable;
+#endif
+}
+#endif /* ERTS_ENABLE_LOCK_COUNT */
diff --git a/erts/emulator/sys/common/erl_check_io.h b/erts/emulator/sys/common/erl_check_io.h
index ee4abeece9..2d3bb98afa 100644
--- a/erts/emulator/sys/common/erl_check_io.h
+++ b/erts/emulator/sys/common/erl_check_io.h
@@ -59,6 +59,11 @@ void erts_init_check_io_nkp(void);
 int erts_check_io_debug_kp(ErtsCheckIoDebugInfo *);
 int erts_check_io_debug_nkp(ErtsCheckIoDebugInfo *);
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_update_cio_locks_kp(int enable);
+void erts_lcnt_update_cio_locks_nkp(int enable);
+#endif
+
 #else /* !ERTS_ENABLE_KERNEL_POLL */
 
 Uint erts_check_io_size(void);
@@ -72,6 +77,10 @@ void erts_check_io_interrupt_timed(int, ErtsMonotonicTime);
 void erts_check_io(int);
 void erts_init_check_io(void);
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void erts_lcnt_update_cio_locks(int enable);
+#endif
+
 #endif
 
 extern erts_smp_atomic_t erts_check_io_time;
diff --git a/erts/emulator/sys/common/erl_mmap.c b/erts/emulator/sys/common/erl_mmap.c
index bb930ff03b..214ed01c82 100644
--- a/erts/emulator/sys/common/erl_mmap.c
+++ b/erts/emulator/sys/common/erl_mmap.c
@@ -2212,9 +2212,11 @@ erts_mmap_init(ErtsMemMapper* mm, ErtsMMapInit *init, int executable)
 	erts_exit(1, "erts_mmap: Failed to open /dev/zero\n");
 #endif
 
-    erts_smp_mtx_init(&mm->mtx, "erts_mmap");
+    erts_smp_mtx_init(&mm->mtx, "erts_mmap", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     if (is_first_call) {
-        erts_mtx_init(&am.init_mutex, "mmap_init_atoms");
+        erts_mtx_init(&am.init_mutex, "mmap_init_atoms", NIL,
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
     }
 
 #ifdef ERTS_HAVE_OS_PHYSICAL_MEMORY_RESERVATION
@@ -2816,7 +2818,8 @@ static void hard_dbg_mseg_init(void)
 {
     ErtsFreeSegDesc_fake* p;
 
-    erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg");
+    erts_mtx_init(&hard_dbg_mseg_mtx, "hard_dbg_mseg", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
     hard_dbg_mseg_tree.root = NULL;
     hard_dbg_mseg_tree.order = ADDR_ORDER;
 
diff --git a/erts/emulator/sys/common/erl_mseg.c b/erts/emulator/sys/common/erl_mseg.c
index b8f0bb7150..d69a79dc2a 100644
--- a/erts/emulator/sys/common/erl_mseg.c
+++ b/erts/emulator/sys/common/erl_mseg.c
@@ -1420,7 +1420,8 @@ erts_mseg_init(ErtsMsegInit_t *init)
 
     atoms_initialized = 0;
 
-    erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms");
+    erts_mtx_init(&init_atoms_mutex, "mseg_init_atoms", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
 #ifdef ERTS_HAVE_EXEC_MMAPPER
     /* Initialize erts_exec_mapper *FIRST*, to increase probability
@@ -1449,7 +1450,8 @@ erts_mseg_init(ErtsMsegInit_t *init)
 	    ma->is_thread_safe = 0;
 	else {
 	    ma->is_thread_safe = 1;
-	    erts_mtx_init(&ma->mtx, "mseg");
+            erts_mtx_init(&ma->mtx, "mseg", make_small(i),
+                ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_ALLOCATOR);
 	}
 
 	ma->is_cache_check_scheduled = 0;
diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c
index 5e7ae8953a..52a8b6a53f 100644
--- a/erts/emulator/sys/common/erl_poll.c
+++ b/erts/emulator/sys/common/erl_poll.c
@@ -336,7 +336,7 @@ static void fatal_error_async_signal_safe(char *error_str);
 
 static int max_fds = -1;
 static ErtsPollSet pollsets;
-static erts_smp_spinlock_t pollsets_lock;
+static erts_smp_mtx_t pollsets_lock;
 
 #if ERTS_POLL_USE_POLL
 
@@ -2583,7 +2583,8 @@ ERTS_POLL_EXPORT(erts_poll_max_fds)(void)
 void
 ERTS_POLL_EXPORT(erts_poll_init)(void)
 {
-    erts_smp_spinlock_init(&pollsets_lock, "pollsets_lock");
+    erts_smp_mtx_init(&pollsets_lock, "pollsets_lock", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     pollsets = NULL;
 
     errno = 0;
@@ -2689,7 +2690,7 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 #endif
 #ifdef ERTS_SMP
     erts_atomic32_init_nob(&ps->polled, 0);
-    erts_smp_mtx_init(&ps->mtx, "pollset");
+    erts_smp_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
 #endif
 #if defined(USE_THREADS) || ERTS_POLL_ASYNC_INTERRUPT_SUPPORT
     erts_atomic32_init_nob(&ps->wakeup_state, (erts_aint32_t) 0);
@@ -2731,10 +2732,10 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void)
 #endif
     erts_smp_atomic_set_nob(&ps->no_of_user_fds, 0); /* Don't count wakeup pipe and fallback fd */
 
-    erts_smp_spin_lock(&pollsets_lock);
+    erts_smp_mtx_lock(&pollsets_lock);
     ps->next = pollsets;
     pollsets = ps;
-    erts_smp_spin_unlock(&pollsets_lock);
+    erts_smp_mtx_unlock(&pollsets_lock);
 
     return ps;
 }
@@ -2795,7 +2796,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
         close(ps->timer_fd);
 #endif
 
-    erts_smp_spin_lock(&pollsets_lock);
+    erts_smp_mtx_lock(&pollsets_lock);
     if (ps == pollsets)
 	pollsets = pollsets->next;
     else {
@@ -2805,7 +2806,7 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps)
 	ASSERT(ps == prev_ps->next);
 	prev_ps->next = ps->next;
     }
-    erts_smp_spin_unlock(&pollsets_lock);
+    erts_smp_mtx_unlock(&pollsets_lock);
 
     erts_free(ERTS_ALC_T_POLLSET, (void *) ps);
 }
@@ -3148,3 +3149,26 @@ print_misc_debug_info(void)
 }
     
 #endif
+
+#ifdef ERTS_ENABLE_LOCK_COUNT
+static void erts_lcnt_enable_pollset_lock_count(ErtsPollSet pollset, int enable) {
+    if(enable) {
+        erts_lcnt_install_new_lock_info(&pollset->mtx.lcnt, "pollset_rm", NIL,
+            ERTS_LOCK_TYPE_MUTEX | ERTS_LOCK_FLAGS_CATEGORY_IO);
+    } else {
+        erts_lcnt_uninstall(&pollset->mtx.lcnt);
+    }
+}
+
+void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable) {
+    ErtsPollSet iterator;
+
+    erts_smp_mtx_lock(&pollsets_lock);
+
+    for(iterator = pollsets; iterator != NULL; iterator = iterator->next) {
+        erts_lcnt_enable_pollset_lock_count(iterator, enable);
+    }
+
+    erts_smp_mtx_unlock(&pollsets_lock);
+}
+#endif
diff --git a/erts/emulator/sys/common/erl_poll.h b/erts/emulator/sys/common/erl_poll.h
index c16122610d..b3b4d79984 100644
--- a/erts/emulator/sys/common/erl_poll.h
+++ b/erts/emulator/sys/common/erl_poll.h
@@ -260,4 +260,8 @@ void		ERTS_POLL_EXPORT(erts_poll_get_selected_events)(ErtsPollSet,
 
 int erts_poll_new_table_len(int old_len, int need_len);
 
+#ifdef ERTS_ENABLE_LOCK_COUNT
+void ERTS_POLL_EXPORT(erts_lcnt_update_pollset_locks)(int enable);
+#endif
+
 #endif /* #ifndef ERL_POLL_H__ */
diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c
index e86258b5e0..de4ec61c47 100644
--- a/erts/emulator/sys/unix/sys.c
+++ b/erts/emulator/sys/unix/sys.c
@@ -1550,7 +1550,8 @@ erl_sys_args(int* argc, char** argv)
 {
     int i, j;
 
-    erts_smp_rwmtx_init(&environ_rwmtx, "environ");
+    erts_smp_rwmtx_init(&environ_rwmtx, "environ", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 
     i = 1;
 
diff --git a/erts/emulator/sys/unix/sys_time.c b/erts/emulator/sys/unix/sys_time.c
index 4f26639703..102ef7bebf 100644
--- a/erts/emulator/sys/unix/sys_time.c
+++ b/erts/emulator/sys/unix/sys_time.c
@@ -304,8 +304,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
 	    erts_sys_time_data__.r.o.os_times =
 		clock_gettime_times_verified;
 #endif
-	    erts_smp_mtx_init(&internal_state.w.f.mtx,
-			      "os_monotonic_time");
+            erts_smp_mtx_init(&internal_state.w.f.mtx, "os_monotonic_time", NIL,
+                ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
 	    internal_state.w.f.last_delivered
 		= clock_gettime_monotonic();
 	    init_resp->os_monotonic_time_info.locked_use = 1;
diff --git a/erts/emulator/sys/win32/erl_poll.c b/erts/emulator/sys/win32/erl_poll.c
index b10fc1e430..8743f83a50 100644
--- a/erts/emulator/sys/win32/erl_poll.c
+++ b/erts/emulator/sys/win32/erl_poll.c
@@ -142,7 +142,8 @@ static erts_mtx_t save_ops_mtx;
 
 static void poll_debug_init(void)
 {
-    erts_mtx_init(&save_ops_mtx, "save_ops_lock");
+    erts_mtx_init(&save_ops_mtx, "save_ops_lock", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_DEBUG);
 }
 
 void poll_debug_set_active_fd(ErtsSysFdType fd)
@@ -677,7 +678,7 @@ static void new_waiter(ErtsPollSet ps)
     w->active_events = 1;
     w->highwater = 1;
     w->total_events = 1;
-    erts_mtx_init(&w->mtx, "pollwaiter");
+    erts_mtx_init(&w->mtx, "pollwaiter", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
 
 
     /*
@@ -1359,7 +1360,7 @@ ErtsPollSet erts_poll_create_pollset(void)
 
     erts_atomic32_init_nob(&ps->wakeup_state, ERTS_POLL_NOT_WOKEN);
 #ifdef ERTS_SMP
-    erts_smp_mtx_init(&ps->mtx, "pollset");
+    erts_smp_mtx_init(&ps->mtx, "pollset", NIL, ERTS_LOCK_FLAGS_CATEGORY_IO);
 #endif
     init_timeout_time(ps);
 
@@ -1411,7 +1412,8 @@ void  erts_poll_init(void)
     HARDTRACEF(("In erts_poll_init"));
     erts_sys_break_event = CreateManualEvent(FALSE);
 
-    erts_mtx_init(&break_waiter_lock,"break_waiter_lock");
+    erts_mtx_init(&break_waiter_lock, "break_waiter_lock", NIL,
+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO);
     break_happened_event = CreateManualEvent(FALSE);
     erts_atomic32_init_nob(&break_waiter_state, 0); 
 
diff --git a/erts/emulator/sys/win32/sys_env.c b/erts/emulator/sys/win32/sys_env.c
index 21ef71ad9a..8fcee1cbb6 100644
--- a/erts/emulator/sys/win32/sys_env.c
+++ b/erts/emulator/sys/win32/sys_env.c
@@ -37,7 +37,8 @@ static erts_smp_rwmtx_t environ_rwmtx;
 void

 erts_sys_env_init(void)

 {

-    erts_smp_rwmtx_init(&environ_rwmtx, "environ");

+    erts_smp_rwmtx_init(&environ_rwmtx, "environ", NIL,

+        ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);

 }

 

 int

diff --git a/erts/emulator/sys/win32/sys_time.c b/erts/emulator/sys/win32/sys_time.c
index e8c67b3928..88131aaa6a 100644
--- a/erts/emulator/sys/win32/sys_time.c
+++ b/erts/emulator/sys/win32/sys_time.c
@@ -300,8 +300,8 @@ sys_init_time(ErtsSysInitTimeResult *init_resp)
     module = GetModuleHandle(kernel_dll_name);
     if (!module) {
     get_tick_count:
-	erts_smp_mtx_init(&internal_state.w.f.mtime_mtx,
-			  "os_monotonic_time");
+        erts_smp_mtx_init(&internal_state.w.f.mtime_mtx, "os_monotonic_time", NIL,
+            ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
 	internal_state.w.f.wrap = 0;
 	internal_state.w.f.last_tick_count = 0;