diff options
Diffstat (limited to 'erts/emulator')
| -rw-r--r-- | erts/emulator/Makefile.in | 1 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_alloc.types | 7 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_bif_info.c | 317 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_dirty_bif.tab | 2 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_drv_thread.c | 14 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_lock_count.c | 832 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_lock_count.h | 937 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_process_lock.c | 126 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_process_lock.h | 177 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_thr_progress.c | 33 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_threads.h | 58 | ||||
| -rw-r--r-- | erts/emulator/beam/io.c | 53 | ||||
| -rw-r--r-- | erts/emulator/beam/utils.c | 1 | ||||
| -rw-r--r-- | erts/emulator/sys/unix/sys.c | 12 | ||||
| -rw-r--r-- | erts/emulator/sys/win32/sys.c | 15 |
15 files changed, 1690 insertions, 895 deletions
diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 254e59f2c7..ecbf76595b 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -116,6 +116,7 @@ ifeq ($(TYPE),lcnt) PURIFY = TYPEMARKER = .lcnt TYPE_FLAGS = @CFLAGS@ -DERTS_ENABLE_LOCK_COUNT +ENABLE_ALLOC_TYPE_VARS += lcnt else ifeq ($(TYPE),frmptr) diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 8a23a1526e..c155630765 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -368,6 +368,13 @@ type SSB SHORT_LIVED PROCESSES ssb +endif ++if lcnt + +type LCNT_CARRIER STANDARD SYSTEM lcnt_lock_info_carrier +type LCNT_VECTOR SHORT_LIVED SYSTEM lcnt_lock_info_vector + ++endif + type DEBUG SHORT_LIVED SYSTEM debugging type DDLL_PROCESS STANDARD SYSTEM ddll_processes diff --git a/erts/emulator/beam/erl_bif_info.c b/erts/emulator/beam/erl_bif_info.c index 5fc70dfc02..8f7de32f12 100644 --- a/erts/emulator/beam/erl_bif_info.c +++ b/erts/emulator/beam/erl_bif_info.c @@ -4430,48 +4430,108 @@ BIF_RETTYPE erts_debug_set_internal_state_2(BIF_ALIST_2) } #ifdef ERTS_ENABLE_LOCK_COUNT + +typedef struct lcnt_lock_info_vector_ { + erts_lcnt_lock_info_t **elements; + size_t size; +} lcnt_lock_info_vector_t; + +static lcnt_lock_info_vector_t lcnt_build_lock_info_vector(erts_lcnt_lock_info_list_t *list) { + erts_lcnt_lock_info_t *iterator; + lcnt_lock_info_vector_t result; + size_t allocated_entries; + + allocated_entries = 64; + result.size = 0; + + result.elements = erts_alloc(ERTS_ALC_T_LCNT_VECTOR, + allocated_entries * sizeof(erts_lcnt_lock_info_t*)); + + iterator = NULL; + while(erts_lcnt_iterate_list(list, &iterator)) { + erts_lcnt_retain_lock_info(iterator); + + result.elements[result.size++] = iterator; + + if(result.size >= allocated_entries) { + allocated_entries *= 2; + + result.elements = erts_realloc(ERTS_ALC_T_LCNT_VECTOR, result.elements, + allocated_entries * sizeof(erts_lcnt_lock_info_t*)); + } + } + + return result; +} + +static void lcnt_destroy_lock_info_vector(lcnt_lock_info_vector_t *vector) { + size_t i; + + for(i = 0; i < vector->size; i++) { + erts_lcnt_release_lock_info(vector->elements[i]); + } + + erts_free(ERTS_ALC_T_LCNT_VECTOR, vector->elements); +} + +/* The size of an integer is not guaranteed to be constant since we're walking + * over live data, and may cross over into bignum territory between size calc + * and the actual build. This takes care of that through always assuming the + * worst, but needs to be fixed up with HRelease once the final term has been + * built. */ +static ERTS_INLINE Eterm bld_unstable_uint64(Uint **hpp, Uint *szp, Uint64 ui) { + Eterm res = THE_NON_VALUE; + + if(szp) { + *szp += ERTS_UINT64_HEAP_SIZE(~((Uint64) 0)); + } + + if(hpp) { + if (IS_USMALL(0, ui)) { + res = make_small(ui); + } else { + res = erts_uint64_to_big(ui, hpp); + } + } + + return res; +} + static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_stats_t *stats, Eterm res) { - Uint tries = 0, colls = 0; - unsigned long timer_s = 0, timer_ns = 0, timer_n = 0; - unsigned int line = 0; unsigned int i; - + const char *file; + Eterm af, uil; Eterm uit, uic; Eterm uits, uitns, uitn; Eterm tt, tstat, tloc, t; Eterm thist, vhist[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; - + /* term: - * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}, - * { .. histogram .. }] - */ + * [{{file, line}, {tries, colls, {seconds, nanoseconds, n_blocks}}, + * { .. histogram .. }] + */ - tries = (Uint) ethr_atomic_read(&stats->tries); - colls = (Uint) ethr_atomic_read(&stats->colls); - - line = stats->line; - timer_s = stats->timer.s; - timer_ns = stats->timer.ns; - timer_n = stats->timer_n; - - af = erts_atom_put((byte *)stats->file, strlen(stats->file), ERTS_ATOM_ENC_LATIN1, 1); - uil = erts_bld_uint( hpp, szp, line); + file = stats->file ? stats->file : "undefined"; + + af = erts_atom_put((byte *)file, strlen(file), ERTS_ATOM_ENC_LATIN1, 1); + uil = erts_bld_uint( hpp, szp, stats->line); tloc = erts_bld_tuple(hpp, szp, 2, af, uil); - - uit = erts_bld_uint( hpp, szp, tries); - uic = erts_bld_uint( hpp, szp, colls); - uits = erts_bld_uint( hpp, szp, timer_s); - uitns = erts_bld_uint( hpp, szp, timer_ns); - uitn = erts_bld_uint( hpp, szp, timer_n); + uit = bld_unstable_uint64(hpp, szp, (Uint)ethr_atomic_read(&stats->attempts)); + uic = bld_unstable_uint64(hpp, szp, (Uint)ethr_atomic_read(&stats->collisions)); + + uits = bld_unstable_uint64(hpp, szp, stats->total_time_waited.s); + uitns = bld_unstable_uint64(hpp, szp, stats->total_time_waited.ns); + uitn = bld_unstable_uint64(hpp, szp, stats->times_waited); tt = erts_bld_tuple(hpp, szp, 3, uits, uitns, uitn); tstat = erts_bld_tuple(hpp, szp, 3, uit, uic, tt); for(i = 0; i < ERTS_LCNT_HISTOGRAM_SLOT_SIZE; i++) { - vhist[i] = erts_bld_uint(hpp, szp, stats->hist.ns[i]); + vhist[i] = bld_unstable_uint64(hpp, szp, stats->wait_time_histogram.ns[i]); } + thist = erts_bld_tuplev(hpp, szp, ERTS_LCNT_HISTOGRAM_SLOT_SIZE, vhist); t = erts_bld_tuple(hpp, szp, 3, tloc, tstat, thist); @@ -4480,150 +4540,165 @@ static Eterm lcnt_build_lock_stats_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_s return res; } -static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_t *lock, Eterm res) { +static Eterm lcnt_pretty_print_lock_id(erts_lcnt_lock_info_t *info) { + Eterm id = info->id; + + if(info->flag & ERTS_LCNT_LT_ALLOC) { + /* Use allocator type names as id's for allocator locks. This blatantly + * assumes that all locks in this category are identified by their + * allocator number. */ + const char *ltype = (const char*)ERTS_ALC_A2AD(signed_val(info->id)); + id = erts_atom_put((byte*)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); + } else if(info->flag & ERTS_LCNT_LT_PROCLOCK) { + /* Use registered names as id's for process locks if available. Thread + * progress is delayed since we may be running on a dirty scheduler. */ + ErtsThrPrgrDelayHandle delay_handle; + Process *process; + + delay_handle = erts_thr_progress_unmanaged_delay(); + + process = erts_proc_lookup(info->id); + if (process && process->common.u.alive.reg) { + id = process->common.u.alive.reg->name; + } + + erts_thr_progress_unmanaged_continue(delay_handle); + } + + return id; +} + +static Eterm lcnt_build_lock_term(Eterm **hpp, Uint *szp, erts_lcnt_lock_info_t *info, Eterm res) { Eterm name, type, id, stats = NIL, t; - Process *proc = NULL; - char *ltype; + const char *ltype; int i; /* term: * [{name, id, type, stats()}] */ - - ASSERT(lock->name); + + ASSERT(info->name); - ltype = erts_lcnt_lock_type(lock->flag); + ltype = erts_lcnt_lock_type(info->flag); ASSERT(ltype); - type = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); - name = erts_atom_put((byte *)lock->name, strlen(lock->name), ERTS_ATOM_ENC_LATIN1, 1); - - if (lock->flag & ERTS_LCNT_LT_ALLOC) { - /* use allocator types names as id's for allocator locks */ - ltype = (char *) ERTS_ALC_A2AD(signed_val(lock->id)); - id = erts_atom_put((byte *)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); - } else if (lock->flag & ERTS_LCNT_LT_PROCLOCK) { - /* use registered names as id's for process locks if available */ - proc = erts_proc_lookup(lock->id); - if (proc && proc->common.u.alive.reg) { - id = proc->common.u.alive.reg->name; - } else { - /* otherwise use process id */ - id = lock->id; - } + type = erts_atom_put((byte*)ltype, strlen(ltype), ERTS_ATOM_ENC_LATIN1, 1); + name = erts_atom_put((byte*)info->name, strlen(info->name), ERTS_ATOM_ENC_LATIN1, 1); + + /* Only attempt to resolve ids when actually emitting the term. This ought + * to be safe since all immediates are the same size. */ + if(hpp != NULL) { + id = lcnt_pretty_print_lock_id(info); } else { - id = lock->id; + id = NIL; } - for (i = 0; i < lock->n_stats; i++) { - stats = lcnt_build_lock_stats_term(hpp, szp, &(lock->stats[i]), stats); + /* info->location_count is ignored since it can be changed at any time. */ + for(i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { + stats = lcnt_build_lock_stats_term(hpp, szp, &(info->location_stats[i]), stats); } t = erts_bld_tuple(hpp, szp, 4, name, id, type, stats); - res = erts_bld_cons( hpp, szp, t, res); + res = erts_bld_cons(hpp, szp, t, res); return res; } -static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_data_t *data, Eterm res) { +static Eterm lcnt_build_result_term(Eterm **hpp, Uint *szp, erts_lcnt_time_t *duration, + lcnt_lock_info_vector_t *current_locks, + lcnt_lock_info_vector_t *deleted_locks, Eterm res) { Eterm dts, dtns, tdt, adur, tdur, aloc, lloc = NIL, tloc; - erts_lcnt_lock_t *lock = NULL; + + size_t i; + char *str_duration = "duration"; char *str_locks = "locks"; - + /* term: * [{'duration', {seconds, nanoseconds}}, {'locks', locks()}] */ - + /* duration tuple */ - dts = erts_bld_uint( hpp, szp, data->duration.s); - dtns = erts_bld_uint( hpp, szp, data->duration.ns); + dts = bld_unstable_uint64(hpp, szp, duration->s); + dtns = bld_unstable_uint64(hpp, szp, duration->ns); tdt = erts_bld_tuple(hpp, szp, 2, dts, dtns); - + adur = erts_atom_put((byte *)str_duration, strlen(str_duration), ERTS_ATOM_ENC_LATIN1, 1); tdur = erts_bld_tuple(hpp, szp, 2, adur, tdt); /* lock tuple */ aloc = erts_atom_put((byte *)str_locks, strlen(str_locks), ERTS_ATOM_ENC_LATIN1, 1); - - for (lock = data->current_locks->head; lock != NULL ; lock = lock->next ) { - lloc = lcnt_build_lock_term(hpp, szp, lock, lloc); + + for(i = 0; i < current_locks->size; i++) { + lloc = lcnt_build_lock_term(hpp, szp, current_locks->elements[i], lloc); } - - for (lock = data->deleted_locks->head; lock != NULL ; lock = lock->next ) { - lloc = lcnt_build_lock_term(hpp, szp, lock, lloc); + + for(i = 0; i < deleted_locks->size; i++) { + lloc = lcnt_build_lock_term(hpp, szp, deleted_locks->elements[i], lloc); } - + tloc = erts_bld_tuple(hpp, szp, 2, aloc, lloc); - - res = erts_bld_cons( hpp, szp, tloc, res); - res = erts_bld_cons( hpp, szp, tdur, res); + + res = erts_bld_cons(hpp, szp, tloc, res); + res = erts_bld_cons(hpp, szp, tdur, res); return res; -} +} + #endif BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) { -#ifdef ERTS_ENABLE_LOCK_COUNT - Eterm res = NIL; -#endif - - +#ifndef ERTS_ENABLE_LOCK_COUNT if (BIF_ARG_1 == am_enabled) { -#ifdef ERTS_ENABLE_LOCK_COUNT - BIF_RET(am_true); -#else - BIF_RET(am_false); -#endif + BIF_RET(am_false); } -#ifdef ERTS_ENABLE_LOCK_COUNT - else if (BIF_ARG_1 == am_info) { - erts_lcnt_data_t *data; - Uint hsize = 0; - Uint *szp; - Eterm* hp; + BIF_ERROR(BIF_P, BADARG); +#else - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + if (BIF_ARG_1 == am_enabled) { + BIF_RET(am_true); + } else if (BIF_ARG_1 == am_info) { + lcnt_lock_info_vector_t current_locks, deleted_locks; + erts_lcnt_data_t data; - erts_lcnt_set_rt_opt(ERTS_LCNT_OPT_SUSPEND); - data = erts_lcnt_get_data(); + Eterm *term_heap_start, *term_heap_end; + Uint term_heap_size = 0; + Eterm result; - /* calculate size */ + data = erts_lcnt_get_data(); - szp = &hsize; - lcnt_build_result_term(NULL, szp, data, NIL); + current_locks = lcnt_build_lock_info_vector(data.current_locks); + deleted_locks = lcnt_build_lock_info_vector(data.deleted_locks); - /* alloc and build */ + lcnt_build_result_term(NULL, &term_heap_size, &data.duration, + ¤t_locks, &deleted_locks, NIL); - hp = HAlloc(BIF_P, hsize); + term_heap_start = HAlloc(BIF_P, term_heap_size); + term_heap_end = term_heap_start; - res = lcnt_build_result_term(&hp, NULL, data, res); - - erts_lcnt_clear_rt_opt(ERTS_LCNT_OPT_SUSPEND); + result = lcnt_build_result_term(&term_heap_end, NULL, + &data.duration, ¤t_locks, &deleted_locks, NIL); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); - - BIF_RET(res); - } else if (BIF_ARG_1 == am_clear) { - erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); - erts_smp_thr_progress_block(); + HRelease(BIF_P, term_heap_start + term_heap_size, term_heap_end); - erts_lcnt_clear_counters(); + lcnt_destroy_lock_info_vector(¤t_locks); + lcnt_destroy_lock_info_vector(&deleted_locks); - erts_smp_thr_progress_unblock(); - erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + BIF_RET(result); + } else if (BIF_ARG_1 == am_clear) { + erts_lcnt_clear_counters(); - BIF_RET(am_ok); + BIF_RET(am_ok); } else if (is_tuple(BIF_ARG_1)) { Eterm* ptr = tuple_val(BIF_ARG_1); if ((arityval(ptr[0]) == 2) && (ptr[2] == am_false || ptr[2] == am_true)) { + Eterm res; + int lock_opt = 0, enable = (ptr[2] == am_true) ? 1 : 0; if (ERTS_IS_ATOM_STR("copy_save", ptr[1])) { lock_opt = ERTS_LCNT_OPT_COPYSAVE; @@ -4631,35 +4706,41 @@ BIF_RETTYPE erts_debug_lock_counters_1(BIF_ALIST_1) lock_opt = ERTS_LCNT_OPT_PROCLOCK; } else if (ERTS_IS_ATOM_STR("port_locks", ptr[1])) { lock_opt = ERTS_LCNT_OPT_PORTLOCK; - } else if (ERTS_IS_ATOM_STR("suspend", ptr[1])) { - lock_opt = ERTS_LCNT_OPT_SUSPEND; } else if (ERTS_IS_ATOM_STR("location", ptr[1])) { lock_opt = ERTS_LCNT_OPT_LOCATION; } else { BIF_ERROR(BIF_P, BADARG); } + if (enable) { + res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false; + } else { + res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false; + } + +#ifdef ERTS_SMP + /* FIXME: make this non-blocking. */ erts_smp_proc_unlock(BIF_P, ERTS_PROC_LOCK_MAIN); erts_smp_thr_progress_block(); +#endif - if (enable) res = erts_lcnt_set_rt_opt(lock_opt) ? am_true : am_false; - else res = erts_lcnt_clear_rt_opt(lock_opt) ? am_true : am_false; - -#ifdef ERTS_SMP if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PORTLOCK) { erts_lcnt_enable_io_lock_count(enable); } else if (res != ptr[2] && lock_opt == ERTS_LCNT_OPT_PROCLOCK) { erts_lcnt_enable_proc_lock_count(enable); } -#endif - erts_smp_thr_progress_unblock(); + +#ifdef ERTS_SMP erts_smp_proc_lock(BIF_P, ERTS_PROC_LOCK_MAIN); + erts_smp_thr_progress_unblock(); +#endif + BIF_RET(res); } - } + } -#endif BIF_ERROR(BIF_P, BADARG); +#endif } static void os_info_init(void) diff --git a/erts/emulator/beam/erl_dirty_bif.tab b/erts/emulator/beam/erl_dirty_bif.tab index 69421dcfcc..94d8589779 100644 --- a/erts/emulator/beam/erl_dirty_bif.tab +++ b/erts/emulator/beam/erl_dirty_bif.tab @@ -46,6 +46,8 @@ dirty-cpu erts_debug:dirty_cpu/2 dirty-io erts_debug:dirty_io/2 +dirty-cpu erts_debug:lock_counters/1 + # --- TEST of Dirty BIF functionality --- # Functions below will execute on dirty schedulers when emulator has # been configured for testing dirty schedulers. This is used for test diff --git a/erts/emulator/beam/erl_drv_thread.c b/erts/emulator/beam/erl_drv_thread.c index 0e6aadf568..3b68abe5d7 100644 --- a/erts/emulator/beam/erl_drv_thread.c +++ b/erts/emulator/beam/erl_drv_thread.c @@ -55,7 +55,7 @@ fatal_error(int err, char *func) struct ErlDrvMutex_ { ethr_mutex mtx; #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; #endif char *name; }; @@ -68,7 +68,7 @@ struct ErlDrvCond_ { struct ErlDrvRWLock_ { ethr_rwmutex rwmtx; #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; #endif char *name; }; @@ -176,7 +176,8 @@ erl_drv_mutex_create(char *name) dmtx->name = no_name; } #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX); + erts_lcnt_init_ref(&dmtx->lcnt); + erts_lcnt_install_new_lock_info(&dmtx->lcnt, dmtx->name, ERTS_LCNT_LT_MUTEX); #endif } return dmtx; @@ -191,7 +192,7 @@ erl_drv_mutex_destroy(ErlDrvMutex *dmtx) #ifdef USE_THREADS int res; #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&dmtx->lcnt); + erts_lcnt_uninstall(&dmtx->lcnt); #endif res = dmtx ? ethr_mutex_destroy(&dmtx->mtx) : EINVAL; if (res != 0) @@ -368,7 +369,8 @@ erl_drv_rwlock_create(char *name) drwlck->name = no_name; } #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX); + erts_lcnt_init_ref(&drwlck->lcnt); + erts_lcnt_install_new_lock_info(&drwlck->lcnt, drwlck->name, ERTS_LCNT_LT_RWMUTEX); #endif } return drwlck; @@ -383,7 +385,7 @@ erl_drv_rwlock_destroy(ErlDrvRWLock *drwlck) #ifdef USE_THREADS int res; #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&drwlck->lcnt); + erts_lcnt_uninstall(&drwlck->lcnt); #endif res = drwlck ? ethr_rwmutex_destroy(&drwlck->rwmtx) : EINVAL; if (res != 0) diff --git a/erts/emulator/beam/erl_lock_count.c b/erts/emulator/beam/erl_lock_count.c index aee9796171..2cf59aa367 100644 --- a/erts/emulator/beam/erl_lock_count.c +++ b/erts/emulator/beam/erl_lock_count.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2008-2016. All Rights Reserved. + * Copyright Ericsson AB 2008-2017. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,51 +18,30 @@ * %CopyrightEnd% */ -/* - * Description: Statistics for locks. - * - * Author: Björn-Egil Dahlberg - * Date: 2008-07-03 - */ - #ifdef HAVE_CONFIG_H # include "config.h" #endif -/* Needed for VxWorks va_arg */ -#include "sys.h" - #ifdef ERTS_ENABLE_LOCK_COUNT +#include "sys.h" + #include "erl_lock_count.h" -#include "ethread.h" -#include "erl_term.h" -#include "atom.h" -#include <stdio.h> +#include "erl_thr_progress.h" -/* globals, dont access these without locks or blocks */ +#define LCNT_MAX_CARRIER_ENTRIES 255 -ethr_mutex lcnt_data_lock; -erts_lcnt_data_t *erts_lcnt_data; -Uint16 erts_lcnt_rt_options; -erts_lcnt_time_t timer_start; -const char *str_undefined = "undefined"; +/* - Exported global - */ -static ethr_tsd_key lcnt_thr_data_key; -static int lcnt_n_thr; -static erts_lcnt_thread_data_t *lcnt_thread_data[2048]; +Uint16 erts_lcnt_rt_options = ERTS_LCNT_OPT_PROCLOCK | ERTS_LCNT_OPT_LOCATION; -/* local functions */ +/* - Locals that are shared with the header implementation - */ -static ERTS_INLINE void lcnt_lock(void) { - ethr_mutex_lock(&lcnt_data_lock); -} +int lcnt_initialization_completed__ = 0; -static ERTS_INLINE void lcnt_unlock(void) { - ethr_mutex_unlock(&lcnt_data_lock); -} +ethr_tsd_key lcnt_thr_data_key__; -const int log2_tab64[64] = { +const int lcnt_log2_tab64__[64] = { 63, 0, 58, 1, 59, 47, 53, 2, 60, 39, 48, 27, 54, 33, 42, 3, 61, 51, 37, 40, 49, 18, 28, 20, @@ -72,99 +51,52 @@ const int log2_tab64[64] = { 56, 45, 25, 31, 35, 16, 9, 12, 44, 24, 15, 8, 23, 7, 6, 5}; -static ERTS_INLINE int lcnt_log2(Uint64 v) { - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - v |= v >> 32; - return log2_tab64[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58]; -} - -static char* lcnt_lock_type(Uint16 flag) { - switch(flag & ERTS_LCNT_LT_ALL) { - case ERTS_LCNT_LT_SPINLOCK: return "spinlock"; - case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock"; - case ERTS_LCNT_LT_MUTEX: return "mutex"; - case ERTS_LCNT_LT_RWMUTEX: return "rw_mutex"; - case ERTS_LCNT_LT_PROCLOCK: return "proclock"; - default: return ""; - } -} +/* - Local variables - */ -static void lcnt_clear_stats(erts_lcnt_lock_stats_t *stats) { - ethr_atomic_set(&stats->tries, 0); - ethr_atomic_set(&stats->colls, 0); - stats->timer.s = 0; - stats->timer.ns = 0; - stats->timer_n = 0; - stats->file = (char *)str_undefined; - stats->line = 0; - sys_memzero(stats->hist.ns, sizeof(stats->hist.ns)); -} +static erts_lcnt_lock_info_list_t lcnt_current_lock_list; +static erts_lcnt_lock_info_list_t lcnt_deleted_lock_list; -static void lcnt_time(erts_lcnt_time_t *time) { - /* - * erts_sys_hrtime() is the highest resolution - * we could find, it may or may not be monotonic... - */ - ErtsMonotonicTime mtime = erts_sys_hrtime(); - time->s = (unsigned long) (mtime / 1000000000LL); - time->ns = (unsigned long) (mtime - 1000000000LL*time->s); -} +static erts_lcnt_time_t lcnt_timer_start; -static void lcnt_time_diff(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) { - long ds; - long dns; +/* local functions */ - ds = t1->s - t0->s; - dns = t1->ns - t0->ns; +static void lcnt_clear_stats(erts_lcnt_lock_info_t *info) { + size_t i; - /* the difference should not be able to get bigger than 1 sec in ns*/ + for(i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { + erts_lcnt_lock_stats_t *stats = &info->location_stats[i]; - if (dns < 0) { - ds -= 1; - dns += 1000000000LL; - } + sys_memzero(&stats->wait_time_histogram, sizeof(stats->wait_time_histogram)); - ASSERT(ds >= 0); + stats->total_time_waited.s = 0; + stats->total_time_waited.ns = 0; - d->s = ds; - d->ns = dns; -} + stats->times_waited = 0; -/* difference d must be non-negative */ + stats->file = NULL; + stats->line = 0; -static void lcnt_time_add(erts_lcnt_time_t *t, erts_lcnt_time_t *d) { - t->s += d->s; - t->ns += d->ns; + ethr_atomic_set(&stats->attempts, 0); + ethr_atomic_set(&stats->collisions, 0); + } - t->s += t->ns / 1000000000LL; - t->ns = t->ns % 1000000000LL; + info->location_count = 1; } -static erts_lcnt_thread_data_t *lcnt_thread_data_alloc(void) { - erts_lcnt_thread_data_t *eltd; +static lcnt_thread_data_t__ *lcnt_thread_data_alloc(void) { + lcnt_thread_data_t__ *eltd = + (lcnt_thread_data_t__*)malloc(sizeof(lcnt_thread_data_t__)); - eltd = (erts_lcnt_thread_data_t*)malloc(sizeof(erts_lcnt_thread_data_t)); - if (!eltd) { - ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); + if(!eltd) { + ERTS_INTERNAL_ERROR("Failed to allocate lcnt thread data."); } + eltd->timer_set = 0; eltd->lock_in_conflict = 0; - eltd->id = lcnt_n_thr++; - /* set thread data to array */ - lcnt_thread_data[eltd->id] = eltd; - return eltd; } -static erts_lcnt_thread_data_t *lcnt_get_thread_data(void) { - return (erts_lcnt_thread_data_t *)ethr_tsd_get(lcnt_thr_data_key); -} - /* debug */ #if 0 @@ -175,472 +107,391 @@ static char* lock_opt(Uint16 flag) { return "--"; } -static void print_lock_x(erts_lcnt_lock_t *lock, Uint16 flag, char *action) { - erts_aint_t w_state, r_state; +static void print_lock_x(erts_lcnt_lock_info_t *info, Uint16 flag, char *action) { + ethr_sint_t w_state, r_state; char *type; - if (strcmp(lock->name, "run_queue") != 0) return; - type = lcnt_lock_type(lock->flag); - r_state = ethr_atomic_read(&lock->r_state); - w_state = ethr_atomic_read(&lock->w_state); + if (strcmp(info->name, "run_queue") != 0) return; + type = erts_lcnt_lock_type(info->flag); + r_state = ethr_atomic_read(&info->r_state); + w_state = ethr_atomic_read(&info->w_state); - if (lock->flag & flag) { + if (info->flag & flag) { erts_fprintf(stderr,"%10s [%24s] [r/w state %4ld/%4ld] %2s id %T\r\n", action, - lock->name, + info->name, r_state, w_state, type, - lock->id); + info->id); } } #endif -static erts_lcnt_lock_stats_t *lcnt_get_lock_stats(erts_lcnt_lock_t *lock, char *file, unsigned int line) { - unsigned int i; - erts_lcnt_lock_stats_t *stats = NULL; - - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) { - for (i = 0; i < lock->n_stats; i++) { - if ((lock->stats[i].file == file) && (lock->stats[i].line == line)) { - return &(lock->stats[i]); - } - } - if (lock->n_stats < ERTS_LCNT_MAX_LOCK_LOCATIONS) { - stats = &lock->stats[lock->n_stats]; - lock->n_stats++; - stats->file = file; - stats->line = line; - return stats; - } - } - return &lock->stats[0]; +/* - List operations - + * + * Info entries are kept in a doubly linked list where each entry is locked + * with its neighbors rather than a global lock. Deletion is rather quick, but + * insertion is still serial since the head becomes a de facto global lock. + * + * We rely on ad-hoc spinlocks to avoid "recursing" into this module. */ + +#define LCNT_SPINLOCK_YIELD_ITERATIONS 50 + +#define LCNT_SPINLOCK_HELPER_INIT \ + Uint failed_spin_count = 0; + +#define LCNT_SPINLOCK_HELPER_YIELD \ + do { \ + failed_spin_count++; \ + if(!(failed_spin_count % LCNT_SPINLOCK_YIELD_ITERATIONS)) { \ + erts_thr_yield(); \ + } else { \ + ERTS_SPIN_BODY; \ + } \ + } while(0) + +static void lcnt_unlock_list_entry(erts_lcnt_lock_info_t *info) { + ethr_atomic32_set_relb(&info->lock, 0); } -static void lcnt_update_stats_hist(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_wait) { - int idx; - unsigned long r; +static int lcnt_try_lock_list_entry(erts_lcnt_lock_info_t *info) { + return ethr_atomic32_cmpxchg_acqb(&info->lock, 1, 0) == 0; +} - if (time_wait->s > 0 || time_wait->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) { - idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1; - } else { - r = time_wait->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT; - if (r) idx = lcnt_log2(r); - else idx = 0; +static void lcnt_lock_list_entry(erts_lcnt_lock_info_t *info) { + LCNT_SPINLOCK_HELPER_INIT; + + while(!lcnt_try_lock_list_entry(info)) { + LCNT_SPINLOCK_HELPER_YIELD; } - hist->ns[idx]++; } -static void lcnt_update_stats(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, - erts_lcnt_time_t *time_wait) { +static void lcnt_lock_list_entry_with_neighbors(erts_lcnt_lock_info_t *info) { + LCNT_SPINLOCK_HELPER_INIT; - ethr_atomic_inc(&stats->tries); + for(;;) { + if(!lcnt_try_lock_list_entry(info)) + goto retry_after_entry_failed; + if(!lcnt_try_lock_list_entry(info->next)) + goto retry_after_next_failed; + if(!lcnt_try_lock_list_entry(info->prev)) + goto retry_after_prev_failed; - if (lock_in_conflict) - ethr_atomic_inc(&stats->colls); + return; - if (time_wait) { - lcnt_time_add(&(stats->timer), time_wait); - stats->timer_n++; - lcnt_update_stats_hist(&stats->hist,time_wait); + retry_after_prev_failed: + lcnt_unlock_list_entry(info->next); + retry_after_next_failed: + lcnt_unlock_list_entry(info); + retry_after_entry_failed: + LCNT_SPINLOCK_HELPER_YIELD; } } -/* interface */ +static void lcnt_unlock_list_entry_with_neighbors(erts_lcnt_lock_info_t *info) { + lcnt_unlock_list_entry(info->prev); + lcnt_unlock_list_entry(info->next); + lcnt_unlock_list_entry(info); +} -void erts_lcnt_init() { - erts_lcnt_thread_data_t *eltd = NULL; +static void lcnt_insert_list_entry(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t *info) { + erts_lcnt_lock_info_t *next, *prev; - /* init lock */ - if (ethr_mutex_init(&lcnt_data_lock) != 0) abort(); + prev = &list->head; - /* init tsd */ - lcnt_n_thr = 0; - ethr_tsd_key_create(&lcnt_thr_data_key, "lcnt_data"); + lcnt_lock_list_entry(prev); - lcnt_lock(); + next = prev->next; - erts_lcnt_rt_options = ERTS_LCNT_OPT_LOCATION | ERTS_LCNT_OPT_PROCLOCK; - eltd = lcnt_thread_data_alloc(); - ethr_tsd_set(lcnt_thr_data_key, eltd); + lcnt_lock_list_entry(next); - /* init lcnt structure */ - erts_lcnt_data = (erts_lcnt_data_t*)malloc(sizeof(erts_lcnt_data_t)); - if (!erts_lcnt_data) { - ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); - } - erts_lcnt_data->current_locks = erts_lcnt_list_init(); - erts_lcnt_data->deleted_locks = erts_lcnt_list_init(); + info->next = next; + info->prev = prev; - lcnt_unlock(); + prev->next = info; + next->prev = info; + lcnt_unlock_list_entry(next); + lcnt_unlock_list_entry(prev); } -void erts_lcnt_late_init() { - /* set start timer and zero statistics */ - erts_lcnt_clear_counters(); - erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler); -} +static void lcnt_insert_list_carrier(erts_lcnt_lock_info_list_t *list, + erts_lcnt_lock_info_carrier_t *carrier) { + erts_lcnt_lock_info_t *next, *prev; + size_t i; -/* list operations */ + for(i = 0; i < carrier->entry_count; i++) { + erts_lcnt_lock_info_t *info = &carrier->entries[i]; -/* BEGIN ASSUMPTION: lcnt_data_lock taken */ + info->prev = &carrier->entries[i - 1]; + info->next = &carrier->entries[i + 1]; + } -erts_lcnt_lock_list_t *erts_lcnt_list_init(void) { - erts_lcnt_lock_list_t *list; + prev = &list->head; - list = (erts_lcnt_lock_list_t*)malloc(sizeof(erts_lcnt_lock_list_t)); - if (!list) { - ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); - } - list->head = NULL; - list->tail = NULL; - list->n = 0; - return list; -} + lcnt_lock_list_entry(prev); -static void lcnt_list_free(erts_lcnt_lock_t *head) { - erts_lcnt_lock_t *lock, *next; + next = prev->next; - lock = head; + lcnt_lock_list_entry(next); - while(lock != NULL) { - next = lock->next; - free(lock); - lock = next; - } + next->prev = &carrier->entries[carrier->entry_count - 1]; + carrier->entries[carrier->entry_count - 1].next = next; + + prev->next = &carrier->entries[0]; + carrier->entries[0].prev = prev; + + lcnt_unlock_list_entry(next); + lcnt_unlock_list_entry(prev); } -void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) { - erts_lcnt_lock_t *tail = NULL; +static void lcnt_init_list(erts_lcnt_lock_info_list_t *list) { + /* Ensure that ref_count operations explode when touching the sentinels in + * DEBUG mode. */ + ethr_atomic_init(&(list->head.ref_count), -1); + ethr_atomic_init(&(list->tail.ref_count), -1); - tail = list->tail; - if (tail) { - tail->next = lock; - lock->prev = tail; - } else { - list->head = lock; - lock->prev = NULL; - ASSERT(!lock->next); - } - lock->next = NULL; - list->tail = lock; + ethr_atomic32_init(&(list->head.lock), 0); + (list->head).next = &list->tail; + (list->head).prev = &list->tail; - list->n++; + ethr_atomic32_init(&(list->tail.lock), 0); + (list->tail).next = &list->head; + (list->tail).prev = &list->head; } -void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock) { - if (lock->next) lock->next->prev = lock->prev; - if (lock->prev) lock->prev->next = lock->next; - if (list->head == lock) list->head = lock->next; - if (list->tail == lock) list->tail = lock->prev; +/* - Carrier operations - */ - lock->prev = NULL; - lock->next = NULL; - list->n--; +int lcnt_thr_progress_unmanaged_delay__(void) { + return erts_thr_progress_unmanaged_delay(); } -/* END ASSUMPTION: lcnt_data_lock taken */ +void lcnt_thr_progress_unmanaged_continue__(int handle) { + return erts_thr_progress_unmanaged_continue(handle); +} -/* lock operations */ +static void lcnt_deallocate_carrier_malloc(erts_lcnt_lock_info_carrier_t *carrier) { + ASSERT(ethr_atomic_read(&carrier->ref_count) == 0); + free(carrier); +} -/* interface to erl_threads.h */ -/* only lock on init and destroy, all others should use atomics */ -void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag ) { - erts_lcnt_init_lock_x(lock, name, flag, NIL); +static void lcnt_deallocate_carrier_erts(erts_lcnt_lock_info_carrier_t *carrier) { + ASSERT(ethr_atomic_read(&carrier->ref_count) == 0); + erts_free(ERTS_ALC_T_LCNT_CARRIER, (void*)carrier); } -void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id) { - int i; +static void lcnt_thr_prg_cleanup_carrier(void *data) { + erts_lcnt_lock_info_carrier_t *carrier = data; + size_t entry_count, i; - if (flag & ERTS_LCNT_LT_DISABLE) { - ERTS_LCNT_CLEAR_FLAG(lock); - return; + /* carrier->entry_count will be replaced with garbage if it's deallocated + * on the final iteration, so we'll tuck it away to get a clean exit. */ + entry_count = carrier->entry_count; + + for(i = 0; i < entry_count; i++) { + ASSERT(ethr_atomic_read(&carrier->ref_count) >= (entry_count - i)); + + erts_lcnt_release_lock_info(&carrier->entries[i]); } +} - lock->next = NULL; - lock->prev = NULL; - lock->flag = flag; - lock->name = name; - lock->id = id; +static void lcnt_schedule_carrier_cleanup(void *data) { + ErtsSchedulerData *esdp = erts_get_scheduler_data(); - ethr_atomic_init(&lock->r_state, 0); - ethr_atomic_init(&lock->w_state, 0); -#ifdef DEBUG - ethr_atomic_init(&lock->flowstate, 0); -#endif + /* We can't issue cleanup jobs on anything other than normal schedulers, so + * we move to the first scheduler if required. */ + + if(!esdp || esdp->type != ERTS_SCHED_NORMAL) { + erts_schedule_misc_aux_work(1, &lcnt_schedule_carrier_cleanup, data); + } else { + erts_lcnt_lock_info_carrier_t *carrier = data; + size_t carrier_size; - lock->n_stats = 1; + carrier_size = sizeof(erts_lcnt_lock_info_carrier_t) + + sizeof(erts_lcnt_lock_info_t) * carrier->entry_count; - for (i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - lcnt_clear_stats(&lock->stats[i]); + erts_schedule_thr_prgr_later_cleanup_op(&lcnt_thr_prg_cleanup_carrier, + data, (ErtsThrPrgrLaterOp*)&carrier->release_entries, carrier_size); } +} - lcnt_lock(); - erts_lcnt_list_insert(erts_lcnt_data->current_locks, lock); - lcnt_unlock(); +static void lcnt_info_deallocate(erts_lcnt_lock_info_t *info) { + lcnt_release_carrier__(info->carrier); } -/* init empty, instead of zero struct - * used by process locks probes - */ -void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock) { - lock->next = NULL; - lock->prev = NULL; - lock->flag = 0; - lock->name = NULL; - lock->id = NIL; - ethr_atomic_init(&lock->r_state, 0); - ethr_atomic_init(&lock->w_state, 0); -#ifdef DEBUG - ethr_atomic_init(&lock->flowstate, 0); -#endif - lock->n_stats = 0; - sys_memzero(lock->stats, sizeof(lock->stats)); -} -/* destroy lock */ -void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock) { - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - lcnt_lock(); - - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) { - erts_lcnt_lock_t *deleted_lock; - /* copy structure and insert the copy */ - deleted_lock = (erts_lcnt_lock_t*)malloc(sizeof(erts_lcnt_lock_t)); - if (!deleted_lock) { - ERTS_INTERNAL_ERROR("Lock counter failed to allocate memory!"); - } - memcpy(deleted_lock, lock, sizeof(erts_lcnt_lock_t)); - deleted_lock->next = NULL; - deleted_lock->prev = NULL; - erts_lcnt_list_insert(erts_lcnt_data->deleted_locks, deleted_lock); - } - /* delete original */ - erts_lcnt_list_delete(erts_lcnt_data->current_locks, lock); - ERTS_LCNT_CLEAR_FLAG(lock); +static void lcnt_info_dispose(erts_lcnt_lock_info_t *info) { + ASSERT(ethr_atomic_read(&info->ref_count) == 0); - lcnt_unlock(); + if(erts_lcnt_rt_options & ERTS_LCNT_OPT_COPYSAVE) { + ethr_atomic_set(&info->ref_count, 1); + + /* Move straight to deallocation the next time around. */ + info->dispose = &lcnt_info_deallocate; + + lcnt_insert_list_entry(&lcnt_deleted_lock_list, info); + } else { + lcnt_info_deallocate(info); + } } -/* lock */ +static void lcnt_lock_info_init_helper(erts_lcnt_lock_info_t *info) { +#ifdef DEBUG + ethr_atomic_init(&info->flowstate, 0); +#endif -void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option) { - erts_aint_t r_state = 0, w_state = 0; - erts_lcnt_thread_data_t *eltd; + ethr_atomic_init(&info->ref_count, 1); + ethr_atomic32_init(&info->lock, 0); - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + ethr_atomic_init(&info->r_state, 0); + ethr_atomic_init(&info->w_state, 0); - eltd = lcnt_get_thread_data(); - ASSERT(eltd); + info->dispose = &lcnt_info_dispose; - w_state = ethr_atomic_read(&lock->w_state); + lcnt_clear_stats(info); +} - if (option & ERTS_LCNT_LO_WRITE) { - r_state = ethr_atomic_read(&lock->r_state); - ethr_atomic_inc( &lock->w_state); - } - if (option & ERTS_LCNT_LO_READ) { - ethr_atomic_inc( &lock->r_state); - } +erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int entry_count) { + erts_lcnt_lock_info_carrier_t *result; + size_t carrier_size, i; + + ASSERT(entry_count > 0 && entry_count <= LCNT_MAX_CARRIER_ENTRIES); - /* we cannot acquire w_lock if either w or r are taken */ - /* we cannot acquire r_lock if w_lock is taken */ + carrier_size = sizeof(erts_lcnt_lock_info_carrier_t) + + sizeof(erts_lcnt_lock_info_t) * entry_count; - if ((w_state > 0) || (r_state > 0)) { - eltd->lock_in_conflict = 1; - if (eltd->timer_set == 0) { - lcnt_time(&eltd->timer); - } - eltd->timer_set++; + if(lcnt_initialization_completed__) { + result = (erts_lcnt_lock_info_carrier_t*)erts_alloc(ERTS_ALC_T_LCNT_CARRIER, carrier_size); + result->deallocate = &lcnt_deallocate_carrier_erts; } else { - eltd->lock_in_conflict = 0; + result = (erts_lcnt_lock_info_carrier_t*)malloc(carrier_size); + result->deallocate = &lcnt_deallocate_carrier_malloc; } -} -void erts_lcnt_lock(erts_lcnt_lock_t *lock) { - erts_aint_t w_state; - erts_lcnt_thread_data_t *eltd; + ethr_atomic_init(&result->ref_count, entry_count); - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + result->entry_count = entry_count; - w_state = ethr_atomic_read(&lock->w_state); - ethr_atomic_inc(&lock->w_state); - eltd = lcnt_get_thread_data(); + for(i = 0; i < entry_count; i++) { + erts_lcnt_lock_info_t *info = &result->entries[i]; - ASSERT(eltd); + lcnt_lock_info_init_helper(info); - if (w_state > 0) { - eltd->lock_in_conflict = 1; - /* only set the timer if nobody else has it - * This should only happen when proc_locks aquires several locks - * 'atomicly'. All other locks will block the thread if w_state > 0 - * i.e. locked. - */ - if (eltd->timer_set == 0) { - lcnt_time(&eltd->timer); - } - eltd->timer_set++; - } else { - eltd->lock_in_conflict = 0; + info->carrier = result; } -} - -/* if a lock wasn't really a lock operation, bad bad process locks */ - -void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock) { - /* should check if this thread was "waiting" */ - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - ethr_atomic_dec(&lock->w_state); + return result; } -/* - * erts_lcnt_lock_post - * - * Used when we get a lock (i.e. directly after a lock operation) - * if the timer was set then we had to wait for the lock - * lock_post will calculate the wait time. - */ +void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carrier) { + ethr_sint_t swapped_carrier; -void erts_lcnt_lock_post(erts_lcnt_lock_t *lock) { - erts_lcnt_lock_post_x(lock, (char*)str_undefined, 0); -} + swapped_carrier = ethr_atomic_cmpxchg_mb(ref, (ethr_sint_t)carrier, (ethr_sint_t)NULL); -void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line) { - erts_lcnt_thread_data_t *eltd; - erts_lcnt_time_t timer; - erts_lcnt_time_t time_wait; - erts_lcnt_lock_stats_t *stats; + if(swapped_carrier != (ethr_sint_t)NULL) { #ifdef DEBUG - erts_aint_t flowstate; + ASSERT(ethr_atomic_read(&carrier->ref_count) == carrier->entry_count); + ethr_atomic_set(&carrier->ref_count, 0); #endif - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; + carrier->deallocate(carrier); + } else { + lcnt_insert_list_carrier(&lcnt_current_lock_list, carrier); + } +} + +void erts_lcnt_uninstall(erts_lcnt_ref_t *ref) { + ethr_sint_t previous_carrier, swapped_carrier; -#ifdef DEBUG - if (!(lock->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc(&lock->flowstate); + previous_carrier = ethr_atomic_read(ref); + swapped_carrier = ethr_atomic_cmpxchg_mb(ref, (ethr_sint_t)NULL, previous_carrier); + + if(previous_carrier && previous_carrier == swapped_carrier) { + lcnt_schedule_carrier_cleanup((void*)previous_carrier); } -#endif +} - eltd = lcnt_get_thread_data(); +/* - Initialization - */ - ASSERT(eltd); +void erts_lcnt_pre_thr_init() { + /* Ensure that the dependency hack mentioned in the header doesn't + * explode at runtime. */ + ERTS_CT_ASSERT(sizeof(LcntThrPrgrLaterOp) >= sizeof(ErtsThrPrgrLaterOp)); + ERTS_CT_ASSERT(ERTS_THR_PRGR_DHANDLE_MANAGED == + (ErtsThrPrgrDelayHandle)LCNT_THR_PRGR_DHANDLE_MANAGED); - /* if lock was in conflict, time it */ - stats = lcnt_get_lock_stats(lock, file, line); - if (eltd->timer_set) { - lcnt_time(&timer); + lcnt_init_list(&lcnt_current_lock_list); + lcnt_init_list(&lcnt_deleted_lock_list); +} - lcnt_time_diff(&time_wait, &timer, &(eltd->timer)); - lcnt_update_stats(stats, eltd->lock_in_conflict, &time_wait); - eltd->timer_set--; - ASSERT(eltd->timer_set >= 0); - } else { - lcnt_update_stats(stats, eltd->lock_in_conflict, NULL); - } +void erts_lcnt_post_thr_init() { + /* ASSUMPTION: this is safe since it runs prior to the creation of other + * threads (Directly after ethread init). */ + ethr_tsd_key_create(&lcnt_thr_data_key__, "lcnt_data"); + + erts_lcnt_thread_setup(); } -/* unlock */ +void erts_lcnt_late_init() { + /* Set start timer and zero all statistics */ + erts_lcnt_clear_counters(); + erts_thr_install_exit_handler(erts_lcnt_thread_exit_handler); -void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_dec(&lock->w_state); - if (option & ERTS_LCNT_LO_READ ) ethr_atomic_dec(&lock->r_state); + /* It's safe to use erts_alloc and thread progress past this point. */ + lcnt_initialization_completed__ = 1; } -void erts_lcnt_unlock(erts_lcnt_lock_t *lock) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; -#ifdef DEBUG - { - erts_aint_t w_state; - erts_aint_t flowstate; - - /* flowstate */ - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 1); - ethr_atomic_dec(&lock->flowstate); - - /* write state */ - w_state = ethr_atomic_read(&lock->w_state); - ASSERT(w_state > 0); - } -#endif - ethr_atomic_dec(&lock->w_state); +void erts_lcnt_thread_setup() { + lcnt_thread_data_t__ *eltd = lcnt_thread_data_alloc(); + + ASSERT(eltd); + + ethr_tsd_set(lcnt_thr_data_key__, eltd); } -/* trylock */ +void erts_lcnt_thread_exit_handler() { + lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__(); -void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option) { - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - /* Determine lock_state via res instead of state */ - if (res != EBUSY) { - if (option & ERTS_LCNT_LO_WRITE) ethr_atomic_inc(&lock->w_state); - if (option & ERTS_LCNT_LO_READ ) ethr_atomic_inc(&lock->r_state); - lcnt_update_stats(&(lock->stats[0]), 0, NULL); - } else { - ethr_atomic_inc(&lock->stats[0].tries); - ethr_atomic_inc(&lock->stats[0].colls); + if (eltd) { + free(eltd); } } +/* - BIF interface - */ -void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res) { - /* Determine lock_state via res instead of state */ - if (erts_lcnt_rt_options & ERTS_LCNT_OPT_SUSPEND) return; - if (ERTS_LCNT_IS_LOCK_INVALID(lock)) return; - if (res != EBUSY) { +void erts_lcnt_retain_lock_info(erts_lcnt_lock_info_t *info) { #ifdef DEBUG - { - erts_aint_t flowstate; - flowstate = ethr_atomic_read(&lock->flowstate); - ASSERT(flowstate == 0); - ethr_atomic_inc( &lock->flowstate); - } + ASSERT(ethr_atomic_inc_read_acqb(&info->ref_count) >= 2); +#else + ethr_atomic_inc_acqb(&info->ref_count); #endif - ethr_atomic_inc(&lock->w_state); - lcnt_update_stats(&(lock->stats[0]), 0, NULL); - } else { - ethr_atomic_inc(&lock->stats[0].tries); - ethr_atomic_inc(&lock->stats[0].colls); - } } -/* thread operations */ +void erts_lcnt_release_lock_info(erts_lcnt_lock_info_t *info) { + ethr_sint_t count; -void erts_lcnt_thread_setup(void) { - erts_lcnt_thread_data_t *eltd; + /* We need to acquire the lock before decrementing ref_count to avoid + * racing with list iteration; there's a short window between reading the + * reference to info and increasing its ref_count. */ + lcnt_lock_list_entry_with_neighbors(info); - lcnt_lock(); - /* lock for thread id global update */ - eltd = lcnt_thread_data_alloc(); - lcnt_unlock(); - ASSERT(eltd); - ethr_tsd_set(lcnt_thr_data_key, eltd); -} + count = ethr_atomic_dec_read(&info->ref_count); -void erts_lcnt_thread_exit_handler() { - erts_lcnt_thread_data_t *eltd; + ASSERT(count >= 0); - eltd = ethr_tsd_get(lcnt_thr_data_key); + if(count > 0) { + lcnt_unlock_list_entry_with_neighbors(info); + } else { + (info->next)->prev = info->prev; + (info->prev)->next = info->next; - if (eltd) { - free(eltd); + lcnt_unlock_list_entry_with_neighbors(info); + + info->dispose(info); } } -/* bindings for bifs */ - Uint16 erts_lcnt_set_rt_opt(Uint16 opt) { Uint16 prev; prev = (erts_lcnt_rt_options & opt); @@ -656,51 +507,72 @@ Uint16 erts_lcnt_clear_rt_opt(Uint16 opt) { } void erts_lcnt_clear_counters(void) { - erts_lcnt_lock_t *lock; - erts_lcnt_lock_list_t *list; - erts_lcnt_lock_stats_t *stats; - int i; + erts_lcnt_lock_info_t *iterator; - lcnt_lock(); + lcnt_time__(&lcnt_timer_start); - list = erts_lcnt_data->current_locks; + iterator = NULL; + while(erts_lcnt_iterate_list(&lcnt_current_lock_list, &iterator)) { + lcnt_clear_stats(iterator); + } - for (lock = list->head; lock != NULL; lock = lock->next) { - for( i = 0; i < ERTS_LCNT_MAX_LOCK_LOCATIONS; i++) { - stats = &lock->stats[i]; - lcnt_clear_stats(stats); - } - lock->n_stats = 1; + iterator = NULL; + while(erts_lcnt_iterate_list(&lcnt_deleted_lock_list, &iterator)) { + erts_lcnt_release_lock_info(iterator); } +} + +erts_lcnt_data_t erts_lcnt_get_data(void) { + erts_lcnt_time_t timer_stop; + erts_lcnt_data_t result; - lock = erts_lcnt_data->deleted_locks->head; - erts_lcnt_data->deleted_locks->head = NULL; - erts_lcnt_data->deleted_locks->tail = NULL; - erts_lcnt_data->deleted_locks->n = 0; + lcnt_time__(&timer_stop); - lcnt_time(&timer_start); + result.timer_start = lcnt_timer_start; - lcnt_unlock(); + result.current_locks = &lcnt_current_lock_list; + result.deleted_locks = &lcnt_deleted_lock_list; - /* free deleted locks */ - lcnt_list_free(lock); + lcnt_time_diff__(&result.duration, &timer_stop, &result.timer_start); + + return result; } -erts_lcnt_data_t *erts_lcnt_get_data(void) { - erts_lcnt_time_t timer_stop; +const char *erts_lcnt_lock_type(Uint16 type) { + switch(type & ERTS_LCNT_LT_ALL) { + case ERTS_LCNT_LT_SPINLOCK: return "spinlock"; + case ERTS_LCNT_LT_RWSPINLOCK: return "rw_spinlock"; + case ERTS_LCNT_LT_MUTEX: return "mutex"; + case ERTS_LCNT_LT_RWMUTEX: return "rw_mutex"; + case ERTS_LCNT_LT_PROCLOCK: return "proclock"; + default: return ""; + } +} - lcnt_lock(); +int erts_lcnt_iterate_list(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t **iterator) { + erts_lcnt_lock_info_t *current, *next; - lcnt_time(&timer_stop); - lcnt_time_diff(&(erts_lcnt_data->duration), &timer_stop, &timer_start); + current = *iterator ? *iterator : &list->head; - lcnt_unlock(); + ASSERT(current != &list->tail); - return erts_lcnt_data; -} + lcnt_lock_list_entry(current); + + next = current->next; + + if(next != &list->tail) { + erts_lcnt_retain_lock_info(next); + } + + lcnt_unlock_list_entry(current); + + if(current != &list->head) { + erts_lcnt_release_lock_info(current); + } + + *iterator = next; -char *erts_lcnt_lock_type(Uint16 type) { - return lcnt_lock_type(type); + return next != &list->tail; } -#endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ +#endif /* #ifdef ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/beam/erl_lock_count.h b/erts/emulator/beam/erl_lock_count.h index 6caffbfe86..c89dfcba47 100644 --- a/erts/emulator/beam/erl_lock_count.h +++ b/erts/emulator/beam/erl_lock_count.h @@ -18,60 +18,56 @@ * %CopyrightEnd% */ -/* - * Description: Statistics for locks. - * - * Author: Björn-Egil Dahlberg - * Date: 2008-07-03 - * Abstract: - * Locks statistics internal representation. - * - * Conceptual representation, - * - set name - * | - id (the unique lock) - * | | - lock type - * | | - statistics - * | | | - location (file and line number) - * | | | - tries - * | | | - collisions (including trylock busy) - * | | | - timer (time spent in waiting for lock) - * | | | - n_timer (collisions excluding trylock busy) - * | | | - histogram - * | | | | - # 0 = log2(lock wait_time ns) - * | | | | - ... - * | | | | - # n = log2(lock wait_time ns) - * - * Each instance of a lock is the unique lock, i.e. set and id in that set. - * For each lock there is a set of statistics with where and what impact - * the lock aqusition had. - * - * Runtime options - * - suspend, used when internal lock-counting can't be applied. For instance - * when allocating a term for the outside and halloc needs to be used. - * Default: off. - * - location, reserved and not used. - * - proclock, disable proclock counting. Used when performance might be an - * issue. Accessible from erts_debug:lock_counters({process_locks, bool()}). - * Default: off. - * - copysave, enable saving of destroyed locks (and thereby its statistics). - * If memory constraints is an issue this need to be disabled. - * Accessible from erts_debug:lock_counters({copy_save, bool()}). - * Default: off. +/** + * @description Statistics for locks. + * @file erl_lock_count.h + * + * @author Björn-Egil Dahlberg + * @author John Högberg + * + * Conceptual representation: * + * - set name + * | - id (the unique lock) + * | | - lock type + * | | - statistics + * | | | - location (file and line number) + * | | | - attempts + * | | | - collisions (including trylock busy) + * | | | - timer (time spent in waiting for lock) + * | | | - n_timer (collisions excluding trylock busy) + * | | | - histogram + * | | | | - # 0 = log2(lock wait_time ns) + * | | | | - ... + * | | | | - # n = log2(lock wait_time ns) + * + * Each instance of a lock is the unique lock, i.e. set and id in that set. + * For each lock there is a set of statistics with where and what impact + * the lock aqusition had. + * + * Runtime options: + * - location, reserved and not used. + * - proclock, disable proclock counting. Used when performance might be an + * issue. Accessible from erts_debug:lock_counters({process_locks, bool()}). + * Default: off. + * - copysave, enable saving of destroyed locks (and thereby its statistics). + * If memory constraints is an issue this need to be disabled. + * Accessible from erts_debug:lock_counters({copy_save, bool()}). + * Default: off. */ -#include "sys.h" - #ifndef ERTS_LOCK_COUNT_H__ #define ERTS_LOCK_COUNT_H__ #ifdef ERTS_ENABLE_LOCK_COUNT #ifndef ERTS_ENABLE_LOCK_POSITION -/* Enable in order for _x variants of mtx functions to be used. */ +/** @brief Controls whether _x variants of mtx functions are used. */ #define ERTS_ENABLE_LOCK_POSITION 1 #endif +#include "sys.h" #include "ethread.h" +#include "erl_term.h" #define ERTS_LCNT_MAX_LOCK_LOCATIONS (10) @@ -106,13 +102,8 @@ | ERTS_LCNT_LT_RWMUTEX \ | ERTS_LCNT_LT_PROCLOCK ) -#define ERTS_LCNT_LOCK_TYPE(lock) ((lock)->flag & ERTS_LCNT_LT_ALL) -#define ERTS_LCNT_IS_LOCK_INVALID(lock) (!((lock)->flag & ERTS_LCNT_LT_ALL)) -#define ERTS_LCNT_CLEAR_FLAG(lock) ((lock)->flag = 0) - -/* runtime options */ +/* Runtime options */ -#define ERTS_LCNT_OPT_SUSPEND (((Uint16) 1) << 0) #define ERTS_LCNT_OPT_LOCATION (((Uint16) 1) << 1) #define ERTS_LCNT_OPT_PROCLOCK (((Uint16) 1) << 2) #define ERTS_LCNT_OPT_PORTLOCK (((Uint16) 1) << 3) @@ -123,116 +114,818 @@ typedef struct { unsigned long ns; } erts_lcnt_time_t; -extern erts_lcnt_time_t timer_start; - typedef struct { - Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; /* log2 array of nano seconds occurences */ + /* @brief log2 array of nano seconds occurences */ + Uint32 ns[ERTS_LCNT_HISTOGRAM_SLOT_SIZE]; } erts_lcnt_hist_t; -typedef struct erts_lcnt_lock_stats_s { - /* "tries" and "colls" needs to be atomic since - * trylock busy does not acquire a lock and there - * is no post action to rectify the situation - */ +typedef struct { + /** @brief In which file the lock was taken. May be NULL. */ + const char *file; + /** @brief Line number in \c file */ + unsigned int line; + + /* "attempts" and "collisions" need to be atomic since try_lock busy does + * not acquire a lock and there is no post action to rectify the + * situation. */ - char *file; /* which file the lock was taken */ - unsigned int line; /* line number in file */ + ethr_atomic_t attempts; + ethr_atomic_t collisions; - ethr_atomic_t tries; /* n tries to get lock */ - ethr_atomic_t colls; /* n collisions of tries to get lock */ + erts_lcnt_time_t total_time_waited; + Uint64 times_waited; - unsigned long timer_n; /* #times waited for lock */ - erts_lcnt_time_t timer; /* total wait time for lock */ - erts_lcnt_hist_t hist; + erts_lcnt_hist_t wait_time_histogram; } erts_lcnt_lock_stats_t; -/* rw locks uses both states, other locks only uses w_state */ -typedef struct erts_lcnt_lock_s { - char *name; /* lock name */ - Uint16 flag; /* lock type */ - Eterm id; /* id if possible */ +typedef struct lcnt_lock_info_t_ { + const char *name; /**< Lock name */ + Uint16 flag; /**< Lock type */ + Eterm id; /**< Id if possible, must be an immediate */ -#ifdef DEBUG + /* The first entry is reserved as a fallback for when location information + * is missing, and when the lock is used in more than (MAX_LOCK_LOCATIONS + * - 1) different places. */ + erts_lcnt_lock_stats_t location_stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; + unsigned int location_count; + + /* -- Everything below is internal to this module ---------------------- */ + + /* Lock states; rw locks uses both states, other locks only uses w_state */ + + /** @brief Write state. 0 = not taken, otherwise n threads waiting */ + ethr_atomic_t w_state; + /** @brief Read state. 0 = not taken, > 0 -> writes will wait */ + ethr_atomic_t r_state; + +#ifdef LCNT_DEBUG_LOCK_FLOW + /** @brief Tracks lock/unlock operations. This will explode if the lock is + * held at the time lock counting is installed. + * + * Avoid enabling existing locks at runtime while running in this + * configuration. */ ethr_atomic_t flowstate; #endif - /* lock states */ - ethr_atomic_t w_state; /* 0 not taken, otherwise n threads waiting */ - ethr_atomic_t r_state; /* 0 not taken, > 0 -> writes will wait */ + struct lcnt_lock_info_t_ *prev; + struct lcnt_lock_info_t_ *next; - /* statistics */ - unsigned int n_stats; - erts_lcnt_lock_stats_t stats[ERTS_LCNT_MAX_LOCK_LOCATIONS]; /* first entry is "undefined"*/ + /** @brief Used in place of erts_refc_t to avoid a circular dependency. */ + ethr_atomic_t ref_count; + ethr_atomic32_t lock; - /* chains for list handling */ - /* data is hold by lcnt_lock */ - struct erts_lcnt_lock_s *prev; - struct erts_lcnt_lock_s *next; -} erts_lcnt_lock_t; + /** @brief Deletion hook called once \c ref_count reaches 0; may defer + * deletion by modifying \c ref_count. */ + void (*dispose)(struct lcnt_lock_info_t_ *); -typedef struct { - erts_lcnt_lock_t *head; - erts_lcnt_lock_t *tail; - unsigned long n; -} erts_lcnt_lock_list_t; + struct lcnt_lock_info_carrier_ *carrier; +} erts_lcnt_lock_info_t; + +typedef struct lcnt_lock_info_list_ { + erts_lcnt_lock_info_t head; + erts_lcnt_lock_info_t tail; +} erts_lcnt_lock_info_list_t; typedef struct { - erts_lcnt_time_t duration; /* time since last clear */ - erts_lcnt_lock_list_t *current_locks; - erts_lcnt_lock_list_t *deleted_locks; + erts_lcnt_time_t timer_start; /**< Time of last clear */ + erts_lcnt_time_t duration; /**< Time since last clear */ + + erts_lcnt_lock_info_list_t *current_locks; + erts_lcnt_lock_info_list_t *deleted_locks; } erts_lcnt_data_t; -typedef struct { - int id; +typedef struct lcnt_lock_info_carrier_ erts_lcnt_lock_info_carrier_t; - erts_lcnt_time_t timer; /* timer */ - int timer_set; /* bool */ - int lock_in_conflict; /* bool */ -} erts_lcnt_thread_data_t; +typedef ethr_atomic_t erts_lcnt_ref_t; -/* globals */ +/* -- Globals -------------------------------------------------------------- */ extern Uint16 erts_lcnt_rt_options; -/* function declerations */ - -void erts_lcnt_init(void); +/* -- Lock operations ------------------------------------------------------ + * + * All of these will nop if there's nothing "installed" on the given reference, + * in order to transparently support enable/disable at runtime. */ + +/** @brief Records that a lock is being acquired. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock(erts_lcnt_ref_t *ref); + +/** @copydoc erts_lcnt_lock + * @param option Notes whether the lock is a read or write lock. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, Uint16 option); + +/** @brief Records that a lock has been acquired. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_post(erts_lcnt_ref_t *ref); + +/** @copydoc erts_lcnt_lock_post. + * @param file The name of the file where the lock was acquired. + * @param line The line at which the lock was acquired. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_post_x(erts_lcnt_ref_t *ref, char *file, unsigned int line); + +/** @brief Records that a lock has been released. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_unlock(erts_lcnt_ref_t *ref); + +/** @copydoc erts_lcnt_unlock_opt + * @param option Whether the lock is a read or write lock. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, Uint16 option); + +/** @brief Rectifies the case where a lock wasn't actually a lock operation. + * + * Only used for process locks at the moment. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_unacquire(erts_lcnt_ref_t *ref); + +/** @brief Records the result of a trylock, placing the queried lock status in + * \c result. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_trylock(erts_lcnt_ref_t *ref, int result); + +/** @copydoc erts_lcnt_trylock + * @param option Whether the lock is a read or write lock. */ +ERTS_GLB_FORCE_INLINE +void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option); + +/* Indexed variants of the standard lock operations, for use when a single + * reference contains many counters (eg. process locks). + * + * erts_lcnt_open_ref must be used to safely extract the installed carrier, + * which must released with erts_lcnt_close_reference on success. + * + * Refer to \c erts_lcnt_lock for example usage. */ + +ERTS_GLB_FORCE_INLINE +int erts_lcnt_open_ref(erts_lcnt_ref_t *ref, int *handle, erts_lcnt_lock_info_carrier_t **result); + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_close_ref(int handle, erts_lcnt_lock_info_carrier_t *carrier); + +ERTS_GLB_INLINE +void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index); +ERTS_GLB_INLINE +void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option); + +ERTS_GLB_INLINE +void erts_lcnt_lock_post_idx(erts_lcnt_lock_info_carrier_t *carrier, int index); +ERTS_GLB_INLINE +void erts_lcnt_lock_post_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, char *file, unsigned int line); + +ERTS_GLB_INLINE +void erts_lcnt_lock_unacquire_idx(erts_lcnt_lock_info_carrier_t *carrier, int index); + +ERTS_GLB_INLINE +void erts_lcnt_unlock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index); +ERTS_GLB_INLINE +void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option); + +ERTS_GLB_INLINE +void erts_lcnt_trylock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result); +ERTS_GLB_INLINE +void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, Uint16 option); + +/* -- Reference operations ------------------------------------------------- */ + +erts_lcnt_lock_info_carrier_t *erts_lcnt_create_lock_info_carrier(int count); + +/** @brief Fills in the name and lock type of the given index. */ +#define erts_lcnt_init_lock_info_idx(carrier, index, name, flag) \ + erts_lcnt_init_lock_info_x_idx(carrier, index, name, flag, NIL) + +/** @copydoc erts_lcnt_install_new_lock_info + * @param id An immediate erlang term with whatever extra data you want to + * identify this lock with. */ +ERTS_GLB_INLINE +void erts_lcnt_init_lock_info_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, + const char *name, Uint16 flag, Eterm id); + +/** @brief Initializes a lock counter reference; this must be called prior to + * using any other functions in this module. */ +#define erts_lcnt_init_ref(ref) ethr_atomic_init(ref, (ethr_sint_t)NULL); + +/** @brief Atomically installs the given lock counters. Nops (and releases the + * provided carrier) if something was already installed. */ +void erts_lcnt_install(erts_lcnt_ref_t *ref, erts_lcnt_lock_info_carrier_t *carrier); + +/** @brief Atomically removes the currently installed lock counters. Nops if + * nothing was installed. */ +void erts_lcnt_uninstall(erts_lcnt_ref_t *ref); + +/** @brief Convenience macro to install a single lock counter of the given + * name and type. */ +#define erts_lcnt_install_new_lock_info(reference, name, flag) \ + erts_lcnt_install_new_lock_info_x(reference, name, flag, NIL) + +/** @copydoc erts_lcnt_install_new_lock_info + * @param id An immediate erlang term with whatever extra data you want to + * identify this lock with. */ +#define erts_lcnt_install_new_lock_info_x(reference, name, flag, id) \ + do { \ + erts_lcnt_lock_info_carrier_t *__carrier; \ + __carrier = erts_lcnt_create_lock_info_carrier(1); \ + erts_lcnt_init_lock_info_x_idx(__carrier, 0, name, flag, id); \ + erts_lcnt_install(reference, __carrier); \ + } while(0) + +/* -- Module initialization ------------------------------------------------ */ + +void erts_lcnt_pre_thr_init(void); +void erts_lcnt_post_thr_init(void); void erts_lcnt_late_init(void); -/* thread operations */ void erts_lcnt_thread_setup(void); void erts_lcnt_thread_exit_handler(void); -/* list operations (local) */ -erts_lcnt_lock_list_t *erts_lcnt_list_init(void); +/* -- BIF interface -------------------------------------------------------- */ -void erts_lcnt_list_insert(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock); -void erts_lcnt_list_delete(erts_lcnt_lock_list_t *list, erts_lcnt_lock_t *lock); - -/* lock operations (global) */ -void erts_lcnt_init_lock(erts_lcnt_lock_t *lock, char *name, Uint16 flag); -void erts_lcnt_init_lock_x(erts_lcnt_lock_t *lock, char *name, Uint16 flag, Eterm id); -void erts_lcnt_init_lock_empty(erts_lcnt_lock_t *lock); -void erts_lcnt_destroy_lock(erts_lcnt_lock_t *lock); +/** @brief Safely iterates through all entries in the given list. + * + * The referenced item will be valid until the next call to + * \c erts_lcnt_iterate_list after which point it may be destroyed; call + * erts_lcnt_retain_lock_info if you wish to hang on to it beyond that point. + * + * Iteration can be cancelled by calling erts_lcnt_release_lock_info on the + * iterator and breaking out of the loop. + * + * @param iterator The iteration variable; set the pointee to NULL to start + * iteration. + * @return 1 while the iterator is valid, 0 at the end of the list. */ +int erts_lcnt_iterate_list(erts_lcnt_lock_info_list_t *list, erts_lcnt_lock_info_t **iterator); -void erts_lcnt_lock(erts_lcnt_lock_t *lock); -void erts_lcnt_lock_opt(erts_lcnt_lock_t *lock, Uint16 option); -void erts_lcnt_lock_post(erts_lcnt_lock_t *lock); -void erts_lcnt_lock_post_x(erts_lcnt_lock_t *lock, char *file, unsigned int line); -void erts_lcnt_lock_unaquire(erts_lcnt_lock_t *lock); +/** @brief Clears the counter state of all locks, and releases all locks + * preserved through ERTS_LCNT_OPT_COPYSAVE (if any). */ +void erts_lcnt_clear_counters(void); -void erts_lcnt_unlock(erts_lcnt_lock_t *lock); -void erts_lcnt_unlock_opt(erts_lcnt_lock_t *lock, Uint16 option); +/** @brief Retrieves the global lock counter state. + * + * Note that the lists may be modified while you're mucking around with them. + * Always use \c erts_lcnt_iterate_list to enumerate them. */ +erts_lcnt_data_t erts_lcnt_get_data(void); -void erts_lcnt_trylock_opt(erts_lcnt_lock_t *lock, int res, Uint16 option); -void erts_lcnt_trylock(erts_lcnt_lock_t *lock, int res); +void erts_lcnt_retain_lock_info(erts_lcnt_lock_info_t *info); +void erts_lcnt_release_lock_info(erts_lcnt_lock_info_t *info); -/* bif interface */ Uint16 erts_lcnt_set_rt_opt(Uint16 opt); Uint16 erts_lcnt_clear_rt_opt(Uint16 opt); -void erts_lcnt_clear_counters(void); -char *erts_lcnt_lock_type(Uint16 type); -erts_lcnt_data_t *erts_lcnt_get_data(void); + +const char *erts_lcnt_lock_type(Uint16 type); + +/* -- Inline implementation ------------------------------------------------ */ + +/* The following is a hack to get the things we need from erl_thr_progress.h, + * which we can't #include without dependency hell breaking loose. + * + * The size of LcntThrPrgrLaterOp and value of the constant are verified at + * compile-time in erts_lcnt_pre_thr_init. */ + +int lcnt_thr_progress_unmanaged_delay__(void); +void lcnt_thr_progress_unmanaged_continue__(int handle); +typedef struct { Uint64 _[4]; } LcntThrPrgrLaterOp; +#define LCNT_THR_PRGR_DHANDLE_MANAGED -1 + +struct lcnt_lock_info_carrier_ { + ethr_atomic_t ref_count; + + void (*deallocate)(struct lcnt_lock_info_carrier_ *); + + LcntThrPrgrLaterOp release_entries; + + unsigned char entry_count; + erts_lcnt_lock_info_t entries[]; +}; + +typedef struct { + erts_lcnt_time_t timer; /* timer */ + int timer_set; /* bool */ + int lock_in_conflict; /* bool */ +} lcnt_thread_data_t__; + +extern ethr_tsd_key lcnt_thr_data_key__; + +/** @brief Some operations (eg erts_alloc or erts_thr_progress_unmanaged_delay) + * are unsafe in the early stages of initialization, so we're using this flag + * to know when we can move over to normal operation. */ +extern int lcnt_initialization_completed__; + +extern const int lcnt_log2_tab64__[]; + +ERTS_GLB_INLINE +int lcnt_log2__(Uint64 v); + +ERTS_GLB_INLINE +void lcnt_update_wait_histogram__(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_waited); + +ERTS_GLB_INLINE +void lcnt_update_stats__(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, erts_lcnt_time_t *time_waited); + +ERTS_GLB_INLINE +erts_lcnt_lock_stats_t *lcnt_get_lock_stats__(erts_lcnt_lock_info_t *info, char *file, unsigned int line); + +ERTS_GLB_INLINE +void lcnt_dec_lock_state__(ethr_atomic_t *l_state); + +ERTS_GLB_INLINE +void lcnt_time__(erts_lcnt_time_t *time); + +ERTS_GLB_INLINE +void lcnt_time_add__(erts_lcnt_time_t *t, erts_lcnt_time_t *d); + +ERTS_GLB_INLINE +void lcnt_time_diff__(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0); + +ERTS_GLB_INLINE +void lcnt_retain_carrier__(erts_lcnt_lock_info_carrier_t *carrier); + +ERTS_GLB_INLINE +void lcnt_release_carrier__(erts_lcnt_lock_info_carrier_t *carrier); + +ERTS_GLB_INLINE +lcnt_thread_data_t__ *lcnt_get_thread_data__(void); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE +void lcnt_time__(erts_lcnt_time_t *time) { + /* + * erts_sys_hrtime() is the highest resolution + * we could find, it may or may not be monotonic... + */ + ErtsMonotonicTime mtime = erts_sys_hrtime(); + time->s = (unsigned long) (mtime / 1000000000LL); + time->ns = (unsigned long) (mtime - 1000000000LL*time->s); +} + +/* difference d must be non-negative */ + +ERTS_GLB_INLINE +void lcnt_time_add__(erts_lcnt_time_t *t, erts_lcnt_time_t *d) { + t->s += d->s; + t->ns += d->ns; + + t->s += t->ns / 1000000000LL; + t->ns = t->ns % 1000000000LL; +} + +ERTS_GLB_INLINE +void lcnt_time_diff__(erts_lcnt_time_t *d, erts_lcnt_time_t *t1, erts_lcnt_time_t *t0) { + long ds; + long dns; + + ds = t1->s - t0->s; + dns = t1->ns - t0->ns; + + /* the difference should not be able to get bigger than 1 sec in ns*/ + + if (dns < 0) { + ds -= 1; + dns += 1000000000LL; + } + + ASSERT(ds >= 0); + + d->s = ds; + d->ns = dns; +} + +ERTS_GLB_INLINE +int lcnt_log2__(Uint64 v) { + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + v |= v >> 32; + + return lcnt_log2_tab64__[((Uint64)((v - (v >> 1))*0x07EDD5E59A4E28C2)) >> 58]; +} + +ERTS_GLB_INLINE +void lcnt_update_wait_histogram__(erts_lcnt_hist_t *hist, erts_lcnt_time_t *time_waited) { + int idx; + + if(time_waited->s > 0 || time_waited->ns > ERTS_LCNT_HISTOGRAM_MAX_NS) { + idx = ERTS_LCNT_HISTOGRAM_SLOT_SIZE - 1; + } else { + unsigned long r = time_waited->ns >> ERTS_LCNT_HISTOGRAM_RSHIFT; + + idx = r ? lcnt_log2__(r) : 0; + } + + hist->ns[idx]++; +} + +ERTS_GLB_INLINE +void lcnt_update_stats__(erts_lcnt_lock_stats_t *stats, int lock_in_conflict, erts_lcnt_time_t *time_waited) { + ethr_atomic_inc(&stats->attempts); + + if(lock_in_conflict) { + ethr_atomic_inc(&stats->collisions); + } + + if(time_waited) { + stats->times_waited++; + + lcnt_time_add__(&stats->total_time_waited, time_waited); + lcnt_update_wait_histogram__(&stats->wait_time_histogram, time_waited); + } +} + +/* If we were installed while the lock was held, r/w_state will be 0 and we + * can't tell which unlock or unacquire operation was the last. To get around + * this we assume that all excess operations go *towards* zero rather than down + * to zero, eventually becoming consistent with the actual state once the lock + * is fully released. + * + * Conflicts might not be counted until the recorded state is fully consistent + * with the actual state, but there should be no other ill effects. */ + +ERTS_GLB_INLINE +void lcnt_dec_lock_state__(ethr_atomic_t *l_state) { + ethr_sint_t state = ethr_atomic_dec_read_acqb(l_state); + + /* We can not assume that state is >= -1 here; unlock and unacquire might + * bring it below -1 and race to increment it back. */ + + if(state < 0) { + ethr_atomic_inc_acqb(l_state); + } +} + +ERTS_GLB_INLINE +erts_lcnt_lock_stats_t *lcnt_get_lock_stats__(erts_lcnt_lock_info_t *info, char *file, unsigned int line) { + ASSERT(info->location_count >= 1 && info->location_count <= ERTS_LCNT_MAX_LOCK_LOCATIONS); + + if(erts_lcnt_rt_options & ERTS_LCNT_OPT_LOCATION) { + unsigned int i; + + for(i = 0; i < info->location_count; i++) { + erts_lcnt_lock_stats_t *stats = &info->location_stats[i]; + + if(stats->file == file && stats->line == line) { + return stats; + } + } + + if(info->location_count < ERTS_LCNT_MAX_LOCK_LOCATIONS) { + erts_lcnt_lock_stats_t *stats = &info->location_stats[info->location_count]; + + stats->file = file; + stats->line = line; + + info->location_count++; + + return stats; + } + } + + return &info->location_stats[0]; +} + +ERTS_GLB_INLINE +lcnt_thread_data_t__ *lcnt_get_thread_data__(void) { + lcnt_thread_data_t__ *eltd = (lcnt_thread_data_t__ *)ethr_tsd_get(lcnt_thr_data_key__); + + ASSERT(eltd); + + return eltd; +} + +ERTS_GLB_FORCE_INLINE +int erts_lcnt_open_ref(erts_lcnt_ref_t *ref, int *handle, erts_lcnt_lock_info_carrier_t **result) { + if(!*ethr_atomic_addr(ref) || !lcnt_initialization_completed__) { + return 0; + } + + (*handle) = lcnt_thr_progress_unmanaged_delay__(); + (*result) = (erts_lcnt_lock_info_carrier_t*)ethr_atomic_read(ref); + + if(*result) { + if(*handle != LCNT_THR_PRGR_DHANDLE_MANAGED) { + lcnt_retain_carrier__(*result); + lcnt_thr_progress_unmanaged_continue__(*handle); + } + + return 1; + } else if(*handle != LCNT_THR_PRGR_DHANDLE_MANAGED) { + lcnt_thr_progress_unmanaged_continue__(*handle); + } + + return 0; +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_close_ref(int handle, erts_lcnt_lock_info_carrier_t *carrier) { + if(handle != LCNT_THR_PRGR_DHANDLE_MANAGED) { + lcnt_release_carrier__(carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock(erts_lcnt_ref_t *ref) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_idx(carrier, 0); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_opt(erts_lcnt_ref_t *ref, Uint16 option) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_opt_idx(carrier, 0, option); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_post(erts_lcnt_ref_t *ref) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_post_idx(carrier, 0); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_post_x(erts_lcnt_ref_t *ref, char *file, unsigned int line) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_post_x_idx(carrier, 0, file, line); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_lock_unacquire(erts_lcnt_ref_t *ref) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_lock_unacquire_idx(carrier, 0); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_unlock(erts_lcnt_ref_t *ref) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_unlock_idx(carrier, 0); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_unlock_opt(erts_lcnt_ref_t *ref, Uint16 option) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_unlock_opt_idx(carrier, 0, option); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_trylock(erts_lcnt_ref_t *ref, int result) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_trylock_idx(carrier, 0, result); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_FORCE_INLINE +void erts_lcnt_trylock_opt(erts_lcnt_ref_t *ref, int result, Uint16 option) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(ref, &handle, &carrier)) { + erts_lcnt_trylock_opt_idx(carrier, 0, result, option); + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) { + erts_lcnt_lock_opt_idx(carrier, index, ERTS_LCNT_LO_WRITE); +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__(); + + ASSERT(index < carrier->entry_count); + + ASSERT((option & ERTS_LCNT_LO_READ) || (option & ERTS_LCNT_LO_WRITE)); + + if(option & ERTS_LCNT_LO_WRITE) { + ethr_sint_t w_state, r_state; + + w_state = ethr_atomic_inc_read(&info->w_state) - 1; + r_state = ethr_atomic_read(&info->r_state); + + /* We cannot acquire w_lock if either w or r are taken */ + eltd->lock_in_conflict = (w_state > 0) || (r_state > 0); + } else { + ethr_sint_t w_state = ethr_atomic_read(&info->w_state); + + /* We cannot acquire r_lock if w_lock is taken */ + eltd->lock_in_conflict = (w_state > 0); + } + + if(option & ERTS_LCNT_LO_READ) { + ethr_atomic_inc(&info->r_state); + } + + if(eltd->lock_in_conflict) { + /* Only set the timer if nobody else has it. This should only happen + * when proc_locks acquires several locks "atomically." All other locks + * will block the thread when locked (w_state > 0) */ + if(eltd->timer_set == 0) { + lcnt_time__(&eltd->timer); + } + + eltd->timer_set++; + } +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_post_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) { + erts_lcnt_lock_post_x_idx(carrier, index, NULL, 0); +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_post_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, char *file, unsigned int line) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + lcnt_thread_data_t__ *eltd = lcnt_get_thread_data__(); + erts_lcnt_lock_stats_t *stats; + + ASSERT(index < carrier->entry_count); + +#ifdef LCNT_DEBUG_LOCK_FLOW + if(!(info->flag & (ERTS_LCNT_LT_RWMUTEX | ERTS_LCNT_LT_RWSPINLOCK))) { + ASSERT(ethr_atomic_inc_read(&info->flowstate) == 1); + } +#endif + + /* If the lock was in conflict, update the time spent waiting. */ + stats = lcnt_get_lock_stats__(info, file, line); + if(eltd->timer_set) { + erts_lcnt_time_t time_wait; + erts_lcnt_time_t timer; + + lcnt_time__(&timer); + + lcnt_time_diff__(&time_wait, &timer, &eltd->timer); + lcnt_update_stats__(stats, eltd->lock_in_conflict, &time_wait); + + eltd->timer_set--; + + ASSERT(eltd->timer_set >= 0); + } else { + lcnt_update_stats__(stats, eltd->lock_in_conflict, NULL); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_unlock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) { +#ifdef LCNT_DEBUG_LOCK_FLOW + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(ethr_atomic_dec_read(&info->flowstate) == 0); +#endif + + ASSERT(index < carrier->entry_count); + + erts_lcnt_unlock_opt_idx(carrier, index, ERTS_LCNT_LO_WRITE); +} + +ERTS_GLB_INLINE +void erts_lcnt_unlock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, Uint16 option) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(index < carrier->entry_count); + + ASSERT((option & ERTS_LCNT_LO_READ) || (option & ERTS_LCNT_LO_WRITE)); + + if(option & ERTS_LCNT_LO_WRITE) { + lcnt_dec_lock_state__(&info->w_state); + } + + if(option & ERTS_LCNT_LO_READ) { + lcnt_dec_lock_state__(&info->r_state); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_lock_unacquire_idx(erts_lcnt_lock_info_carrier_t *carrier, int index) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(index < carrier->entry_count); + + lcnt_dec_lock_state__(&info->w_state); +} + +ERTS_GLB_INLINE +void erts_lcnt_trylock_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result) { +#ifdef LCNT_DEBUG_LOCK_FLOW + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(result == EBUSY || ethr_atomic_inc_read(&info->flowstate) == 1); +#endif + + ASSERT(index < carrier->entry_count); + + erts_lcnt_trylock_opt_idx(carrier, index, result, ERTS_LCNT_LO_WRITE); +} + +ERTS_GLB_INLINE +void erts_lcnt_trylock_opt_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, int result, Uint16 option) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(index < carrier->entry_count); + + ASSERT((option & ERTS_LCNT_LO_READ) || (option & ERTS_LCNT_LO_WRITE)); + + if(result != EBUSY) { + if(option & ERTS_LCNT_LO_WRITE) { + ethr_atomic_inc(&info->w_state); + } + + if(option & ERTS_LCNT_LO_READ) { + ethr_atomic_inc(&info->r_state); + } + + lcnt_update_stats__(&info->location_stats[0], 0, NULL); + } else { + ethr_atomic_inc(&info->location_stats[0].attempts); + ethr_atomic_inc(&info->location_stats[0].collisions); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_init_lock_info_x_idx(erts_lcnt_lock_info_carrier_t *carrier, int index, + const char *name, Uint16 flag, Eterm id) { + erts_lcnt_lock_info_t *info = &carrier->entries[index]; + + ASSERT(is_immed(id)); + + info->flag = flag; + info->name = name; + info->id = id; +} + +ERTS_GLB_INLINE +void lcnt_retain_carrier__(erts_lcnt_lock_info_carrier_t *carrier) { +#ifdef DEBUG + ASSERT(ethr_atomic_inc_read_acqb(&carrier->ref_count) >= 2); +#else + ethr_atomic_inc_acqb(&carrier->ref_count); +#endif +} + +ERTS_GLB_INLINE +void lcnt_release_carrier__(erts_lcnt_lock_info_carrier_t *carrier) { + ethr_sint_t count = ethr_atomic_dec_read_relb(&carrier->ref_count); + + ASSERT(count >= 0); + + if(count == 0) { + carrier->deallocate(carrier); + } +} + +#endif #endif /* ifdef ERTS_ENABLE_LOCK_COUNT */ #endif /* ifndef ERTS_LOCK_COUNT_H__ */ diff --git a/erts/emulator/beam/erl_process_lock.c b/erts/emulator/beam/erl_process_lock.c index c0e7380ed0..59577bf422 100644 --- a/erts/emulator/beam/erl_process_lock.c +++ b/erts/emulator/beam/erl_process_lock.c @@ -944,7 +944,7 @@ erts_pid2proc_opt(Process *c_p, erts_proc_inc_refc(proc); #if ERTS_PROC_LOCK_OWN_IMPL && defined(ERTS_ENABLE_LOCK_COUNT) - erts_lcnt_proc_lock_unaquire(&proc->lock, lcnt_locks); + erts_lcnt_proc_lock_unacquire(&proc->lock, lcnt_locks); #endif managed = dhndl == ERTS_THR_PRGR_DHANDLE_MANAGED; @@ -1127,114 +1127,52 @@ erts_proc_lock_fin(Process *p) void erts_lcnt_enable_proc_lock_count(int enable) { int ix, max = erts_ptab_max(&erts_proc); Process *proc = NULL; - for (ix = 0; ix < max; ++ix) { - if ((proc = erts_pix2proc(ix)) != NULL) + for(ix = 0; ix < max; ++ix) { + if((proc = erts_pix2proc(ix)) != NULL) { lcnt_enable_proc_lock_count(proc, enable); + } } /* for all processes */ } void erts_lcnt_proc_lock_init(Process *p) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) { - erts_lcnt_init_lock_empty(&(p->lock.lcnt_main)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_link)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_msgq)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_btm)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_status)); - erts_lcnt_init_lock_empty(&(p->lock.lcnt_trace)); - } else { /* now the common case */ - Eterm pid = (p->common.id != ERTS_INVALID_PID) ? p->common.id : NIL; - erts_lcnt_init_lock_x(&(p->lock.lcnt_main), "proc_main", ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_link), "proc_link", ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_msgq), "proc_msgq", ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_btm), "proc_btm", ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_status),"proc_status",ERTS_LCNT_LT_PROCLOCK, pid); - erts_lcnt_init_lock_x(&(p->lock.lcnt_trace), "proc_trace", ERTS_LCNT_LT_PROCLOCK, pid); - } /* the lock names should really be aligned to four characters */ + erts_lcnt_init_ref(&p->lock.lcnt_carrier); + + if (erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK) { + lcnt_enable_proc_lock_count(p, 1); + } } /* logic reversed */ void erts_lcnt_proc_lock_destroy(Process *p) { - erts_lcnt_destroy_lock(&(p->lock.lcnt_main)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_link)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_msgq)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_btm)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_status)); - erts_lcnt_destroy_lock(&(p->lock.lcnt_trace)); + erts_lcnt_uninstall(&p->lock.lcnt_carrier); } static void lcnt_enable_proc_lock_count(Process *proc, int enable) { if (enable) { - if (!ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { - erts_lcnt_proc_lock_init(proc); - } - } - else { - if (ERTS_LCNT_LOCK_TYPE(&(proc->lock.lcnt_main))) { - erts_lcnt_proc_lock_destroy(proc); - } - } -} - -void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock(&(lock->lcnt_main)); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock(&(lock->lcnt_link)); } - if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock(&(lock->lcnt_msgq)); } - if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock(&(lock->lcnt_btm)); } - if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock(&(lock->lcnt_status)); } - if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock(&(lock->lcnt_trace)); } -} - -void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, - char *file, unsigned int line) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { - erts_lcnt_lock_post_x(&(lock->lcnt_main), file, line); - } - if (locks & ERTS_PROC_LOCK_LINK) { - erts_lcnt_lock_post_x(&(lock->lcnt_link), file, line); - } - if (locks & ERTS_PROC_LOCK_MSGQ) { - erts_lcnt_lock_post_x(&(lock->lcnt_msgq), file, line); - } - if (locks & ERTS_PROC_LOCK_BTM) { - erts_lcnt_lock_post_x(&(lock->lcnt_btm), file, line); - } - if (locks & ERTS_PROC_LOCK_STATUS) { - erts_lcnt_lock_post_x(&(lock->lcnt_status), file, line); - } - if (locks & ERTS_PROC_LOCK_TRACE) { - erts_lcnt_lock_post_x(&(lock->lcnt_trace), file, line); + erts_lcnt_lock_info_carrier_t *carrier; + Eterm pid; + + carrier = erts_lcnt_create_lock_info_carrier(ERTS_LCNT_PROCLOCK_COUNT); + pid = (proc->common.id != ERTS_INVALID_PID) ? proc->common.id : NIL; + + erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, + "proc_main", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, + "proc_link", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, + "proc_msgq", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM, + "proc_btm", ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS, + "proc_status",ERTS_LCNT_LT_PROCLOCK, pid); + erts_lcnt_init_lock_info_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE, + "proc_trace", ERTS_LCNT_LT_PROCLOCK, pid); + + erts_lcnt_install(&proc->lock.lcnt_carrier, carrier); + } else { + erts_lcnt_proc_lock_destroy(proc); } } -void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_lock_unaquire(&(lock->lcnt_main)); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_lock_unaquire(&(lock->lcnt_link)); } - if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_lock_unaquire(&(lock->lcnt_msgq)); } - if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_lock_unaquire(&(lock->lcnt_btm)); } - if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_lock_unaquire(&(lock->lcnt_status)); } - if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_lock_unaquire(&(lock->lcnt_trace)); } -} - -void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_unlock(&(lock->lcnt_main)); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_unlock(&(lock->lcnt_link)); } - if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_unlock(&(lock->lcnt_msgq)); } - if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_unlock(&(lock->lcnt_btm)); } - if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_unlock(&(lock->lcnt_status)); } - if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_unlock(&(lock->lcnt_trace)); } -} -void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) { - if (!(erts_lcnt_rt_options & ERTS_LCNT_OPT_PROCLOCK)) return; - if (locks & ERTS_PROC_LOCK_MAIN) { erts_lcnt_trylock(&(lock->lcnt_main), res); } - if (locks & ERTS_PROC_LOCK_LINK) { erts_lcnt_trylock(&(lock->lcnt_link), res); } - if (locks & ERTS_PROC_LOCK_MSGQ) { erts_lcnt_trylock(&(lock->lcnt_msgq), res); } - if (locks & ERTS_PROC_LOCK_BTM) { erts_lcnt_trylock(&(lock->lcnt_btm), res); } - if (locks & ERTS_PROC_LOCK_STATUS) { erts_lcnt_trylock(&(lock->lcnt_status), res); } - if (locks & ERTS_PROC_LOCK_TRACE) { erts_lcnt_trylock(&(lock->lcnt_trace), res); } -} /* reversed logic */ #endif /* ERTS_ENABLE_LOCK_COUNT */ diff --git a/erts/emulator/beam/erl_process_lock.h b/erts/emulator/beam/erl_process_lock.h index 6e704b185d..b4a5ef3f28 100644 --- a/erts/emulator/beam/erl_process_lock.h +++ b/erts/emulator/beam/erl_process_lock.h @@ -78,13 +78,19 @@ typedef struct erts_proc_lock_t_ { ErtsProcLocks flags; #endif erts_tse_t *queue[ERTS_PROC_LOCK_MAX_BIT+1]; -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt_main; - erts_lcnt_lock_t lcnt_link; - erts_lcnt_lock_t lcnt_msgq; - erts_lcnt_lock_t lcnt_btm; - erts_lcnt_lock_t lcnt_status; - erts_lcnt_lock_t lcnt_trace; +#if defined(ERTS_ENABLE_LOCK_COUNT) && !ERTS_PROC_LOCK_RAW_MUTEX_IMPL + /* Each erts_mtx_t has its own lock counter ^ */ + + #define ERTS_LCNT_PROCLOCK_IDX_MAIN 0 + #define ERTS_LCNT_PROCLOCK_IDX_LINK 1 + #define ERTS_LCNT_PROCLOCK_IDX_MSGQ 2 + #define ERTS_LCNT_PROCLOCK_IDX_BTM 3 + #define ERTS_LCNT_PROCLOCK_IDX_STATUS 4 + #define ERTS_LCNT_PROCLOCK_IDX_TRACE 5 + + #define ERTS_LCNT_PROCLOCK_COUNT 6 + + erts_lcnt_ref_t lcnt_carrier; #endif #elif ERTS_PROC_LOCK_RAW_MUTEX_IMPL erts_mtx_t main; @@ -245,14 +251,169 @@ typedef struct erts_proc_lock_t_ { void erts_lcnt_proc_lock_init(Process *p); void erts_lcnt_proc_lock_destroy(Process *p); + +ERTS_GLB_INLINE void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks); +ERTS_GLB_INLINE void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, char *file, unsigned int line); -void erts_lcnt_proc_lock_unaquire(erts_proc_lock_t *lock, ErtsProcLocks locks); +ERTS_GLB_INLINE +void erts_lcnt_proc_lock_unacquire(erts_proc_lock_t *lock, ErtsProcLocks locks); +ERTS_GLB_INLINE void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks); +ERTS_GLB_INLINE void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res); void erts_lcnt_enable_proc_lock_count(int enable); +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +ERTS_GLB_INLINE +void erts_lcnt_proc_lock(erts_proc_lock_t *lock, ErtsProcLocks locks) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_lock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE); + } + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_proc_lock_post_x(erts_proc_lock_t *lock, ErtsProcLocks locks, + char *file, unsigned int line) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, file, line); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, file, line); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, file, line); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM, file, line); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS, file, line); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_lock_post_x_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE, file, line); + } + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_proc_lock_unacquire(erts_proc_lock_t *lock, ErtsProcLocks locks) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_lock_unacquire_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE); + } + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_proc_unlock(erts_proc_lock_t *lock, ErtsProcLocks locks) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_unlock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE); + } + + erts_lcnt_close_ref(handle, carrier); + } +} + +ERTS_GLB_INLINE +void erts_lcnt_proc_trylock(erts_proc_lock_t *lock, ErtsProcLocks locks, int res) { + erts_lcnt_lock_info_carrier_t *carrier; + int handle; + + if(erts_lcnt_open_ref(&lock->lcnt_carrier, &handle, &carrier)) { + if (locks & ERTS_PROC_LOCK_MAIN) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MAIN, res); + } + if (locks & ERTS_PROC_LOCK_LINK) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_LINK, res); + } + if (locks & ERTS_PROC_LOCK_MSGQ) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_MSGQ, res); + } + if (locks & ERTS_PROC_LOCK_BTM) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_BTM, res); + } + if (locks & ERTS_PROC_LOCK_STATUS) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_STATUS, res); + } + if (locks & ERTS_PROC_LOCK_TRACE) { + erts_lcnt_trylock_idx(carrier, ERTS_LCNT_PROCLOCK_IDX_TRACE, res); + } + + erts_lcnt_close_ref(handle, carrier); + } +} /* reversed logic */ + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ #endif /* ERTS_ENABLE_LOCK_COUNT*/ diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 700ed90def..2a9f276e02 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -321,13 +321,23 @@ tmp_thr_prgr_data(ErtsSchedulerData *esdp) ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(esdp); if (!tpd) { - /* - * We only allocate the part up to the wakeup_request field - * which is the first field only used by registered threads - */ - tpd = erts_alloc(ERTS_ALC_T_T_THR_PRGR_DATA, - offsetof(ErtsThrPrgrData, wakeup_request)); - init_tmp_thr_prgr_data(tpd); + /* + * We only allocate the part up to the wakeup_request field which is + * the first field only used by registered threads + */ + size_t alloc_size = offsetof(ErtsThrPrgrData, wakeup_request); + + /* We may land here as a result of unmanaged_delay being called from + * the lock counting module, which in turn might be called from within + * the allocator, so we use plain malloc to avoid deadlocks. */ + tpd = +#ifdef ERTS_ENABLE_LOCK_COUNT + malloc(alloc_size); +#else + erts_alloc(ERTS_ALC_T_T_THR_PRGR_DATA, alloc_size); +#endif + + init_tmp_thr_prgr_data(tpd); } return tpd; @@ -337,8 +347,13 @@ static ERTS_INLINE void return_tmp_thr_prgr_data(ErtsThrPrgrData *tpd) { if (tpd->is_temporary) { - erts_tsd_set(erts_thr_prgr_data_key__, NULL); - erts_free(ERTS_ALC_T_T_THR_PRGR_DATA, tpd); + erts_tsd_set(erts_thr_prgr_data_key__, NULL); + +#ifdef ERTS_ENABLE_LOCK_COUNT + free(tpd); +#else + erts_free(ERTS_ALC_T_T_THR_PRGR_DATA, tpd); +#endif } } diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 28ff5d3a42..f7b53f5ef3 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -307,7 +307,7 @@ typedef struct { erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; #endif } erts_mtx_t; @@ -320,7 +320,7 @@ typedef struct { erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; #endif } erts_rwmtx_t; @@ -365,7 +365,7 @@ typedef struct { erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; #endif } erts_spinlock_t; @@ -376,7 +376,7 @@ typedef struct { erts_lc_lock_t lc; #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_lock_t lcnt; + erts_lcnt_ref_t lcnt; #endif } erts_rwlock_t; @@ -2169,7 +2169,8 @@ erts_mtx_init_x(erts_mtx_t *mtx, char *name, Eterm extra) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); + erts_lcnt_init_ref(&mtx->lcnt); + erts_lcnt_install_new_lock_info_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); #endif #endif } @@ -2185,7 +2186,10 @@ erts_mtx_init_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); + erts_lcnt_init_ref(&mtx->lcnt); + if(!(opt & ERTS_LCNT_LT_DISABLE)) { // Incorrect, but preserves the old interface. + erts_lcnt_install_new_lock_info_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); + } #endif #endif } @@ -2202,7 +2206,10 @@ erts_mtx_init_locked_x_opt(erts_mtx_t *mtx, char *name, Eterm extra, Uint16 opt) erts_lc_init_lock_x(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX | opt, extra); + erts_lcnt_init_ref(&mtx->lcnt); + if(!(opt & ERTS_LCNT_LT_DISABLE)) { // Incorrect, but preserves the old interface. + erts_lcnt_install_new_lock_info_x(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX, extra); + } #endif ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -2225,7 +2232,8 @@ erts_mtx_init(erts_mtx_t *mtx, char *name) erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); + erts_lcnt_init_ref(&mtx->lcnt); + erts_lcnt_install_new_lock_info(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); #endif #endif } @@ -2241,7 +2249,8 @@ erts_mtx_init_locked(erts_mtx_t *mtx, char *name) erts_lc_init_lock(&mtx->lc, name, ERTS_LC_FLG_LT_MUTEX); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); + erts_lcnt_init_ref(&mtx->lcnt); + erts_lcnt_install_new_lock_info(&mtx->lcnt, name, ERTS_LCNT_LT_MUTEX); #endif ethr_mutex_lock(&mtx->mtx); #ifdef ERTS_ENABLE_LOCK_CHECK @@ -2262,7 +2271,7 @@ erts_mtx_destroy(erts_mtx_t *mtx) erts_lc_destroy_lock(&mtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&mtx->lcnt); + erts_lcnt_uninstall(&mtx->lcnt); #endif res = ethr_mutex_destroy(&mtx->mtx); if (res != 0) { @@ -2481,10 +2490,11 @@ erts_rwmtx_init_opt_x(erts_rwmtx_t *rwmtx, erts_lc_init_lock_x(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_init_ref(&rwmtx->lcnt); if (name && name[0] == '\0') - erts_lcnt_init_lock_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra); + erts_lcnt_install_new_lock_info_x(&rwmtx->lcnt, NULL, ERTS_LCNT_LT_RWMUTEX, extra); else - erts_lcnt_init_lock_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra); + erts_lcnt_install_new_lock_info_x(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX, extra); #endif #endif } @@ -2510,7 +2520,8 @@ erts_rwmtx_init_opt(erts_rwmtx_t *rwmtx, erts_lc_init_lock(&rwmtx->lc, name, ERTS_LC_FLG_LT_RWMUTEX); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX); + erts_lcnt_init_ref(&rwmtx->lcnt); + erts_lcnt_install_new_lock_info(&rwmtx->lcnt, name, ERTS_LCNT_LT_RWMUTEX); #endif #endif } @@ -2530,7 +2541,7 @@ erts_rwmtx_destroy(erts_rwmtx_t *rwmtx) erts_lc_destroy_lock(&rwmtx->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&rwmtx->lcnt); + erts_lcnt_uninstall(&rwmtx->lcnt); #endif res = ethr_rwmutex_destroy(&rwmtx->rwmtx); if (res != 0) { @@ -3085,7 +3096,8 @@ erts_spinlock_init_x(erts_spinlock_t *lock, char *name, Eterm extra) erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra); + erts_lcnt_init_ref(&lock->lcnt); + erts_lcnt_install_new_lock_info_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK, extra); #endif #else (void)lock; @@ -3104,7 +3116,8 @@ erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, Eterm extra, erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK|opt, extra); + erts_lcnt_init_ref(&lock->lcnt); + erts_lcnt_install_new_lock_info_x(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK | opt, extra); #endif #else (void)lock; @@ -3123,7 +3136,8 @@ erts_spinlock_init(erts_spinlock_t *lock, char *name) erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_SPINLOCK); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK); + erts_lcnt_init_ref(&lock->lcnt); + erts_lcnt_install_new_lock_info(&lock->lcnt, name, ERTS_LCNT_LT_SPINLOCK); #endif #else (void)lock; @@ -3139,7 +3153,7 @@ erts_spinlock_destroy(erts_spinlock_t *lock) erts_lc_destroy_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&lock->lcnt); + erts_lcnt_uninstall(&lock->lcnt); #endif res = ethr_spinlock_destroy(&lock->slck); if (res != 0) { @@ -3228,7 +3242,8 @@ erts_rwlock_init_x(erts_rwlock_t *lock, char *name, Eterm extra) erts_lc_init_lock_x(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK, extra); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra); + erts_lcnt_init_ref(&lock->lcnt); + erts_lcnt_install_new_lock_info_x(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK, extra); #endif #else (void)lock; @@ -3246,7 +3261,8 @@ erts_rwlock_init(erts_rwlock_t *lock, char *name) erts_lc_init_lock(&lock->lc, name, ERTS_LC_FLG_LT_RWSPINLOCK); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init_lock(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK); + erts_lcnt_init_ref(&lock->lcnt); + erts_lcnt_install_new_lock_info(&lock->lcnt, name, ERTS_LCNT_LT_RWSPINLOCK); #endif #else (void)lock; @@ -3262,7 +3278,7 @@ erts_rwlock_destroy(erts_rwlock_t *lock) erts_lc_destroy_lock(&lock->lc); #endif #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_destroy_lock(&lock->lcnt); + erts_lcnt_uninstall(&lock->lcnt); #endif res = ethr_rwlock_destroy(&lock->rwlck); if (res != 0) { diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index d25e53ada0..11cb2dfddf 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -3461,39 +3461,34 @@ void erts_init_io(int port_tab_size, static ERTS_INLINE void lcnt_enable_drv_lock_count(erts_driver_t *dp, int enable) { if (dp->lock) { - if (enable) - erts_lcnt_init_lock_x(&dp->lock->lcnt, - "driver_lock", - ERTS_LCNT_LT_MUTEX, - erts_atom_put((byte*)dp->name, - sys_strlen(dp->name), - ERTS_ATOM_ENC_LATIN1, - 1)); - - else - erts_lcnt_destroy_lock(&dp->lock->lcnt); - - } + if (enable) { + Eterm name_as_atom = erts_atom_put((byte*)dp->name, sys_strlen(dp->name), + ERTS_ATOM_ENC_LATIN1, 1); + erts_lcnt_install_new_lock_info_x(&dp->lock->lcnt, + "driver_lock", ERTS_LCNT_LT_MUTEX, name_as_atom); + } else { + erts_lcnt_uninstall(&dp->lock->lcnt); + } + } } static ERTS_INLINE void lcnt_enable_port_lock_count(Port *prt, int enable) { erts_aint32_t state = erts_atomic32_read_nob(&prt->state); - if (!enable) { - erts_lcnt_destroy_lock(&prt->sched.mtx.lcnt); - if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) - erts_lcnt_destroy_lock(&prt->lock->lcnt); - } - else { - erts_lcnt_init_lock_x(&prt->sched.mtx.lcnt, - "port_sched_lock", - ERTS_LCNT_LT_MUTEX, - prt->common.id); - if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) - erts_lcnt_init_lock_x(&prt->lock->lcnt, - "port_lock", - ERTS_LCNT_LT_MUTEX, - prt->common.id); + + if(enable) { + erts_lcnt_install_new_lock_info_x(&prt->sched.mtx.lcnt, + "port_sched_lock", ERTS_LCNT_LT_MUTEX,prt->common.id); + + if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { + erts_lcnt_install_new_lock_info_x(&prt->lock->lcnt, + "port_lock", ERTS_LCNT_LT_MUTEX, prt->common.id); + } + } else { + erts_lcnt_uninstall(&prt->sched.mtx.lcnt); + if (state & ERTS_PORT_SFLG_PORT_SPECIFIC_LOCK) { + erts_lcnt_uninstall(&prt->lock->lcnt); + } } } @@ -3701,7 +3696,7 @@ deliver_result(Port *prt, Eterm sender, Eterm pid, Eterm res) ERTS_SMP_CHK_NO_PROC_LOCKS; ASSERT(!prt || prt->common.id == sender); -#ifdef ERTS_SMP +#if defined(ERTS_SMP) && defined(ERTS_ENABLE_LOCK_CHECK) ASSERT(!prt || erts_lc_is_port_locked(prt)); #endif diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index 457cada745..0fb25c2082 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -42,6 +42,7 @@ #include "dist.h" #include "erl_printf.h" #include "erl_threads.h" +#include "erl_lock_count.h" #include "erl_smp.h" #include "erl_time.h" #include "erl_thr_progress.h" diff --git a/erts/emulator/sys/unix/sys.c b/erts/emulator/sys/unix/sys.c index 0079912b10..e86258b5e0 100644 --- a/erts/emulator/sys/unix/sys.c +++ b/erts/emulator/sys/unix/sys.c @@ -438,14 +438,18 @@ erts_sys_pre_init(void) /* After creation in parent */ eid.thread_create_parent_func = thr_create_cleanup, +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_pre_thr_init(); +#endif + erts_thr_init(&eid); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init(); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_post_thr_init(); #endif -#ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init(); +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init(); #endif #endif /* USE_THREADS */ diff --git a/erts/emulator/sys/win32/sys.c b/erts/emulator/sys/win32/sys.c index 28019e306c..15c59109b1 100644 --- a/erts/emulator/sys/win32/sys.c +++ b/erts/emulator/sys/win32/sys.c @@ -3195,15 +3195,22 @@ erts_sys_pre_init(void) /* After creation in parent */ eid.thread_create_parent_func = thr_create_cleanup; - erts_thr_init(&eid); -#ifdef ERTS_ENABLE_LOCK_CHECK - erts_lc_init(); +#ifdef ERTS_ENABLE_LOCK_COUNT + erts_lcnt_pre_thr_init(); #endif + + erts_thr_init(&eid); + #ifdef ERTS_ENABLE_LOCK_COUNT - erts_lcnt_init(); + erts_lcnt_post_thr_init(); #endif + +#ifdef ERTS_ENABLE_LOCK_CHECK + erts_lc_init(); #endif +#endif /* USE_THREADS */ + erts_init_sys_time_sup(); erts_smp_atomic_init_nob(&sys_misc_mem_sz, 0); |