diff options
| -rw-r--r-- | erts/emulator/sys/common/erl_check_io.c | 198 |
1 files changed, 104 insertions, 94 deletions
diff --git a/erts/emulator/sys/common/erl_check_io.c b/erts/emulator/sys/common/erl_check_io.c index a5a851d122..52584e4246 100644 --- a/erts/emulator/sys/common/erl_check_io.c +++ b/erts/emulator/sys/common/erl_check_io.c @@ -156,15 +156,36 @@ struct removed_fd { }; +struct drv_ev_state_shared { + /* The layout of this struct must be independent of kp/nkp compilation */ + #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS -static int max_fds = -1; + int max_fds; #endif #define DRV_EV_STATE_LOCK_CNT 128 -static union { - erts_mtx_t lck; - byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))]; -}drv_ev_state_locks[DRV_EV_STATE_LOCK_CNT]; + union { + erts_mtx_t lck; + byte _cache_line_alignment[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(erts_mtx_t))]; + }locks[DRV_EV_STATE_LOCK_CNT]; + +#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS + erts_atomic_t len; + ErtsDrvEventState *v; + erts_mtx_t grow_lock; /* prevent lock-hogging of racing growers */ +#else + SafeHash tab; + int num_prealloc; + ErtsDrvEventState *prealloc_first; + erts_spinlock_t prealloc_lock; +#endif +}; + +#ifndef ERTS_KERNEL_POLL_VERSION +struct drv_ev_state_shared drv_ev_state; +#else +extern struct drv_ev_state_shared drv_ev_state; +#endif static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd) { @@ -172,26 +193,16 @@ static ERTS_INLINE erts_mtx_t* fd_mtx(ErtsSysFdType fd) # ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS hash ^= (hash >> 9); # endif - return &drv_ev_state_locks[hash % DRV_EV_STATE_LOCK_CNT].lck; + return &drv_ev_state.locks[hash % DRV_EV_STATE_LOCK_CNT].lck; } -#ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - -static erts_atomic_t drv_ev_state_len; -static ErtsDrvEventState *drv_ev_state; -static erts_mtx_t drv_ev_state_grow_lock; /* prevent lock-hogging of racing growers */ - -#else -static SafeHash drv_ev_state_tab; -static int num_state_prealloc; -static ErtsDrvEventState *state_prealloc_first; -erts_spinlock_t state_prealloc_lock; +#ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS static ERTS_INLINE ErtsDrvEventState *hash_get_drv_ev_state(ErtsSysFdType fd) { ErtsDrvEventState tmpl; tmpl.fd = fd; - return (ErtsDrvEventState *) safe_hash_get(&drv_ev_state_tab, (void *) &tmpl); + return (ErtsDrvEventState *) safe_hash_get(&drv_ev_state.tab, (void *) &tmpl); } static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd) @@ -205,13 +216,13 @@ static ERTS_INLINE ErtsDrvEventState* hash_new_drv_ev_state(ErtsSysFdType fd) tmpl.remove_cnt = 0; tmpl.type = ERTS_EV_TYPE_NONE; tmpl.flags = 0; - return (ErtsDrvEventState *) safe_hash_put(&drv_ev_state_tab, (void *) &tmpl); + return (ErtsDrvEventState *) safe_hash_put(&drv_ev_state.tab, (void *) &tmpl); } static ERTS_INLINE void hash_erase_drv_ev_state(ErtsDrvEventState *state) { ASSERT(state->remove_cnt == 0); - safe_hash_erase(&drv_ev_state_tab, (void *) state); + safe_hash_erase(&drv_ev_state.tab, (void *) state); } #endif /* !ERTS_SYS_CONTINOUS_FD_NUMBERS */ @@ -356,7 +367,7 @@ forget_removed(struct pollset_info* psi) fd = fdlp->fd; mtx = fd_mtx(fd); erts_mtx_lock(mtx); - state = &drv_ev_state[(int) fd]; + state = &drv_ev_state.v[(int) fd]; #else state = fdlp->state; fd = state->fd; @@ -426,40 +437,40 @@ grow_drv_ev_state(int min_ix) int old_len; int new_len; - erts_mtx_lock(&drv_ev_state_grow_lock); - old_len = erts_atomic_read_nob(&drv_ev_state_len); + erts_mtx_lock(&drv_ev_state.grow_lock); + old_len = erts_atomic_read_nob(&drv_ev_state.len); if (min_ix >= old_len) { new_len = erts_poll_new_table_len(old_len, min_ix + 1); - if (new_len > max_fds) - new_len = max_fds; + if (new_len > drv_ev_state.max_fds) + new_len = drv_ev_state.max_fds; for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { /* lock all fd's */ - erts_mtx_lock(&drv_ev_state_locks[i].lck); + erts_mtx_lock(&drv_ev_state.locks[i].lck); } - drv_ev_state = (drv_ev_state + drv_ev_state.v = (drv_ev_state.v ? erts_realloc(ERTS_ALC_T_DRV_EV_STATE, - drv_ev_state, + drv_ev_state.v, sizeof(ErtsDrvEventState)*new_len) : erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState)*new_len)); for (i = old_len; i < new_len; i++) { - drv_ev_state[i].fd = (ErtsSysFdType) i; - drv_ev_state[i].driver.select = NULL; - drv_ev_state[i].driver.stop.drv_ptr = NULL; - drv_ev_state[i].driver.nif = NULL; - drv_ev_state[i].events = 0; - drv_ev_state[i].remove_cnt = 0; - drv_ev_state[i].type = ERTS_EV_TYPE_NONE; - drv_ev_state[i].flags = 0; + drv_ev_state.v[i].fd = (ErtsSysFdType) i; + drv_ev_state.v[i].driver.select = NULL; + drv_ev_state.v[i].driver.stop.drv_ptr = NULL; + drv_ev_state.v[i].driver.nif = NULL; + drv_ev_state.v[i].events = 0; + drv_ev_state.v[i].remove_cnt = 0; + drv_ev_state.v[i].type = ERTS_EV_TYPE_NONE; + drv_ev_state.v[i].flags = 0; } - erts_atomic_set_nob(&drv_ev_state_len, new_len); + erts_atomic_set_nob(&drv_ev_state.len, new_len); for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { - erts_mtx_unlock(&drv_ev_state_locks[i].lck); + erts_mtx_unlock(&drv_ev_state.locks[i].lck); } } /*else already grown by racing thread */ - erts_mtx_unlock(&drv_ev_state_grow_lock); + erts_mtx_unlock(&drv_ev_state.grow_lock); } #endif /* ERTS_SYS_CONTINOUS_FD_NUMBERS */ @@ -623,7 +634,7 @@ check_cleanup_active_fd(ErtsSysFdType fd, erts_mtx_lock(mtx); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; + state = &drv_ev_state.v[(int) fd]; #else state = hash_get_drv_ev_state(fd); /* may be NULL! */ if (state) @@ -867,11 +878,11 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) { + if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) { if (fd < 0) { return -1; } - if (fd >= max_fds) { + if (fd >= drv_ev_state.max_fds) { drv_select_large_fd_error(ix, fd, mode, on); return -1; } @@ -882,7 +893,7 @@ ERTS_CIO_EXPORT(driver_select)(ErlDrvPort ix, erts_mtx_lock(fd_mtx(fd)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; + state = &drv_ev_state.v[(int) fd]; #else state = hash_get_drv_ev_state(fd); /* may be NULL! */ #endif @@ -1096,11 +1107,11 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, ASSERT(!(resource->monitors && resource->monitors->is_dying)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state_len)) { + if ((unsigned)fd >= (unsigned)erts_atomic_read_nob(&drv_ev_state.len)) { if (fd < 0) { return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; } - if (fd >= max_fds) { + if (fd >= drv_ev_state.max_fds) { nif_select_large_fd_error(fd, mode, resource, ref); return INT_MIN | ERL_NIF_SELECT_INVALID_EVENT; } @@ -1111,7 +1122,7 @@ ERTS_CIO_EXPORT(enif_select)(ErlNifEnv* env, erts_mtx_lock(fd_mtx(fd)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[(int) fd]; + state = &drv_ev_state.v[(int) fd]; #else state = hash_get_drv_ev_state(fd); /* may be NULL! */ #endif @@ -1500,7 +1511,7 @@ large_fd_error_common(erts_dsprintf_buf_t *dsbufp, ErtsSysFdType fd) { erts_dsprintf(dsbufp, "fd=%d is larger than the largest allowed fd=%d\n", - (int) fd, max_fds - 1); + (int) fd, drv_ev_state.max_fds - 1); } static void @@ -1828,7 +1839,7 @@ ERTS_CIO_EXPORT(erts_check_io)(int do_wait) erts_mtx_lock(fd_mtx(fd)); #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - state = &drv_ev_state[ (int) fd]; + state = &drv_ev_state.v[ (int) fd]; #else state = hash_get_drv_ev_state(fd); if (!state) { @@ -2058,16 +2069,16 @@ static int drv_ev_state_cmp(void *des1, void *des2) static void *drv_ev_state_alloc(void *des_tmpl) { ErtsDrvEventState *evstate; - erts_spin_lock(&state_prealloc_lock); - if (state_prealloc_first == NULL) { - erts_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + if (drv_ev_state.prealloc_first == NULL) { + erts_spin_unlock(&drv_ev_state.prealloc_lock); evstate = (ErtsDrvEventState *) erts_alloc(ERTS_ALC_T_DRV_EV_STATE, sizeof(ErtsDrvEventState)); } else { - evstate = state_prealloc_first; - state_prealloc_first = (ErtsDrvEventState *) evstate->hb.next; - --num_state_prealloc; - erts_spin_unlock(&state_prealloc_lock); + evstate = drv_ev_state.prealloc_first; + drv_ev_state.prealloc_first = (ErtsDrvEventState *) evstate->hb.next; + --drv_ev_state.num_prealloc; + erts_spin_unlock(&drv_ev_state.prealloc_lock); } /* XXX: Already valid data if prealloced, could ignore template! */ *evstate = *((ErtsDrvEventState *) des_tmpl); @@ -2077,11 +2088,11 @@ static void *drv_ev_state_alloc(void *des_tmpl) static void drv_ev_state_free(void *des) { - erts_spin_lock(&state_prealloc_lock); - ((ErtsDrvEventState *) des)->hb.next = &state_prealloc_first->hb; - state_prealloc_first = (ErtsDrvEventState *) des; - ++num_state_prealloc; - erts_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + ((ErtsDrvEventState *) des)->hb.next = &drv_ev_state.prealloc_first->hb; + drv_ev_state.prealloc_first = (ErtsDrvEventState *) des; + ++drv_ev_state.num_prealloc; + erts_spin_unlock(&drv_ev_state.prealloc_lock); } #endif @@ -2158,17 +2169,17 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) init_removed_fd_alloc(); erts_atomic_init_nob(&pollset.removed_list, (erts_aint_t)NULL); { - int i; - for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { - erts_mtx_init(&drv_ev_state_locks[i].lck, "drv_ev_state", make_small(i), + int i; + for (i=0; i<DRV_EV_STATE_LOCK_CNT; i++) { + erts_mtx_init(&drv_ev_state.locks[i].lck, "drv_ev_state", make_small(i), ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); - } + } } #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - max_fds = ERTS_CIO_POLL_MAX_FDS(); - erts_atomic_init_nob(&drv_ev_state_len, 0); - drv_ev_state = NULL; - erts_mtx_init(&drv_ev_state_grow_lock, "drv_ev_state_grow", NIL, + drv_ev_state.max_fds = ERTS_CIO_POLL_MAX_FDS(); + erts_atomic_init_nob(&drv_ev_state.len, 0); + drv_ev_state.v = NULL; + erts_mtx_init(&drv_ev_state.grow_lock, "drv_ev_state_grow", NIL, ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); #else { @@ -2177,12 +2188,11 @@ ERTS_CIO_EXPORT(erts_init_check_io)(void) hf.cmp = &drv_ev_state_cmp; hf.alloc = &drv_ev_state_alloc; hf.free = &drv_ev_state_free; - num_state_prealloc = 0; - state_prealloc_first = NULL; - erts_spinlock_init(&state_prealloc_lock,"state_prealloc", NIL, - ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); - - safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state_tab, "drv_ev_state_tab", + drv_ev_state.num_prealloc = 0; + drv_ev_state.prealloc_first = NULL; + erts_spinlock_init(&drv_ev_state.prealloc_lock, "state_prealloc", NIL, + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_IO); + safe_hash_init(ERTS_ALC_T_DRV_EV_STATE, &drv_ev_state.tab, "drv_ev_state_tab", ERTS_LOCK_FLAGS_CATEGORY_IO, DRV_EV_STATE_HTAB_SIZE, hf); } #endif @@ -2192,7 +2202,7 @@ int ERTS_CIO_EXPORT(erts_check_io_max_files)(void) { #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - return max_fds; + return drv_ev_state.max_fds; #else return ERTS_POLL_EXPORT(erts_poll_max_fds)(); #endif @@ -2206,17 +2216,17 @@ ERTS_CIO_EXPORT(erts_check_io_size)(void) ERTS_CIO_POLL_INFO(pollset.ps, &pi); res = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len); + res += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len); #else - res += safe_hash_table_sz(&drv_ev_state_tab); + res += safe_hash_table_sz(&drv_ev_state.tab); { SafeHashInfo hi; - safe_hash_get_info(&hi, &drv_ev_state_tab); + safe_hash_get_info(&hi, &drv_ev_state.tab); res += hi.objs * sizeof(ErtsDrvEventState); } - erts_spin_lock(&state_prealloc_lock); - res += num_state_prealloc * sizeof(ErtsDrvEventState); - erts_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + res += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState); + erts_spin_unlock(&drv_ev_state.prealloc_lock); #endif return res; } @@ -2248,17 +2258,17 @@ ERTS_CIO_EXPORT(erts_check_io_info)(void *proc) memory_size = pi.memory_size; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state_len); + memory_size += sizeof(ErtsDrvEventState) * erts_atomic_read_nob(&drv_ev_state.len); #else - memory_size += safe_hash_table_sz(&drv_ev_state_tab); + memory_size += safe_hash_table_sz(&drv_ev_state.tab); { SafeHashInfo hi; - safe_hash_get_info(&hi, &drv_ev_state_tab); + safe_hash_get_info(&hi, &drv_ev_state.tab); memory_size += hi.objs * sizeof(ErtsDrvEventState); } - erts_spin_lock(&state_prealloc_lock); - memory_size += num_state_prealloc * sizeof(ErtsDrvEventState); - erts_spin_unlock(&state_prealloc_lock); + erts_spin_lock(&drv_ev_state.prealloc_lock); + memory_size += drv_ev_state.num_prealloc * sizeof(ErtsDrvEventState); + erts_spin_unlock(&drv_ev_state.prealloc_lock); #endif hpp = NULL; @@ -2632,8 +2642,8 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) erts_thr_progress_block(); /* stop the world to avoid messy locking */ #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*max_fds); - ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollset.ps, counters.epep, max_fds); + counters.epep = erts_alloc(ERTS_ALC_T_TMP, sizeof(ErtsPollEvents)*drv_ev_state.max_fds); + ERTS_POLL_EXPORT(erts_poll_get_selected_events)(pollset.ps, counters.epep, drv_ev_state.max_fds); counters.internal_fds = 0; #endif counters.used_fds = 0; @@ -2642,16 +2652,16 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) counters.no_enif_select_structs = 0; #ifdef ERTS_SYS_CONTINOUS_FD_NUMBERS - len = erts_atomic_read_nob(&drv_ev_state_len); + len = erts_atomic_read_nob(&drv_ev_state.len); for (fd = 0; fd < len; fd++) { - doit_erts_check_io_debug((void *) &drv_ev_state[fd], (void *) &counters); + doit_erts_check_io_debug((void *) &drv_ev_state.v[fd], (void *) &counters); } - for ( ; fd < max_fds; fd++) { + for ( ; fd < drv_ev_state.max_fds; fd++) { null_des.fd = fd; doit_erts_check_io_debug((void *) &null_des, (void *) &counters); } #else - safe_hash_for_each(&drv_ev_state_tab, &doit_erts_check_io_debug, (void *) &counters); + safe_hash_for_each(&drv_ev_state.tab, &doit_erts_check_io_debug, (void *) &counters); #endif erts_thr_progress_unblock(); @@ -2679,7 +2689,7 @@ ERTS_CIO_EXPORT(erts_check_io_debug)(ErtsCheckIoDebugInfo *ciodip) #ifdef ERTS_ENABLE_LOCK_COUNT void ERTS_CIO_EXPORT(erts_lcnt_update_cio_locks)(int enable) { #ifndef ERTS_SYS_CONTINOUS_FD_NUMBERS - erts_lcnt_enable_hash_lock_count(&drv_ev_state_tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable); + erts_lcnt_enable_hash_lock_count(&drv_ev_state.tab, ERTS_LOCK_FLAGS_CATEGORY_IO, enable); #else (void)enable; #endif |