diff options
Diffstat (limited to 'erts/emulator')
| -rw-r--r-- | erts/emulator/beam/erl_alloc.c | 6 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_alloc.types | 1 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_alloc_util.c | 41 | ||||
| -rw-r--r-- | erts/emulator/beam/erl_ptab.c | 2 | ||||
| -rw-r--r-- | erts/emulator/beam/io.c | 2 | ||||
| -rw-r--r-- | erts/emulator/sys/common/erl_poll.c | 166 | ||||
| -rw-r--r-- | erts/emulator/test/port_SUITE.erl | 2 |
7 files changed, 176 insertions, 44 deletions
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index a547191d6d..86575ccb9b 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -240,6 +240,10 @@ do { \ sys_memcpy((void *) (IP), (void *) &aui__, sizeof(struct au_init)); \ } while (0) +#if ERTS_ALC_DEFAULT_ACUL \ + || ERTS_ALC_DEFAULT_ACUL_LL_ALLOC \ + || ERTS_ALC_DEFAULT_ACUL_EHEAP_ALLOC + static ERTS_INLINE void set_default_acul(struct au_init *ip, int acul) { @@ -249,6 +253,8 @@ set_default_acul(struct au_init *ip, int acul) ip->init.util.acul = acul; } +#endif + static void set_default_sl_alloc_opts(struct au_init *ip) { diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 095ad24387..7e3b3c707d 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -397,6 +397,7 @@ type POLLSET_UPDREQ SHORT_LIVED SYSTEM pollset_update_req type POLL_FDS LONG_LIVED SYSTEM poll_fds type POLL_RES_EVS LONG_LIVED SYSTEM poll_result_events type FD_STATUS LONG_LIVED SYSTEM fd_status +type SELECT_FDS LONG_LIVED SYSTEM select_fds +if unix diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index b19e603a5f..bf8a37c71b 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1753,7 +1753,8 @@ handle_delayed_dealloc(Allctr_t *allctr, if (IS_FREE_LAST_MBC_BLK(blk)) { /* * A multiblock carrier that previously has been migrated away - * from us and now is back to be deallocated... + * from us and now is back to be deallocated. For more info + * see schedule_dealloc_carrier(). * * Note that we cannot use FBLK_TO_MBC(blk) since it * data has been overwritten by the queue. @@ -1761,6 +1762,12 @@ handle_delayed_dealloc(Allctr_t *allctr, Carrier_t *crr = FIRST_BLK_TO_MBC(allctr, blk); ERTS_ALC_CPOOL_ASSERT(ERTS_ALC_IS_CPOOL_ENABLED(allctr)); ERTS_ALC_CPOOL_ASSERT(allctr == crr->cpool.orig_allctr); + ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) + != (erts_smp_atomic_read_nob(&crr->allctr) + & ~FLG_MASK)); + + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); + schedule_dealloc_carrier(allctr, crr); } else { @@ -3001,7 +3008,7 @@ check_pending_dealloc_carrier(Allctr_t *allctr, static void schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) { - Allctr_t *used_allctr; + Allctr_t *orig_allctr; int check_pending_dealloc; erts_aint_t max_size; @@ -3010,25 +3017,37 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) return; } - used_allctr = crr->cpool.orig_allctr; + orig_allctr = crr->cpool.orig_allctr; - if (allctr != used_allctr) { + if (allctr != orig_allctr) { Block_t *blk = MBC_TO_FIRST_BLK(allctr, crr); - int cinit = used_allctr->dd.ix - allctr->dd.ix; + int cinit = orig_allctr->dd.ix - allctr->dd.ix; /* - * Receiver will recognize that this is a carrier to - * deallocate since the block is an mbc block that - * is free and last in carrier... + * We send the carrier to its origin for deallocation. + * This in order: + * - not to complicate things for the thread specific + * instances of mseg_alloc, and + * - to ensure that we always only reuse empty carriers + * originating from our own thread specific mseg_alloc + * instance which is beneficial on NUMA systems. + * + * The receiver will recognize that this is a carrier to + * deallocate (and not a block which is the common case) + * since the block is an mbc block that is free and last + * in the carrier. */ ERTS_ALC_CPOOL_ASSERT(IS_FREE_LAST_MBC_BLK(blk)); ERTS_ALC_CPOOL_ASSERT(IS_MBC_FIRST_ABLK(allctr, blk)); ERTS_ALC_CPOOL_ASSERT(crr == FBLK_TO_MBC(blk)); - ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(used_allctr, blk)); + ERTS_ALC_CPOOL_ASSERT(crr == FIRST_BLK_TO_MBC(allctr, blk)); + ERTS_ALC_CPOOL_ASSERT(((erts_aint_t) allctr) + == (erts_smp_atomic_read_nob(&crr->allctr) + & ~FLG_MASK)); - if (ddq_enqueue(&used_allctr->dd.q, BLK2UMEM(blk), cinit)) - erts_alloc_notify_delayed_dealloc(used_allctr->ix); + if (ddq_enqueue(&orig_allctr->dd.q, BLK2UMEM(blk), cinit)) + erts_alloc_notify_delayed_dealloc(orig_allctr->ix); return; } diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index 8da135b2c8..f7b5f25eac 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -1435,7 +1435,7 @@ erts_ptab_test_next_id(ErtsPTab *ptab, int set, Uint next) aid_ix = max_ix; else aid_ix--; - ASSERT((aid_ix & max_ix) == (((Uint32) erts_atomic32_read_nob(&ptab->vola.tile.fid_ix)) & max_ix)); + ASSERT((aid_ix & max_ix) == (((Uint32) erts_smp_atomic32_read_nob(&ptab->vola.tile.fid_ix)) & max_ix)); #endif } diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 01e130bd64..c1e66b59af 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -4842,7 +4842,7 @@ erts_stale_drv_select(Eterm port, if (drv_port == ERTS_INVALID_ERL_DRV_PORT) { Port *prt = erts_port_lookup_raw(port); if (prt) - drv_port = ERTS_Port2ErlDrvPort(port); + drv_port = ERTS_Port2ErlDrvPort(prt); else drv_port = ERTS_INVALID_ERL_DRV_PORT; } diff --git a/erts/emulator/sys/common/erl_poll.c b/erts/emulator/sys/common/erl_poll.c index a523d67158..7d292a304a 100644 --- a/erts/emulator/sys/common/erl_poll.c +++ b/erts/emulator/sys/common/erl_poll.c @@ -40,6 +40,13 @@ # include "config.h" #endif +#if defined(__DARWIN__) || defined(__APPLE__) && defined(__MACH__) +/* Setting _DARWIN_UNLIMITED_SELECT before including sys/select.h enables + * the version of select() that does not place a limit on the fd_set. + */ +# define _DARWIN_UNLIMITED_SELECT +#endif + #ifndef WANT_NONBLOCKING # define WANT_NONBLOCKING #endif @@ -90,6 +97,52 @@ #define HARD_DEBUG #endif +#ifdef _DARWIN_UNLIMITED_SELECT +typedef struct { + size_t sz; + fd_set* ptr; +}ERTS_fd_set; +# define ERTS_FD_CLR(fd, fds) FD_CLR((fd), (fds)->ptr) +# define ERTS_FD_SET(fd, fds) FD_SET((fd), (fds)->ptr) +# define ERTS_FD_ISSET(fd,fds) FD_ISSET((fd), (fds)->ptr) +# define ERTS_FD_ZERO(fds) memset((fds)->ptr, 0, (fds)->sz) +# define ERTS_FD_SIZE(n) ((((n)+NFDBITS-1)/NFDBITS)*sizeof(fd_mask)) + +static void ERTS_FD_COPY(ERTS_fd_set *src, ERTS_fd_set *dst) +{ + if (dst->sz != src->sz) { + dst->ptr = dst->ptr + ? erts_realloc(ERTS_ALC_T_SELECT_FDS, dst->ptr, src->sz) + : erts_alloc(ERTS_ALC_T_SELECT_FDS, src->sz); + dst->sz = src->sz; + } + memcpy(dst->ptr, src->ptr, src->sz); +} + +static ERTS_INLINE +int ERTS_SELECT(int nfds, ERTS_fd_set *readfds, ERTS_fd_set *writefds, + ERTS_fd_set *exceptfds, struct timeval *timeout) +{ + ASSERT(!readfds || readfds->sz >= nfds); + ASSERT(!writefds || writefds->sz >= nfds); + ASSERT(!exceptfds); + return select(nfds, + (readfds ? readfds->ptr : NULL ), + (writefds ? writefds->ptr : NULL), + NULL, + timeout); +} + +#else /* !_DARWIN_UNLIMITED_SELECT */ +# define ERTS_fd_set fd_set +# define ERTS_FD_CLR FD_CLR +# define ERTS_FD_ISSET FD_ISSET +# define ERTS_FD_SET FD_SET +# define ERTS_FD_ZERO FD_ZERO +# define ERTS_FD_COPY(src,dst) (*(dst) = *(src)) +# define ERTS_SELECT select +#endif + #define ERTS_POLL_USE_BATCH_UPDATE_POLLSET (ERTS_POLL_USE_DEVPOLL \ || ERTS_POLL_USE_KQUEUE) #define ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE \ @@ -244,10 +297,10 @@ struct ErtsPollSet_ { #if ERTS_POLL_USE_FALLBACK int no_select_fds; #endif - fd_set input_fds; - fd_set res_input_fds; - fd_set output_fds; - fd_set res_output_fds; + ERTS_fd_set input_fds; + ERTS_fd_set res_input_fds; + ERTS_fd_set output_fds; + ERTS_fd_set res_output_fds; #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE ErtsPollSetUpdateRequestsBlock update_requests; @@ -624,6 +677,33 @@ grow_poll_fds(ErtsPollSet ps, int min_ix) } #endif +#ifdef _DARWIN_UNLIMITED_SELECT +static void +grow_select_fds(int fd, ERTS_fd_set* fds) +{ + int new_len = ERTS_POLL_EXPORT(erts_poll_get_table_len)(fd + 1); + if (new_len > max_fds) + new_len = max_fds; + new_len = ERTS_FD_SIZE(new_len); + fds->ptr = fds->sz + ? erts_realloc(ERTS_ALC_T_SELECT_FDS, fds->ptr, new_len) + : erts_alloc(ERTS_ALC_T_SELECT_FDS, new_len); + memset((char*)fds->ptr + fds->sz, 0, new_len - fds->sz); + fds->sz = new_len; +} +static ERTS_INLINE void +ensure_select_fds(int fd, ERTS_fd_set* in, ERTS_fd_set* out) +{ + ASSERT(in->sz == out->sz); + if (ERTS_FD_SIZE(fd+1) > in->sz) { + grow_select_fds(fd, in); + grow_select_fds(fd, out); + } +} +#else +# define ensure_select_fds(fd, in, out) do {} while(0) +#endif /* _DARWIN_UNLIMITED_SELECT */ + static void grow_fds_status(ErtsPollSet ps, int min_fd) { @@ -1290,22 +1370,23 @@ static int update_pollset(ErtsPollSet ps, int fd) #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ { ErtsPollEvents events = ps->fds_status[fd].events; + ensure_select_fds(fd, &ps->input_fds, &ps->output_fds); if ((ERTS_POLL_EV_IN & events) != (ERTS_POLL_EV_IN & ps->fds_status[fd].used_events)) { if (ERTS_POLL_EV_IN & events) { - FD_SET(fd, &ps->input_fds); + ERTS_FD_SET(fd, &ps->input_fds); } else { - FD_CLR(fd, &ps->input_fds); + ERTS_FD_CLR(fd, &ps->input_fds); } } if ((ERTS_POLL_EV_OUT & events) != (ERTS_POLL_EV_OUT & ps->fds_status[fd].used_events)) { if (ERTS_POLL_EV_OUT & events) { - FD_SET(fd, &ps->output_fds); + ERTS_FD_SET(fd, &ps->output_fds); } else { - FD_CLR(fd, &ps->output_fds); + ERTS_FD_CLR(fd, &ps->output_fds); } } @@ -1789,7 +1870,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, while (fd < end_fd && res < max_res) { pr[res].events = (ErtsPollEvents) 0; - if (FD_ISSET(fd, &ps->res_input_fds)) { + if (ERTS_FD_ISSET(fd, &ps->res_input_fds)) { #if ERTS_POLL_USE_FALLBACK if (fd == ps->kp_fd) { res += get_kp_results(ps, &pr[res], max_res-res); @@ -1805,7 +1886,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, #endif pr[res].events |= ERTS_POLL_EV_IN; } - if (FD_ISSET(fd, &ps->res_output_fds)) + if (ERTS_FD_ISSET(fd, &ps->res_output_fds)) pr[res].events |= ERTS_POLL_EV_OUT; if (pr[res].events) { pr[res].fd = fd; @@ -1832,24 +1913,23 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, while (fd < end_fd && res < max_res) { if (ps->fds_status[fd].events) { int sres; - fd_set *iset = NULL; - fd_set *oset = NULL; + ERTS_fd_set *iset = NULL; + ERTS_fd_set *oset = NULL; if (ps->fds_status[fd].events & ERTS_POLL_EV_IN) { iset = &ps->res_input_fds; - FD_ZERO(iset); - FD_SET(fd, iset); + ERTS_FD_ZERO(iset); + ERTS_FD_SET(fd, iset); } if (ps->fds_status[fd].events & ERTS_POLL_EV_OUT) { oset = &ps->res_output_fds; - FD_ZERO(oset); - FD_SET(fd, oset); - + ERTS_FD_ZERO(oset); + ERTS_FD_SET(fd, oset); } do { /* Initiate 'tv' each time; select() may modify it */ SysTimeval tv = {0, 0}; - sres = select(ps->max_fd+1, iset, oset, NULL, &tv); + sres = ERTS_SELECT(ps->max_fd+1, iset, oset, NULL, &tv); } while (sres < 0 && errno == EINTR); if (sres < 0) { #if ERTS_POLL_USE_FALLBACK @@ -1873,7 +1953,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, } else if (sres > 0) { pr[res].fd = fd; - if (iset && FD_ISSET(fd, iset)) { + if (iset && ERTS_FD_ISSET(fd, iset)) { #if ERTS_POLL_USE_FALLBACK if (fd == ps->kp_fd) { res += get_kp_results(ps, @@ -1891,7 +1971,7 @@ save_poll_result(ErtsPollSet ps, ErtsPollResFd pr[], int max_res, #endif pr[res].events |= ERTS_POLL_EV_IN; } - if (oset && FD_ISSET(fd, oset)) { + if (oset && ERTS_FD_ISSET(fd, oset)) { pr[res].events |= ERTS_POLL_EV_OUT; } ASSERT(pr[res].events); @@ -1992,14 +2072,14 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res) #elif ERTS_POLL_USE_SELECT /* --- select ------------------------------ */ SysTimeval to = *tv; - ps->res_input_fds = ps->input_fds; - ps->res_output_fds = ps->output_fds; - + ERTS_FD_COPY(&ps->input_fds, &ps->res_input_fds); + ERTS_FD_COPY(&ps->output_fds, &ps->res_output_fds); + #ifdef ERTS_SMP if (to.tv_sec || to.tv_usec) erts_thr_progress_prepare_wait(NULL); #endif - res = select(ps->max_fd + 1, + res = ERTS_SELECT(ps->max_fd + 1, &ps->res_input_fds, &ps->res_output_fds, NULL, @@ -2027,7 +2107,7 @@ check_fd_events(ErtsPollSet ps, SysTimeval *tv, int max_res) ERTS_POLLSET_LOCK(ps); handle_update_requests(ps); ERTS_POLLSET_UNLOCK(ps); - res = select(ps->max_fd + 1, + res = ERTS_SELECT(ps->max_fd + 1, &ps->res_input_fds, &ps->res_output_fds, NULL, @@ -2233,7 +2313,8 @@ ERTS_POLL_EXPORT(erts_poll_init)(void) max_fds = OPEN_MAX; #endif -#if ERTS_POLL_USE_SELECT && defined(FD_SETSIZE) +#if ERTS_POLL_USE_SELECT && defined(FD_SETSIZE) && \ + !defined(_DARWIN_UNLIMITED_SELECT) if (max_fds > FD_SETSIZE) max_fds = FD_SETSIZE; #endif @@ -2301,10 +2382,21 @@ ERTS_POLL_EXPORT(erts_poll_create_pollset)(void) #if ERTS_POLL_USE_FALLBACK ps->no_select_fds = 0; #endif - FD_ZERO(&ps->input_fds); - FD_ZERO(&ps->res_input_fds); - FD_ZERO(&ps->output_fds); - FD_ZERO(&ps->res_output_fds); +#ifdef _DARWIN_UNLIMITED_SELECT + ps->input_fds.sz = 0; + ps->input_fds.ptr = NULL; + ps->res_input_fds.sz = 0; + ps->res_input_fds.ptr = NULL; + ps->output_fds.sz = 0; + ps->output_fds.ptr = NULL; + ps->res_output_fds.sz = 0; + ps->res_output_fds.ptr = NULL; +#else + ERTS_FD_ZERO(&ps->input_fds); + ERTS_FD_ZERO(&ps->res_input_fds); + ERTS_FD_ZERO(&ps->output_fds); + ERTS_FD_ZERO(&ps->res_output_fds); +#endif #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE ps->update_requests.next = NULL; @@ -2382,6 +2474,16 @@ ERTS_POLL_EXPORT(erts_poll_destroy_pollset)(ErtsPollSet ps) if (ps->poll_fds) erts_free(ERTS_ALC_T_POLL_FDS, (void *) ps->poll_fds); #elif ERTS_POLL_USE_SELECT +#ifdef _DARWIN_UNLIMITED_SELECT + if (ps->input_fds.ptr) + erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->input_fds.ptr); + if (ps->res_input_fds.ptr) + erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_input_fds.ptr); + if (ps->output_fds.ptr) + erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->output_fds.ptr); + if (ps->res_output_fds.ptr) + erts_free(ERTS_ALC_T_SELECT_FDS, (void *) ps->res_output_fds.ptr); +#endif #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE { @@ -2445,6 +2547,10 @@ ERTS_POLL_EXPORT(erts_poll_info)(ErtsPollSet ps, ErtsPollInfo *pip) #if ERTS_POLL_USE_POLL size += ps->poll_fds_len*sizeof(struct pollfd); #elif ERTS_POLL_USE_SELECT +#ifdef _DARWIN_UNLIMITED_SELECT + size += ps->input_fds.sz + ps->res_input_fds.sz + + ps->output_fds.sz + ps->res_output_fds.sz; +#endif #endif #if ERTS_POLL_USE_UPDATE_REQUESTS_QUEUE diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index e467e844b3..ced8b41c4b 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -628,7 +628,7 @@ iter_max_ports(Config) when is_list(Config) -> iter_max_ports_test(Config) -> - Dog = test_server:timetrap(test_server:minutes(20)), + Dog = test_server:timetrap(test_server:minutes(30)), PortTest = port_test(Config), Command = lists:concat([PortTest, " -h0 -q"]), Iters = case os:type() of |