diff options
Diffstat (limited to 'erts/emulator/beam/erl_async.c')
| -rw-r--r-- | erts/emulator/beam/erl_async.c | 165 |
1 files changed, 61 insertions, 104 deletions
diff --git a/erts/emulator/beam/erl_async.c b/erts/emulator/beam/erl_async.c index decae6b2ca..3ceb2fd368 100644 --- a/erts/emulator/beam/erl_async.c +++ b/erts/emulator/beam/erl_async.c @@ -1,18 +1,19 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2000-2013. All Rights Reserved. + * Copyright Ericsson AB 2000-2016. All Rights Reserved. * - * The contents of this file are subject to the Erlang Public License, - * Version 1.1, (the "License"); you may not use this file except in - * compliance with the License. You should have received a copy of the - * Erlang Public License along with this software. If not, it can be - * retrieved online at http://www.erlang.org/. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * Software distributed under the License is distributed on an "AS IS" - * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See - * the License for the specific language governing rights and limitations - * under the License. + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * * %CopyrightEnd% */ @@ -27,14 +28,12 @@ #include "erl_thr_queue.h" #include "erl_async.h" #include "dtrace-wrapper.h" +#include "lttng-wrapper.h" #define ERTS_MAX_ASYNC_READY_CALLS_IN_SEQ 20 #define ERTS_ASYNC_PRINT_JOB 0 -#if !defined(ERTS_SMP) && defined(USE_THREADS) && !ERTS_USE_ASYNC_READY_Q -# error "Need async ready queue in non-smp case" -#endif typedef struct _erl_async { DE_Handle* hndl; /* The DE_Handle is needed when port is gone */ @@ -44,16 +43,13 @@ typedef struct _erl_async { ErlDrvPDL pdl; void (*async_invoke)(void*); void (*async_free)(void*); -#if ERTS_USE_ASYNC_READY_Q Uint sched_id; union { ErtsThrQPrepEnQ_t *prep_enq; ErtsThrQFinDeQ_t fin_deq; } q; -#endif } ErtsAsync; -#if ERTS_USE_ASYNC_READY_Q /* * We can do without the enqueue mutex since it isn't needed for @@ -92,7 +88,6 @@ typedef union { char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncReadyQ))]; } ErtsAlgndAsyncReadyQ; -#endif /* ERTS_USE_ASYNC_READY_Q */ typedef struct { ErtsThrQ_t thr_q; @@ -117,12 +112,10 @@ typedef struct { char align__[ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncInit))]; } init; ErtsAlgndAsyncQ *queue; -#if ERTS_USE_ASYNC_READY_Q ErtsAlgndAsyncReadyQ *ready_queue; -#endif } ErtsAsyncData; -#if defined(USE_THREADS) && defined(USE_VM_PROBES) +#if defined(USE_VM_PROBES) /* * Some compilers, e.g. GCC 4.2.1 and -O3, will optimize away DTrace @@ -138,15 +131,6 @@ int erts_async_thread_suggested_stack_size; /* Initialized by erl_init.c */ static ErtsAsyncData *async; -#ifndef USE_THREADS - -void -erts_init_async(void) -{ - -} - -#else static void *async_main(void *); @@ -156,7 +140,6 @@ async_q(int i) return &async->queue[i].aq; } -#if ERTS_USE_ASYNC_READY_Q static ERTS_INLINE ErtsAsyncReadyQ * async_ready_q(Uint sched_id) @@ -164,27 +147,21 @@ async_ready_q(Uint sched_id) return &async->ready_queue[((int)sched_id)-1].arq; } -#endif - void erts_init_async(void) { async = NULL; if (erts_async_max_threads > 0) { -#if ERTS_USE_ASYNC_READY_Q ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; -#endif erts_thr_opts_t thr_opts = ERTS_THR_OPTS_DEFAULT_INITER; - char *ptr; + char *ptr, thr_name[16]; size_t tot_size = 0; int i; tot_size += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); tot_size += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; -#if ERTS_USE_ASYNC_READY_Q tot_size += sizeof(ErtsAlgndAsyncReadyQ)*erts_no_schedulers; -#endif ptr = erts_alloc_permanent_cache_aligned(ERTS_ALC_T_ASYNC_DATA, tot_size); @@ -193,14 +170,14 @@ erts_init_async(void) ptr += ERTS_ALC_CACHE_LINE_ALIGN_SIZE(sizeof(ErtsAsyncData)); async->init.data.no_initialized = 0; - erts_mtx_init(&async->init.data.mtx, "async_init_mtx"); + erts_mtx_init(&async->init.data.mtx, "async_init_mtx", NIL, + ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); erts_cnd_init(&async->init.data.cnd); erts_atomic_init_nob(&async->init.data.id, 0); async->queue = (ErtsAlgndAsyncQ *) ptr; ptr += sizeof(ErtsAlgndAsyncQ)*erts_async_max_threads; -#if ERTS_USE_ASYNC_READY_Q qinit.live.queue = ERTS_THR_Q_LIVE_LONG; qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; @@ -212,14 +189,14 @@ erts_init_async(void) for (i = 1; i <= erts_no_schedulers; i++) { ErtsAsyncReadyQ *arq = async_ready_q(i); #if ERTS_USE_ASYNC_READY_ENQ_MTX - erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx"); + erts_mtx_init(&arq->x.data.enq_mtx, "async_enq_mtx", make_small(i), + ERTS_LOCK_FLAGS_PROPERTY_STATIC | ERTS_LOCK_FLAGS_CATEGORY_SCHEDULER); #endif erts_thr_q_finalize_dequeue_state_init(&arq->fin_deq); qinit.arg = (void *) (SWord) i; erts_thr_q_initialize(&arq->thr_q, &qinit); } -#endif /* Create async threads... */ @@ -227,23 +204,16 @@ erts_init_async(void) thr_opts.suggested_stack_size = erts_async_thread_suggested_stack_size; -#ifdef ETHR_HAVE_THREAD_NAMES - thr_opts.name = malloc(sizeof(char)*(strlen("async_XXXX")+1)); -#endif + thr_opts.name = thr_name; for (i = 0; i < erts_async_max_threads; i++) { ErtsAsyncQ *aq = async_q(i); -#ifdef ETHR_HAVE_THREAD_NAMES - sprintf(thr_opts.name, "async_%d", i+1); -#endif + erts_snprintf(thr_opts.name, 16, "async_%d", i+1); erts_thr_create(&aq->thr_id, async_main, (void*) aq, &thr_opts); } -#ifdef ETHR_HAVE_THREAD_NAMES - free(thr_opts.name); -#endif /* Wait for async threads to initialize... */ erts_mtx_lock(&async->init.data.mtx); @@ -257,7 +227,6 @@ erts_init_async(void) } } -#if ERTS_USE_ASYNC_READY_Q void * erts_get_async_ready_queue(Uint sched_id) @@ -265,7 +234,6 @@ erts_get_async_ready_queue(Uint sched_id) return (void *) async ? async_ready_q(sched_id) : NULL; } -#endif static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) { @@ -274,10 +242,8 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) #endif if (is_internal_port(a->port)) { -#if ERTS_USE_ASYNC_READY_Q ErtsAsyncReadyQ *arq = async_ready_q(a->sched_id); a->q.prep_enq = erts_thr_q_prepare_enqueue(&arq->thr_q); -#endif /* make sure the driver will stay around */ if (a->hndl) erts_ddll_reference_referenced_driver(a->hndl); @@ -288,6 +254,13 @@ static ERTS_INLINE void async_add(ErtsAsync *a, ErtsAsyncQ* q) #endif erts_thr_q_enqueue(&q->thr_q, a); +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(aio_pool_put)) { + lttng_decl_portbuf(port_str); + lttng_portid_to_str(a->port, port_str); + LTTNG2(aio_pool_put, port_str, -1); + } +#endif #ifdef USE_VM_PROBES if (DTRACE_ENABLED(aio_pool_add)) { DTRACE_CHARBUF(port_str, 16); @@ -306,10 +279,8 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, erts_tse_t *tse, ErtsThrQPrepEnQ_t **prep_enq) { -#if ERTS_USE_ASYNC_READY_Q int saved_fin_deq = 0; ErtsThrQFinDeQ_t fin_deq; -#endif #ifdef USE_VM_PROBES int len; #endif @@ -318,11 +289,17 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, ErtsAsync *a = (ErtsAsync *) erts_thr_q_dequeue(q); if (a) { -#if ERTS_USE_ASYNC_READY_Q *prep_enq = a->q.prep_enq; erts_thr_q_get_finalize_dequeue_data(q, &a->q.fin_deq); if (saved_fin_deq) erts_thr_q_append_finalize_dequeue_data(&a->q.fin_deq, &fin_deq); +#ifdef USE_LTTNG_VM_TRACEPOINTS + if (LTTNG_ENABLED(aio_pool_get)) { + lttng_decl_portbuf(port_str); + int length = erts_thr_q_length_dirty(q); + lttng_portid_to_str(a->port, port_str); + LTTNG2(aio_pool_get, port_str, length); + } #endif #ifdef USE_VM_PROBES if (DTRACE_ENABLED(aio_pool_get)) { @@ -343,7 +320,6 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, erts_tse_reset(tse); -#if ERTS_USE_ASYNC_READY_Q chk_fin_deq: if (erts_thr_q_get_finalize_dequeue_data(q, &tmp_fin_deq)) { if (!saved_fin_deq) { @@ -353,13 +329,11 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, erts_thr_q_append_finalize_dequeue_data(&fin_deq, &tmp_fin_deq); } -#endif switch (erts_thr_q_inspect(q, 1)) { case ERTS_THR_Q_DIRTY: break; case ERTS_THR_Q_NEED_THR_PRGR: -#ifdef ERTS_SMP { ErtsThrPrgrVal prgr = erts_thr_q_need_thr_progress(q); erts_thr_progress_wakeup(NULL, prgr); @@ -371,17 +345,14 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, erts_tse_wait(tse); break; } -#endif case ERTS_THR_Q_CLEAN: -#if ERTS_USE_ASYNC_READY_Q if (saved_fin_deq) { if (erts_thr_q_finalize_dequeue(&fin_deq)) goto chk_fin_deq; else saved_fin_deq = 0; } -#endif erts_tse_wait(tse); break; @@ -397,29 +368,26 @@ static ERTS_INLINE ErtsAsync *async_get(ErtsThrQ_t *q, static ERTS_INLINE void call_async_ready(ErtsAsync *a) { -#if ERTS_USE_ASYNC_READY_Q Port *p = erts_id2port_sflgs(a->port, NULL, 0, ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); -#else - Port *p = erts_thr_id2port_sflgs(a->port, - ERTS_PORT_SFLGS_INVALID_DRIVER_LOOKUP); -#endif if (!p) { - if (a->async_free) + if (a->async_free) { + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT); a->async_free(a->async_data); + ERTS_MSACC_POP_STATE(); + } } else { if (async_ready(p, a->async_data)) { - if (a->async_free) + if (a->async_free) { + ERTS_MSACC_PUSH_AND_SET_STATE(ERTS_MSACC_STATE_PORT); a->async_free(a->async_data); + ERTS_MSACC_POP_STATE(); + } } -#if ERTS_USE_ASYNC_READY_Q erts_port_release(p); -#else - erts_thr_port_release(p); -#endif } if (a->pdl) driver_pdl_dec_refc(a->pdl); @@ -429,7 +397,6 @@ static ERTS_INLINE void call_async_ready(ErtsAsync *a) static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq) { -#if ERTS_USE_ASYNC_READY_Q ErtsAsyncReadyQ *arq; #if ERTS_ASYNC_PRINT_JOB @@ -448,12 +415,6 @@ static ERTS_INLINE void async_reply(ErtsAsync *a, ErtsThrQPrepEnQ_t *prep_enq) erts_mtx_unlock(&arq->x.data.enq_mtx); #endif -#else /* ERTS_USE_ASYNC_READY_Q */ - - call_async_ready(a); - erts_free(ERTS_ALC_T_ASYNC, (void *) a); - -#endif /* ERTS_USE_ASYNC_READY_Q */ } @@ -467,7 +428,8 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) { ErtsThrQInit_t qinit = ERTS_THR_Q_INIT_DEFAULT; erts_tse_t *tse = erts_tse_fetch(); -#ifdef ERTS_SMP + ERTS_DECLARE_DUMMY(Uint no); + ErtsThrPrgrCallbacks callbacks; callbacks.arg = (void *) tse; @@ -476,24 +438,23 @@ static erts_tse_t *async_thread_init(ErtsAsyncQ *aq) callbacks.wait = NULL; erts_thr_progress_register_unmanaged_thread(&callbacks); -#endif qinit.live.queue = ERTS_THR_Q_LIVE_LONG; qinit.live.objects = ERTS_THR_Q_LIVE_SHORT; qinit.arg = (void *) tse; qinit.notify = async_wakeup; -#if ERTS_USE_ASYNC_READY_Q qinit.auto_finalize_dequeue = 0; -#endif erts_thr_q_initialize(&aq->thr_q, &qinit); /* Inform main thread that we are done initializing... */ erts_mtx_lock(&async->init.data.mtx); - async->init.data.no_initialized++; + no = async->init.data.no_initialized++; erts_cnd_signal(&async->init.data.cnd); erts_mtx_unlock(&async->init.data.mtx); + erts_msacc_init_thread("async", no, 0); + return tse; } @@ -501,6 +462,7 @@ static void *async_main(void* arg) { ErtsAsyncQ *aq = (ErtsAsyncQ *) arg; erts_tse_t *tse = async_thread_init(aq); + ERTS_MSACC_DECLARE_CACHE(); while (1) { ErtsThrQPrepEnQ_t *prep_enq; @@ -508,11 +470,14 @@ static void *async_main(void* arg) if (is_nil(a->port)) break; /* Time to die */ + ERTS_MSACC_UPDATE_CACHE(); + #if ERTS_ASYNC_PRINT_JOB erts_fprintf(stderr, "<- %ld\n", a->async_id); #endif - + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); a->async_invoke(a->async_data); + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_OTHER); async_reply(a, prep_enq); } @@ -520,12 +485,10 @@ static void *async_main(void* arg) return NULL; } -#endif /* USE_THREADS */ void erts_exit_flush_async(void) { -#ifdef USE_THREADS int i; ErtsAsync a; a.port = NIL; @@ -539,11 +502,8 @@ erts_exit_flush_async(void) async_add(&a, async_q(i)); for (i = 0; i < erts_async_max_threads; i++) erts_thr_join(async->queue[i].aq.thr_id, NULL); -#endif } -#if defined(USE_THREADS) && ERTS_USE_ASYNC_READY_Q - int erts_check_async_ready(void *varq) { ErtsAsyncReadyQ *arq = (ErtsAsyncReadyQ *) varq; @@ -584,18 +544,15 @@ int erts_async_ready_clean(void *varq, void *val) case ERTS_THR_Q_DIRTY: return ERTS_ASYNC_READY_DIRTY; case ERTS_THR_Q_NEED_THR_PRGR: -#ifdef ERTS_SMP *((ErtsThrPrgrVal *) val) = erts_thr_q_need_thr_progress(&arq->thr_q); return ERTS_ASYNC_READY_NEED_THR_PRGR; -#endif case ERTS_THR_Q_CLEAN: break; } return ERTS_ASYNC_READY_CLEAN; } -#endif /* ** Generate a fair async key prom an ErlDrvPort @@ -633,25 +590,22 @@ long driver_async(ErlDrvPort ix, unsigned int* key, Port* prt; long id; unsigned int qix; -#if ERTS_USE_ASYNC_READY_Q Uint sched_id; + ERTS_MSACC_PUSH_STATE(); sched_id = erts_get_scheduler_id(); if (!sched_id) sched_id = 1; -#endif prt = erts_drvport2port(ix); if (prt == ERTS_INVALID_ERL_DRV_PORT) return -1; - ERTS_SMP_LC_ASSERT(erts_lc_is_port_locked(prt)); + ERTS_LC_ASSERT(erts_lc_is_port_locked(prt)); a = (ErtsAsync*) erts_alloc(ERTS_ALC_T_ASYNC, sizeof(ErtsAsync)); -#if ERTS_USE_ASYNC_READY_Q a->sched_id = sched_id; -#endif a->hndl = (DE_Handle*)prt->drv_ptr->handle; a->port = prt->common.id; a->pdl = NULL; @@ -681,7 +635,6 @@ long driver_async(ErlDrvPort ix, unsigned int* key, (*key % erts_async_max_threads) : 0; *key = qix; } -#ifdef USE_THREADS if (erts_async_max_threads > 0) { if (prt->port_data_lock) { driver_pdl_inc_refc(prt->port_data_lock); @@ -690,13 +643,17 @@ long driver_async(ErlDrvPort ix, unsigned int* key, async_add(a, async_q(qix)); return id; } -#endif - + + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); (*a->async_invoke)(a->async_data); + ERTS_MSACC_POP_STATE(); if (async_ready(prt, a->async_data)) { - if (a->async_free != NULL) + if (a->async_free != NULL) { + ERTS_MSACC_SET_STATE_CACHED(ERTS_MSACC_STATE_PORT); (*a->async_free)(a->async_data); + ERTS_MSACC_POP_STATE(); + } } erts_free(ERTS_ALC_T_ASYNC, (void *) a); |