From cfb7f3fcffad43647be5b9b818310b44003b97ab Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 16 Feb 2012 02:38:50 +0100 Subject: Misc memory barrier fixes - Document barrier semantics - Introduce ddrb suffix on atomic ops - Barrier macros for both non-SMP and SMP case - Make the thread progress API a bit more intuitive --- erts/emulator/beam/break.c | 7 +- erts/emulator/beam/erl_alloc.c | 6 +- erts/emulator/beam/erl_alloc.h | 9 +- erts/emulator/beam/erl_alloc_util.c | 31 ++- erts/emulator/beam/erl_alloc_util.h | 6 +- erts/emulator/beam/erl_db.c | 6 +- erts/emulator/beam/erl_db_hash.c | 8 +- erts/emulator/beam/erl_process.c | 52 ++++- erts/emulator/beam/erl_process.h | 5 +- erts/emulator/beam/erl_sched_spec_pre_alloc.c | 3 +- erts/emulator/beam/erl_smp.h | 87 ++++++- erts/emulator/beam/erl_thr_progress.c | 69 +++--- erts/emulator/beam/erl_thr_progress.h | 99 ++++++-- erts/emulator/beam/erl_thr_queue.c | 5 +- erts/emulator/beam/erl_threads.h | 319 +++++++++++++++++++++++++- 15 files changed, 601 insertions(+), 111 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c index 0d3b6a4dba..6f5020dc14 100644 --- a/erts/emulator/beam/break.c +++ b/erts/emulator/beam/break.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -646,6 +646,9 @@ bin_check(void) void erl_crash_dump_v(char *file, int line, char* fmt, va_list args) { +#ifdef ERTS_SMP + ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */ +#endif int fd; time_t now; size_t dumpnamebufsize = MAXPATHLEN; @@ -663,7 +666,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args) * We do not release system again. We expect an exit() or abort() after * dump has been written. */ - erts_thr_progress_fatal_error_block(60000); + erts_thr_progress_fatal_error_block(60000, &tpd_buf); /* Either worked or not... */ /* Allow us to pass certain places without locking... */ diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 4d02a67d54..df27186680 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -1577,9 +1577,11 @@ erts_alloc_register_scheduler(void *vesdp) } } +#ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *more_work) { ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp; @@ -1599,10 +1601,12 @@ erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, erts_alcu_check_delayed_dealloc(allctr, 1, need_thr_progress, + thr_prgr_p, more_work); } } } +#endif erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs) diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h index 991061c48e..e475f9d8a2 100644 --- a/erts/emulator/beam/erl_alloc.h +++ b/erts/emulator/beam/erl_alloc.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -21,6 +21,10 @@ #define ERL_ALLOC_H__ #include "erl_alloc_types.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY +#define ERL_THR_PROGRESS_TSD_TYPE_ONLY +#include "erl_thr_progress.h" +#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY #include "erl_alloc_util.h" #ifdef USE_THREADS #include "erl_threads.h" @@ -132,9 +136,12 @@ typedef struct { extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1]; void erts_alloc_register_scheduler(void *vesdp); +#ifdef ERTS_SMP void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *more_work); +#endif erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs); __decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint) diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index c32938bdff..0e719f4a5a 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -949,7 +949,6 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq) ERTS_THR_MEMORY_BARRIER; else { ddq->head.next.unref_end = (ErtsAllctrDDBlock_t *) ilast; - ERTS_THR_MEMORY_BARRIER; ddq->head.next.thr_progress = erts_thr_progress_later(); erts_atomic32_set_relb(&ddq->tail.data.um_refc_ix, um_refc_ix); @@ -961,12 +960,24 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq) return 1; } +static ERTS_INLINE void +store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq) +{ + if (!ddq->head.next.thr_progress_reached + && (*prev_val == ERTS_THR_PRGR_INVALID + || erts_thr_progress_cmp(ddq->head.next.thr_progress, + *prev_val) < 0)) { + *prev_val = ddq->head.next.thr_progress; + } +} + static ERTS_INLINE int handle_delayed_dealloc(Allctr_t *allctr, int allctr_locked, int use_limit, int ops_limit, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *need_more_work) { int need_thr_prgr = 0; @@ -1008,8 +1019,12 @@ handle_delayed_dealloc(Allctr_t *allctr, if (have_checked_incoming) break; need_thr_prgr = ddq_check_incoming(ddq); - if (need_thr_progress) + if (need_thr_progress) { *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, ddq); + + } have_checked_incoming = 1; goto dequeue; } @@ -1067,6 +1082,8 @@ handle_delayed_dealloc(Allctr_t *allctr, if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) { need_thr_prgr = ddq_check_incoming(ddq); *need_thr_progress |= need_thr_prgr; + if (need_thr_prgr) + store_earliest_thr_prgr(thr_prgr_p, ddq); } if (allctr->thread_safe && !allctr_locked) @@ -1086,25 +1103,27 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, Allctr_t *allctr, void *ptr) #endif +#ifdef ERTS_SMP void erts_alcu_check_delayed_dealloc(Allctr_t *allctr, int limit, int *need_thr_progress, + ErtsThrPrgrVal *thr_prgr_p, int *more_work) { -#ifdef ERTS_SMP handle_delayed_dealloc(allctr, 0, limit, ERTS_ALCU_DD_OPS_LIM_HIGH, need_thr_progress, + thr_prgr_p, more_work); -#endif } +#endif #define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \ handle_delayed_dealloc((Allctr), (Locked), 1, \ - ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL) + ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL) /* Multi block carrier alloc/realloc/free ... */ diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index fc1eddb116..cedf4ccf85 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2002-2011. All Rights Reserved. + * Copyright Ericsson AB 2002-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -170,7 +170,9 @@ Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *); void erts_alcu_init(AlcUInit_t *); void erts_alcu_current_size(Allctr_t *, AllctrSize_t *, ErtsAlcUFixInfo_t *, int); -void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, int *); +#ifdef ERTS_SMP +void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *); +#endif erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t); #endif diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index eb89baf1c9..51bdf53823 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -251,7 +251,7 @@ free_dbtable(DbTable* tb) ASSERT(is_immed(tb->common.heir_data)); erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable)); ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable)); - ERTS_THR_MEMORY_BARRIER; + ERTS_SMP_MEMORY_BARRIER; } #ifdef ERTS_SMP @@ -3816,7 +3816,7 @@ void db_info(int to, void *to_arg, int show) /* Called by break handler */ Uint erts_get_ets_misc_mem_size(void) { - ERTS_THR_MEMORY_BARRIER; + ERTS_SMP_MEMORY_BARRIER; /* Memory not allocated in ets_alloc */ return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size); } diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 038a667b06..c726be5fb4 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1998-2011. All Rights Reserved. + * Copyright Ericsson AB 1998-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -105,11 +105,7 @@ #define NSEG_2 256 /* Size of second segment table */ #define NSEG_INC 128 /* Number of segments to grow after that */ -#ifdef ETHR_ORDERED_READ_DEPEND -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_nob(&(tb)->segtab)) -#else -#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_rb(&(tb)->segtab)) -#endif +#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_ddrb(&(tb)->segtab)) #define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive)) #define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems)) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index b8c6b64fc0..57ef7bb8b8 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -709,6 +709,27 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs) return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs); } +#ifdef ERTS_SMP + +static ERTS_INLINE void +thr_prgr_current_reset(ErtsAuxWorkData *awdp) +{ + awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID; +} + +static ERTS_INLINE ErtsThrPrgrVal +thr_prgr_current(ErtsAuxWorkData *awdp) +{ + ErtsThrPrgrVal current = awdp->current_thr_prgr; + if (current == ERTS_THR_PRGR_INVALID) { + current = erts_thr_progress_current(); + awdp->current_thr_prgr = current; + } + return current; +} + +#endif + typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t; struct erts_misc_aux_work_t_ { void (*func)(void *); @@ -805,7 +826,8 @@ static erts_aint32_t handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { - if (!erts_thr_progress_has_reached(awdp->misc.thr_prgr)) + if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp), + awdp->misc.thr_prgr)) return aux_work; unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); @@ -909,7 +931,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, #ifdef ERTS_SMP if (awdp->async_ready.need_thr_prgr - && !erts_thr_progress_has_reached(awdp->async_ready.thr_prgr)) { + && !erts_thr_progress_has_reached_this(thr_prgr_current(awdp), + awdp->async_ready.thr_prgr)) { return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; } @@ -970,11 +993,13 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; int need_thr_progress = 0; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; int more_work = 0; unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, + &wakeup, &more_work); if (more_work) { if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD) @@ -986,9 +1011,12 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) } if (need_thr_progress) { + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later_than(thr_prgr_current(awdp)); + awdp->dd.thr_prgr = wakeup; set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); - awdp->dd.thr_prgr = erts_thr_progress_later(); - erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + awdp->dd.thr_prgr = wakeup; + erts_thr_progress_wakeup(awdp->esdp, wakeup); } else if (awdp->dd.completed_callback) { awdp->dd.completed_callback(awdp->dd.completed_arg); @@ -1004,8 +1032,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) ErtsSchedulerSleepInfo *ssi; int need_thr_progress; int more_work; + ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID; + ErtsThrPrgrVal current = thr_prgr_current(awdp); - if (!erts_thr_progress_has_reached(awdp->dd.thr_prgr)) + if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr)) return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR; ssi = awdp->ssi; @@ -1014,6 +1044,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp, &need_thr_progress, + &wakeup, &more_work); if (more_work) { set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD); @@ -1023,8 +1054,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) } if (need_thr_progress) { - awdp->dd.thr_prgr = erts_thr_progress_later(); - erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr); + if (wakeup == ERTS_THR_PRGR_INVALID) + wakeup = erts_thr_progress_later_than(current); + awdp->dd.thr_prgr = wakeup; + erts_thr_progress_wakeup(awdp->esdp, wakeup); } else { unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR); @@ -1154,6 +1187,9 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) static ERTS_INLINE erts_aint32_t handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { +#ifdef ERTS_SMP + thr_prgr_current_reset(awdp); +#endif /* * Handlers are *only* allowed to modify flags in return value * and ssi flags that are explicity handled by the handler. diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index a51b380bb0..ad32dc11f9 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 1996-2011. All Rights Reserved. + * Copyright Ericsson AB 1996-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -397,6 +397,9 @@ typedef struct { int sched_id; ErtsSchedulerData *esdp; ErtsSchedulerSleepInfo *ssi; +#ifdef ERTS_SMP + ErtsThrPrgrVal current_thr_prgr; +#endif struct { int ix; #ifdef ERTS_SMP diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c index a7ccea7403..bff9d246a3 100644 --- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c +++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -227,7 +227,6 @@ fetch_remote(erts_sspa_chunk_header_t *chdr, int max) ERTS_THR_MEMORY_BARRIER; else { chdr->head.next.unref_end = (erts_sspa_blk_t *) ilast; - ERTS_THR_MEMORY_BARRIER; chdr->head.next.thr_progress = erts_thr_progress_later(); erts_atomic32_set_relb(&chdr->tail.data.um_refc_ix, um_refc_ix); diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index 63179dfad4..a32e9d9d7c 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2005-2011. All Rights Reserved. + * Copyright Ericsson AB 2005-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -61,6 +61,11 @@ typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ +#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER +#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER +#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER +#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER + #else /* #ifdef ERTS_SMP */ #define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0} @@ -95,6 +100,11 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t; typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t; #endif +#define ERTS_SMP_MEMORY_BARRIER +#define ERTS_SMP_WRITE_MEMORY_BARRIER +#define ERTS_SMP_READ_MEMORY_BARRIER +#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER + #endif /* #ifdef ERTS_SMP */ ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id); @@ -206,13 +216,8 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */ /* - * Functions implementing atomic operations with with no (nob), - * full (mb), acquire (acqb), release (relb), read (rb), and - * write (wb) memory barriers. - * - * If SMP support has been disabled, they are mapped to functions - * that performs the same operation, but aren't atomic and don't - * imply memory barriers. + * See "Documentation of atomics and memory barriers" at the top + * of erl_threads.h for info on atomics. */ #ifdef ERTS_SMP @@ -239,6 +244,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb #define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb +#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb +#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb +#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb +#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb + #define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb #define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb #define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb @@ -307,6 +317,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb #define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb +#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb +#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb +#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb +#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb +#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb +#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb +#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb +#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb +#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb +#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb +#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb +#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb +#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb + #define erts_smp_atomic_init_rb erts_atomic_init_rb #define erts_smp_atomic_set_rb erts_atomic_set_rb #define erts_smp_atomic_read_rb erts_atomic_read_rb @@ -393,6 +417,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb #define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb +#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb +#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb +#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb +#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb +#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb +#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb +#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb +#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb +#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb +#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb +#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb +#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb +#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb + #define erts_smp_atomic32_init_rb erts_atomic32_init_rb #define erts_smp_atomic32_set_rb erts_atomic32_set_rb #define erts_smp_atomic32_read_rb erts_atomic32_read_rb @@ -445,6 +483,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read #define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg +#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init +#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set +#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read +#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg + #define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init #define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set #define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read @@ -513,6 +556,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic_xchg_relb erts_no_atomic_xchg #define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_smp_atomic_init_ddrb erts_no_atomic_set +#define erts_smp_atomic_set_ddrb erts_no_atomic_set +#define erts_smp_atomic_read_ddrb erts_no_atomic_read +#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read +#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read +#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc +#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec +#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read +#define erts_smp_atomic_add_ddrb erts_no_atomic_add +#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor +#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band +#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg +#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg + #define erts_smp_atomic_init_rb erts_no_atomic_set #define erts_smp_atomic_set_rb erts_no_atomic_set #define erts_smp_atomic_read_rb erts_no_atomic_read @@ -599,6 +656,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set +#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set +#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read +#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read +#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read +#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc +#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec +#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read +#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add +#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor +#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band +#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg +#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg + #define erts_smp_atomic32_init_rb erts_no_atomic32_set #define erts_smp_atomic32_set_rb erts_no_atomic32_set #define erts_smp_atomic32_read_rb erts_no_atomic32_read diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 75f8209c3b..9ef83746c5 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -33,10 +33,11 @@ * This module keeps track of the progress of a set of managed threads. Only * threads that behave well can be allowed to be managed. A managed thread * should update its thread progress frequently. Currently only scheduler - * threads and the aux_thread are managed threads. We typically do not want - * any async threads as managed threads since they cannot guarantee a - * frequent update of thread progress, since they execute user implemented - * driver code. + * threads, the system-message-dispatcher threads, and the aux-thread are + * managed threads. We typically do not want any async threads as managed + * threads since they cannot guarantee a frequent update of thread progress, + * since they execute user implemented driver code that is assumed to be + * time consuming. * * erts_thr_progress_current() returns the global current thread progress * value of managed threads. I.e., the latest progress value that all @@ -112,8 +113,10 @@ * * On 32-bit systems we therefore need a double word atomic. */ - +#undef read_acqb #define read_acqb erts_thr_prgr_read_acqb__ +#undef read_nob +#define read_nob erts_thr_prgr_read_nob__ #ifdef ARCH_64 @@ -129,12 +132,6 @@ set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) erts_atomic_set_nob(atmc, val); } -static ERTS_INLINE ErtsThrPrgrVal -read_nob(ERTS_THR_PRGR_ATOMIC *atmc) -{ - return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); -} - static ERTS_INLINE void init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { @@ -143,52 +140,44 @@ init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) #else -#undef dw_sint_to_val -#define dw_sint_to_val erts_thr_prgr_dw_sint_to_val__ +#undef dw_aint_to_val +#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__ static void -val_to_dw_sint(ethr_dw_sint_t *dw_sint, ErtsThrPrgrVal val) +val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val) { #ifdef ETHR_SU_DW_NAINT_T__ - dw_sint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; + dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; #else - dw_sint->sint[ETHR_DW_SINT_LOW_WORD] - = (ethr_sint_t) (val & 0xffffffff); - dw_sint->sint[ETHR_DW_SINT_HIGH_WORD] - = (ethr_sint_t) ((val >> 32) & 0xffffffff); + dw_aint->sint[ERTS_DW_AINT_LOW_WORD] + = (erts_aint_t) (val & 0xffffffff); + dw_aint->sint[ERTS_DW_AINT_HIGH_WORD] + = (erts_aint_t) ((val >> 32) & 0xffffffff); #endif } static ERTS_INLINE void set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - ethr_dw_sint_t dw_sint; - val_to_dw_sint(&dw_sint, val); - erts_dw_atomic_set_mb(atmc, &dw_sint); + erts_dw_aint_t dw_aint; + val_to_dw_aint(&dw_aint, val); + erts_dw_atomic_set_mb(atmc, &dw_aint); } static ERTS_INLINE void set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - ethr_dw_sint_t dw_sint; - val_to_dw_sint(&dw_sint, val); - erts_dw_atomic_set_nob(atmc, &dw_sint); -} - -static ERTS_INLINE ErtsThrPrgrVal -read_nob(ERTS_THR_PRGR_ATOMIC *atmc) -{ - ethr_dw_sint_t dw_sint; - erts_dw_atomic_read_nob(atmc, &dw_sint); - return erts_thr_prgr_dw_sint_to_val__(&dw_sint); + erts_dw_aint_t dw_aint; + val_to_dw_aint(&dw_aint, val); + erts_dw_atomic_set_nob(atmc, &dw_aint); } static ERTS_INLINE void init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - ethr_dw_sint_t dw_sint; - val_to_dw_sint(&dw_sint, val); - erts_dw_atomic_init_nob(atmc, &dw_sint); + erts_dw_aint_t dw_aint; + val_to_dw_aint(&dw_aint, val); + erts_dw_atomic_init_nob(atmc, &dw_aint); } #endif @@ -1222,9 +1211,9 @@ erts_thr_progress_block(void) } void -erts_thr_progress_fatal_error_block(SWord timeout) +erts_thr_progress_fatal_error_block(SWord timeout, + ErtsThrPrgrData *tmp_tpd_bufp) { - ErtsThrPrgrData tpd_buf; ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL); erts_aint32_t bc; SWord time_left = timeout; @@ -1248,7 +1237,7 @@ erts_thr_progress_fatal_error_block(SWord timeout) * since we never complete an unblock after a fatal error * block. */ - tpd = &tpd_buf; + tpd = tmp_tpd_bufp; init_tmp_thr_prgr_data(tpd); } diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index 68d14174b9..1bbc49993e 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -44,7 +44,6 @@ #define erts_smp_thr_progress_unblock erts_thr_progress_unblock #define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking -void erts_thr_progress_fatal_error_block(SWord timeout); void erts_thr_progress_block(void); void erts_thr_progress_unblock(void); int erts_thr_progress_is_blocking(void); @@ -73,6 +72,10 @@ typedef struct { ErtsThrPrgrVal current; } previous; } ErtsThrPrgrData; + +void erts_thr_progress_fatal_error_block(SWord timeout, + ErtsThrPrgrData *tmp_tpd_bufp); + #endif /* ERTS_SMP */ #endif @@ -86,6 +89,7 @@ typedef struct { #ifdef ERTS_SMP #define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0)) +#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0)) extern erts_tsd_key_t erts_thr_prgr_data_key__; @@ -127,48 +131,81 @@ void erts_thr_progress_dbg_print_state(void); #ifdef ARCH_32 #define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t -ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint); #endif +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void); +ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later_than(ErtsThrPrgrVal val); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(void); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void); ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2); +ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val); +ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2); ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); #if ERTS_GLB_INLINE_INCL_FUNC_DEF #ifdef ARCH_64 +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); +} + ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) { return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc); } +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc); +} + #else /* ARCH_32 */ ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint) +erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint) { #ifdef ETHR_SU_DW_NAINT_T__ - return (ErtsThrPrgrVal) dw_sint->dw_sint; + return (ErtsThrPrgrVal) dw_aint->dw_sint; #else ErtsThrPrgrVal res; - res = (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]); + res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]); res <<= 32; - res |= (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_LOW_WORD]); + res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]); return res; #endif } +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + erts_dw_aint_t dw_aint; + erts_dw_atomic_read_nob(atmc, &dw_aint); + return erts_thr_prgr_dw_aint_to_val__(&dw_aint); +} + ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) { - ethr_dw_sint_t dw_sint; - erts_dw_atomic_read_acqb(atmc, &dw_sint); - return erts_thr_prgr_dw_sint_to_val__(&dw_sint); + erts_dw_aint_t dw_aint; + erts_dw_atomic_read_acqb(atmc, &dw_aint); + return erts_thr_prgr_dw_aint_to_val__(&dw_aint); +} + +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) +{ + erts_dw_aint_t dw_aint; + erts_dw_atomic_read_mb(atmc, &dw_aint); + return erts_thr_prgr_dw_aint_to_val__(&dw_aint); } #endif @@ -181,9 +218,9 @@ erts_thr_progress_is_managed_thread(void) } ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_progress_later(void) +erts_thr_progress_later_than(ErtsThrPrgrVal val) { - ErtsThrPrgrVal val = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); + ERTS_THR_MEMORY_BARRIER; if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2))) return ((ErtsThrPrgrVal) 0); else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1))) @@ -192,10 +229,20 @@ erts_thr_progress_later(void) return val + ((ErtsThrPrgrVal) 2); } +ERTS_GLB_INLINE ErtsThrPrgrVal +erts_thr_progress_later(void) +{ + ErtsThrPrgrVal val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current); + return erts_thr_progress_later_than(val); +} + ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void) { - return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); + if (erts_thr_progress_is_managed_thread()) + return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current); + else + return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); } ERTS_GLB_INLINE int @@ -217,13 +264,29 @@ erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0) } ERTS_GLB_INLINE int -erts_thr_progress_has_reached(ErtsThrPrgrVal val) +erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val) +{ + if (this == val) + return 1; + return erts_thr_progress_has_passed__(this, val); +} + +ERTS_GLB_INLINE int +erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2) { - ErtsThrPrgrVal current; - current = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current); - if (current == val) + if (val1 == val2) + return 0; + if (erts_thr_progress_has_passed__(val1, val2)) return 1; - return erts_thr_progress_has_passed__(current, val); + else + return -1; +} + +ERTS_GLB_INLINE int +erts_thr_progress_has_reached(ErtsThrPrgrVal val) +{ + ErtsThrPrgrVal current = erts_thr_progress_current(); + return erts_thr_progress_has_reached_this(current, val); } #endif diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c index efb8c635d7..70949ece76 100644 --- a/erts/emulator/beam/erl_thr_queue.c +++ b/erts/emulator/beam/erl_thr_queue.c @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -418,10 +418,9 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify) } if (q->head.unref_end == (ErtsThrQElement_t *) ilast) - ERTS_THR_MEMORY_BARRIER; + ERTS_SMP_MEMORY_BARRIER; else { q->head.next.unref_end = (ErtsThrQElement_t *) ilast; - ERTS_THR_MEMORY_BARRIER; #ifdef ERTS_SMP q->head.next.thr_progress = erts_thr_progress_later(); #endif diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 065e7077c0..ee47c98009 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2001-2011. All Rights Reserved. + * Copyright Ericsson AB 2001-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -25,6 +25,235 @@ #ifndef ERL_THREAD_H__ #define ERL_THREAD_H__ +/* + * --- Documentation of atomics and memory barriers -------------------------- + * + * The following explicit memory barriers exist: + * + * - ERTS_THR_MEMORY_BARRIER + * Full memory barrier. Orders both loads, and stores. No + * load or store is allowed to be reordered over the + * barrier. + * - ERTS_THR_WRITE_MEMORY_BARRIER + * Write barrier. Orders *only* stores. These are not + * allowed to be reordered over the barrier. + * - ERTS_THR_READ_MEMORY_BARRIER + * Read barrier. Orders *only* loads. These are not + * allowed to be reordered over the barrier. + * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER + * Data dependency read barrier. Orders *only* loads + * according to data dependency across the barrier. + * + * If thread support has been disabled, these barriers will become no-ops. + * + * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will + * be enabled only in the SMP enabled runtime system. + * + * --- Atomic operations --- + * + * Atomics operations exist for 32-bit, word size, and double word size + * integers. Function prototypes are listed below. + * + * Each function implementing an atomic operation exist with the following + * implied memory barrier semantics. Not all combinations are useful, but + * all of them exist for simplicity. is suffix in function name: + * + * - - Description + * + * - mb - Full memory barrier. Orders both loads, and + * stores before, and after the atomic operation. + * No load or store is allowed to be reordered + * over the atomic operation. + * - relb - Release barrier. Orders both loads, and + * stores appearing *before* the atomic + * operation. These are not allowed to be + * reordered over the atomic operation. + * - acqb - Acquire barrier. Orders both loads, and stores + * appearing *after* the atomic operation. These + * are not allowed to be reordered over the + * atomic operation. + * - wb - Write barrier. Orders *only* stores. These are + * not allowed to be reordered over the barrier. + * Store in atomic operation is ordered *after* + * the barrier. + * - rb - Read barrier. Orders *only* loads. These are + * not allowed to be reordered over the barrier. + * Load in atomic operation is ordered *before* + * the barrier. + * - ddrb - Data dependency read barrier. Orders *only* + * loads according to data dependency across the + * barrier. Load in atomic operation is ordered + * before the barrier. + * + * If thread support has been disabled, these functions are mapped to + * functions that performs the same operation, but aren't atomic + * and don't imply any memory barriers. + * + * If the atomic operations are prefixed with erts_smp_ instead of only + * erts_ the atomic operations will only be atomic in the SMP enabled + * runtime system, and will be mapped to non-atomic operations without + * memory barriers in the runtime system without SMP support. Atomic + * operations with erts_smp_ prefix should use the atomic types + * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t + * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The + * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t + * are the same. + * + * --- 32-bit atomic operations --- + * + * The following 32-bit atomic operations exist. should be + * replaced with a supported memory barrier (see above). Note + * that sizeof(erts_atomic32_t) might be larger than 4! + * + * + * Initialize (not necessarily the same as the set operation): + * void erts_atomic32_init_(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Set value: + * void erts_atomic32_set_(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Read; returns current value: + * erts_aint32_t erts_atomic32_read_(erts_atomic32_t *atmc); + * + * Increment; returns resulting value: + * erts_aint32_t erts_atomic32_inc_read_(erts_atomic32_t *atmc); + * + * Decrement; returns resulting value: + * erts_aint32_t erts_atomic32_dec_read_(erts_atomic32_t *atmc); + * + * Increment: + * void erts_atomic32_inc_(erts_atomic32_t *atmc); + * + * Decrement: + * void erts_atomic32_dec_(erts_atomic32_t *atmc); + * + * Add value; returns resulting value: + * erts_aint32_t erts_atomic32_add_read_(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Add value: + * void erts_atomic32_add_(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Bitwise-or; returns previous value: + * erts_aint32_t erts_atomic32_read_bor_(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Bitwise-and; returns previous value: + * erts_aint32_t erts_atomic32_read_band_(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Exchange; returns previous value: + * erts_aint32_t erts_atomic32_xchg_(erts_atomic32_t *atmc, + * erts_aint32_t val); + * + * Compare and exchange; returns previous or current value. If + * returned value equals 'exp' the value was changed to 'new'; + * otherwise not: + * erts_aint32_t erts_atomic32_cmpxchg_(erts_atomic32_t *a, + * erts_aint32_t new, + * erts_aint32_t exp); + * + * --- Word size atomic operations --- + * + * The following word size (same size as sizeof(void *)) atomic + * operations exist. should be replaced with a supported + * memory barrier (see above). Note that sizeof(erts_atomic_t) + * might be larger than sizeof(void *)! + * + * Initialize (not necessarily the same as the set operation): + * void erts_atomic_init_(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Set value; + * void erts_atomic_set_(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Read; returns current value: + * erts_aint_t erts_atomic_read_(erts_atomic_t *atmc); + * + * Increment; returns resulting value: + * erts_aint_t erts_atomic_inc_read_(erts_atomic_t *atmc); + * + * Decrement; returns resulting value: + * erts_aint_t erts_atomic_dec_read_(erts_atomic_t *atmc); + * + * Increment: + * void erts_atomic_inc_(erts_atomic_t *atmc); + * + * Decrement: + * void erts_atomic_dec_(erts_atomic_t *atmc); + * + * Add value; returns resulting value: + * erts_aint_t erts_atomic_add_read_(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Add value: + * void erts_atomic_add_(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Bitwise-or; returns previous value: + * erts_aint_t erts_atomic_read_bor_(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Bitwise-and; returns previous value: + * erts_aint_t erts_atomic_read_band_(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Exchange; returns previous value: + * erts_aint_t erts_atomic_xchg_(erts_atomic_t *atmc, + * erts_aint_t val); + * + * Compare and exchange; returns previous or current value. If + * returned value equals 'exp' the value was changed to 'new'; + * otherwise not: + * erts_aint_t erts_atomic_cmpxchg_(erts_atomic_t *a, + * erts_aint_t new, + * erts_aint_t exp); + * + * --- Double word size atomic operations --- + * + * The following double word atomic operations exist. should be + * replaced with a supported memory barrier (see above). + * + * Note that sizeof(erts_dw_atomic_t) usually is larger than + * 2*sizeof(void *)! + * + * The erts_dw_aint_t data type should be accessed as if it was defined + * like this: + * + * typedef struct { + * erts_aint_t sint[2]; + * } erts_dw_aint_t; + * + * Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least + * significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'. + * + * + * Initialize (not necessarily the same as the set operation): + * void erts_dw_atomic_init_(erts_dw_atomic_t *var, + * erts_dw_aint_t *val); + * + * Set; value is written into 'val': + * void erts_dw_atomic_set_(erts_dw_atomic_t *var, + * erts_dw_aint_t *val); + * + * Read; value is written into 'val': + * void erts_dw_atomic_read_(erts_dw_atomic_t *var, + * erts_dw_aint_t *val); + * + * Compare and exchange; returns a value != 0 if exchange was + * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act' + * contains the same value as in memory when the function is called, + * 'new' is written to memory; otherwise, not. If exchange was not + * made, 'exp_act' contains the actual value in memory: + * int erts_dw_atomic_cmpxchg_(erts_dw_atomic_t *var, + * erts_dw_aint_t *new, + * erts_dw_aint_t *exp_act); + */ + #define ERTS_SPIN_BODY ETHR_SPIN_BODY #include "sys.h" @@ -52,6 +281,9 @@ typedef Sint32 erts_no_atomic32_t; #endif #define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER +#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER +#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER +#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER #ifdef ERTS_ENABLE_LOCK_COUNT #define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__) @@ -113,6 +345,9 @@ typedef ethr_ts_event erts_tse_t; #define erts_aint32_t ethr_sint32_t #define erts_atomic32_t ethr_atomic32_t +#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD +#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD + /* spinlock */ typedef struct { ethr_spinlock_t slck; @@ -149,6 +384,9 @@ __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *); #else /* #ifdef USE_THREADS */ #define ERTS_THR_MEMORY_BARRIER +#define ERTS_THR_WRITE_MEMORY_BARRIER +#define ERTS_THR_READ_MEMORY_BARRIER +#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER #define ERTS_THR_OPTS_DEFAULT_INITER 0 typedef int erts_thr_opts_t; @@ -361,18 +599,13 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set, ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #endif /* #ifdef HAVE_ETHR_SIG_FUNCS */ +#ifdef USE_THREADS + /* - * Functions implementing atomic operations with with no (nob), - * full (mb), acquire (acqb), release (relb), read (rb), and - * write (wb) memory barriers. - * - * If thread support has been disabled, they are mapped to - * functions that performs the same operation, but aren't atomic - * and don't imply memory barriers. + * See "Documentation of atomics and memory barriers" at the top + * of this file for info on atomics. */ -#ifdef USE_THREADS - /* Double word size atomics */ #define erts_dw_atomic_init_nob ethr_dw_atomic_init @@ -395,6 +628,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb #define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb +#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb +#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb +#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb +#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb + #define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb #define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb #define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb @@ -463,6 +701,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_relb ethr_atomic_xchg_relb #define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb +#define erts_atomic_init_ddrb ethr_atomic_init_ddrb +#define erts_atomic_set_ddrb ethr_atomic_set_ddrb +#define erts_atomic_read_ddrb ethr_atomic_read_ddrb +#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb +#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb +#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb +#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb +#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb +#define erts_atomic_add_ddrb ethr_atomic_add_ddrb +#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb +#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb +#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb +#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb + #define erts_atomic_init_rb ethr_atomic_init_rb #define erts_atomic_set_rb ethr_atomic_set_rb #define erts_atomic_read_rb ethr_atomic_read_rb @@ -549,6 +801,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb #define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb +#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb +#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb +#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb +#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb +#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb +#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb +#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb +#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb +#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb +#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb +#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb +#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb +#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb + #define erts_atomic32_init_rb ethr_atomic32_init_rb #define erts_atomic32_set_rb ethr_atomic32_set_rb #define erts_atomic32_read_rb ethr_atomic32_read_rb @@ -601,6 +867,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_dw_atomic_read_relb erts_no_dw_atomic_read #define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg +#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init +#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set +#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read +#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg + #define erts_dw_atomic_init_rb erts_no_dw_atomic_init #define erts_dw_atomic_set_rb erts_no_dw_atomic_set #define erts_dw_atomic_read_rb erts_no_dw_atomic_read @@ -669,6 +940,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic_xchg_relb erts_no_atomic_xchg #define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg +#define erts_atomic_init_ddrb erts_no_atomic_set +#define erts_atomic_set_ddrb erts_no_atomic_set +#define erts_atomic_read_ddrb erts_no_atomic_read +#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read +#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read +#define erts_atomic_inc_ddrb erts_no_atomic_inc +#define erts_atomic_dec_ddrb erts_no_atomic_dec +#define erts_atomic_add_read_ddrb erts_no_atomic_add_read +#define erts_atomic_add_ddrb erts_no_atomic_add +#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor +#define erts_atomic_read_band_ddrb erts_no_atomic_read_band +#define erts_atomic_xchg_ddrb erts_no_atomic_xchg +#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg + #define erts_atomic_init_rb erts_no_atomic_set #define erts_atomic_set_rb erts_no_atomic_set #define erts_atomic_read_rb erts_no_atomic_read @@ -755,6 +1040,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig); #define erts_atomic32_xchg_relb erts_no_atomic32_xchg #define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg +#define erts_atomic32_init_ddrb erts_no_atomic32_set +#define erts_atomic32_set_ddrb erts_no_atomic32_set +#define erts_atomic32_read_ddrb erts_no_atomic32_read +#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read +#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read +#define erts_atomic32_inc_ddrb erts_no_atomic32_inc +#define erts_atomic32_dec_ddrb erts_no_atomic32_dec +#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read +#define erts_atomic32_add_ddrb erts_no_atomic32_add +#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor +#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band +#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg +#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg + #define erts_atomic32_init_rb erts_no_atomic32_set #define erts_atomic32_set_rb erts_no_atomic32_set #define erts_atomic32_read_rb erts_no_atomic32_read -- cgit v1.2.3 From a393026eb77ce06711c931bf68eed12d9f8ba9c9 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 16 Feb 2012 15:55:03 +0100 Subject: Reduce thread progress read operations in handle_aux_work() --- erts/emulator/beam/erl_process.c | 171 ++++++++++++++++++++------------------- erts/emulator/beam/erl_process.h | 33 ++++---- 2 files changed, 103 insertions(+), 101 deletions(-) (limited to 'erts/emulator') diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 57ef7bb8b8..b0f82e4c11 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -348,40 +348,25 @@ dbg_chk_aux_work_val(erts_aint32_t value) { erts_aint32_t valid = 0; -#ifdef ERTS_SSI_AUX_WORK_SET_TMO valid |= ERTS_SSI_AUX_WORK_SET_TMO; -#endif -#ifdef ERTS_SSI_AUX_WORK_CHECK_CHILDREN - valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; -#endif -#ifdef ERTS_SSI_AUX_WORK_MISC valid |= ERTS_SSI_AUX_WORK_MISC; -#endif -#ifdef ERTS_SSI_AUX_WORK_MISC_THR_PRGR - valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; -#endif -#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY - valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; -#endif -#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN - valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; -#endif - -#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM; -#endif -#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC; +#if ERTS_USE_ASYNC_READY_Q + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY; + valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN; #endif -#ifdef ERTS_SSI_AUX_WORK_DD +#ifdef ERTS_SMP + valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR; valid |= ERTS_SSI_AUX_WORK_DD; -#endif -#ifdef ERTS_SSI_AUX_WORK_DD valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR; #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +#if HAVE_ERTS_MSEG valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK; #endif +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN; +#endif if (~valid & value) erl_exit(ERTS_ABORT_EXIT, @@ -802,7 +787,7 @@ misc_aux_work_clean(ErtsThrQ_t *q, return aux_work; } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_misc_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -822,13 +807,13 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp, #ifdef ERTS_SMP -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp), awdp->misc.thr_prgr)) - return aux_work; + return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR; unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR); @@ -901,7 +886,7 @@ erts_notify_check_async_ready_queue(void *vno) ERTS_SSI_AUX_WORK_ASYNC_READY); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_async_ready(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -923,7 +908,7 @@ handle_async_ready(ErtsAuxWorkData *awdp, | ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_async_ready_clean(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { @@ -960,7 +945,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp, #endif -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; @@ -988,7 +973,7 @@ erts_alloc_notify_delayed_dealloc(int ix) ERTS_SSI_AUX_WORK_DD); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi = awdp->ssi; @@ -1026,7 +1011,7 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) return aux_work & ~ERTS_SSI_AUX_WORK_DD; } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { ErtsSchedulerSleepInfo *ssi; @@ -1154,7 +1139,7 @@ erts_smp_notify_check_children_needed(void) ERTS_SSI_AUX_WORK_CHECK_CHILDREN); } -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN); @@ -1164,9 +1149,9 @@ handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK +#if HAVE_ERTS_MSEG -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK); @@ -1176,7 +1161,7 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) #endif -static erts_aint32_t +static ERTS_INLINE erts_aint32_t handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) { unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO); @@ -1184,72 +1169,92 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO; } -static ERTS_INLINE erts_aint32_t -handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work) +static erts_aint32_t +handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work) { +#undef HANDLE_AUX_WORK +#define HANDLE_AUX_WORK(FLG, HNDLR) \ + ignore |= FLG; \ + if (aux_work & FLG) { \ + aux_work = HNDLR(awdp, aux_work); \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + if (!(aux_work & ~ignore)) { \ + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \ + return aux_work; \ + } \ + } + + erts_aint32_t aux_work = orig_aux_work; + erts_aint32_t ignore = 0; + #ifdef ERTS_SMP thr_prgr_current_reset(awdp); #endif + + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + ASSERT(aux_work); + /* * Handlers are *only* allowed to modify flags in return value * and ssi flags that are explicity handled by the handler. * Handlers are, e.g., not allowed to read the ssi flag field and * then unconditionally return that value. + * + * Flag field returned should only contain flags for work that + * can continue immediately. + */ + + /* + * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative + * eachother. Most frequent first. */ - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - if (aux_work & ERTS_SSI_AUX_WORK_SET_TMO) { - aux_work = handle_setup_aux_work_timer(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } #ifdef ERTS_SMP - if (aux_work & ERTS_SSI_AUX_WORK_MISC_THR_PRGR) { - aux_work = handle_misc_aux_work_thr_prgr(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD, + handle_delayed_dealloc); + /* DD must be before DD_THR_PRGR */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR, + handle_delayed_dealloc_thr_prgr); #endif - if (aux_work & ERTS_SSI_AUX_WORK_MISC) { - aux_work = handle_misc_aux_work(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + + HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM + | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC), + handle_fix_alloc); + #if ERTS_USE_ASYNC_READY_Q - if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY) { - aux_work = handle_async_ready(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } - if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) { - aux_work = handle_async_ready_clean(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } -#endif -#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN - if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) { - aux_work = handle_check_children(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY, + handle_async_ready); + /* ASYNC_READY must be before ASYNC_READY_CLEAN */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN, + handle_async_ready_clean); #endif - if (aux_work & (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM - | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)) { - aux_work = handle_fix_alloc(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + #ifdef ERTS_SMP - if (aux_work & ERTS_SSI_AUX_WORK_DD) { - aux_work = handle_delayed_dealloc(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } - if (aux_work & ERTS_SSI_AUX_WORK_DD_THR_PRGR) { - aux_work = handle_delayed_dealloc_thr_prgr(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR, + handle_misc_aux_work_thr_prgr); #endif -#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK - if (aux_work & ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK) { - aux_work = handle_mseg_cache_check(awdp, aux_work); - ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); - } + /* MISC_THR_PRGR must be before MISC */ + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC, + handle_misc_aux_work); + +#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN, + handle_check_children); +#endif + + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO, + handle_setup_aux_work_timer); + +#if HAVE_ERTS_MSEG + HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK, + handle_mseg_cache_check); #endif + ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); + return aux_work; + +#undef HANDLE_AUX_WORK + } typedef struct { diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index ad32dc11f9..c4f0ced294 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -248,25 +248,22 @@ typedef enum { | ERTS_SSI_FLG_WAITING \ | ERTS_SSI_FLG_SUSPENDED) -#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 0) -#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 1) -#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 2) -#ifdef ERTS_SMP -#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 3) -#endif -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4) -#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 5) -#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6) -#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7) -#ifdef ERTS_SMP -#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 8) -#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 9) -#endif -#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) +/* + * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative + * eachother. Most frequent - lowest bit number. + */ -#if !HAVE_ERTS_MSEG -# undef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK -#endif +#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 0) +#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 1) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 2) +#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 3) +#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 4) +#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 5) +#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 6) +#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 7) +#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 8) +#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 9) +#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10) typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo; -- cgit v1.2.3