aboutsummaryrefslogtreecommitdiffstats
path: root/erts/emulator
diff options
context:
space:
mode:
Diffstat (limited to 'erts/emulator')
-rw-r--r--erts/emulator/beam/break.c7
-rw-r--r--erts/emulator/beam/erl_alloc.c6
-rw-r--r--erts/emulator/beam/erl_alloc.h9
-rw-r--r--erts/emulator/beam/erl_alloc_util.c31
-rw-r--r--erts/emulator/beam/erl_alloc_util.h6
-rw-r--r--erts/emulator/beam/erl_db.c6
-rw-r--r--erts/emulator/beam/erl_db_hash.c8
-rw-r--r--erts/emulator/beam/erl_process.c223
-rw-r--r--erts/emulator/beam/erl_process.h38
-rw-r--r--erts/emulator/beam/erl_sched_spec_pre_alloc.c3
-rw-r--r--erts/emulator/beam/erl_smp.h87
-rw-r--r--erts/emulator/beam/erl_thr_progress.c69
-rw-r--r--erts/emulator/beam/erl_thr_progress.h99
-rw-r--r--erts/emulator/beam/erl_thr_queue.c5
-rw-r--r--erts/emulator/beam/erl_threads.h319
15 files changed, 704 insertions, 212 deletions
diff --git a/erts/emulator/beam/break.c b/erts/emulator/beam/break.c
index 0d3b6a4dba..6f5020dc14 100644
--- a/erts/emulator/beam/break.c
+++ b/erts/emulator/beam/break.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -646,6 +646,9 @@ bin_check(void)
void
erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
{
+#ifdef ERTS_SMP
+ ErtsThrPrgrData tpd_buf; /* in case we aren't a managed thread... */
+#endif
int fd;
time_t now;
size_t dumpnamebufsize = MAXPATHLEN;
@@ -663,7 +666,7 @@ erl_crash_dump_v(char *file, int line, char* fmt, va_list args)
* We do not release system again. We expect an exit() or abort() after
* dump has been written.
*/
- erts_thr_progress_fatal_error_block(60000);
+ erts_thr_progress_fatal_error_block(60000, &tpd_buf);
/* Either worked or not... */
/* Allow us to pass certain places without locking... */
diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c
index 4d02a67d54..df27186680 100644
--- a/erts/emulator/beam/erl_alloc.c
+++ b/erts/emulator/beam/erl_alloc.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -1577,9 +1577,11 @@ erts_alloc_register_scheduler(void *vesdp)
}
}
+#ifdef ERTS_SMP
void
erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
int *need_thr_progress,
+ ErtsThrPrgrVal *thr_prgr_p,
int *more_work)
{
ErtsSchedulerData *esdp = (ErtsSchedulerData *) vesdp;
@@ -1599,10 +1601,12 @@ erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
erts_alcu_check_delayed_dealloc(allctr,
1,
need_thr_progress,
+ thr_prgr_p,
more_work);
}
}
}
+#endif
erts_aint32_t
erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs)
diff --git a/erts/emulator/beam/erl_alloc.h b/erts/emulator/beam/erl_alloc.h
index 991061c48e..e475f9d8a2 100644
--- a/erts/emulator/beam/erl_alloc.h
+++ b/erts/emulator/beam/erl_alloc.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -21,6 +21,10 @@
#define ERL_ALLOC_H__
#include "erl_alloc_types.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#define ERL_THR_PROGRESS_TSD_TYPE_ONLY
+#include "erl_thr_progress.h"
+#undef ERL_THR_PROGRESS_TSD_TYPE_ONLY
#include "erl_alloc_util.h"
#ifdef USE_THREADS
#include "erl_threads.h"
@@ -132,9 +136,12 @@ typedef struct {
extern ErtsAllocatorThrSpec_t erts_allctr_thr_spec[ERTS_ALC_A_MAX+1];
void erts_alloc_register_scheduler(void *vesdp);
+#ifdef ERTS_SMP
void erts_alloc_scheduler_handle_delayed_dealloc(void *vesdp,
int *need_thr_progress,
+ ErtsThrPrgrVal *thr_prgr_p,
int *more_work);
+#endif
erts_aint32_t erts_alloc_fix_alloc_shrink(int ix, erts_aint32_t flgs);
__decl_noreturn void erts_alloc_enomem(ErtsAlcType_t,Uint)
diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c
index 2e1a92f61d..e0d525bdde 100644
--- a/erts/emulator/beam/erl_alloc_util.c
+++ b/erts/emulator/beam/erl_alloc_util.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -949,7 +949,6 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq)
ERTS_THR_MEMORY_BARRIER;
else {
ddq->head.next.unref_end = (ErtsAllctrDDBlock_t *) ilast;
- ERTS_THR_MEMORY_BARRIER;
ddq->head.next.thr_progress = erts_thr_progress_later();
erts_atomic32_set_relb(&ddq->tail.data.um_refc_ix,
um_refc_ix);
@@ -961,12 +960,24 @@ ddq_check_incoming(ErtsAllctrDDQueue_t *ddq)
return 1;
}
+static ERTS_INLINE void
+store_earliest_thr_prgr(ErtsThrPrgrVal *prev_val, ErtsAllctrDDQueue_t *ddq)
+{
+ if (!ddq->head.next.thr_progress_reached
+ && (*prev_val == ERTS_THR_PRGR_INVALID
+ || erts_thr_progress_cmp(ddq->head.next.thr_progress,
+ *prev_val) < 0)) {
+ *prev_val = ddq->head.next.thr_progress;
+ }
+}
+
static ERTS_INLINE int
handle_delayed_dealloc(Allctr_t *allctr,
int allctr_locked,
int use_limit,
int ops_limit,
int *need_thr_progress,
+ ErtsThrPrgrVal *thr_prgr_p,
int *need_more_work)
{
int need_thr_prgr = 0;
@@ -1008,8 +1019,12 @@ handle_delayed_dealloc(Allctr_t *allctr,
if (have_checked_incoming)
break;
need_thr_prgr = ddq_check_incoming(ddq);
- if (need_thr_progress)
+ if (need_thr_progress) {
*need_thr_progress |= need_thr_prgr;
+ if (need_thr_prgr)
+ store_earliest_thr_prgr(thr_prgr_p, ddq);
+
+ }
have_checked_incoming = 1;
goto dequeue;
}
@@ -1067,6 +1082,8 @@ handle_delayed_dealloc(Allctr_t *allctr,
if (need_thr_progress && !(need_thr_prgr | need_mr_wrk)) {
need_thr_prgr = ddq_check_incoming(ddq);
*need_thr_progress |= need_thr_prgr;
+ if (need_thr_prgr)
+ store_earliest_thr_prgr(thr_prgr_p, ddq);
}
if (allctr->thread_safe && !allctr_locked)
@@ -1086,25 +1103,27 @@ enqueue_dealloc_other_instance(ErtsAlcType_t type, Allctr_t *allctr, void *ptr)
#endif
+#ifdef ERTS_SMP
void
erts_alcu_check_delayed_dealloc(Allctr_t *allctr,
int limit,
int *need_thr_progress,
+ ErtsThrPrgrVal *thr_prgr_p,
int *more_work)
{
-#ifdef ERTS_SMP
handle_delayed_dealloc(allctr,
0,
limit,
ERTS_ALCU_DD_OPS_LIM_HIGH,
need_thr_progress,
+ thr_prgr_p,
more_work);
-#endif
}
+#endif
#define ERTS_ALCU_HANDLE_DD_IN_OP(Allctr, Locked) \
handle_delayed_dealloc((Allctr), (Locked), 1, \
- ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL)
+ ERTS_ALCU_DD_OPS_LIM_LOW, NULL, NULL, NULL)
/* Multi block carrier alloc/realloc/free ... */
diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h
index fc1eddb116..cedf4ccf85 100644
--- a/erts/emulator/beam/erl_alloc_util.h
+++ b/erts/emulator/beam/erl_alloc_util.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2002-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2002-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -170,7 +170,9 @@ Eterm erts_alcu_info(Allctr_t *, int, int *, void *, Uint **, Uint *);
void erts_alcu_init(AlcUInit_t *);
void erts_alcu_current_size(Allctr_t *, AllctrSize_t *,
ErtsAlcUFixInfo_t *, int);
-void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, int *);
+#ifdef ERTS_SMP
+void erts_alcu_check_delayed_dealloc(Allctr_t *, int, int *, ErtsThrPrgrVal *, int *);
+#endif
erts_aint32_t erts_alcu_fix_alloc_shrink(Allctr_t *, erts_aint32_t);
#endif
diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c
index eb89baf1c9..51bdf53823 100644
--- a/erts/emulator/beam/erl_db.c
+++ b/erts/emulator/beam/erl_db.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -251,7 +251,7 @@ free_dbtable(DbTable* tb)
ASSERT(is_immed(tb->common.heir_data));
erts_db_free(ERTS_ALC_T_DB_TABLE, tb, (void *) tb, sizeof(DbTable));
ERTS_ETS_MISC_MEM_ADD(-sizeof(DbTable));
- ERTS_THR_MEMORY_BARRIER;
+ ERTS_SMP_MEMORY_BARRIER;
}
#ifdef ERTS_SMP
@@ -3816,7 +3816,7 @@ void db_info(int to, void *to_arg, int show) /* Called by break handler */
Uint
erts_get_ets_misc_mem_size(void)
{
- ERTS_THR_MEMORY_BARRIER;
+ ERTS_SMP_MEMORY_BARRIER;
/* Memory not allocated in ets_alloc */
return (Uint) erts_smp_atomic_read_nob(&erts_ets_misc_mem_size);
}
diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c
index 038a667b06..c726be5fb4 100644
--- a/erts/emulator/beam/erl_db_hash.c
+++ b/erts/emulator/beam/erl_db_hash.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1998-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1998-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -105,11 +105,7 @@
#define NSEG_2 256 /* Size of second segment table */
#define NSEG_INC 128 /* Number of segments to grow after that */
-#ifdef ETHR_ORDERED_READ_DEPEND
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_nob(&(tb)->segtab))
-#else
-#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_rb(&(tb)->segtab))
-#endif
+#define SEGTAB(tb) ((struct segment**)erts_smp_atomic_read_ddrb(&(tb)->segtab))
#define NACTIVE(tb) ((int)erts_smp_atomic_read_nob(&(tb)->nactive))
#define NITEMS(tb) ((int)erts_smp_atomic_read_nob(&(tb)->common.nitems))
diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c
index eedbb35486..138acfeb2c 100644
--- a/erts/emulator/beam/erl_process.c
+++ b/erts/emulator/beam/erl_process.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -347,40 +347,25 @@ dbg_chk_aux_work_val(erts_aint32_t value)
{
erts_aint32_t valid = 0;
-#ifdef ERTS_SSI_AUX_WORK_SET_TMO
valid |= ERTS_SSI_AUX_WORK_SET_TMO;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_CHECK_CHILDREN
- valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_MISC
valid |= ERTS_SSI_AUX_WORK_MISC;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_MISC_THR_PRGR
- valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY
- valid |= ERTS_SSI_AUX_WORK_ASYNC_READY;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN
- valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
-#endif
-
-#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC
valid |= ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC;
+#if ERTS_USE_ASYNC_READY_Q
+ valid |= ERTS_SSI_AUX_WORK_ASYNC_READY;
+ valid |= ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
#endif
-#ifdef ERTS_SSI_AUX_WORK_DD
+#ifdef ERTS_SMP
+ valid |= ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
valid |= ERTS_SSI_AUX_WORK_DD;
-#endif
-#ifdef ERTS_SSI_AUX_WORK_DD
valid |= ERTS_SSI_AUX_WORK_DD_THR_PRGR;
#endif
-#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
+#if HAVE_ERTS_MSEG
valid |= ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK;
#endif
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+ valid |= ERTS_SSI_AUX_WORK_CHECK_CHILDREN;
+#endif
if (~valid & value)
erl_exit(ERTS_ABORT_EXIT,
@@ -911,6 +896,27 @@ unset_aux_work_flags(ErtsSchedulerSleepInfo *ssi, erts_aint32_t flgs)
return erts_atomic32_read_band_nob(&ssi->aux_work, ~flgs);
}
+#ifdef ERTS_SMP
+
+static ERTS_INLINE void
+thr_prgr_current_reset(ErtsAuxWorkData *awdp)
+{
+ awdp->current_thr_prgr = ERTS_THR_PRGR_INVALID;
+}
+
+static ERTS_INLINE ErtsThrPrgrVal
+thr_prgr_current(ErtsAuxWorkData *awdp)
+{
+ ErtsThrPrgrVal current = awdp->current_thr_prgr;
+ if (current == ERTS_THR_PRGR_INVALID) {
+ current = erts_thr_progress_current();
+ awdp->current_thr_prgr = current;
+ }
+ return current;
+}
+
+#endif
+
typedef struct erts_misc_aux_work_t_ erts_misc_aux_work_t;
struct erts_misc_aux_work_t_ {
void (*func)(void *);
@@ -983,7 +989,7 @@ misc_aux_work_clean(ErtsThrQ_t *q,
return aux_work;
}
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_misc_aux_work(ErtsAuxWorkData *awdp,
erts_aint32_t aux_work)
{
@@ -1003,12 +1009,13 @@ handle_misc_aux_work(ErtsAuxWorkData *awdp,
#ifdef ERTS_SMP
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_misc_aux_work_thr_prgr(ErtsAuxWorkData *awdp,
erts_aint32_t aux_work)
{
- if (!erts_thr_progress_has_reached(awdp->misc.thr_prgr))
- return aux_work;
+ if (!erts_thr_progress_has_reached_this(thr_prgr_current(awdp),
+ awdp->misc.thr_prgr))
+ return aux_work & ~ERTS_SSI_AUX_WORK_MISC_THR_PRGR;
unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MISC_THR_PRGR);
@@ -1081,7 +1088,7 @@ erts_notify_check_async_ready_queue(void *vno)
ERTS_SSI_AUX_WORK_ASYNC_READY);
}
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_async_ready(ErtsAuxWorkData *awdp,
erts_aint32_t aux_work)
{
@@ -1103,7 +1110,7 @@ handle_async_ready(ErtsAuxWorkData *awdp,
| ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN);
}
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_async_ready_clean(ErtsAuxWorkData *awdp,
erts_aint32_t aux_work)
{
@@ -1111,7 +1118,8 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
#ifdef ERTS_SMP
if (awdp->async_ready.need_thr_prgr
- && !erts_thr_progress_has_reached(awdp->async_ready.thr_prgr)) {
+ && !erts_thr_progress_has_reached_this(thr_prgr_current(awdp),
+ awdp->async_ready.thr_prgr)) {
return aux_work & ~ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN;
}
@@ -1139,7 +1147,7 @@ handle_async_ready_clean(ErtsAuxWorkData *awdp,
#endif
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_fix_alloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
{
ErtsSchedulerSleepInfo *ssi = awdp->ssi;
@@ -1167,16 +1175,18 @@ erts_alloc_notify_delayed_dealloc(int ix)
ERTS_SSI_AUX_WORK_DD);
}
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
{
ErtsSchedulerSleepInfo *ssi = awdp->ssi;
int need_thr_progress = 0;
+ ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
int more_work = 0;
unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
&need_thr_progress,
+ &wakeup,
&more_work);
if (more_work) {
if (set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD)
@@ -1188,9 +1198,12 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
}
if (need_thr_progress) {
+ if (wakeup == ERTS_THR_PRGR_INVALID)
+ wakeup = erts_thr_progress_later_than(thr_prgr_current(awdp));
+ awdp->dd.thr_prgr = wakeup;
set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
- awdp->dd.thr_prgr = erts_thr_progress_later();
- erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr);
+ awdp->dd.thr_prgr = wakeup;
+ erts_thr_progress_wakeup(awdp->esdp, wakeup);
}
else if (awdp->dd.completed_callback) {
awdp->dd.completed_callback(awdp->dd.completed_arg);
@@ -1200,14 +1213,16 @@ handle_delayed_dealloc(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
return aux_work & ~ERTS_SSI_AUX_WORK_DD;
}
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
{
ErtsSchedulerSleepInfo *ssi;
int need_thr_progress;
int more_work;
+ ErtsThrPrgrVal wakeup = ERTS_THR_PRGR_INVALID;
+ ErtsThrPrgrVal current = thr_prgr_current(awdp);
- if (!erts_thr_progress_has_reached(awdp->dd.thr_prgr))
+ if (!erts_thr_progress_has_reached_this(current, awdp->dd.thr_prgr))
return aux_work & ~ERTS_SSI_AUX_WORK_DD_THR_PRGR;
ssi = awdp->ssi;
@@ -1216,6 +1231,7 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
erts_alloc_scheduler_handle_delayed_dealloc((void *) awdp->esdp,
&need_thr_progress,
+ &wakeup,
&more_work);
if (more_work) {
set_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD);
@@ -1225,8 +1241,10 @@ handle_delayed_dealloc_thr_prgr(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
}
if (need_thr_progress) {
- awdp->dd.thr_prgr = erts_thr_progress_later();
- erts_thr_progress_wakeup(awdp->esdp, awdp->dd.thr_prgr);
+ if (wakeup == ERTS_THR_PRGR_INVALID)
+ wakeup = erts_thr_progress_later_than(current);
+ awdp->dd.thr_prgr = wakeup;
+ erts_thr_progress_wakeup(awdp->esdp, wakeup);
}
else {
unset_aux_work_flags(ssi, ERTS_SSI_AUX_WORK_DD_THR_PRGR);
@@ -1323,7 +1341,7 @@ erts_smp_notify_check_children_needed(void)
ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
}
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
{
unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_CHECK_CHILDREN);
@@ -1333,9 +1351,9 @@ handle_check_children(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
#endif
-#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
+#if HAVE_ERTS_MSEG
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
{
unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK);
@@ -1345,7 +1363,7 @@ handle_mseg_cache_check(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
#endif
-static erts_aint32_t
+static ERTS_INLINE erts_aint32_t
handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
{
unset_aux_work_flags(awdp->ssi, ERTS_SSI_AUX_WORK_SET_TMO);
@@ -1353,69 +1371,92 @@ handle_setup_aux_work_timer(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
return aux_work & ~ERTS_SSI_AUX_WORK_SET_TMO;
}
-static ERTS_INLINE erts_aint32_t
-handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t aux_work)
+static erts_aint32_t
+handle_aux_work(ErtsAuxWorkData *awdp, erts_aint32_t orig_aux_work)
{
+#undef HANDLE_AUX_WORK
+#define HANDLE_AUX_WORK(FLG, HNDLR) \
+ ignore |= FLG; \
+ if (aux_work & FLG) { \
+ aux_work = HNDLR(awdp, aux_work); \
+ ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
+ if (!(aux_work & ~ignore)) { \
+ ERTS_DBG_CHK_AUX_WORK_VAL(aux_work); \
+ return aux_work; \
+ } \
+ }
+
+ erts_aint32_t aux_work = orig_aux_work;
+ erts_aint32_t ignore = 0;
+
+#ifdef ERTS_SMP
+ thr_prgr_current_reset(awdp);
+#endif
+
+ ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
+ ASSERT(aux_work);
+
/*
* Handlers are *only* allowed to modify flags in return value
* and ssi flags that are explicity handled by the handler.
* Handlers are, e.g., not allowed to read the ssi flag field and
* then unconditionally return that value.
+ *
+ * Flag field returned should only contain flags for work that
+ * can continue immediately.
+ */
+
+ /*
+ * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
+ * eachother. Most frequent first.
*/
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- if (aux_work & ERTS_SSI_AUX_WORK_SET_TMO) {
- aux_work = handle_setup_aux_work_timer(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
#ifdef ERTS_SMP
- if (aux_work & ERTS_SSI_AUX_WORK_MISC_THR_PRGR) {
- aux_work = handle_misc_aux_work_thr_prgr(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD,
+ handle_delayed_dealloc);
+ /* DD must be before DD_THR_PRGR */
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_DD_THR_PRGR,
+ handle_delayed_dealloc_thr_prgr);
#endif
- if (aux_work & ERTS_SSI_AUX_WORK_MISC) {
- aux_work = handle_misc_aux_work(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
+
+ HANDLE_AUX_WORK((ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
+ | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC),
+ handle_fix_alloc);
+
#if ERTS_USE_ASYNC_READY_Q
- if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY) {
- aux_work = handle_async_ready(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
- if (aux_work & ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN) {
- aux_work = handle_async_ready_clean(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY,
+ handle_async_ready);
+ /* ASYNC_READY must be before ASYNC_READY_CLEAN */
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN,
+ handle_async_ready_clean);
#endif
-#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
- if (aux_work & ERTS_SSI_AUX_WORK_CHECK_CHILDREN) {
- aux_work = handle_check_children(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
-#endif
- if (aux_work & (ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM
- | ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC)) {
- aux_work = handle_fix_alloc(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
+
#ifdef ERTS_SMP
- if (aux_work & ERTS_SSI_AUX_WORK_DD) {
- aux_work = handle_delayed_dealloc(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
- if (aux_work & ERTS_SSI_AUX_WORK_DD_THR_PRGR) {
- aux_work = handle_delayed_dealloc_thr_prgr(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC_THR_PRGR,
+ handle_misc_aux_work_thr_prgr);
#endif
-#ifdef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
- if (aux_work & ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK) {
- aux_work = handle_mseg_cache_check(awdp, aux_work);
- ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
- }
+ /* MISC_THR_PRGR must be before MISC */
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MISC,
+ handle_misc_aux_work);
+
+#ifdef ERTS_SMP_SCHEDULERS_NEED_TO_CHECK_CHILDREN
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_CHECK_CHILDREN,
+ handle_check_children);
+#endif
+
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_SET_TMO,
+ handle_setup_aux_work_timer);
+
+#if HAVE_ERTS_MSEG
+ HANDLE_AUX_WORK(ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK,
+ handle_mseg_cache_check);
#endif
+
ERTS_DBG_CHK_AUX_WORK_VAL(aux_work);
+
return aux_work;
+
+#undef HANDLE_AUX_WORK
+
}
typedef struct {
diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h
index 173b7df69d..c23810f15a 100644
--- a/erts/emulator/beam/erl_process.h
+++ b/erts/emulator/beam/erl_process.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 1996-2011. All Rights Reserved.
+ * Copyright Ericsson AB 1996-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -248,25 +248,22 @@ typedef enum {
| ERTS_SSI_FLG_WAITING \
| ERTS_SSI_FLG_SUSPENDED)
-#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 0)
-#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 1)
-#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 2)
-#ifdef ERTS_SMP
-#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 3)
-#endif
-#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 4)
-#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 5)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 6)
-#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 7)
-#ifdef ERTS_SMP
-#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 8)
-#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 9)
-#endif
-#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10)
+/*
+ * Keep ERTS_SSI_AUX_WORK flags in expected frequency order relative
+ * eachother. Most frequent - lowest bit number.
+ */
-#if !HAVE_ERTS_MSEG
-# undef ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK
-#endif
+#define ERTS_SSI_AUX_WORK_DD (((erts_aint32_t) 1) << 0)
+#define ERTS_SSI_AUX_WORK_DD_THR_PRGR (((erts_aint32_t) 1) << 1)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_DEALLOC (((erts_aint32_t) 1) << 2)
+#define ERTS_SSI_AUX_WORK_FIX_ALLOC_LOWER_LIM (((erts_aint32_t) 1) << 3)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY (((erts_aint32_t) 1) << 4)
+#define ERTS_SSI_AUX_WORK_ASYNC_READY_CLEAN (((erts_aint32_t) 1) << 5)
+#define ERTS_SSI_AUX_WORK_MISC_THR_PRGR (((erts_aint32_t) 1) << 6)
+#define ERTS_SSI_AUX_WORK_MISC (((erts_aint32_t) 1) << 7)
+#define ERTS_SSI_AUX_WORK_CHECK_CHILDREN (((erts_aint32_t) 1) << 8)
+#define ERTS_SSI_AUX_WORK_SET_TMO (((erts_aint32_t) 1) << 9)
+#define ERTS_SSI_AUX_WORK_MSEG_CACHE_CHECK (((erts_aint32_t) 1) << 10)
typedef struct ErtsSchedulerSleepInfo_ ErtsSchedulerSleepInfo;
@@ -407,6 +404,9 @@ typedef struct {
int sched_id;
ErtsSchedulerData *esdp;
ErtsSchedulerSleepInfo *ssi;
+#ifdef ERTS_SMP
+ ErtsThrPrgrVal current_thr_prgr;
+#endif
struct {
int ix;
#ifdef ERTS_SMP
diff --git a/erts/emulator/beam/erl_sched_spec_pre_alloc.c b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
index a7ccea7403..bff9d246a3 100644
--- a/erts/emulator/beam/erl_sched_spec_pre_alloc.c
+++ b/erts/emulator/beam/erl_sched_spec_pre_alloc.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -227,7 +227,6 @@ fetch_remote(erts_sspa_chunk_header_t *chdr, int max)
ERTS_THR_MEMORY_BARRIER;
else {
chdr->head.next.unref_end = (erts_sspa_blk_t *) ilast;
- ERTS_THR_MEMORY_BARRIER;
chdr->head.next.thr_progress = erts_thr_progress_later();
erts_atomic32_set_relb(&chdr->tail.data.um_refc_ix,
um_refc_ix);
diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h
index 63179dfad4..a32e9d9d7c 100644
--- a/erts/emulator/beam/erl_smp.h
+++ b/erts/emulator/beam/erl_smp.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2005-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2005-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -61,6 +61,11 @@ typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
+#define ERTS_SMP_MEMORY_BARRIER ERTS_THR_MEMORY_BARRIER
+#define ERTS_SMP_WRITE_MEMORY_BARRIER ERTS_THR_WRITE_MEMORY_BARRIER
+#define ERTS_SMP_READ_MEMORY_BARRIER ERTS_THR_READ_MEMORY_BARRIER
+#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+
#else /* #ifdef ERTS_SMP */
#define ERTS_SMP_THR_OPTS_DEFAULT_INITER {0}
@@ -95,6 +100,11 @@ typedef struct { int gcc_is_buggy; } erts_smp_spinlock_t;
typedef struct { int gcc_is_buggy; } erts_smp_rwlock_t;
#endif
+#define ERTS_SMP_MEMORY_BARRIER
+#define ERTS_SMP_WRITE_MEMORY_BARRIER
+#define ERTS_SMP_READ_MEMORY_BARRIER
+#define ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+
#endif /* #ifdef ERTS_SMP */
ERTS_GLB_INLINE void erts_smp_thr_init(erts_smp_thr_init_data_t *id);
@@ -206,13 +216,8 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
/*
- * Functions implementing atomic operations with with no (nob),
- * full (mb), acquire (acqb), release (relb), read (rb), and
- * write (wb) memory barriers.
- *
- * If SMP support has been disabled, they are mapped to functions
- * that performs the same operation, but aren't atomic and don't
- * imply memory barriers.
+ * See "Documentation of atomics and memory barriers" at the top
+ * of erl_threads.h for info on atomics.
*/
#ifdef ERTS_SMP
@@ -239,6 +244,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb
+#define erts_smp_dw_atomic_init_ddrb erts_dw_atomic_init_ddrb
+#define erts_smp_dw_atomic_set_ddrb erts_dw_atomic_set_ddrb
+#define erts_smp_dw_atomic_read_ddrb erts_dw_atomic_read_ddrb
+#define erts_smp_dw_atomic_cmpxchg_ddrb erts_dw_atomic_cmpxchg_ddrb
+
#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
@@ -307,6 +317,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
+#define erts_smp_atomic_init_ddrb erts_atomic_init_ddrb
+#define erts_smp_atomic_set_ddrb erts_atomic_set_ddrb
+#define erts_smp_atomic_read_ddrb erts_atomic_read_ddrb
+#define erts_smp_atomic_inc_read_ddrb erts_atomic_inc_read_ddrb
+#define erts_smp_atomic_dec_read_ddrb erts_atomic_dec_read_ddrb
+#define erts_smp_atomic_inc_ddrb erts_atomic_inc_ddrb
+#define erts_smp_atomic_dec_ddrb erts_atomic_dec_ddrb
+#define erts_smp_atomic_add_read_ddrb erts_atomic_add_read_ddrb
+#define erts_smp_atomic_add_ddrb erts_atomic_add_ddrb
+#define erts_smp_atomic_read_bor_ddrb erts_atomic_read_bor_ddrb
+#define erts_smp_atomic_read_band_ddrb erts_atomic_read_band_ddrb
+#define erts_smp_atomic_xchg_ddrb erts_atomic_xchg_ddrb
+#define erts_smp_atomic_cmpxchg_ddrb erts_atomic_cmpxchg_ddrb
+
#define erts_smp_atomic_init_rb erts_atomic_init_rb
#define erts_smp_atomic_set_rb erts_atomic_set_rb
#define erts_smp_atomic_read_rb erts_atomic_read_rb
@@ -393,6 +417,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
+#define erts_smp_atomic32_init_ddrb erts_atomic32_init_ddrb
+#define erts_smp_atomic32_set_ddrb erts_atomic32_set_ddrb
+#define erts_smp_atomic32_read_ddrb erts_atomic32_read_ddrb
+#define erts_smp_atomic32_inc_read_ddrb erts_atomic32_inc_read_ddrb
+#define erts_smp_atomic32_dec_read_ddrb erts_atomic32_dec_read_ddrb
+#define erts_smp_atomic32_inc_ddrb erts_atomic32_inc_ddrb
+#define erts_smp_atomic32_dec_ddrb erts_atomic32_dec_ddrb
+#define erts_smp_atomic32_add_read_ddrb erts_atomic32_add_read_ddrb
+#define erts_smp_atomic32_add_ddrb erts_atomic32_add_ddrb
+#define erts_smp_atomic32_read_bor_ddrb erts_atomic32_read_bor_ddrb
+#define erts_smp_atomic32_read_band_ddrb erts_atomic32_read_band_ddrb
+#define erts_smp_atomic32_xchg_ddrb erts_atomic32_xchg_ddrb
+#define erts_smp_atomic32_cmpxchg_ddrb erts_atomic32_cmpxchg_ddrb
+
#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
@@ -445,6 +483,11 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
+#define erts_smp_dw_atomic_init_ddrb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_ddrb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_ddrb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
+
#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
@@ -513,6 +556,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+#define erts_smp_atomic_init_ddrb erts_no_atomic_set
+#define erts_smp_atomic_set_ddrb erts_no_atomic_set
+#define erts_smp_atomic_read_ddrb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_ddrb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_ddrb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_ddrb erts_no_atomic_inc
+#define erts_smp_atomic_dec_ddrb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_ddrb erts_no_atomic_add_read
+#define erts_smp_atomic_add_ddrb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_ddrb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_ddrb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_ddrb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+
#define erts_smp_atomic_init_rb erts_no_atomic_set
#define erts_smp_atomic_set_rb erts_no_atomic_set
#define erts_smp_atomic_read_rb erts_no_atomic_read
@@ -599,6 +656,20 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+#define erts_smp_atomic32_init_ddrb erts_no_atomic32_set
+#define erts_smp_atomic32_set_ddrb erts_no_atomic32_set
+#define erts_smp_atomic32_read_ddrb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_ddrb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_ddrb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_ddrb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_ddrb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_ddrb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_ddrb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+
#define erts_smp_atomic32_init_rb erts_no_atomic32_set
#define erts_smp_atomic32_set_rb erts_no_atomic32_set
#define erts_smp_atomic32_read_rb erts_no_atomic32_read
diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c
index 75f8209c3b..9ef83746c5 100644
--- a/erts/emulator/beam/erl_thr_progress.c
+++ b/erts/emulator/beam/erl_thr_progress.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -33,10 +33,11 @@
* This module keeps track of the progress of a set of managed threads. Only
* threads that behave well can be allowed to be managed. A managed thread
* should update its thread progress frequently. Currently only scheduler
- * threads and the aux_thread are managed threads. We typically do not want
- * any async threads as managed threads since they cannot guarantee a
- * frequent update of thread progress, since they execute user implemented
- * driver code.
+ * threads, the system-message-dispatcher threads, and the aux-thread are
+ * managed threads. We typically do not want any async threads as managed
+ * threads since they cannot guarantee a frequent update of thread progress,
+ * since they execute user implemented driver code that is assumed to be
+ * time consuming.
*
* erts_thr_progress_current() returns the global current thread progress
* value of managed threads. I.e., the latest progress value that all
@@ -112,8 +113,10 @@
*
* On 32-bit systems we therefore need a double word atomic.
*/
-
+#undef read_acqb
#define read_acqb erts_thr_prgr_read_acqb__
+#undef read_nob
+#define read_nob erts_thr_prgr_read_nob__
#ifdef ARCH_64
@@ -129,12 +132,6 @@ set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
erts_atomic_set_nob(atmc, val);
}
-static ERTS_INLINE ErtsThrPrgrVal
-read_nob(ERTS_THR_PRGR_ATOMIC *atmc)
-{
- return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc);
-}
-
static ERTS_INLINE void
init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
{
@@ -143,52 +140,44 @@ init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
#else
-#undef dw_sint_to_val
-#define dw_sint_to_val erts_thr_prgr_dw_sint_to_val__
+#undef dw_aint_to_val
+#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__
static void
-val_to_dw_sint(ethr_dw_sint_t *dw_sint, ErtsThrPrgrVal val)
+val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val)
{
#ifdef ETHR_SU_DW_NAINT_T__
- dw_sint->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
+ dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val;
#else
- dw_sint->sint[ETHR_DW_SINT_LOW_WORD]
- = (ethr_sint_t) (val & 0xffffffff);
- dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]
- = (ethr_sint_t) ((val >> 32) & 0xffffffff);
+ dw_aint->sint[ERTS_DW_AINT_LOW_WORD]
+ = (erts_aint_t) (val & 0xffffffff);
+ dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]
+ = (erts_aint_t) ((val >> 32) & 0xffffffff);
#endif
}
static ERTS_INLINE void
set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
{
- ethr_dw_sint_t dw_sint;
- val_to_dw_sint(&dw_sint, val);
- erts_dw_atomic_set_mb(atmc, &dw_sint);
+ erts_dw_aint_t dw_aint;
+ val_to_dw_aint(&dw_aint, val);
+ erts_dw_atomic_set_mb(atmc, &dw_aint);
}
static ERTS_INLINE void
set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
{
- ethr_dw_sint_t dw_sint;
- val_to_dw_sint(&dw_sint, val);
- erts_dw_atomic_set_nob(atmc, &dw_sint);
-}
-
-static ERTS_INLINE ErtsThrPrgrVal
-read_nob(ERTS_THR_PRGR_ATOMIC *atmc)
-{
- ethr_dw_sint_t dw_sint;
- erts_dw_atomic_read_nob(atmc, &dw_sint);
- return erts_thr_prgr_dw_sint_to_val__(&dw_sint);
+ erts_dw_aint_t dw_aint;
+ val_to_dw_aint(&dw_aint, val);
+ erts_dw_atomic_set_nob(atmc, &dw_aint);
}
static ERTS_INLINE void
init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val)
{
- ethr_dw_sint_t dw_sint;
- val_to_dw_sint(&dw_sint, val);
- erts_dw_atomic_init_nob(atmc, &dw_sint);
+ erts_dw_aint_t dw_aint;
+ val_to_dw_aint(&dw_aint, val);
+ erts_dw_atomic_init_nob(atmc, &dw_aint);
}
#endif
@@ -1222,9 +1211,9 @@ erts_thr_progress_block(void)
}
void
-erts_thr_progress_fatal_error_block(SWord timeout)
+erts_thr_progress_fatal_error_block(SWord timeout,
+ ErtsThrPrgrData *tmp_tpd_bufp)
{
- ErtsThrPrgrData tpd_buf;
ErtsThrPrgrData *tpd = perhaps_thr_prgr_data(NULL);
erts_aint32_t bc;
SWord time_left = timeout;
@@ -1248,7 +1237,7 @@ erts_thr_progress_fatal_error_block(SWord timeout)
* since we never complete an unblock after a fatal error
* block.
*/
- tpd = &tpd_buf;
+ tpd = tmp_tpd_bufp;
init_tmp_thr_prgr_data(tpd);
}
diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h
index 68d14174b9..1bbc49993e 100644
--- a/erts/emulator/beam/erl_thr_progress.h
+++ b/erts/emulator/beam/erl_thr_progress.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -44,7 +44,6 @@
#define erts_smp_thr_progress_unblock erts_thr_progress_unblock
#define erts_smp_thr_progress_is_blocking erts_thr_progress_is_blocking
-void erts_thr_progress_fatal_error_block(SWord timeout);
void erts_thr_progress_block(void);
void erts_thr_progress_unblock(void);
int erts_thr_progress_is_blocking(void);
@@ -73,6 +72,10 @@ typedef struct {
ErtsThrPrgrVal current;
} previous;
} ErtsThrPrgrData;
+
+void erts_thr_progress_fatal_error_block(SWord timeout,
+ ErtsThrPrgrData *tmp_tpd_bufp);
+
#endif /* ERTS_SMP */
#endif
@@ -86,6 +89,7 @@ typedef struct {
#ifdef ERTS_SMP
#define ERTS_THR_PRGR_VAL_WAITING (~((ErtsThrPrgrVal) 0))
+#define ERTS_THR_PRGR_INVALID (~((ErtsThrPrgrVal) 0))
extern erts_tsd_key_t erts_thr_prgr_data_key__;
@@ -127,14 +131,19 @@ void erts_thr_progress_dbg_print_state(void);
#ifdef ARCH_32
#define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t
-ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint);
#endif
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc);
ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void);
+ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later_than(ErtsThrPrgrVal val);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_later(void);
ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_progress_current(void);
ERTS_GLB_INLINE int erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
+ERTS_GLB_INLINE int erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val);
+ERTS_GLB_INLINE int erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2);
ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -142,33 +151,61 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val);
#ifdef ARCH_64
ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+ return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc);
}
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+ return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc);
+}
+
#else /* ARCH_32 */
ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_prgr_dw_sint_to_val__(ethr_dw_sint_t *dw_sint)
+erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint)
{
#ifdef ETHR_SU_DW_NAINT_T__
- return (ErtsThrPrgrVal) dw_sint->dw_sint;
+ return (ErtsThrPrgrVal) dw_aint->dw_sint;
#else
ErtsThrPrgrVal res;
- res = (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_HIGH_WORD]);
+ res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]);
res <<= 32;
- res |= (ErtsThrPrgrVal) ((Uint32) dw_sint->sint[ETHR_DW_SINT_LOW_WORD]);
+ res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]);
return res;
#endif
}
ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+ erts_dw_aint_t dw_aint;
+ erts_dw_atomic_read_nob(atmc, &dw_aint);
+ return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc)
{
- ethr_dw_sint_t dw_sint;
- erts_dw_atomic_read_acqb(atmc, &dw_sint);
- return erts_thr_prgr_dw_sint_to_val__(&dw_sint);
+ erts_dw_aint_t dw_aint;
+ erts_dw_atomic_read_acqb(atmc, &dw_aint);
+ return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc)
+{
+ erts_dw_aint_t dw_aint;
+ erts_dw_atomic_read_mb(atmc, &dw_aint);
+ return erts_thr_prgr_dw_aint_to_val__(&dw_aint);
}
#endif
@@ -181,9 +218,9 @@ erts_thr_progress_is_managed_thread(void)
}
ERTS_GLB_INLINE ErtsThrPrgrVal
-erts_thr_progress_later(void)
+erts_thr_progress_later_than(ErtsThrPrgrVal val)
{
- ErtsThrPrgrVal val = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
+ ERTS_THR_MEMORY_BARRIER;
if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)2)))
return ((ErtsThrPrgrVal) 0);
else if (val == (ERTS_THR_PRGR_VAL_WAITING-((ErtsThrPrgrVal)1)))
@@ -193,9 +230,19 @@ erts_thr_progress_later(void)
}
ERTS_GLB_INLINE ErtsThrPrgrVal
+erts_thr_progress_later(void)
+{
+ ErtsThrPrgrVal val = erts_thr_prgr_read_mb__(&erts_thr_prgr__.current);
+ return erts_thr_progress_later_than(val);
+}
+
+ERTS_GLB_INLINE ErtsThrPrgrVal
erts_thr_progress_current(void)
{
- return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
+ if (erts_thr_progress_is_managed_thread())
+ return erts_thr_prgr_read_nob__(&erts_thr_prgr__.current);
+ else
+ return erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
}
ERTS_GLB_INLINE int
@@ -217,13 +264,29 @@ erts_thr_progress_has_passed__(ErtsThrPrgrVal val1, ErtsThrPrgrVal val0)
}
ERTS_GLB_INLINE int
-erts_thr_progress_has_reached(ErtsThrPrgrVal val)
+erts_thr_progress_has_reached_this(ErtsThrPrgrVal this, ErtsThrPrgrVal val)
+{
+ if (this == val)
+ return 1;
+ return erts_thr_progress_has_passed__(this, val);
+}
+
+ERTS_GLB_INLINE int
+erts_thr_progress_cmp(ErtsThrPrgrVal val1, ErtsThrPrgrVal val2)
{
- ErtsThrPrgrVal current;
- current = erts_thr_prgr_read_acqb__(&erts_thr_prgr__.current);
- if (current == val)
+ if (val1 == val2)
+ return 0;
+ if (erts_thr_progress_has_passed__(val1, val2))
return 1;
- return erts_thr_progress_has_passed__(current, val);
+ else
+ return -1;
+}
+
+ERTS_GLB_INLINE int
+erts_thr_progress_has_reached(ErtsThrPrgrVal val)
+{
+ ErtsThrPrgrVal current = erts_thr_progress_current();
+ return erts_thr_progress_has_reached_this(current, val);
}
#endif
diff --git a/erts/emulator/beam/erl_thr_queue.c b/erts/emulator/beam/erl_thr_queue.c
index efb8c635d7..70949ece76 100644
--- a/erts/emulator/beam/erl_thr_queue.c
+++ b/erts/emulator/beam/erl_thr_queue.c
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2011. All Rights Reserved.
+ * Copyright Ericsson AB 2011-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -418,10 +418,9 @@ clean(ErtsThrQ_t *q, int max_ops, int do_notify)
}
if (q->head.unref_end == (ErtsThrQElement_t *) ilast)
- ERTS_THR_MEMORY_BARRIER;
+ ERTS_SMP_MEMORY_BARRIER;
else {
q->head.next.unref_end = (ErtsThrQElement_t *) ilast;
- ERTS_THR_MEMORY_BARRIER;
#ifdef ERTS_SMP
q->head.next.thr_progress = erts_thr_progress_later();
#endif
diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h
index 065e7077c0..ee47c98009 100644
--- a/erts/emulator/beam/erl_threads.h
+++ b/erts/emulator/beam/erl_threads.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2001-2011. All Rights Reserved.
+ * Copyright Ericsson AB 2001-2012. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -25,6 +25,235 @@
#ifndef ERL_THREAD_H__
#define ERL_THREAD_H__
+/*
+ * --- Documentation of atomics and memory barriers --------------------------
+ *
+ * The following explicit memory barriers exist:
+ *
+ * - ERTS_THR_MEMORY_BARRIER
+ * Full memory barrier. Orders both loads, and stores. No
+ * load or store is allowed to be reordered over the
+ * barrier.
+ * - ERTS_THR_WRITE_MEMORY_BARRIER
+ * Write barrier. Orders *only* stores. These are not
+ * allowed to be reordered over the barrier.
+ * - ERTS_THR_READ_MEMORY_BARRIER
+ * Read barrier. Orders *only* loads. These are not
+ * allowed to be reordered over the barrier.
+ * - ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
+ * Data dependency read barrier. Orders *only* loads
+ * according to data dependency across the barrier.
+ *
+ * If thread support has been disabled, these barriers will become no-ops.
+ *
+ * If the prefix ERTS_THR_ is replaced with ERTS_SMP_, the barriers will
+ * be enabled only in the SMP enabled runtime system.
+ *
+ * --- Atomic operations ---
+ *
+ * Atomics operations exist for 32-bit, word size, and double word size
+ * integers. Function prototypes are listed below.
+ *
+ * Each function implementing an atomic operation exist with the following
+ * implied memory barrier semantics. Not all combinations are useful, but
+ * all of them exist for simplicity. <B> is suffix in function name:
+ *
+ * - <B> - Description
+ *
+ * - mb - Full memory barrier. Orders both loads, and
+ * stores before, and after the atomic operation.
+ * No load or store is allowed to be reordered
+ * over the atomic operation.
+ * - relb - Release barrier. Orders both loads, and
+ * stores appearing *before* the atomic
+ * operation. These are not allowed to be
+ * reordered over the atomic operation.
+ * - acqb - Acquire barrier. Orders both loads, and stores
+ * appearing *after* the atomic operation. These
+ * are not allowed to be reordered over the
+ * atomic operation.
+ * - wb - Write barrier. Orders *only* stores. These are
+ * not allowed to be reordered over the barrier.
+ * Store in atomic operation is ordered *after*
+ * the barrier.
+ * - rb - Read barrier. Orders *only* loads. These are
+ * not allowed to be reordered over the barrier.
+ * Load in atomic operation is ordered *before*
+ * the barrier.
+ * - ddrb - Data dependency read barrier. Orders *only*
+ * loads according to data dependency across the
+ * barrier. Load in atomic operation is ordered
+ * before the barrier.
+ *
+ * If thread support has been disabled, these functions are mapped to
+ * functions that performs the same operation, but aren't atomic
+ * and don't imply any memory barriers.
+ *
+ * If the atomic operations are prefixed with erts_smp_ instead of only
+ * erts_ the atomic operations will only be atomic in the SMP enabled
+ * runtime system, and will be mapped to non-atomic operations without
+ * memory barriers in the runtime system without SMP support. Atomic
+ * operations with erts_smp_ prefix should use the atomic types
+ * erts_smp_atomic32_t, erts_smp_atomic_t, and erts_smp_dw_atomic_t
+ * instead of erts_atomic32_t, erts_atomic_t, and erts_dw_atomic_t. The
+ * integer data types erts_aint32_t, erts_aint_t, and erts_dw_atomic_t
+ * are the same.
+ *
+ * --- 32-bit atomic operations ---
+ *
+ * The following 32-bit atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above). Note
+ * that sizeof(erts_atomic32_t) might be larger than 4!
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ * void erts_atomic32_init_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Set value:
+ * void erts_atomic32_set_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Read; returns current value:
+ * erts_aint32_t erts_atomic32_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment; returns resulting value:
+ * erts_aint32_t erts_atomic32_inc_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ * erts_aint32_t erts_atomic32_dec_read_<B>(erts_atomic32_t *atmc);
+ *
+ * Increment:
+ * void erts_atomic32_inc_<B>(erts_atomic32_t *atmc);
+ *
+ * Decrement:
+ * void erts_atomic32_dec_<B>(erts_atomic32_t *atmc);
+ *
+ * Add value; returns resulting value:
+ * erts_aint32_t erts_atomic32_add_read_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Add value:
+ * void erts_atomic32_add_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Bitwise-or; returns previous value:
+ * erts_aint32_t erts_atomic32_read_bor_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Bitwise-and; returns previous value:
+ * erts_aint32_t erts_atomic32_read_band_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Exchange; returns previous value:
+ * erts_aint32_t erts_atomic32_xchg_<B>(erts_atomic32_t *atmc,
+ * erts_aint32_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ * erts_aint32_t erts_atomic32_cmpxchg_<B>(erts_atomic32_t *a,
+ * erts_aint32_t new,
+ * erts_aint32_t exp);
+ *
+ * --- Word size atomic operations ---
+ *
+ * The following word size (same size as sizeof(void *)) atomic
+ * operations exist. <B> should be replaced with a supported
+ * memory barrier (see above). Note that sizeof(erts_atomic_t)
+ * might be larger than sizeof(void *)!
+ *
+ * Initialize (not necessarily the same as the set operation):
+ * void erts_atomic_init_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Set value;
+ * void erts_atomic_set_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Read; returns current value:
+ * erts_aint_t erts_atomic_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment; returns resulting value:
+ * erts_aint_t erts_atomic_inc_read_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement; returns resulting value:
+ * erts_aint_t erts_atomic_dec_read_<B>(erts_atomic_t *atmc);
+ *
+ * Increment:
+ * void erts_atomic_inc_<B>(erts_atomic_t *atmc);
+ *
+ * Decrement:
+ * void erts_atomic_dec_<B>(erts_atomic_t *atmc);
+ *
+ * Add value; returns resulting value:
+ * erts_aint_t erts_atomic_add_read_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Add value:
+ * void erts_atomic_add_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Bitwise-or; returns previous value:
+ * erts_aint_t erts_atomic_read_bor_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Bitwise-and; returns previous value:
+ * erts_aint_t erts_atomic_read_band_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Exchange; returns previous value:
+ * erts_aint_t erts_atomic_xchg_<B>(erts_atomic_t *atmc,
+ * erts_aint_t val);
+ *
+ * Compare and exchange; returns previous or current value. If
+ * returned value equals 'exp' the value was changed to 'new';
+ * otherwise not:
+ * erts_aint_t erts_atomic_cmpxchg_<B>(erts_atomic_t *a,
+ * erts_aint_t new,
+ * erts_aint_t exp);
+ *
+ * --- Double word size atomic operations ---
+ *
+ * The following double word atomic operations exist. <B> should be
+ * replaced with a supported memory barrier (see above).
+ *
+ * Note that sizeof(erts_dw_atomic_t) usually is larger than
+ * 2*sizeof(void *)!
+ *
+ * The erts_dw_aint_t data type should be accessed as if it was defined
+ * like this:
+ *
+ * typedef struct {
+ * erts_aint_t sint[2];
+ * } erts_dw_aint_t;
+ *
+ * Most significant word is 'sint[ERTS_DW_AINT_HIGH_WORD]' and least
+ * significant word is 'sint[ERTS_DW_AINT_LOW_WORD]'.
+ *
+ *
+ * Initialize (not necessarily the same as the set operation):
+ * void erts_dw_atomic_init_<B>(erts_dw_atomic_t *var,
+ * erts_dw_aint_t *val);
+ *
+ * Set; value is written into 'val':
+ * void erts_dw_atomic_set_<B>(erts_dw_atomic_t *var,
+ * erts_dw_aint_t *val);
+ *
+ * Read; value is written into 'val':
+ * void erts_dw_atomic_read_<B>(erts_dw_atomic_t *var,
+ * erts_dw_aint_t *val);
+ *
+ * Compare and exchange; returns a value != 0 if exchange was
+ * made; otherwise 0. 'new_val' contains new value to set. If 'exp_act'
+ * contains the same value as in memory when the function is called,
+ * 'new' is written to memory; otherwise, not. If exchange was not
+ * made, 'exp_act' contains the actual value in memory:
+ * int erts_dw_atomic_cmpxchg_<B>(erts_dw_atomic_t *var,
+ * erts_dw_aint_t *new,
+ * erts_dw_aint_t *exp_act);
+ */
+
#define ERTS_SPIN_BODY ETHR_SPIN_BODY
#include "sys.h"
@@ -52,6 +281,9 @@ typedef Sint32 erts_no_atomic32_t;
#endif
#define ERTS_THR_MEMORY_BARRIER ETHR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER ETHR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER ETHR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER ETHR_READ_DEPEND_MEMORY_BARRIER
#ifdef ERTS_ENABLE_LOCK_COUNT
#define erts_mtx_lock(L) erts_mtx_lock_x(L, __FILE__, __LINE__)
@@ -113,6 +345,9 @@ typedef ethr_ts_event erts_tse_t;
#define erts_aint32_t ethr_sint32_t
#define erts_atomic32_t ethr_atomic32_t
+#define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD
+#define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD
+
/* spinlock */
typedef struct {
ethr_spinlock_t slck;
@@ -149,6 +384,9 @@ __decl_noreturn void __noreturn erts_thr_fatal_error(int, char *);
#else /* #ifdef USE_THREADS */
#define ERTS_THR_MEMORY_BARRIER
+#define ERTS_THR_WRITE_MEMORY_BARRIER
+#define ERTS_THR_READ_MEMORY_BARRIER
+#define ERTS_THR_DATA_DEPENDENCY_READ_MEMORY_BARRIER
#define ERTS_THR_OPTS_DEFAULT_INITER 0
typedef int erts_thr_opts_t;
@@ -361,18 +599,13 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
+#ifdef USE_THREADS
+
/*
- * Functions implementing atomic operations with with no (nob),
- * full (mb), acquire (acqb), release (relb), read (rb), and
- * write (wb) memory barriers.
- *
- * If thread support has been disabled, they are mapped to
- * functions that performs the same operation, but aren't atomic
- * and don't imply memory barriers.
+ * See "Documentation of atomics and memory barriers" at the top
+ * of this file for info on atomics.
*/
-#ifdef USE_THREADS
-
/* Double word size atomics */
#define erts_dw_atomic_init_nob ethr_dw_atomic_init
@@ -395,6 +628,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb
+#define erts_dw_atomic_init_ddrb ethr_dw_atomic_init_ddrb
+#define erts_dw_atomic_set_ddrb ethr_dw_atomic_set_ddrb
+#define erts_dw_atomic_read_ddrb ethr_dw_atomic_read_ddrb
+#define erts_dw_atomic_cmpxchg_ddrb ethr_dw_atomic_cmpxchg_ddrb
+
#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
@@ -463,6 +701,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb
+#define erts_atomic_init_ddrb ethr_atomic_init_ddrb
+#define erts_atomic_set_ddrb ethr_atomic_set_ddrb
+#define erts_atomic_read_ddrb ethr_atomic_read_ddrb
+#define erts_atomic_inc_read_ddrb ethr_atomic_inc_read_ddrb
+#define erts_atomic_dec_read_ddrb ethr_atomic_dec_read_ddrb
+#define erts_atomic_inc_ddrb ethr_atomic_inc_ddrb
+#define erts_atomic_dec_ddrb ethr_atomic_dec_ddrb
+#define erts_atomic_add_read_ddrb ethr_atomic_add_read_ddrb
+#define erts_atomic_add_ddrb ethr_atomic_add_ddrb
+#define erts_atomic_read_bor_ddrb ethr_atomic_read_bor_ddrb
+#define erts_atomic_read_band_ddrb ethr_atomic_read_band_ddrb
+#define erts_atomic_xchg_ddrb ethr_atomic_xchg_ddrb
+#define erts_atomic_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb
+
#define erts_atomic_init_rb ethr_atomic_init_rb
#define erts_atomic_set_rb ethr_atomic_set_rb
#define erts_atomic_read_rb ethr_atomic_read_rb
@@ -549,6 +801,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb
+#define erts_atomic32_init_ddrb ethr_atomic32_init_ddrb
+#define erts_atomic32_set_ddrb ethr_atomic32_set_ddrb
+#define erts_atomic32_read_ddrb ethr_atomic32_read_ddrb
+#define erts_atomic32_inc_read_ddrb ethr_atomic32_inc_read_ddrb
+#define erts_atomic32_dec_read_ddrb ethr_atomic32_dec_read_ddrb
+#define erts_atomic32_inc_ddrb ethr_atomic32_inc_ddrb
+#define erts_atomic32_dec_ddrb ethr_atomic32_dec_ddrb
+#define erts_atomic32_add_read_ddrb ethr_atomic32_add_read_ddrb
+#define erts_atomic32_add_ddrb ethr_atomic32_add_ddrb
+#define erts_atomic32_read_bor_ddrb ethr_atomic32_read_bor_ddrb
+#define erts_atomic32_read_band_ddrb ethr_atomic32_read_band_ddrb
+#define erts_atomic32_xchg_ddrb ethr_atomic32_xchg_ddrb
+#define erts_atomic32_cmpxchg_ddrb ethr_atomic32_cmpxchg_ddrb
+
#define erts_atomic32_init_rb ethr_atomic32_init_rb
#define erts_atomic32_set_rb ethr_atomic32_set_rb
#define erts_atomic32_read_rb ethr_atomic32_read_rb
@@ -601,6 +867,11 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
+#define erts_dw_atomic_init_ddrb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_ddrb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_ddrb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_ddrb erts_no_dw_atomic_cmpxchg
+
#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
@@ -669,6 +940,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic_xchg_relb erts_no_atomic_xchg
#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+#define erts_atomic_init_ddrb erts_no_atomic_set
+#define erts_atomic_set_ddrb erts_no_atomic_set
+#define erts_atomic_read_ddrb erts_no_atomic_read
+#define erts_atomic_inc_read_ddrb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_ddrb erts_no_atomic_dec_read
+#define erts_atomic_inc_ddrb erts_no_atomic_inc
+#define erts_atomic_dec_ddrb erts_no_atomic_dec
+#define erts_atomic_add_read_ddrb erts_no_atomic_add_read
+#define erts_atomic_add_ddrb erts_no_atomic_add
+#define erts_atomic_read_bor_ddrb erts_no_atomic_read_bor
+#define erts_atomic_read_band_ddrb erts_no_atomic_read_band
+#define erts_atomic_xchg_ddrb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_ddrb erts_no_atomic_cmpxchg
+
#define erts_atomic_init_rb erts_no_atomic_set
#define erts_atomic_set_rb erts_no_atomic_set
#define erts_atomic_read_rb erts_no_atomic_read
@@ -755,6 +1040,20 @@ ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+#define erts_atomic32_init_ddrb erts_no_atomic32_set
+#define erts_atomic32_set_ddrb erts_no_atomic32_set
+#define erts_atomic32_read_ddrb erts_no_atomic32_read
+#define erts_atomic32_inc_read_ddrb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_ddrb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_ddrb erts_no_atomic32_inc
+#define erts_atomic32_dec_ddrb erts_no_atomic32_dec
+#define erts_atomic32_add_read_ddrb erts_no_atomic32_add_read
+#define erts_atomic32_add_ddrb erts_no_atomic32_add
+#define erts_atomic32_read_bor_ddrb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_ddrb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_ddrb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_ddrb erts_no_atomic32_cmpxchg
+
#define erts_atomic32_init_rb erts_no_atomic32_set
#define erts_atomic32_set_rb erts_no_atomic32_set
#define erts_atomic32_read_rb erts_no_atomic32_read
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-09 02:23:41 +0000