diff options
Diffstat (limited to 'erts/emulator/beam')
28 files changed, 1133 insertions, 493 deletions
diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c index e9f5fd798b..3af7c43abf 100644 --- a/erts/emulator/beam/beam_emu.c +++ b/erts/emulator/beam/beam_emu.c @@ -5142,7 +5142,7 @@ get_map_elements_fail: #ifndef NO_JUMP_TABLE #ifdef ERTS_OPCODE_COUNTER_SUPPORT /* Are tables correctly generated by beam_makeops? */ - ASSERT(sizeof(counting_opcodes) == sizeof(opcodes)); + ERTS_CT_ASSERT(sizeof(counting_opcodes) == sizeof(opcodes)); #ifdef DEBUG counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y); #endif diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 55ac778475..aea3224342 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -600,6 +600,8 @@ bif maps:values/1 bif erts_internal:cmp_term/2 +bif ets:take/2 + # # New in 17.1 # diff --git a/erts/emulator/beam/erl_alloc.c b/erts/emulator/beam/erl_alloc.c index 90cd227fae..f2bceff4eb 100644 --- a/erts/emulator/beam/erl_alloc.c +++ b/erts/emulator/beam/erl_alloc.c @@ -3939,7 +3939,7 @@ static Uint install_debug_functions(void) { int i; - ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); + ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs)); diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index e3172dc4fb..2f277690e4 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -1442,7 +1442,7 @@ get_pref_allctr(void *extra) pref_ix = ERTS_ALC_GET_THR_IX(); - ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); + ERTS_CT_ASSERT(sizeof(UWord) == sizeof(Allctr_t *)); ASSERT(0 <= pref_ix && pref_ix < tspec->size); return tspec->allctr[pref_ix]; @@ -1861,7 +1861,7 @@ handle_delayed_dealloc(Allctr_t *allctr, * if this carrier is pulled from dc_list by cpool_fetch() */ ERTS_ALC_CPOOL_ASSERT(FBLK_TO_MBC(blk) != crr); - ERTS_ALC_CPOOL_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*)); + ERTS_CT_ASSERT(sizeof(ErtsAllctrDDBlock_t) == sizeof(void*)); #ifdef MBC_ABLK_OFFSET_BITS blk->u.carrier = crr; #else @@ -5942,7 +5942,7 @@ erts_alcu_init(AlcUInit_t *init) erts_atomic_init_nob(&sentinel->prev, (erts_aint_t) sentinel); } #endif - ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ + ERTS_CT_ASSERT(SBC_BLK_SZ_MASK == MBC_FBLK_SZ_MASK); /* see BLK_SZ */ #if HAVE_ERTS_MSEG ASSERT(erts_mseg_unit_size() == ERTS_SACRR_UNIT_SZ); max_mseg_carriers = init->mmc; diff --git a/erts/emulator/beam/erl_bif_trace.c b/erts/emulator/beam/erl_bif_trace.c index 06fbbea123..f5e582b1c5 100644 --- a/erts/emulator/beam/erl_bif_trace.c +++ b/erts/emulator/beam/erl_bif_trace.c @@ -651,7 +651,7 @@ Eterm trace_3(BIF_ALIST_3) if (pid_spec == am_all) { if (on) { if (!erts_cpu_timestamp) { -#ifdef HAVE_CLOCK_GETTIME +#ifdef HAVE_CLOCK_GETTIME_CPU_TIME /* Perhaps clock_gettime was found during config on a different machine than this. We check @@ -678,7 +678,7 @@ Eterm trace_3(BIF_ALIST_3) if (erts_start_now_cpu() < 0) { goto error; } -#endif /* HAVE_CLOCK_GETTIME */ +#endif /* HAVE_CLOCK_GETTIME_CPU_TIME */ erts_cpu_timestamp = !0; } } diff --git a/erts/emulator/beam/erl_db.c b/erts/emulator/beam/erl_db.c index 0d75dbbc04..4806befd99 100644 --- a/erts/emulator/beam/erl_db.c +++ b/erts/emulator/beam/erl_db.c @@ -753,6 +753,31 @@ BIF_RETTYPE ets_prev_2(BIF_ALIST_2) BIF_RET(ret); } +/* +** take(Tab, Key) +*/ +BIF_RETTYPE ets_take_2(BIF_ALIST_2) +{ + DbTable* tb; +#ifdef DEBUG + int cret; +#endif + Eterm ret; + CHECK_TABLES(); + + tb = db_get_table(BIF_P, BIF_ARG_1, DB_WRITE, LCK_WRITE_REC); + if (!tb) { + BIF_ERROR(BIF_P, BADARG); + } +#ifdef DEBUG + cret = +#endif + tb->common.meth->db_take(BIF_P, tb, BIF_ARG_2, &ret); + ASSERT(cret == DB_ERROR_NONE); + db_unlock(tb, LCK_WRITE_REC); + BIF_RET(ret); +} + /* ** update_element(Tab, Key, {Pos, Value}) ** update_element(Tab, Key, [{Pos, Value}]) diff --git a/erts/emulator/beam/erl_db_hash.c b/erts/emulator/beam/erl_db_hash.c index 82d1f894ae..c2157457a0 100644 --- a/erts/emulator/beam/erl_db_hash.c +++ b/erts/emulator/beam/erl_db_hash.c @@ -382,7 +382,7 @@ static HashDbTerm* search_list(DbTableHash* tb, Eterm key, static void shrink(DbTableHash* tb, int nactive); static void grow(DbTableHash* tb, int nactive); static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, - DbTableHash*); + Uint sz, DbTableHash*); static int analyze_pattern(DbTableHash *tb, Eterm pattern, struct mp_info *mpi); @@ -426,6 +426,7 @@ static int db_select_count_continue_hash(Process *p, DbTable *tbl, static int db_select_delete_continue_hash(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_take_hash(Process *, DbTable *, Eterm, Eterm *); static void db_print_hash(int to, void *to_arg, int show, @@ -536,6 +537,7 @@ DbTableMethod db_hash = db_select_delete_continue_hash, db_select_count_hash, db_select_count_continue_hash, + db_take_hash, db_delete_all_objects_hash, db_free_table_hash, db_free_table_continue_hash, @@ -860,34 +862,49 @@ Ldone: return ret; } +static Eterm +get_term_list(Process *p, DbTableHash *tb, Eterm key, HashValue hval, + HashDbTerm *b1, HashDbTerm **bend) +{ + HashDbTerm* b2 = b1->next; + Eterm copy; + Uint sz = b1->dbterm.size + 2; + + if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { + while (b2 && has_key(tb, b2, key, hval)) { + if (b2->hvalue != INVALID_HASH) + sz += b2->dbterm.size + 2; + + b2 = b2->next; + } + } + copy = build_term_list(p, b1, b2, sz, tb); + CHECK_TABLES(); + if (bend) { + *bend = b2; + } + return copy; +} + int db_get_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) { DbTableHash *tb = &tbl->hash; HashValue hval; int ix; - HashDbTerm* b1; + HashDbTerm* b; erts_smp_rwmtx_t* lck; hval = MAKE_HASH(key); lck = RLOCK_HASH(tb,hval); ix = hash_to_ix(tb, hval); - b1 = BUCKET(tb, ix); - - while(b1 != 0) { - if (has_live_key(tb,b1,key,hval)) { - HashDbTerm* b2 = b1->next; - Eterm copy; + b = BUCKET(tb, ix); - if (tb->common.status & (DB_BAG | DB_DUPLICATE_BAG)) { - while(b2 != NULL && has_key(tb,b2,key,hval)) - b2 = b2->next; - } - copy = build_term_list(p, b1, b2, tb); - CHECK_TABLES(); - *ret = copy; + while(b != 0) { + if (has_live_key(tb, b, key, hval)) { + *ret = get_term_list(p, tb, key, hval, b, NULL); goto done; } - b1 = b1->next; + b = b->next; } *ret = NIL; done: @@ -1221,7 +1238,7 @@ static int db_slot_hash(Process *p, DbTable *tbl, Eterm slot_term, Eterm *ret) lck = RLOCK_HASH(tb, slot); nactive = NACTIVE(tb); if (slot < nactive) { - *ret = build_term_list(p, BUCKET(tb, slot), 0, tb); + *ret = build_term_list(p, BUCKET(tb, slot), NULL, 0, tb); retval = DB_ERROR_NONE; } else if (slot == nactive) { @@ -2050,6 +2067,46 @@ trap: } +static int db_take_hash(Process *p, DbTable *tbl, Eterm key, Eterm *ret) +{ + DbTableHash *tb = &tbl->hash; + HashDbTerm **bp, *b; + HashValue hval = MAKE_HASH(key); + erts_smp_rwmtx_t *lck = WLOCK_HASH(tb, hval); + int ix = hash_to_ix(tb, hval); + int nitems_diff = 0; + + *ret = NIL; + for (bp = &BUCKET(tb, ix), b = *bp; b; bp = &b->next, b = b->next) { + if (has_live_key(tb, b, key, hval)) { + HashDbTerm *bend; + + *ret = get_term_list(p, tb, key, hval, b, &bend); + while (b != bend) { + --nitems_diff; + if (nitems_diff == -1 && IS_FIXED(tb)) { + /* Pseudo remove (no need to keep several of same key) */ + add_fixed_deletion(tb, ix); + bp = &b->next; + b->hvalue = INVALID_HASH; + b = b->next; + } else { + *bp = b->next; + free_term(tb, b); + b = *bp; + } + } + break; + } + } + WUNLOCK_HASH(lck); + if (nitems_diff) { + erts_smp_atomic_add_nob(&tb->common.nitems, nitems_diff); + try_shrink(tb); + } + return DB_ERROR_NONE; +} + /* ** Other interface routines (not directly coupled to one bif) */ @@ -2492,23 +2549,23 @@ static int free_seg(DbTableHash *tb, int free_records) ** Copy terms from ptr1 until ptr2 ** works for ptr1 == ptr2 == 0 => [] ** or ptr2 == 0 +** sz is either precalculated heap size or 0 if not known */ static Eterm build_term_list(Process* p, HashDbTerm* ptr1, HashDbTerm* ptr2, - DbTableHash* tb) + Uint sz, DbTableHash* tb) { - int sz = 0; HashDbTerm* ptr; Eterm list = NIL; Eterm copy; Eterm *hp, *hend; - ptr = ptr1; - while(ptr != ptr2) { - - if (ptr->hvalue != INVALID_HASH) - sz += ptr->dbterm.size + 2; - - ptr = ptr->next; + if (!sz) { + ptr = ptr1; + while(ptr != ptr2) { + if (ptr->hvalue != INVALID_HASH) + sz += ptr->dbterm.size + 2; + ptr = ptr->next; + } } hp = HAlloc(p, sz); diff --git a/erts/emulator/beam/erl_db_tree.c b/erts/emulator/beam/erl_db_tree.c index a62a83a928..720c0659c3 100644 --- a/erts/emulator/beam/erl_db_tree.c +++ b/erts/emulator/beam/erl_db_tree.c @@ -383,6 +383,7 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, Eterm pattern, Eterm *ret); static int db_select_delete_continue_tree(Process *p, DbTable *tbl, Eterm continuation, Eterm *ret); +static int db_take_tree(Process *, DbTable *, Eterm, Eterm *); static void db_print_tree(int to, void *to_arg, int show, DbTable *tbl); static int db_free_table_tree(DbTable *tbl); @@ -431,6 +432,7 @@ DbTableMethod db_tree = db_select_delete_continue_tree, db_select_count_tree, db_select_count_continue_tree, + db_take_tree, db_delete_all_objects_tree, db_free_table_tree, db_free_table_continue_tree, @@ -1722,6 +1724,28 @@ static int db_select_delete_tree(Process *p, DbTable *tbl, } +static int db_take_tree(Process *p, DbTable *tbl, Eterm key, Eterm *ret) +{ + DbTableTree *tb = &tbl->tree; + TreeDbTerm *this; + + *ret = NIL; + this = linkout_tree(tb, key, NULL); + if (this) { + Eterm copy, *hp, *hend; + + hp = HAlloc(p, this->dbterm.size + 2); + hend = hp + this->dbterm.size + 2; + copy = db_copy_object_from_ets(&tb->common, + &this->dbterm, &hp, &MSO(p)); + *ret = CONS(hp, copy, NIL); + hp += 2; + HRelease(p, hend, hp); + free_term(tb, this); + } + return DB_ERROR_NONE; +} + /* ** Other interface routines (not directly coupled to one bif) */ diff --git a/erts/emulator/beam/erl_db_util.c b/erts/emulator/beam/erl_db_util.c index b9fd3b208e..7eb80e3bb1 100644 --- a/erts/emulator/beam/erl_db_util.c +++ b/erts/emulator/beam/erl_db_util.c @@ -3254,34 +3254,38 @@ int db_is_variable(Eterm obj) /* return 1 if obj contains a variable or underscore */ /* return 0 if obj is fully ground */ -int db_has_variable(Eterm obj) -{ - switch(obj & _TAG_PRIMARY_MASK) { - case TAG_PRIMARY_LIST: { - while (is_list(obj)) { - if (db_has_variable(CAR(list_val(obj)))) +int db_has_variable(Eterm node) { + DECLARE_ESTACK(s); + + ESTACK_PUSH(s,node); + while (!ESTACK_ISEMPTY(s)) { + node = ESTACK_POP(s); + switch(node & _TAG_PRIMARY_MASK) { + case TAG_PRIMARY_LIST: + while (is_list(node)) { + ESTACK_PUSH(s,CAR(list_val(node))); + node = CDR(list_val(node)); + } + ESTACK_PUSH(s,node); /* Non wellformed list or [] */ + break; + case TAG_PRIMARY_BOXED: + if (is_tuple(node)) { + Eterm *tuple = tuple_val(node); + int arity = arityval(*tuple); + while(arity--) { + ESTACK_PUSH(s,*(++tuple)); + } + } + break; + case TAG_PRIMARY_IMMED1: + if (node == am_Underscore || db_is_variable(node) >= 0) { + DESTROY_ESTACK(s); return 1; - obj = CDR(list_val(obj)); - } - return(db_has_variable(obj)); /* Non wellformed list or [] */ - } - case TAG_PRIMARY_BOXED: - if (!is_tuple(obj)) { - return 0; - } else { - Eterm *tuple = tuple_val(obj); - int arity = arityval(*tuple++); - while(arity--) { - if (db_has_variable(*tuple)) - return 1; - tuple++; } - return(0); + break; } - case TAG_PRIMARY_IMMED1: - if (obj == am_Underscore || db_is_variable(obj) >= 0) - return 1; } + DESTROY_ESTACK(s); return 0; } diff --git a/erts/emulator/beam/erl_db_util.h b/erts/emulator/beam/erl_db_util.h index 328b19dfc9..5ace93c8ed 100644 --- a/erts/emulator/beam/erl_db_util.h +++ b/erts/emulator/beam/erl_db_util.h @@ -165,6 +165,7 @@ typedef struct db_table_method DbTable* tb, /* [in out] */ Eterm continuation, Eterm* ret); + int (*db_take)(Process *, DbTable *, Eterm, Eterm *); int (*db_delete_all_objects)(Process* p, DbTable* db /* [in out] */ ); diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c index 5f78a7b532..d1a7ee113b 100644 --- a/erts/emulator/beam/erl_gc.c +++ b/erts/emulator/beam/erl_gc.c @@ -175,15 +175,15 @@ erts_init_gc(void) int i = 0, ix; Sint max_heap_size = 0; - ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); - ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); - ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); - ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size)); - ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size)); - ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size)); - ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); - ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); - ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); + ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(struct erl_off_heap_header,thing_word)); + ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ErlFunThing,thing_word)); + ERTS_CT_ASSERT(offsetof(ProcBin,thing_word) == offsetof(ExternalThing,header)); + ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(struct erl_off_heap_header,size)); + ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlSubBin,size)); + ERTS_CT_ASSERT(offsetof(ProcBin,size) == offsetof(ErlHeapBin,size)); + ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(struct erl_off_heap_header,next)); + ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ErlFunThing,next)); + ERTS_CT_ASSERT(offsetof(ProcBin,next) == offsetof(ExternalThing,next)); erts_test_long_gc_sleep = 0; diff --git a/erts/emulator/beam/erl_instrument.c b/erts/emulator/beam/erl_instrument.c index df7c443387..da85b86c87 100644 --- a/erts/emulator/beam/erl_instrument.c +++ b/erts/emulator/beam/erl_instrument.c @@ -1226,7 +1226,7 @@ erts_instr_init(int stat, int map_stat) mem_anchor = NULL; /* Install instrumentation functions */ - ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); + ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *)real_allctrs,(void *)erts_allctrs,sizeof(erts_allctrs)); diff --git a/erts/emulator/beam/erl_mtrace.c b/erts/emulator/beam/erl_mtrace.c index c8bb126687..fa1bde1c87 100644 --- a/erts/emulator/beam/erl_mtrace.c +++ b/erts/emulator/beam/erl_mtrace.c @@ -627,7 +627,7 @@ erts_mtrace_install_wrapper_functions(void) if (erts_mtrace_enabled) { int i; /* Install trace functions */ - ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); + ERTS_CT_ASSERT(sizeof(erts_allctrs) == sizeof(real_allctrs)); sys_memcpy((void *) real_allctrs, (void *) erts_allctrs, diff --git a/erts/emulator/beam/erl_printf_term.c b/erts/emulator/beam/erl_printf_term.c index 74e38c13df..c982dc2080 100644 --- a/erts/emulator/beam/erl_printf_term.c +++ b/erts/emulator/beam/erl_printf_term.c @@ -594,7 +594,7 @@ erts_printf_term(fmtfn_t fn, void* arg, ErlPfEterm term, long precision, ErlPfEterm* term_base) { int res; - ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm)); + ERTS_CT_ASSERT(sizeof(ErlPfEterm) == sizeof(Eterm)); res = print_term(fn, arg, (Eterm)term, &precision, (Eterm*)term_base); if (res < 0) diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 1ec931e312..ba09ee57c2 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -715,72 +715,24 @@ sched_wall_time_ts(void) #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT -#ifdef ARCH_64 - -static ERTS_INLINE Uint64 -aschedtime_read(ErtsAtomicSchedTime *var) -{ - return (Uint64) erts_atomic_read_nob((erts_atomic_t *) var); -} - -static ERTS_INLINE void -aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val) -{ - erts_atomic_set_nob((erts_atomic_t *) var, (erts_aint_t) val); -} - -static ERTS_INLINE void -aschedtime_init(ErtsAtomicSchedTime *var) -{ - erts_atomic_init_nob((erts_atomic_t *) var, (erts_aint_t) 0); -} - -#elif defined(ARCH_32) - static ERTS_INLINE Uint64 aschedtime_read(ErtsAtomicSchedTime *var) { - erts_dw_aint_t dw; - erts_dw_atomic_read_nob((erts_dw_atomic_t *) var, &dw); -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw.dw_sint; -#else - { - Uint64 res; - res = (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw.sint[ERTS_DW_AINT_LOW_WORD]); - return res; - } -#endif + return (Uint64) erts_atomic64_read_nob((erts_atomic64_t *) var); } static ERTS_INLINE void aschedtime_set(ErtsAtomicSchedTime *var, Uint64 val) { - erts_dw_aint_t dw; -#ifdef ETHR_SU_DW_NAINT_T__ - dw.dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); - dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif - erts_dw_atomic_set_nob((erts_dw_atomic_t *) var, &dw); + erts_atomic64_set_nob((erts_atomic64_t *) var, (erts_aint64_t) val); } static ERTS_INLINE void aschedtime_init(ErtsAtomicSchedTime *var) { - erts_dw_aint_t dw; - dw.sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) 0; - dw.sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) 0; - erts_dw_atomic_init_nob((erts_dw_atomic_t *) var, &dw); + erts_atomic64_init_nob((erts_atomic64_t *) var, (erts_aint64_t) 0); } -#else -# error :-/ -#endif - #define ERTS_GET_AVG_MAX_UNLOCKED_TRY 50 #define ERTS_SCHED_AVG_UTIL_WRITE_MARKER (~((Uint64) 0)) @@ -2365,7 +2317,6 @@ erts_active_schedulers(void) ERTS_ATOMIC_FOREACH_RUNQ(rq, as -= abs(rq->waiting)); - ASSERT(as >= 0); return as; } @@ -10466,7 +10417,7 @@ alloc_process(ErtsRunQueue *rq, erts_aint32_t state) init_arg.run_queue = rq; init_arg.state = state; - ASSERT(((char *) p) == ((char *) &p->common)); + ERTS_CT_ASSERT(offsetof(Process,common) == 0); if (!erts_ptab_new_element(&erts_proc, &p->common, diff --git a/erts/emulator/beam/erl_process.h b/erts/emulator/beam/erl_process.h index a12b577656..d12ac792af 100644 --- a/erts/emulator/beam/erl_process.h +++ b/erts/emulator/beam/erl_process.h @@ -358,13 +358,7 @@ typedef struct { #undef ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT #define ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT_OPT -#ifdef ARCH_64 -typedef erts_atomic_t ErtsAtomicSchedTime; -#elif defined(ARCH_32) -typedef erts_dw_atomic_t ErtsAtomicSchedTime; -#else -# error :-/ -#endif +typedef erts_atomic64_t ErtsAtomicSchedTime; #if ERTS_HAVE_SCHED_UTIL_BALANCING_SUPPORT typedef struct { diff --git a/erts/emulator/beam/erl_ptab.c b/erts/emulator/beam/erl_ptab.c index eabf016081..02943ee683 100644 --- a/erts/emulator/beam/erl_ptab.c +++ b/erts/emulator/beam/erl_ptab.c @@ -280,124 +280,38 @@ struct ErtsPTabListBifData_ { }; -#ifdef ARCH_32 - -static ERTS_INLINE Uint64 -dw_aint_to_uint64(erts_dw_aint_t *dw) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw->dw_sint; -#else - Uint64 res; - res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif -} - -static void -unint64_to_dw_aint(erts_dw_aint_t *dw, Uint64 val) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - dw->dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw->sint[ERTS_DW_AINT_LOW_WORD] = (erts_aint_t) (val & 0xffffffff); - dw->sint[ERTS_DW_AINT_HIGH_WORD] = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif -} - static ERTS_INLINE void last_data_init_nob(ErtsPTab *ptab, Uint64 val) { - erts_dw_aint_t dw; - unint64_to_dw_aint(&dw, val); - erts_smp_dw_atomic_init_nob(&ptab->vola.tile.last_data, &dw); + erts_smp_atomic64_init_nob(&ptab->vola.tile.last_data, (erts_aint64_t) val); } static ERTS_INLINE void last_data_set_relb(ErtsPTab *ptab, Uint64 val) { - erts_dw_aint_t dw; - unint64_to_dw_aint(&dw, val); - erts_smp_dw_atomic_set_relb(&ptab->vola.tile.last_data, &dw); + erts_smp_atomic64_set_relb(&ptab->vola.tile.last_data, (erts_aint64_t) val); } static ERTS_INLINE Uint64 last_data_read_nob(ErtsPTab *ptab) { - erts_dw_aint_t dw; - erts_smp_dw_atomic_read_nob(&ptab->vola.tile.last_data, &dw); - return dw_aint_to_uint64(&dw); + return (Uint64) erts_smp_atomic64_read_nob(&ptab->vola.tile.last_data); } static ERTS_INLINE Uint64 last_data_read_acqb(ErtsPTab *ptab) { - erts_dw_aint_t dw; - erts_smp_dw_atomic_read_acqb(&ptab->vola.tile.last_data, &dw); - return dw_aint_to_uint64(&dw); + return (Uint64) erts_smp_atomic64_read_acqb(&ptab->vola.tile.last_data); } static ERTS_INLINE Uint64 last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp) { - erts_dw_aint_t dw_new, dw_xchg; - - unint64_to_dw_aint(&dw_new, new); - unint64_to_dw_aint(&dw_xchg, exp); - - if (erts_smp_dw_atomic_cmpxchg_relb(&ptab->vola.tile.last_data, - &dw_new, - &dw_xchg)) - return exp; - else - return dw_aint_to_uint64(&dw_xchg); -} - -#elif defined(ARCH_64) - -union { - erts_smp_atomic_t pid_data; - char align[ERTS_CACHE_LINE_SIZE]; -} last erts_align_attribute(ERTS_CACHE_LINE_SIZE); - -static ERTS_INLINE void -last_data_init_nob(ErtsPTab *ptab, Uint64 val) -{ - erts_smp_atomic_init_nob(&ptab->vola.tile.last_data, (erts_aint_t) val); + return (Uint64) erts_smp_atomic64_cmpxchg_relb(&ptab->vola.tile.last_data, + (erts_aint64_t) new, + (erts_aint64_t) exp); } -static ERTS_INLINE void -last_data_set_relb(ErtsPTab *ptab, Uint64 val) -{ - erts_smp_atomic_set_relb(&ptab->vola.tile.last_data, (erts_aint_t) val); -} - -static ERTS_INLINE Uint64 -last_data_read_nob(ErtsPTab *ptab) -{ - return (Uint64) erts_smp_atomic_read_nob(&ptab->vola.tile.last_data); -} - -static ERTS_INLINE Uint64 -last_data_read_acqb(ErtsPTab *ptab) -{ - return (Uint64) erts_smp_atomic_read_acqb(&ptab->vola.tile.last_data); -} - -static ERTS_INLINE Uint64 -last_data_cmpxchg_relb(ErtsPTab *ptab, Uint64 new, Uint64 exp) -{ - return (Uint64) erts_smp_atomic_cmpxchg_relb(&ptab->vola.tile.last_data, - (erts_aint_t) new, - (erts_aint_t) exp); -} - -#else -# error "Not 64-bit, nor 32-bit architecture..." -#endif - static ERTS_INLINE int last_data_cmp(Uint64 ld1, Uint64 ld2) { diff --git a/erts/emulator/beam/erl_ptab.h b/erts/emulator/beam/erl_ptab.h index e3e05f14af..876241159b 100644 --- a/erts/emulator/beam/erl_ptab.h +++ b/erts/emulator/beam/erl_ptab.h @@ -88,11 +88,7 @@ typedef struct { } ErtsPTabListData; typedef struct { -#ifdef ARCH_32 - erts_smp_dw_atomic_t last_data; -#else - erts_smp_atomic_t last_data; -#endif + erts_smp_atomic64_t last_data; erts_smp_atomic32_t count; erts_smp_atomic32_t aid_ix; erts_smp_atomic32_t fid_ix; diff --git a/erts/emulator/beam/erl_smp.h b/erts/emulator/beam/erl_smp.h index c38ef47d87..6c40edeb3e 100644 --- a/erts/emulator/beam/erl_smp.h +++ b/erts/emulator/beam/erl_smp.h @@ -60,6 +60,7 @@ typedef erts_tsd_key_t erts_smp_tsd_key_t; #define erts_smp_dw_atomic_t erts_dw_atomic_t #define erts_smp_atomic_t erts_atomic_t #define erts_smp_atomic32_t erts_atomic32_t +#define erts_smp_atomic64_t erts_atomic64_t typedef erts_spinlock_t erts_smp_spinlock_t; typedef erts_rwlock_t erts_smp_rwlock_t; void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */ @@ -95,6 +96,7 @@ typedef int erts_smp_tsd_key_t; #define erts_smp_dw_atomic_t erts_no_dw_atomic_t #define erts_smp_atomic_t erts_no_atomic_t #define erts_smp_atomic32_t erts_no_atomic32_t +#define erts_smp_atomic64_t erts_no_atomic64_t #if __GNUC__ > 2 typedef struct { } erts_smp_spinlock_t; typedef struct { } erts_smp_rwlock_t; @@ -489,6 +491,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_set_dirty erts_atomic32_set_dirty #define erts_smp_atomic32_read_dirty erts_atomic32_read_dirty +/* 64-bit atomics */ + +#define erts_smp_atomic64_init_nob erts_atomic64_init_nob +#define erts_smp_atomic64_set_nob erts_atomic64_set_nob +#define erts_smp_atomic64_read_nob erts_atomic64_read_nob +#define erts_smp_atomic64_inc_read_nob erts_atomic64_inc_read_nob +#define erts_smp_atomic64_dec_read_nob erts_atomic64_dec_read_nob +#define erts_smp_atomic64_inc_nob erts_atomic64_inc_nob +#define erts_smp_atomic64_dec_nob erts_atomic64_dec_nob +#define erts_smp_atomic64_add_read_nob erts_atomic64_add_read_nob +#define erts_smp_atomic64_add_nob erts_atomic64_add_nob +#define erts_smp_atomic64_read_bor_nob erts_atomic64_read_bor_nob +#define erts_smp_atomic64_read_band_nob erts_atomic64_read_band_nob +#define erts_smp_atomic64_xchg_nob erts_atomic64_xchg_nob +#define erts_smp_atomic64_cmpxchg_nob erts_atomic64_cmpxchg_nob +#define erts_smp_atomic64_read_bset_nob erts_atomic64_read_bset_nob + +#define erts_smp_atomic64_init_mb erts_atomic64_init_mb +#define erts_smp_atomic64_set_mb erts_atomic64_set_mb +#define erts_smp_atomic64_read_mb erts_atomic64_read_mb +#define erts_smp_atomic64_inc_read_mb erts_atomic64_inc_read_mb +#define erts_smp_atomic64_dec_read_mb erts_atomic64_dec_read_mb +#define erts_smp_atomic64_inc_mb erts_atomic64_inc_mb +#define erts_smp_atomic64_dec_mb erts_atomic64_dec_mb +#define erts_smp_atomic64_add_read_mb erts_atomic64_add_read_mb +#define erts_smp_atomic64_add_mb erts_atomic64_add_mb +#define erts_smp_atomic64_read_bor_mb erts_atomic64_read_bor_mb +#define erts_smp_atomic64_read_band_mb erts_atomic64_read_band_mb +#define erts_smp_atomic64_xchg_mb erts_atomic64_xchg_mb +#define erts_smp_atomic64_cmpxchg_mb erts_atomic64_cmpxchg_mb +#define erts_smp_atomic64_read_bset_mb erts_atomic64_read_bset_mb + +#define erts_smp_atomic64_init_acqb erts_atomic64_init_acqb +#define erts_smp_atomic64_set_acqb erts_atomic64_set_acqb +#define erts_smp_atomic64_read_acqb erts_atomic64_read_acqb +#define erts_smp_atomic64_inc_read_acqb erts_atomic64_inc_read_acqb +#define erts_smp_atomic64_dec_read_acqb erts_atomic64_dec_read_acqb +#define erts_smp_atomic64_inc_acqb erts_atomic64_inc_acqb +#define erts_smp_atomic64_dec_acqb erts_atomic64_dec_acqb +#define erts_smp_atomic64_add_read_acqb erts_atomic64_add_read_acqb +#define erts_smp_atomic64_add_acqb erts_atomic64_add_acqb +#define erts_smp_atomic64_read_bor_acqb erts_atomic64_read_bor_acqb +#define erts_smp_atomic64_read_band_acqb erts_atomic64_read_band_acqb +#define erts_smp_atomic64_xchg_acqb erts_atomic64_xchg_acqb +#define erts_smp_atomic64_cmpxchg_acqb erts_atomic64_cmpxchg_acqb +#define erts_smp_atomic64_read_bset_acqb erts_atomic64_read_bset_acqb + +#define erts_smp_atomic64_init_relb erts_atomic64_init_relb +#define erts_smp_atomic64_set_relb erts_atomic64_set_relb +#define erts_smp_atomic64_read_relb erts_atomic64_read_relb +#define erts_smp_atomic64_inc_read_relb erts_atomic64_inc_read_relb +#define erts_smp_atomic64_dec_read_relb erts_atomic64_dec_read_relb +#define erts_smp_atomic64_inc_relb erts_atomic64_inc_relb +#define erts_smp_atomic64_dec_relb erts_atomic64_dec_relb +#define erts_smp_atomic64_add_read_relb erts_atomic64_add_read_relb +#define erts_smp_atomic64_add_relb erts_atomic64_add_relb +#define erts_smp_atomic64_read_bor_relb erts_atomic64_read_bor_relb +#define erts_smp_atomic64_read_band_relb erts_atomic64_read_band_relb +#define erts_smp_atomic64_xchg_relb erts_atomic64_xchg_relb +#define erts_smp_atomic64_cmpxchg_relb erts_atomic64_cmpxchg_relb +#define erts_smp_atomic64_read_bset_relb erts_atomic64_read_bset_relb + +#define erts_smp_atomic64_init_ddrb erts_atomic64_init_ddrb +#define erts_smp_atomic64_set_ddrb erts_atomic64_set_ddrb +#define erts_smp_atomic64_read_ddrb erts_atomic64_read_ddrb +#define erts_smp_atomic64_inc_read_ddrb erts_atomic64_inc_read_ddrb +#define erts_smp_atomic64_dec_read_ddrb erts_atomic64_dec_read_ddrb +#define erts_smp_atomic64_inc_ddrb erts_atomic64_inc_ddrb +#define erts_smp_atomic64_dec_ddrb erts_atomic64_dec_ddrb +#define erts_smp_atomic64_add_read_ddrb erts_atomic64_add_read_ddrb +#define erts_smp_atomic64_add_ddrb erts_atomic64_add_ddrb +#define erts_smp_atomic64_read_bor_ddrb erts_atomic64_read_bor_ddrb +#define erts_smp_atomic64_read_band_ddrb erts_atomic64_read_band_ddrb +#define erts_smp_atomic64_xchg_ddrb erts_atomic64_xchg_ddrb +#define erts_smp_atomic64_cmpxchg_ddrb erts_atomic64_cmpxchg_ddrb +#define erts_smp_atomic64_read_bset_ddrb erts_atomic64_read_bset_ddrb + +#define erts_smp_atomic64_init_rb erts_atomic64_init_rb +#define erts_smp_atomic64_set_rb erts_atomic64_set_rb +#define erts_smp_atomic64_read_rb erts_atomic64_read_rb +#define erts_smp_atomic64_inc_read_rb erts_atomic64_inc_read_rb +#define erts_smp_atomic64_dec_read_rb erts_atomic64_dec_read_rb +#define erts_smp_atomic64_inc_rb erts_atomic64_inc_rb +#define erts_smp_atomic64_dec_rb erts_atomic64_dec_rb +#define erts_smp_atomic64_add_read_rb erts_atomic64_add_read_rb +#define erts_smp_atomic64_add_rb erts_atomic64_add_rb +#define erts_smp_atomic64_read_bor_rb erts_atomic64_read_bor_rb +#define erts_smp_atomic64_read_band_rb erts_atomic64_read_band_rb +#define erts_smp_atomic64_xchg_rb erts_atomic64_xchg_rb +#define erts_smp_atomic64_cmpxchg_rb erts_atomic64_cmpxchg_rb +#define erts_smp_atomic64_read_bset_rb erts_atomic64_read_bset_rb + +#define erts_smp_atomic64_init_wb erts_atomic64_init_wb +#define erts_smp_atomic64_set_wb erts_atomic64_set_wb +#define erts_smp_atomic64_read_wb erts_atomic64_read_wb +#define erts_smp_atomic64_inc_read_wb erts_atomic64_inc_read_wb +#define erts_smp_atomic64_dec_read_wb erts_atomic64_dec_read_wb +#define erts_smp_atomic64_inc_wb erts_atomic64_inc_wb +#define erts_smp_atomic64_dec_wb erts_atomic64_dec_wb +#define erts_smp_atomic64_add_read_wb erts_atomic64_add_read_wb +#define erts_smp_atomic64_add_wb erts_atomic64_add_wb +#define erts_smp_atomic64_read_bor_wb erts_atomic64_read_bor_wb +#define erts_smp_atomic64_read_band_wb erts_atomic64_read_band_wb +#define erts_smp_atomic64_xchg_wb erts_atomic64_xchg_wb +#define erts_smp_atomic64_cmpxchg_wb erts_atomic64_cmpxchg_wb +#define erts_smp_atomic64_read_bset_wb erts_atomic64_read_bset_wb + +#define erts_smp_atomic64_set_dirty erts_atomic64_set_dirty +#define erts_smp_atomic64_read_dirty erts_atomic64_read_dirty + #else /* !ERTS_SMP */ /* Double word size atomics */ @@ -751,6 +863,116 @@ ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig); #define erts_smp_atomic32_set_dirty erts_no_atomic32_set #define erts_smp_atomic32_read_dirty erts_no_atomic32_read +/* 64-bit atomics */ + +#define erts_smp_atomic64_init_nob erts_no_atomic64_set +#define erts_smp_atomic64_set_nob erts_no_atomic64_set +#define erts_smp_atomic64_read_nob erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_nob erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_nob erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_nob erts_no_atomic64_inc +#define erts_smp_atomic64_dec_nob erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_nob erts_no_atomic64_add_read +#define erts_smp_atomic64_add_nob erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_nob erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_nob erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_nob erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_nob erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_mb erts_no_atomic64_set +#define erts_smp_atomic64_set_mb erts_no_atomic64_set +#define erts_smp_atomic64_read_mb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_mb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_mb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_mb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_mb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_mb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_mb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_mb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_mb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_mb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_mb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_acqb erts_no_atomic64_set +#define erts_smp_atomic64_set_acqb erts_no_atomic64_set +#define erts_smp_atomic64_read_acqb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_acqb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_acqb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_acqb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_acqb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_acqb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_acqb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_acqb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_acqb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_acqb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_acqb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_relb erts_no_atomic64_set +#define erts_smp_atomic64_set_relb erts_no_atomic64_set +#define erts_smp_atomic64_read_relb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_relb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_relb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_relb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_relb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_relb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_relb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_relb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_relb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_relb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_relb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_ddrb erts_no_atomic64_set +#define erts_smp_atomic64_set_ddrb erts_no_atomic64_set +#define erts_smp_atomic64_read_ddrb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_ddrb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_ddrb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_ddrb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_ddrb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_ddrb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_ddrb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_ddrb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_ddrb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_ddrb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_ddrb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_rb erts_no_atomic64_set +#define erts_smp_atomic64_set_rb erts_no_atomic64_set +#define erts_smp_atomic64_read_rb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_rb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_rb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_rb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_rb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_rb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_rb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_rb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_rb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_rb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_rb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_init_wb erts_no_atomic64_set +#define erts_smp_atomic64_set_wb erts_no_atomic64_set +#define erts_smp_atomic64_read_wb erts_no_atomic64_read +#define erts_smp_atomic64_inc_read_wb erts_no_atomic64_inc_read +#define erts_smp_atomic64_dec_read_wb erts_no_atomic64_dec_read +#define erts_smp_atomic64_inc_wb erts_no_atomic64_inc +#define erts_smp_atomic64_dec_wb erts_no_atomic64_dec +#define erts_smp_atomic64_add_read_wb erts_no_atomic64_add_read +#define erts_smp_atomic64_add_wb erts_no_atomic64_add +#define erts_smp_atomic64_read_bor_wb erts_no_atomic64_read_bor +#define erts_smp_atomic64_read_band_wb erts_no_atomic64_read_band +#define erts_smp_atomic64_xchg_wb erts_no_atomic64_xchg +#define erts_smp_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg +#define erts_smp_atomic64_read_bset_wb erts_no_atomic64_read_bset + +#define erts_smp_atomic64_set_dirty erts_no_atomic64_set +#define erts_smp_atomic64_read_dirty erts_no_atomic64_read + #endif /* !ERTS_SMP */ #if ERTS_GLB_INLINE_INCL_FUNC_DEF diff --git a/erts/emulator/beam/erl_thr_progress.c b/erts/emulator/beam/erl_thr_progress.c index 3dca2aba10..c2365c5cf7 100644 --- a/erts/emulator/beam/erl_thr_progress.c +++ b/erts/emulator/beam/erl_thr_progress.c @@ -115,70 +115,24 @@ #undef read_nob #define read_nob erts_thr_prgr_read_nob__ -#ifdef ARCH_64 - static ERTS_INLINE void set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - erts_atomic_set_mb(atmc, val); + erts_atomic64_set_mb(atmc, (erts_aint64_t) val); } static ERTS_INLINE void set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - erts_atomic_set_nob(atmc, val); + erts_atomic64_set_nob(atmc, (erts_aint64_t) val); } static ERTS_INLINE void init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) { - erts_atomic_init_nob(atmc, val); -} - -#else - -#undef dw_aint_to_val -#define dw_aint_to_val erts_thr_prgr_dw_aint_to_val__ - -static void -val_to_dw_aint(erts_dw_aint_t *dw_aint, ErtsThrPrgrVal val) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - dw_aint->dw_sint = (ETHR_SU_DW_NAINT_T__) val; -#else - dw_aint->sint[ERTS_DW_AINT_LOW_WORD] - = (erts_aint_t) (val & 0xffffffff); - dw_aint->sint[ERTS_DW_AINT_HIGH_WORD] - = (erts_aint_t) ((val >> 32) & 0xffffffff); -#endif + erts_atomic64_init_nob(atmc, (erts_aint64_t) val); } -static ERTS_INLINE void -set_mb(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) -{ - erts_dw_aint_t dw_aint; - val_to_dw_aint(&dw_aint, val); - erts_dw_atomic_set_mb(atmc, &dw_aint); -} - -static ERTS_INLINE void -set_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) -{ - erts_dw_aint_t dw_aint; - val_to_dw_aint(&dw_aint, val); - erts_dw_atomic_set_nob(atmc, &dw_aint); -} - -static ERTS_INLINE void -init_nob(ERTS_THR_PRGR_ATOMIC *atmc, ErtsThrPrgrVal val) -{ - erts_dw_aint_t dw_aint; - val_to_dw_aint(&dw_aint, val); - erts_dw_atomic_init_nob(atmc, &dw_aint); -} - -#endif - /* #define ERTS_THR_PROGRESS_STATE_DEBUG */ #ifdef ERTS_THR_PROGRESS_STATE_DEBUG diff --git a/erts/emulator/beam/erl_thr_progress.h b/erts/emulator/beam/erl_thr_progress.h index 85b4eeb0eb..cf11c4e114 100644 --- a/erts/emulator/beam/erl_thr_progress.h +++ b/erts/emulator/beam/erl_thr_progress.h @@ -115,11 +115,7 @@ struct ErtsThrPrgrLaterOp_ { extern erts_tsd_key_t erts_thr_prgr_data_key__; -#ifdef ARCH_64 -# define ERTS_THR_PRGR_ATOMIC erts_atomic_t -#else /* ARCH_32 */ -# define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t -#endif +#define ERTS_THR_PRGR_ATOMIC erts_atomic64_t typedef struct { void *arg; @@ -158,10 +154,6 @@ void erts_thr_progress_unmanaged_continue__(int umrefc_ix); void erts_thr_progress_dbg_print_state(void); -#ifdef ARCH_32 -#define ERTS_THR_PRGR_ATOMIC erts_dw_atomic_t -ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint); -#endif ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc); ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc); @@ -184,68 +176,24 @@ ERTS_GLB_INLINE int erts_thr_progress_has_reached(ErtsThrPrgrVal val); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ARCH_64 - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) -{ - return (ErtsThrPrgrVal) erts_atomic_read_nob(atmc); -} - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) -{ - return (ErtsThrPrgrVal) erts_atomic_read_acqb(atmc); -} - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) -{ - return (ErtsThrPrgrVal) erts_atomic_read_mb(atmc); -} - -#else /* ARCH_32 */ - -ERTS_GLB_INLINE ErtsThrPrgrVal -erts_thr_prgr_dw_aint_to_val__(erts_dw_aint_t *dw_aint) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (ErtsThrPrgrVal) dw_aint->dw_sint; -#else - ErtsThrPrgrVal res; - res = (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (ErtsThrPrgrVal) ((Uint32) dw_aint->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif -} - ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_nob__(ERTS_THR_PRGR_ATOMIC *atmc) { - erts_dw_aint_t dw_aint; - erts_dw_atomic_read_nob(atmc, &dw_aint); - return erts_thr_prgr_dw_aint_to_val__(&dw_aint); + return (ErtsThrPrgrVal) erts_atomic64_read_nob(atmc); } ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_acqb__(ERTS_THR_PRGR_ATOMIC *atmc) { - erts_dw_aint_t dw_aint; - erts_dw_atomic_read_acqb(atmc, &dw_aint); - return erts_thr_prgr_dw_aint_to_val__(&dw_aint); + return (ErtsThrPrgrVal) erts_atomic64_read_acqb(atmc); } ERTS_GLB_INLINE ErtsThrPrgrVal erts_thr_prgr_read_mb__(ERTS_THR_PRGR_ATOMIC *atmc) { - erts_dw_aint_t dw_aint; - erts_dw_atomic_read_mb(atmc, &dw_aint); - return erts_thr_prgr_dw_aint_to_val__(&dw_aint); + return (ErtsThrPrgrVal) erts_atomic64_read_mb(atmc); } -#endif - ERTS_GLB_INLINE int erts_thr_progress_is_managed_thread(void) { diff --git a/erts/emulator/beam/erl_threads.h b/erts/emulator/beam/erl_threads.h index 463271252d..1fd800d524 100644 --- a/erts/emulator/beam/erl_threads.h +++ b/erts/emulator/beam/erl_threads.h @@ -344,6 +344,16 @@ typedef ethr_ts_event erts_tse_t; #define erts_aint32_t ethr_sint32_t #define erts_atomic32_t ethr_atomic32_t +#if defined(ARCH_32) +# define erts_atomic64_t ethr_dw_atomic_t +# define erts_aint64_t ethr_sint64_t +#elif defined(ARCH_64) +# define erts_atomic64_t ethr_atomic_t +# define erts_aint64_t ethr_sint_t +#else +# error "Not supported architecture" +#endif + #define ERTS_DW_AINT_HIGH_WORD ETHR_DW_SINT_HIGH_WORD #define ERTS_DW_AINT_LOW_WORD ETHR_DW_SINT_LOW_WORD @@ -414,10 +424,12 @@ typedef int erts_tse_t; typedef struct { SWord sint[2]; } erts_dw_aint_t; typedef SWord erts_aint_t; typedef Sint32 erts_aint32_t; +typedef Sint64 erts_aint64_t; #define erts_dw_atomic_t erts_dw_aint_t #define erts_atomic_t erts_aint_t #define erts_atomic32_t erts_aint32_t +#define erts_atomic64_t erts_aint64_t #if __GNUC__ > 2 typedef struct { } erts_spinlock_t; @@ -446,6 +458,7 @@ typedef struct { int gcc_is_buggy; } erts_rwlock_t; #define erts_no_dw_atomic_t erts_dw_aint_t #define erts_no_atomic_t erts_aint_t #define erts_no_atomic32_t erts_aint32_t +#define erts_no_atomic64_t erts_aint64_t #define ERTS_AINT_NULL ((erts_aint_t) NULL) @@ -571,6 +584,29 @@ ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bset(erts_no_atomic32_t *var, erts_aint32_t mask, erts_aint32_t set); +ERTS_GLB_INLINE void erts_no_atomic64_set(erts_no_atomic64_t *var, + erts_aint64_t i); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read(erts_no_atomic64_t *var); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_inc_read(erts_no_atomic64_t *incp); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_dec_read(erts_no_atomic64_t *decp); +ERTS_GLB_INLINE void erts_no_atomic64_inc(erts_no_atomic64_t *incp); +ERTS_GLB_INLINE void erts_no_atomic64_dec(erts_no_atomic64_t *decp); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_add_read(erts_no_atomic64_t *addp, + erts_aint64_t i); +ERTS_GLB_INLINE void erts_no_atomic64_add(erts_no_atomic64_t *addp, + erts_aint64_t i); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bor(erts_no_atomic64_t *var, + erts_aint64_t mask); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_band(erts_no_atomic64_t *var, + erts_aint64_t mask); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new, + erts_aint64_t expected); +ERTS_GLB_INLINE erts_aint64_t erts_no_atomic64_read_bset(erts_no_atomic64_t *var, + erts_aint64_t mask, + erts_aint64_t set); ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock, char *name, @@ -1207,6 +1243,441 @@ erts_atomic32_read_dirty(erts_atomic32_t *var) #endif +/* 64-bit atomics */ + +#if defined(ARCH_64) + +#define erts_atomic64_init_nob ethr_atomic_init +#define erts_atomic64_set_nob ethr_atomic_set +#define erts_atomic64_read_nob ethr_atomic_read +#define erts_atomic64_inc_read_nob ethr_atomic_inc_read +#define erts_atomic64_dec_read_nob ethr_atomic_dec_read +#define erts_atomic64_inc_nob ethr_atomic_inc +#define erts_atomic64_dec_nob ethr_atomic_dec +#define erts_atomic64_add_read_nob ethr_atomic_add_read +#define erts_atomic64_add_nob ethr_atomic_add +#define erts_atomic64_read_bor_nob ethr_atomic_read_bor +#define erts_atomic64_read_band_nob ethr_atomic_read_band +#define erts_atomic64_xchg_nob ethr_atomic_xchg +#define erts_atomic64_cmpxchg_nob ethr_atomic_cmpxchg +#define erts_atomic64_read_bset_nob erts_atomic_read_bset_nob + +#define erts_atomic64_init_mb ethr_atomic_init_mb +#define erts_atomic64_set_mb ethr_atomic_set_mb +#define erts_atomic64_read_mb ethr_atomic_read_mb +#define erts_atomic64_inc_read_mb ethr_atomic_inc_read_mb +#define erts_atomic64_dec_read_mb ethr_atomic_dec_read_mb +#define erts_atomic64_inc_mb ethr_atomic_inc_mb +#define erts_atomic64_dec_mb ethr_atomic_dec_mb +#define erts_atomic64_add_read_mb ethr_atomic_add_read_mb +#define erts_atomic64_add_mb ethr_atomic_add_mb +#define erts_atomic64_read_bor_mb ethr_atomic_read_bor_mb +#define erts_atomic64_read_band_mb ethr_atomic_read_band_mb +#define erts_atomic64_xchg_mb ethr_atomic_xchg_mb +#define erts_atomic64_cmpxchg_mb ethr_atomic_cmpxchg_mb +#define erts_atomic64_read_bset_mb erts_atomic_read_bset_mb + +#define erts_atomic64_init_acqb ethr_atomic_init_acqb +#define erts_atomic64_set_acqb ethr_atomic_set_acqb +#define erts_atomic64_read_acqb ethr_atomic_read_acqb +#define erts_atomic64_inc_read_acqb ethr_atomic_inc_read_acqb +#define erts_atomic64_dec_read_acqb ethr_atomic_dec_read_acqb +#define erts_atomic64_inc_acqb ethr_atomic_inc_acqb +#define erts_atomic64_dec_acqb ethr_atomic_dec_acqb +#define erts_atomic64_add_read_acqb ethr_atomic_add_read_acqb +#define erts_atomic64_add_acqb ethr_atomic_add_acqb +#define erts_atomic64_read_bor_acqb ethr_atomic_read_bor_acqb +#define erts_atomic64_read_band_acqb ethr_atomic_read_band_acqb +#define erts_atomic64_xchg_acqb ethr_atomic_xchg_acqb +#define erts_atomic64_cmpxchg_acqb ethr_atomic_cmpxchg_acqb +#define erts_atomic64_read_bset_acqb erts_atomic_read_bset_acqb + +#define erts_atomic64_init_relb ethr_atomic_init_relb +#define erts_atomic64_set_relb ethr_atomic_set_relb +#define erts_atomic64_read_relb ethr_atomic_read_relb +#define erts_atomic64_inc_read_relb ethr_atomic_inc_read_relb +#define erts_atomic64_dec_read_relb ethr_atomic_dec_read_relb +#define erts_atomic64_inc_relb ethr_atomic_inc_relb +#define erts_atomic64_dec_relb ethr_atomic_dec_relb +#define erts_atomic64_add_read_relb ethr_atomic_add_read_relb +#define erts_atomic64_add_relb ethr_atomic_add_relb +#define erts_atomic64_read_bor_relb ethr_atomic_read_bor_relb +#define erts_atomic64_read_band_relb ethr_atomic_read_band_relb +#define erts_atomic64_xchg_relb ethr_atomic_xchg_relb +#define erts_atomic64_cmpxchg_relb ethr_atomic_cmpxchg_relb +#define erts_atomic64_read_bset_relb erts_atomic_read_bset_relb + +#define erts_atomic64_init_ddrb ethr_atomic_init_ddrb +#define erts_atomic64_set_ddrb ethr_atomic_set_ddrb +#define erts_atomic64_read_ddrb ethr_atomic_read_ddrb +#define erts_atomic64_inc_read_ddrb ethr_atomic_inc_read_ddrb +#define erts_atomic64_dec_read_ddrb ethr_atomic_dec_read_ddrb +#define erts_atomic64_inc_ddrb ethr_atomic_inc_ddrb +#define erts_atomic64_dec_ddrb ethr_atomic_dec_ddrb +#define erts_atomic64_add_read_ddrb ethr_atomic_add_read_ddrb +#define erts_atomic64_add_ddrb ethr_atomic_add_ddrb +#define erts_atomic64_read_bor_ddrb ethr_atomic_read_bor_ddrb +#define erts_atomic64_read_band_ddrb ethr_atomic_read_band_ddrb +#define erts_atomic64_xchg_ddrb ethr_atomic_xchg_ddrb +#define erts_atomic64_cmpxchg_ddrb ethr_atomic_cmpxchg_ddrb +#define erts_atomic64_read_bset_ddrb erts_atomic_read_bset_ddrb + +#define erts_atomic64_init_rb ethr_atomic_init_rb +#define erts_atomic64_set_rb ethr_atomic_set_rb +#define erts_atomic64_read_rb ethr_atomic_read_rb +#define erts_atomic64_inc_read_rb ethr_atomic_inc_read_rb +#define erts_atomic64_dec_read_rb ethr_atomic_dec_read_rb +#define erts_atomic64_inc_rb ethr_atomic_inc_rb +#define erts_atomic64_dec_rb ethr_atomic_dec_rb +#define erts_atomic64_add_read_rb ethr_atomic_add_read_rb +#define erts_atomic64_add_rb ethr_atomic_add_rb +#define erts_atomic64_read_bor_rb ethr_atomic_read_bor_rb +#define erts_atomic64_read_band_rb ethr_atomic_read_band_rb +#define erts_atomic64_xchg_rb ethr_atomic_xchg_rb +#define erts_atomic64_cmpxchg_rb ethr_atomic_cmpxchg_rb +#define erts_atomic64_read_bset_rb erts_atomic_read_bset_rb + +#define erts_atomic64_init_wb ethr_atomic_init_wb +#define erts_atomic64_set_wb ethr_atomic_set_wb +#define erts_atomic64_read_wb ethr_atomic_read_wb +#define erts_atomic64_inc_read_wb ethr_atomic_inc_read_wb +#define erts_atomic64_dec_read_wb ethr_atomic_dec_read_wb +#define erts_atomic64_inc_wb ethr_atomic_inc_wb +#define erts_atomic64_dec_wb ethr_atomic_dec_wb +#define erts_atomic64_add_read_wb ethr_atomic_add_read_wb +#define erts_atomic64_add_wb ethr_atomic_add_wb +#define erts_atomic64_read_bor_wb ethr_atomic_read_bor_wb +#define erts_atomic64_read_band_wb ethr_atomic_read_band_wb +#define erts_atomic64_xchg_wb ethr_atomic_xchg_wb +#define erts_atomic64_cmpxchg_wb ethr_atomic_cmpxchg_wb +#define erts_atomic64_read_bset_wb erts_atomic_read_bset_wb + +#define erts_atomic64_set_dirty erts_atomic_set_dirty +#define erts_atomic64_read_dirty erts_atomic_read_dirty + +#elif defined(ARCH_32) + +#undef ERTS_ATOMIC64_OPS_DECL__ + +#define ERTS_ATOMIC64_OPS_DECL__(BARRIER) \ +ERTS_GLB_INLINE void \ +erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE void \ +erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE void \ +erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE void \ +erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE void \ +erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t new, \ + erts_aint64_t exp); \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t mask, \ + erts_aint64_t set) + +ERTS_ATOMIC64_OPS_DECL__(nob); +ERTS_ATOMIC64_OPS_DECL__(mb); +ERTS_ATOMIC64_OPS_DECL__(acqb); +ERTS_ATOMIC64_OPS_DECL__(relb); +ERTS_ATOMIC64_OPS_DECL__(ddrb); +ERTS_ATOMIC64_OPS_DECL__(rb); +ERTS_ATOMIC64_OPS_DECL__(wb); + +#undef ERTS_ATOMIC64_OPS_DECL__ + +ERTS_GLB_INLINE void +erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val); +ERTS_GLB_INLINE erts_aint64_t +erts_atomic64_read_dirty(erts_atomic64_t *var); + +#if ERTS_GLB_INLINE_INCL_FUNC_DEF + +/* + * The ethr_dw_atomic_*_nob() functions below + * are here to make it possible for the + * ERTS_ATOMIC64_OPS_IMPL__() to map erts + * barriers to ethread barriers... + */ +static ERTS_INLINE void +ethr_dw_atomic_init_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_init(var, val); +} + +static ERTS_INLINE void +ethr_dw_atomic_set_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_set(var, val); +} + +static ERTS_INLINE void +ethr_dw_atomic_read_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *val) +{ + ethr_dw_atomic_read(var, val); +} + +static ERTS_INLINE int +ethr_dw_atomic_cmpxchg_nob(ethr_dw_atomic_t *var, + ethr_dw_sint_t *new, + ethr_dw_sint_t *xchg) +{ + return ethr_dw_atomic_cmpxchg(var, new, xchg); +} + +#undef ERTS_ATOMIC64_OPS_IMPL__ +#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ +#undef ERTS_DW_SINT_TO_AINT64__ +#undef ERTS_AINT64_TO_DW_SINT__ + +#ifdef ETHR_SU_DW_NAINT_T__ +#define ERTS_DW_SINT_TO_AINT64__(DW) \ + ((erts_aint64_t) DW.dw_sint) +#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ + (DW.dw_sint = (ETHR_SU_DW_NAINT_T__) AINT64) +#else /* !ETHR_SU_DW_NAINT_T__ */ +#define ERTS_DW_SINT_TO_AINT64__(DW) \ + ((((erts_aint64_t) DW.sint[ETHR_DW_SINT_HIGH_WORD]) << 32) \ + | (((erts_aint64_t) DW.sint[ETHR_DW_SINT_LOW_WORD]) \ + & ((erts_aint64_t) 0xffffffff))) +#define ERTS_AINT64_TO_DW_SINT__(DW, AINT64) \ + do { \ + DW.sint[ETHR_DW_SINT_LOW_WORD] = \ + (ethr_sint_t) (AINT64 & 0xffffffff); \ + DW.sint[ETHR_DW_SINT_HIGH_WORD] = \ + (ethr_sint_t) ((AINT64 >> 32) & 0xffffffff); \ + } while (0) +#endif /* !ETHR_SU_DW_NAINT_T__ */ + +#define ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(CmpXchgOp, \ + AVarP, XchgVar, NewVar, \ + ModificationCode) \ +do { \ + ethr_dw_sint_t dw_xchg__, dw_new__; \ + ethr_dw_atomic_read(AVarP, &dw_xchg__); \ + do { \ + XchgVar = ERTS_DW_SINT_TO_AINT64__(dw_xchg__); \ + { \ + ModificationCode; \ + } \ + ERTS_AINT64_TO_DW_SINT__(dw_new__, NewVar); \ + } while (!CmpXchgOp((AVarP), &dw_new__, &dw_xchg__)); \ +} while (0) + +#define ERTS_ATOMIC64_OPS_IMPL__(BARRIER) \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_init_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + ethr_dw_sint_t dw; \ + ERTS_AINT64_TO_DW_SINT__(dw, val); \ + ethr_dw_atomic_init_ ## BARRIER(var, &dw); \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_set_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + ethr_dw_sint_t dw; \ + ERTS_AINT64_TO_DW_SINT__(dw, val); \ + ethr_dw_atomic_set_ ## BARRIER(var, &dw); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + ethr_dw_sint_t dw; \ + ethr_dw_atomic_read_ ## BARRIER(var, &dw); \ + return ERTS_DW_SINT_TO_AINT64__(dw); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_inc_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + 1); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_dec_read_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg - 1); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_inc_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + 1); \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_dec_ ## BARRIER(erts_atomic64_t *var) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg - 1); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_add_read_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + val); \ + return new; \ +} \ + \ +ERTS_GLB_INLINE void \ +erts_atomic64_add_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg + val); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bor_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg | val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_band_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = xchg & val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_xchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t val) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + new = val); \ + return xchg; \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_cmpxchg_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t new, \ + erts_aint64_t exp) \ +{ \ + ethr_dw_sint_t dw_xchg, dw_new; \ + ERTS_AINT64_TO_DW_SINT__(dw_xchg, exp); \ + ERTS_AINT64_TO_DW_SINT__(dw_new, new); \ + if (ethr_dw_atomic_cmpxchg_ ## BARRIER(var, &dw_new, &dw_xchg)) \ + return exp; \ + return ERTS_DW_SINT_TO_AINT64__(dw_xchg); \ +} \ + \ +ERTS_GLB_INLINE erts_aint64_t \ +erts_atomic64_read_bset_ ## BARRIER(erts_atomic64_t *var, \ + erts_aint64_t mask, \ + erts_aint64_t set) \ +{ \ + erts_aint64_t xchg, new; \ + ERTS_ATOMIC64_DW_CMPXCHG_IMPL__(ethr_dw_atomic_cmpxchg_ ## BARRIER, \ + var, xchg, new, \ + { \ + new = xchg & ~mask; \ + new |= mask & set; \ + }); \ + return xchg; \ +} + +ERTS_ATOMIC64_OPS_IMPL__(nob) +ERTS_ATOMIC64_OPS_IMPL__(mb) +ERTS_ATOMIC64_OPS_IMPL__(acqb) +ERTS_ATOMIC64_OPS_IMPL__(relb) +ERTS_ATOMIC64_OPS_IMPL__(ddrb) +ERTS_ATOMIC64_OPS_IMPL__(rb) +ERTS_ATOMIC64_OPS_IMPL__(wb) + +#undef ERTS_ATOMIC64_OPS_IMPL__ +#undef ERTS_ATOMIC64_DW_CMPXCHG_IMPL__ + +ERTS_GLB_INLINE void +erts_atomic64_set_dirty(erts_atomic64_t *var, erts_aint64_t val) +{ + ethr_sint_t *sint = ethr_dw_atomic_addr(var); + ethr_dw_sint_t dw; + ERTS_AINT64_TO_DW_SINT__(dw, val); + sint[0] = dw.sint[0]; + sint[1] = dw.sint[1]; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_atomic64_read_dirty(erts_atomic64_t *var) +{ + ethr_sint_t *sint; + ethr_dw_sint_t dw; + sint = ethr_dw_atomic_addr(var); + dw.sint[0] = sint[0]; + dw.sint[1] = sint[1]; + return ERTS_DW_SINT_TO_AINT64__(dw); +} + +#undef ERTS_DW_SINT_TO_AINT64__ +#undef ERTS_AINT64_TO_DW_SINT__ + +#endif /* ERTS_GLB_INLINE_INCL_FUNC_DEF */ + +#endif /* ARCH_32 */ + #else /* !USE_THREADS */ /* Double word size atomics */ @@ -1469,6 +1940,116 @@ erts_atomic32_read_dirty(erts_atomic32_t *var) #define erts_atomic32_set_dirty erts_no_atomic32_set #define erts_atomic32_read_dirty erts_no_atomic32_read +/* 64-bit atomics */ + +#define erts_atomic64_init_nob erts_no_atomic64_set +#define erts_atomic64_set_nob erts_no_atomic64_set +#define erts_atomic64_read_nob erts_no_atomic64_read +#define erts_atomic64_inc_read_nob erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_nob erts_no_atomic64_dec_read +#define erts_atomic64_inc_nob erts_no_atomic64_inc +#define erts_atomic64_dec_nob erts_no_atomic64_dec +#define erts_atomic64_add_read_nob erts_no_atomic64_add_read +#define erts_atomic64_add_nob erts_no_atomic64_add +#define erts_atomic64_read_bor_nob erts_no_atomic64_read_bor +#define erts_atomic64_read_band_nob erts_no_atomic64_read_band +#define erts_atomic64_xchg_nob erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_nob erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_nob erts_no_atomic64_read_bset + +#define erts_atomic64_init_mb erts_no_atomic64_set +#define erts_atomic64_set_mb erts_no_atomic64_set +#define erts_atomic64_read_mb erts_no_atomic64_read +#define erts_atomic64_inc_read_mb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_mb erts_no_atomic64_dec_read +#define erts_atomic64_inc_mb erts_no_atomic64_inc +#define erts_atomic64_dec_mb erts_no_atomic64_dec +#define erts_atomic64_add_read_mb erts_no_atomic64_add_read +#define erts_atomic64_add_mb erts_no_atomic64_add +#define erts_atomic64_read_bor_mb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_mb erts_no_atomic64_read_band +#define erts_atomic64_xchg_mb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_mb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_mb erts_no_atomic64_read_bset + +#define erts_atomic64_init_acqb erts_no_atomic64_set +#define erts_atomic64_set_acqb erts_no_atomic64_set +#define erts_atomic64_read_acqb erts_no_atomic64_read +#define erts_atomic64_inc_read_acqb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_acqb erts_no_atomic64_dec_read +#define erts_atomic64_inc_acqb erts_no_atomic64_inc +#define erts_atomic64_dec_acqb erts_no_atomic64_dec +#define erts_atomic64_add_read_acqb erts_no_atomic64_add_read +#define erts_atomic64_add_acqb erts_no_atomic64_add +#define erts_atomic64_read_bor_acqb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_acqb erts_no_atomic64_read_band +#define erts_atomic64_xchg_acqb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_acqb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_acqb erts_no_atomic64_read_bset + +#define erts_atomic64_init_relb erts_no_atomic64_set +#define erts_atomic64_set_relb erts_no_atomic64_set +#define erts_atomic64_read_relb erts_no_atomic64_read +#define erts_atomic64_inc_read_relb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_relb erts_no_atomic64_dec_read +#define erts_atomic64_inc_relb erts_no_atomic64_inc +#define erts_atomic64_dec_relb erts_no_atomic64_dec +#define erts_atomic64_add_read_relb erts_no_atomic64_add_read +#define erts_atomic64_add_relb erts_no_atomic64_add +#define erts_atomic64_read_bor_relb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_relb erts_no_atomic64_read_band +#define erts_atomic64_xchg_relb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_relb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_relb erts_no_atomic64_read_bset + +#define erts_atomic64_init_ddrb erts_no_atomic64_set +#define erts_atomic64_set_ddrb erts_no_atomic64_set +#define erts_atomic64_read_ddrb erts_no_atomic64_read +#define erts_atomic64_inc_read_ddrb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_ddrb erts_no_atomic64_dec_read +#define erts_atomic64_inc_ddrb erts_no_atomic64_inc +#define erts_atomic64_dec_ddrb erts_no_atomic64_dec +#define erts_atomic64_add_read_ddrb erts_no_atomic64_add_read +#define erts_atomic64_add_ddrb erts_no_atomic64_add +#define erts_atomic64_read_bor_ddrb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_ddrb erts_no_atomic64_read_band +#define erts_atomic64_xchg_ddrb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_ddrb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_ddrb erts_no_atomic64_read_bset + +#define erts_atomic64_init_rb erts_no_atomic64_set +#define erts_atomic64_set_rb erts_no_atomic64_set +#define erts_atomic64_read_rb erts_no_atomic64_read +#define erts_atomic64_inc_read_rb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_rb erts_no_atomic64_dec_read +#define erts_atomic64_inc_rb erts_no_atomic64_inc +#define erts_atomic64_dec_rb erts_no_atomic64_dec +#define erts_atomic64_add_read_rb erts_no_atomic64_add_read +#define erts_atomic64_add_rb erts_no_atomic64_add +#define erts_atomic64_read_bor_rb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_rb erts_no_atomic64_read_band +#define erts_atomic64_xchg_rb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_rb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_rb erts_no_atomic64_read_bset + +#define erts_atomic64_init_wb erts_no_atomic64_set +#define erts_atomic64_set_wb erts_no_atomic64_set +#define erts_atomic64_read_wb erts_no_atomic64_read +#define erts_atomic64_inc_read_wb erts_no_atomic64_inc_read +#define erts_atomic64_dec_read_wb erts_no_atomic64_dec_read +#define erts_atomic64_inc_wb erts_no_atomic64_inc +#define erts_atomic64_dec_wb erts_no_atomic64_dec +#define erts_atomic64_add_read_wb erts_no_atomic64_add_read +#define erts_atomic64_add_wb erts_no_atomic64_add +#define erts_atomic64_read_bor_wb erts_no_atomic64_read_bor +#define erts_atomic64_read_band_wb erts_no_atomic64_read_band +#define erts_atomic64_xchg_wb erts_no_atomic64_xchg +#define erts_atomic64_cmpxchg_wb erts_no_atomic64_cmpxchg +#define erts_atomic64_read_bset_wb erts_no_atomic64_read_bset + +#define erts_atomic64_set_dirty erts_no_atomic64_set +#define erts_atomic64_read_dirty erts_no_atomic64_read + #endif /* !USE_THREADS */ #if ERTS_GLB_INLINE_INCL_FUNC_DEF @@ -2400,6 +2981,104 @@ erts_no_atomic32_read_bset(erts_no_atomic32_t *var, return old; } +/* atomic64 */ + +ERTS_GLB_INLINE void +erts_no_atomic64_set(erts_no_atomic64_t *var, erts_aint64_t i) +{ + *var = i; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read(erts_no_atomic64_t *var) +{ + return *var; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_inc_read(erts_no_atomic64_t *incp) +{ + return ++(*incp); +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_dec_read(erts_no_atomic64_t *decp) +{ + return --(*decp); +} + +ERTS_GLB_INLINE void +erts_no_atomic64_inc(erts_no_atomic64_t *incp) +{ + ++(*incp); +} + +ERTS_GLB_INLINE void +erts_no_atomic64_dec(erts_no_atomic64_t *decp) +{ + --(*decp); +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_add_read(erts_no_atomic64_t *addp, erts_aint64_t i) +{ + return *addp += i; +} + +ERTS_GLB_INLINE void +erts_no_atomic64_add(erts_no_atomic64_t *addp, erts_aint64_t i) +{ + *addp += i; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_bor(erts_no_atomic64_t *var, erts_aint64_t mask) +{ + erts_aint64_t old; + old = *var; + *var |= mask; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_band(erts_no_atomic64_t *var, erts_aint64_t mask) +{ + erts_aint64_t old; + old = *var; + *var &= mask; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_xchg(erts_no_atomic64_t *xchgp, erts_aint64_t new) +{ + erts_aint64_t old = *xchgp; + *xchgp = new; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_cmpxchg(erts_no_atomic64_t *xchgp, + erts_aint64_t new, + erts_aint64_t expected) +{ + erts_aint64_t old = *xchgp; + if (old == expected) + *xchgp = new; + return old; +} + +ERTS_GLB_INLINE erts_aint64_t +erts_no_atomic64_read_bset(erts_no_atomic64_t *var, + erts_aint64_t mask, + erts_aint64_t set) +{ + erts_aint64_t old = *var; + *var &= ~mask; + *var |= (mask & set); + return old; +} + /* spinlock */ ERTS_GLB_INLINE void diff --git a/erts/emulator/beam/erl_time.h b/erts/emulator/beam/erl_time.h index 4bbdcaa3e3..7ed1a395ad 100644 --- a/erts/emulator/beam/erl_time.h +++ b/erts/emulator/beam/erl_time.h @@ -107,7 +107,7 @@ ERTS_GLB_INLINE void erts_do_time_add(erts_short_time_t elapsed) /* time_sup */ -#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME)) +#if (defined(HAVE_GETHRVTIME) || defined(HAVE_CLOCK_GETTIME_CPU_TIME)) # ifndef HAVE_ERTS_NOW_CPU # define HAVE_ERTS_NOW_CPU # ifdef HAVE_GETHRVTIME diff --git a/erts/emulator/beam/erl_trace.c b/erts/emulator/beam/erl_trace.c index 15648d6056..2f9969b0e7 100644 --- a/erts/emulator/beam/erl_trace.c +++ b/erts/emulator/beam/erl_trace.c @@ -2225,7 +2225,7 @@ trace_gc(Process *p, Eterm what) Eterm* limit; #endif - ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm)); + ERTS_CT_ASSERT(sizeof(values)/sizeof(*values) == sizeof(tags)/sizeof(Eterm)); UseTmpHeap(LOCAL_HEAP_SIZE,p); diff --git a/erts/emulator/beam/erl_utils.h b/erts/emulator/beam/erl_utils.h index 0807649ea1..c32f8fd61c 100644 --- a/erts/emulator/beam/erl_utils.h +++ b/erts/emulator/beam/erl_utils.h @@ -32,11 +32,7 @@ typedef struct { #endif union { Uint64 not_atomic; -#ifdef ARCH_64 - erts_atomic_t atomic; -#else - erts_dw_atomic_t atomic; -#endif + erts_atomic64_t atomic; } counter; } erts_interval_t; @@ -50,9 +46,6 @@ Uint64 erts_ensure_later_interval_nob(erts_interval_t *, Uint64); Uint64 erts_ensure_later_interval_acqb(erts_interval_t *, Uint64); Uint64 erts_smp_ensure_later_interval_nob(erts_interval_t *, Uint64); Uint64 erts_smp_ensure_later_interval_acqb(erts_interval_t *, Uint64); -#ifdef ARCH_32 -ERTS_GLB_INLINE Uint64 erts_interval_dw_aint_to_val__(erts_dw_aint_t *); -#endif ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *); ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *); ERTS_GLB_INLINE Uint64 erts_current_interval_nob(erts_interval_t *); @@ -62,46 +55,16 @@ ERTS_GLB_INLINE Uint64 erts_smp_current_interval_acqb(erts_interval_t *); #if ERTS_GLB_INLINE_INCL_FUNC_DEF -#ifdef ARCH_32 - -ERTS_GLB_INLINE Uint64 -erts_interval_dw_aint_to_val__(erts_dw_aint_t *dw) -{ -#ifdef ETHR_SU_DW_NAINT_T__ - return (Uint64) dw->dw_sint; -#else - Uint64 res; - res = (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_HIGH_WORD]); - res <<= 32; - res |= (Uint64) ((Uint32) dw->sint[ERTS_DW_AINT_LOW_WORD]); - return res; -#endif -} - -#endif - ERTS_GLB_INLINE Uint64 erts_current_interval_nob__(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_read_nob(&icp->counter.atomic); -#else - erts_dw_aint_t dw; - erts_dw_atomic_read_nob(&icp->counter.atomic, &dw); - return erts_interval_dw_aint_to_val__(&dw); -#endif + return (Uint64) erts_atomic64_read_nob(&icp->counter.atomic); } ERTS_GLB_INLINE Uint64 erts_current_interval_acqb__(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); -#else - erts_dw_aint_t dw; - erts_dw_atomic_read_acqb(&icp->counter.atomic, &dw); - return erts_interval_dw_aint_to_val__(&dw); -#endif + return (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic); } ERTS_GLB_INLINE Uint64 diff --git a/erts/emulator/beam/io.c b/erts/emulator/beam/io.c index 4d262ff022..d46ac9b336 100644 --- a/erts/emulator/beam/io.c +++ b/erts/emulator/beam/io.c @@ -391,7 +391,7 @@ static Port *create_port(char *name, /* Set default tracing */ erts_get_default_tracing(&ERTS_TRACE_FLAGS(prt), &ERTS_TRACER_PROC(prt)); - ASSERT(((char *) prt) == ((char *) &prt->common)); + ERTS_CT_ASSERT(offsetof(Port,common) == 0); #if !ERTS_PORT_INIT_INSTR_NEED_ID /* @@ -6698,7 +6698,7 @@ static void ref_to_driver_monitor(Eterm ref, ErlDrvMonitor *mon) { RefThing *refp; ASSERT(is_internal_ref(ref)); - ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor)); + ERTS_CT_ASSERT(sizeof(RefThing) <= sizeof(ErlDrvMonitor)); refp = ref_thing_ptr(ref); memset(mon,0,sizeof(ErlDrvMonitor)); memcpy(mon,refp,sizeof(RefThing)); diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index 9227ce74a2..828f5b427a 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -189,6 +189,22 @@ __decl_noreturn void __noreturn erl_assert_error(const char* expr, const char *f #endif /* + * Compile time assert + * (the actual compiler error msg can be a bit confusing) + */ +#if ERTS_AT_LEAST_GCC_VSN__(3,1,1) +# define ERTS_CT_ASSERT(e) \ + do { \ + enum { compile_time_assert__ = __builtin_choose_expr((e),0,(void)0) }; \ + } while(0) +#else +# define ERTS_CT_ASSERT(e) \ + do { \ + enum { compile_time_assert__ = 1/(e) }; \ + } while (0) +#endif + +/* * Microsoft C/C++: We certainly want to use stdarg.h and prototypes. * But MSC doesn't define __STDC__, unless we compile with the -Za * flag (strict ANSI C, no Microsoft extension). Compiling with -Za diff --git a/erts/emulator/beam/utils.c b/erts/emulator/beam/utils.c index e03cd22070..b341c4d949 100644 --- a/erts/emulator/beam/utils.c +++ b/erts/emulator/beam/utils.c @@ -4214,19 +4214,7 @@ void erts_silence_warn_unused_result(long unused) void erts_interval_init(erts_interval_t *icp) { -#ifdef ARCH_64 - erts_atomic_init_nob(&icp->counter.atomic, 0); -#else - erts_dw_aint_t dw; -#ifdef ETHR_SU_DW_NAINT_T__ - dw.dw_sint = 0; -#else - dw.sint[ERTS_DW_AINT_HIGH_WORD] = 0; - dw.sint[ERTS_DW_AINT_LOW_WORD] = 0; -#endif - erts_dw_atomic_init_nob(&icp->counter.atomic, &dw); - -#endif + erts_atomic64_init_nob(&icp->counter.atomic, 0); #ifdef DEBUG icp->smp_api = 0; #endif @@ -4248,55 +4236,13 @@ erts_smp_interval_init(erts_interval_t *icp) static ERTS_INLINE Uint64 step_interval_nob(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - } -#endif + return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic); } static ERTS_INLINE Uint64 step_interval_relb(erts_interval_t *icp) { -#ifdef ARCH_64 - return (Uint64) erts_atomic_inc_read_relb(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_relb(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - } -#endif + return (Uint64) erts_atomic64_inc_read_relb(&icp->counter.atomic); } @@ -4304,38 +4250,10 @@ static ERTS_INLINE Uint64 ensure_later_interval_nob(erts_interval_t *icp, Uint64 ic) { Uint64 curr_ic; -#ifdef ARCH_64 - curr_ic = (Uint64) erts_atomic_read_nob(&icp->counter.atomic); + curr_ic = (Uint64) erts_atomic64_read_nob(&icp->counter.atomic); if (curr_ic > ic) return curr_ic; - return (Uint64) erts_atomic_inc_read_nob(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_nob(&icp->counter.atomic, &exp); - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_nob(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - } -#endif + return (Uint64) erts_atomic64_inc_read_nob(&icp->counter.atomic); } @@ -4343,38 +4261,10 @@ static ERTS_INLINE Uint64 ensure_later_interval_acqb(erts_interval_t *icp, Uint64 ic) { Uint64 curr_ic; -#ifdef ARCH_64 - curr_ic = (Uint64) erts_atomic_read_acqb(&icp->counter.atomic); + curr_ic = (Uint64) erts_atomic64_read_acqb(&icp->counter.atomic); if (curr_ic > ic) return curr_ic; - return (Uint64) erts_atomic_inc_read_acqb(&icp->counter.atomic); -#else - erts_dw_aint_t exp; - - erts_dw_atomic_read_acqb(&icp->counter.atomic, &exp); - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - - while (1) { - erts_dw_aint_t new = exp; - -#ifdef ETHR_SU_DW_NAINT_T__ - new.dw_sint++; -#else - new.sint[ERTS_DW_AINT_LOW_WORD]++; - if (new.sint[ERTS_DW_AINT_LOW_WORD] == 0) - new.sint[ERTS_DW_AINT_HIGH_WORD]++; -#endif - - if (erts_dw_atomic_cmpxchg_acqb(&icp->counter.atomic, &new, &exp)) - return erts_interval_dw_aint_to_val__(&new); - - curr_ic = erts_interval_dw_aint_to_val__(&exp); - if (curr_ic > ic) - return curr_ic; - } -#endif + return (Uint64) erts_atomic64_inc_read_acqb(&icp->counter.atomic); } Uint64 |