From cfb7f3fcffad43647be5b9b818310b44003b97ab Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Thu, 16 Feb 2012 02:38:50 +0100 Subject: Misc memory barrier fixes - Document barrier semantics - Introduce ddrb suffix on atomic ops - Barrier macros for both non-SMP and SMP case - Make the thread progress API a bit more intuitive --- erts/lib_src/common/ethr_atomics.c | 303 +++++++++++++++++++++++++++++++++++- erts/lib_src/utils/make_atomics_api | 285 ++++++++++++++++++++------------- 2 files changed, 477 insertions(+), 111 deletions(-) (limited to 'erts/lib_src') diff --git a/erts/lib_src/common/ethr_atomics.c b/erts/lib_src/common/ethr_atomics.c index 5796bdc22e..e4213e1eef 100644 --- a/erts/lib_src/common/ethr_atomics.c +++ b/erts/lib_src/common/ethr_atomics.c @@ -10,7 +10,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2011. All Rights Reserved. + * Copyright Ericsson AB 2011-2012. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -64,12 +64,31 @@ * Appart from a function implementing the atomic operation * with unspecified memory barrier semantics, there are * functions implementing each operation with the following - * memory barrier semantics: - * - rb (read barrier) - * - wb (write barrier) - * - acqb (acquire barrier) - * - relb (release barrier) - * - mb (full memory barrier) + * implied memory barrier semantics: + * - mb - Full memory barrier. Orders both loads, and + * stores before, and after the atomic operation. + * No load or store is allowed to be reordered + * over the atomic operation. + * - relb - Release barrier. Orders both loads, and + * stores appearing *before* the atomic + * operation. These are not allowed to be + * reordered over the atomic operation. + * - acqb - Acquire barrier. Orders both loads, and stores + * appearing *after* the atomic operation. These + * are not allowed to be reordered over the + * atomic operation. + * - wb - Write barrier. Orders *only* stores. These are + * not allowed to be reordered over the barrier. + * Store in atomic operation is ordered *after* + * the barrier. + * - rb - Read barrier. Orders *only* loads. These are + * not allowed to be reordered over the barrier. + * Load in atomic operation is ordered *before* + * the barrier. + * - ddrb - Data dependency read barrier. Orders *only* + * loads according to data dependency across the + * barrier. Load in atomic operation is ordered + * before the barrier. * * We implement all of these operation/barrier * combinations, regardless of whether they are useful @@ -542,6 +561,15 @@ int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_s } #endif +int ETHR_DW_ATOMIC_FUNC__(cmpxchg_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ETHR_DW_ATOMIC_FUNC__(cmpxchg)(var, val, old_val); +#else + return ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(var, val, old_val); +#endif +} + int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val) { int res; @@ -756,6 +784,15 @@ void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) } #endif +void ETHR_DW_ATOMIC_FUNC__(set_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ETHR_DW_ATOMIC_FUNC__(set)(var, val); +#else + ETHR_DW_ATOMIC_FUNC__(set_rb)(var, val); +#endif +} + void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) { ETHR_ASSERT(!ethr_not_inited__); @@ -910,6 +947,15 @@ void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) } #endif +void ETHR_DW_ATOMIC_FUNC__(read_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ETHR_DW_ATOMIC_FUNC__(read)(var, val); +#else + ETHR_DW_ATOMIC_FUNC__(read_rb)(var, val); +#endif +} + void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) { ETHR_ASSERT(!ethr_not_inited__); @@ -1061,6 +1107,15 @@ void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) } #endif +void ETHR_DW_ATOMIC_FUNC__(init_ddrb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ETHR_DW_ATOMIC_FUNC__(init)(var, val); +#else + ETHR_DW_ATOMIC_FUNC__(init_rb)(var, val); +#endif +} + void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val) { ETHR_ASSERT(var); @@ -1221,6 +1276,15 @@ ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t return res; } +ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic_cmpxchg(var, val, old_val); +#else + return ethr_atomic_cmpxchg_rb(var, val, old_val); +#endif +} + ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val) { ethr_sint_t res; @@ -1332,6 +1396,15 @@ ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val) return res; } +ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic_xchg(var, val); +#else + return ethr_atomic_xchg_rb(var, val); +#endif +} + ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val) { ethr_sint_t res; @@ -1437,6 +1510,15 @@ void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val) } +void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic_set(var, val); +#else + ethr_atomic_set_rb(var, val); +#endif +} + void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val) { ETHR_ASSERT(!ethr_not_inited__); @@ -1536,6 +1618,15 @@ void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val) } +void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic_init(var, val); +#else + ethr_atomic_init_rb(var, val); +#endif +} + void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val) { ETHR_ASSERT(var); @@ -1632,6 +1723,15 @@ ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val) return res; } +ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic_add_read(var, val); +#else + return ethr_atomic_add_read_rb(var, val); +#endif +} + ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val) { ethr_sint_t res; @@ -1738,6 +1838,15 @@ ethr_sint_t ethr_atomic_read(ethr_atomic_t *var) return res; } +ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic_read(var); +#else + return ethr_atomic_read_rb(var); +#endif +} + ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var) { ethr_sint_t res; @@ -1843,6 +1952,15 @@ ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var) return res; } +ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic_inc_read(var); +#else + return ethr_atomic_inc_read_rb(var); +#endif +} + ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var) { ethr_sint_t res; @@ -1949,6 +2067,15 @@ ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var) return res; } +ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic_dec_read(var); +#else + return ethr_atomic_dec_read_rb(var); +#endif +} + ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var) { ethr_sint_t res; @@ -2054,6 +2181,15 @@ void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val) } +void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic_add(var, val); +#else + ethr_atomic_add_rb(var, val); +#endif +} + void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val) { ETHR_ASSERT(!ethr_not_inited__); @@ -2154,6 +2290,15 @@ void ethr_atomic_inc(ethr_atomic_t *var) } +void ethr_atomic_inc_ddrb(ethr_atomic_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic_inc(var); +#else + ethr_atomic_inc_rb(var); +#endif +} + void ethr_atomic_inc_rb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); @@ -2254,6 +2399,15 @@ void ethr_atomic_dec(ethr_atomic_t *var) } +void ethr_atomic_dec_ddrb(ethr_atomic_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic_dec(var); +#else + ethr_atomic_dec_rb(var); +#endif +} + void ethr_atomic_dec_rb(ethr_atomic_t *var) { ETHR_ASSERT(!ethr_not_inited__); @@ -2355,6 +2509,15 @@ ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val) return res; } +ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic_read_band(var, val); +#else + return ethr_atomic_read_band_rb(var, val); +#endif +} + ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val) { ethr_sint_t res; @@ -2461,6 +2624,15 @@ ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val) return res; } +ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic_read_bor(var, val); +#else + return ethr_atomic_read_bor_rb(var, val); +#endif +} + ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val) { ethr_sint_t res; @@ -2587,6 +2759,15 @@ ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, eth return res; } +ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic32_cmpxchg(var, val, old_val); +#else + return ethr_atomic32_cmpxchg_rb(var, val, old_val); +#endif +} + ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val) { ethr_sint32_t res; @@ -2675,6 +2856,15 @@ ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val) return res; } +ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic32_xchg(var, val); +#else + return ethr_atomic32_xchg_rb(var, val); +#endif +} + ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val) { ethr_sint32_t res; @@ -2762,6 +2952,15 @@ void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val) } +void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic32_set(var, val); +#else + ethr_atomic32_set_rb(var, val); +#endif +} + void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val) { ETHR_ASSERT(!ethr_not_inited__); @@ -2843,6 +3042,15 @@ void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val) } +void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic32_init(var, val); +#else + ethr_atomic32_init_rb(var, val); +#endif +} + void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val) { ETHR_ASSERT(var); @@ -2921,6 +3129,15 @@ ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val) return res; } +ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic32_add_read(var, val); +#else + return ethr_atomic32_add_read_rb(var, val); +#endif +} + ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val) { ethr_sint32_t res; @@ -3009,6 +3226,15 @@ ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var) return res; } +ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic32_read(var); +#else + return ethr_atomic32_read_rb(var); +#endif +} + ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var) { ethr_sint32_t res; @@ -3097,6 +3323,15 @@ ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var) return res; } +ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic32_inc_read(var); +#else + return ethr_atomic32_inc_read_rb(var); +#endif +} + ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var) { ethr_sint32_t res; @@ -3185,6 +3420,15 @@ ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var) return res; } +ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic32_dec_read(var); +#else + return ethr_atomic32_dec_read_rb(var); +#endif +} + ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var) { ethr_sint32_t res; @@ -3272,6 +3516,15 @@ void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val) } +void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic32_add(var, val); +#else + ethr_atomic32_add_rb(var, val); +#endif +} + void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val) { ETHR_ASSERT(!ethr_not_inited__); @@ -3354,6 +3607,15 @@ void ethr_atomic32_inc(ethr_atomic32_t *var) } +void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic32_inc(var); +#else + ethr_atomic32_inc_rb(var); +#endif +} + void ethr_atomic32_inc_rb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); @@ -3436,6 +3698,15 @@ void ethr_atomic32_dec(ethr_atomic32_t *var) } +void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + ethr_atomic32_dec(var); +#else + ethr_atomic32_dec_rb(var); +#endif +} + void ethr_atomic32_dec_rb(ethr_atomic32_t *var) { ETHR_ASSERT(!ethr_not_inited__); @@ -3519,6 +3790,15 @@ ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val) return res; } +ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic32_read_band(var, val); +#else + return ethr_atomic32_read_band_rb(var, val); +#endif +} + ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val) { ethr_sint32_t res; @@ -3607,6 +3887,15 @@ ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val) return res; } +ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val) +{ +#ifdef ETHR_ORDERED_READ_DEPEND + return ethr_atomic32_read_bor(var, val); +#else + return ethr_atomic32_read_bor_rb(var, val); +#endif +} + ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val) { ethr_sint32_t res; diff --git a/erts/lib_src/utils/make_atomics_api b/erts/lib_src/utils/make_atomics_api index f4e71c7618..d8b1a56100 100755 --- a/erts/lib_src/utils/make_atomics_api +++ b/erts/lib_src/utils/make_atomics_api @@ -4,7 +4,7 @@ %% %% %CopyrightBegin% %% -%% Copyright Ericsson AB 2011. All Rights Reserved. +%% Copyright Ericsson AB 2011-2012. All Rights Reserved. %% %% The contents of this file are subject to the Erlang Public License, %% Version 1.1, (the "License"); you may not use this file except in @@ -50,7 +50,9 @@ -define(DW_RTCHK_MACRO, "ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__"). %% Barrier versions we implement --define(BARRIERS, [none, rb, wb, acqb, relb, mb]). +-define(BARRIERS, [none, ddrb, rb, wb, acqb, relb, mb]). +-define(NON_NATIVE_BARRIERS, [ddrb]). +-define(NATIVE_BARRIERS, (?BARRIERS -- ?NON_NATIVE_BARRIERS)). -define(ATOMIC_SIZES, ["dword", "word", "32"]). @@ -381,7 +383,6 @@ do_func_header(#atomic_context{dw = true, cmpxchg, Inline, Func) -> [Inline, "int ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ", ", AintT, " *", Arg3, ")"]. - xbarriers(_Op, none, _NB) -> {"", ""}; @@ -455,7 +456,7 @@ try_barrier_order_first(mb) -> try_barrier_order(B) -> First = try_barrier_order_first(B), - First ++ (?BARRIERS -- First). + First ++ (?NATIVE_BARRIERS -- First). native_barrier_op(#atomic_context{'NATMC' = NATMC} = AC, If, ExtraDecl, Op, B, NB, TypeCasts) -> NOpStr = opstr(native(Op)), @@ -571,12 +572,12 @@ do_cmpxchg_fallback_define(#atomic_context{'NATMC' = NATMC, NoneTryBarrierOrder = try_barrier_order(none), %% First a sanity check [" -#if (", NotDefCMPXCHG(hd(?BARRIERS)) , +#if (", NotDefCMPXCHG(hd(?NATIVE_BARRIERS)) , lists:map(fun (B) -> [" \\ && ", NotDefCMPXCHG(B)] end, - tl(?BARRIERS)), ") + tl(?NATIVE_BARRIERS)), ") # error \"No native cmpxchg() op available\" #endif @@ -744,7 +745,7 @@ translate_have_defs(#atomic_context{dw = DW, 'NATMC' = NATMC}) -> ] end] end, - ?BARRIERS) + ?NATIVE_BARRIERS) end, case DW of true -> ?DW_ATOMIC_OPS; @@ -801,6 +802,46 @@ rtchk_fallback_call(Return, #atomic_context{dw = DW, false -> [RetVar, " ="] end, [?DW_FUNC_MACRO, "(", opstr(Op), op_barrier_ext(B), ")"], Arg1, Arg2, Arg3, ""). +non_native_barrier(B) -> + lists:member(B, ?NON_NATIVE_BARRIERS). + +non_native_barrier_impl(AC, Type, Macro, Op, B) -> + [" +", func_header(AC, Type, Macro, Op, B), " +{", + case B of + ddrb -> + [" +#ifdef ETHR_ORDERED_READ_DEPEND + ", func_call(AC, Type, Macro, Op, none, true), " +#else + ", func_call(AC, Type, Macro, Op, rb, true), " +#endif +" + ] + end, + "} +" + ]. + +func_call(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, _Macro, Op, B, RetStatement) -> + func_call(AC, Op, ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"], RetStatement); +func_call(#atomic_context{atomic = Atomic} = AC, implementation, false, Op, B, RetStatement) -> + func_call(AC, Op, [Atomic, "_", opstr(Op), op_barrier_ext(B)], RetStatement); +func_call(AC, implementation, Macro, Op, B, RetStatement) -> + func_call(AC, Op, [Macro, "(", opstr(Op), op_barrier_ext(B), ")"], RetStatement). + +func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, Op, Func, true) -> + op_call(Op, DW, case is_return_op(AC, Op) of + true -> "return"; + false -> "" + end, Func, Arg1, Arg2, Arg3, ""); +func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar} = AC, Op, Func, false) -> + op_call(Op, DW, case is_return_op(AC, Op) of + true -> [RetVar, " = "]; + false -> "" + end, Func, Arg1, Arg2, Arg3, ""). + make_implementations(#atomic_context{dw = DW, ret_type = RetType, ret_var = RetVar, @@ -858,73 +899,78 @@ make_implementations(#atomic_context{dw = DW, ", lists:map(fun (B) -> - TryBarriers = try_barrier_order(B), - [" + case non_native_barrier(B) of + true -> + non_native_barrier_impl(AC, inline_implementation, false, Op, B); + false -> + TryBarriers = try_barrier_order(B), + [" ", func_header(AC, inline_implementation, false, Op, B), " { ", - case is_return_op(AC, Op) of - true -> - [" ", RetType, " ", RetVar, ";\n"]; - _ -> "" - end, - case DW of - true -> - [RtchkBegin, - "\n", - su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)), - lists:map(fun (NB) -> - su_dw_native_barrier_op(AC, "#elif", Op, B, NB) - end, - tl(TryBarriers)), - lists:map(fun (NB) -> - dw_native_barrier_op(AC, "#elif", "", Op, B, NB) - end, - TryBarriers), - case simple_fallback(AC, Op, B) of - "" -> - %% No simple fallback available; - %% use cmpxchg() fallbacks... - [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B), - cmpxchg_fallbacks(AC, false, Op, B), - "#else + case is_return_op(AC, Op) of + true -> + [" ", RetType, " ", RetVar, ";\n"]; + _ -> "" + end, + case DW of + true -> + [RtchkBegin, + "\n", + su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)), + lists:map(fun (NB) -> + su_dw_native_barrier_op(AC, "#elif", Op, B, NB) + end, + tl(TryBarriers)), + lists:map(fun (NB) -> + dw_native_barrier_op(AC, "#elif", "", Op, B, NB) + end, + TryBarriers), + case simple_fallback(AC, Op, B) of + "" -> + %% No simple fallback available; + %% use cmpxchg() fallbacks... + [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B), + cmpxchg_fallbacks(AC, false, Op, B), + "#else #error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\" #endif " - ]; - SimpleFallback -> - ["#else\n", SimpleFallback, "#endif\n"] - end, - RtchkEnd(false, Op, B), "\n"]; - false -> - [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true), - lists:map(fun (NB) -> - native_barrier_op(AC, "#elif", "", Op, B, NB, true) - end, - tl(TryBarriers)), - case simple_fallback(AC, Op, B) of - "" -> - %% No simple fallback available; - %% use cmpxchg() fallbacks... - [cmpxchg_fallbacks(AC, false, Op, B), - "#else + ]; + SimpleFallback -> + ["#else\n", SimpleFallback, "#endif\n"] + end, + RtchkEnd(false, Op, B), "\n"]; + false -> + [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true), + lists:map(fun (NB) -> + native_barrier_op(AC, "#elif", "", Op, B, NB, true) + end, + tl(TryBarriers)), + case simple_fallback(AC, Op, B) of + "" -> + %% No simple fallback available; + %% use cmpxchg() fallbacks... + [cmpxchg_fallbacks(AC, false, Op, B), + "#else #error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\" #endif " - ]; - SimpleFallback -> - ["#else\n", SimpleFallback, "#endif\n"] - end] - end, - case is_return_op(AC, Op) of - true -> - [" return ", RetVar, ";\n"]; - false -> - "" - end, - "}\n"] + ]; + SimpleFallback -> + ["#else\n", SimpleFallback, "#endif\n"] + end] + end, + case is_return_op(AC, Op) of + true -> + [" return ", RetVar, ";\n"]; + false -> + "" + end, + "}\n"] + end end, - ?BARRIERS)] + ?NATIVE_BARRIERS ++ ?NON_NATIVE_BARRIERS)] %% non-native needs to be after native... end, case DW of true -> ?DW_ATOMIC_OPS; @@ -1159,33 +1205,38 @@ int ethr_have_native_dw_atomic(void) ", lists:map(fun (B) -> - ["\n", - func_header(AC, implementation, - case DW of - true -> ?DW_FUNC_MACRO; - false -> false - end, Op, B), - "\n{\n", - case is_return_op(AC, Op) of - true -> [" ", RetType, " ", RetVar, ";\n"]; - false -> "" - end, - case Op of - init -> ""; - _ -> [" ETHR_ASSERT(!ethr_not_inited__);\n"] - end, - [" ETHR_ASSERT(", Arg1, ");\n"], - make_native_impl_op(AC, Op, B), - make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), - make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), - case is_return_op(AC, Op) of - true -> [" return ", RetVar, ";" - ]; - false -> - "" - end, - "\n}\n", - make_symbol_to_fallback_impl(AC, Op, B)] + Macro = case DW of + true -> ?DW_FUNC_MACRO; + false -> false + end, + case non_native_barrier(B) of + true -> + non_native_barrier_impl(AC, implementation, Macro, Op, B); + false -> + ["\n", + func_header(AC, implementation, Macro, Op, B), + "\n{\n", + case is_return_op(AC, Op) of + true -> [" ", RetType, " ", RetVar, ";\n"]; + false -> "" + end, + case Op of + init -> ""; + _ -> [" ETHR_ASSERT(!ethr_not_inited__);\n"] + end, + [" ETHR_ASSERT(", Arg1, ");\n"], + make_native_impl_op(AC, Op, B), + make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), + make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), + case is_return_op(AC, Op) of + true -> [" return ", RetVar, ";" + ]; + false -> + "" + end, + "\n}\n", + make_symbol_to_fallback_impl(AC, Op, B)] + end end, ?BARRIERS)] end, @@ -1233,7 +1284,7 @@ static char *native_", DW, "atomic", Bits, "_ops[] = {", #endif" ] end, - ?BARRIERS) + ?NATIVE_BARRIERS) end, case NBits of "dw" -> ?DW_ATOMIC_OPS; @@ -1390,25 +1441,51 @@ comments() -> * Appart from a function implementing the atomic operation * with unspecified memory barrier semantics, there are * functions implementing each operation with the following - * memory barrier semantics: -", + * implied memory barrier semantics:", lists:map(fun (none) -> ""; - (rb) -> - [" * - rb (read barrier)\n"]; - (wb) -> - [" * - wb (write barrier)\n"]; + (mb) -> + [" + * - mb - Full memory barrier. Orders both loads, and + * stores before, and after the atomic operation. + * No load or store is allowed to be reordered + * over the atomic operation."]; (acqb) -> - [" * - acqb (acquire barrier)\n"]; + [" + * - acqb - Acquire barrier. Orders both loads, and stores + * appearing *after* the atomic operation. These + * are not allowed to be reordered over the + * atomic operation."]; (relb) -> - [" * - relb (release barrier)\n"]; - (mb) -> - [" * - mb (full memory barrier)\n"]; + [" + * - relb - Release barrier. Orders both loads, and + * stores appearing *before* the atomic + * operation. These are not allowed to be + * reordered over the atomic operation."]; + (rb) -> + [" + * - rb - Read barrier. Orders *only* loads. These are + * not allowed to be reordered over the barrier. + * Load in atomic operation is ordered *before* + * the barrier. "]; + (ddrb) -> + [" + * - ddrb - Data dependency read barrier. Orders *only* + * loads according to data dependency across the + * barrier. Load in atomic operation is ordered + * before the barrier."]; + (wb) -> + [" + * - wb - Write barrier. Orders *only* stores. These are + * not allowed to be reordered over the barrier. + * Store in atomic operation is ordered *after* + * the barrier."]; (B) -> [" * - ", a2l(B), "\n"] end, - ?BARRIERS), - " * + lists:reverse(?BARRIERS)), + " + * * We implement all of these operation/barrier * combinations, regardless of whether they are useful * or not (some of them are useless). -- cgit v1.2.3