#!/usr/bin/env escript %% -*- erlang -*- %% %% %CopyrightBegin% %% %% Copyright Ericsson AB 2011-2016. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. %% You may obtain a copy of the License at %% %% http://www.apache.org/licenses/LICENSE-2.0 %% %% Unless required by applicable law or agreed to in writing, software %% distributed under the License is distributed on an "AS IS" BASIS, %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. %% See the License for the specific language governing permissions and %% limitations under the License. %% %% %CopyrightEnd% %% -mode(compile). %%%------------------------------------------------------------------- %%% @author Rickard Green %%% @copyright (C) 2011, Rickard Green %%% @doc %%% Generation of the ethread atomic API %%% @end %%% Created : 17 Jan 2011 by Rickard Green %%%------------------------------------------------------------------- -define(H_FILE, "erts/include/internal/ethr_atomics.h"). -define(C_FILE, "erts/lib_src/common/ethr_atomics.c"). %% These order constraints are important: %% - 'cmpxchg' needs to appear before 'read' %% - 'xchg' needs to apper before 'set' %% - 'set' needs to apper before 'init' %% - 'add_read' needs to apper before 'add', 'inc_read', and 'dec_read' %% - 'inc_read' needs to apper before and 'inc' %% - 'dec_read' needs to apper before and 'dec' -define(ATOMIC_OPS, [cmpxchg, xchg, set, init, add_read, read, inc_read, dec_read, add, inc, dec, read_band, read_bor]). -define(DW_ATOMIC_OPS, [cmpxchg, set, read, init]). -define(DW_FUNC_MACRO, "ETHR_DW_ATOMIC_FUNC__"). -define(DW_RTCHK_MACRO, "ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__"). %% Barrier versions we implement -define(BARRIERS, [none, ddrb, rb, wb, acqb, relb, mb]). -define(NON_NATIVE_BARRIERS, [ddrb]). -define(NATIVE_BARRIERS, (?BARRIERS -- ?NON_NATIVE_BARRIERS)). -define(ATOMIC_SIZES, ["dword", "word", "32"]). -define(HAVE_NATIVE_ATOMIC, "ETHR_HAVE_ETHR_NATIVE_ATOMIC"). -define(SU_DW_SINT_FIELD, "dw_sint"). -define(DW_SINT_FIELD, "sint"). %% Fallback -define(ETHR_ATMC_FLLBK_ADDR_BITS, "10"). -define(ETHR_ATMC_FLLBK_ADDR_SHIFT, "6"). -record(atomic_context, {dw, amc_fallback, ret_type, ret_var, arg1, arg2, arg3, have_native_atomic_ops, atomic, atomic_t, addr_aint_t, aint_t, naint_t, 'NATMC', 'ATMC', unusual_val}). atomic_context("dword") -> #atomic_context{dw = true, amc_fallback = true, ret_type = "int", ret_var = "res", arg1 = "var", arg2 = "val", arg3 = "old_val", have_native_atomic_ops = "ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS", atomic = "ethr_dw_atomic", atomic_t = "ethr_dw_atomic_t", addr_aint_t = "ethr_sint_t", aint_t = "ethr_dw_sint_t", naint_t = "ETHR_SU_DW_NAINT_T__", 'NATMC' = "DW_NATMC", 'ATMC' = "DW_ATMC", unusual_val = "ETHR_UNUSUAL_SINT_VAL__"}; atomic_context(Size) -> {SizeSuffix, HaveSize, AMC} = case Size of "word" -> {"", "WORD_SZ", true}; _ -> {Size, Size++"BIT", false} end, AintT = ["ethr_sint", SizeSuffix, "_t"], #atomic_context{dw = false, amc_fallback = AMC, ret_type = AintT, ret_var = "res", arg1 = "var", arg2 = "val", arg3 = "old_val", have_native_atomic_ops = ["ETHR_HAVE_", HaveSize, "_NATIVE_ATOMIC_OPS"], atomic = ["ethr_atomic", SizeSuffix], atomic_t = ["ethr_atomic", SizeSuffix, "_t"], addr_aint_t = AintT, aint_t = AintT, naint_t = ["ETHR_NAINT", SizeSuffix, "_T__"], 'NATMC' = ["NATMC", SizeSuffix], 'ATMC' = ["ATMC", SizeSuffix], unusual_val = ["ETHR_UNUSUAL_SINT", SizeSuffix, "_VAL__"]}. -record(op_context, {ret, var, val1, val2}). -define(POTENTIAL_NBITS, ["64", "32"]). is_return_op(#atomic_context{dw = false}, add) -> false; is_return_op(#atomic_context{dw = false}, inc) -> false; is_return_op(#atomic_context{dw = false}, dec) -> false; is_return_op(#atomic_context{dw = true}, read) -> false; is_return_op(_AC, init) -> false; is_return_op(_AC, set) -> false; is_return_op(_AC, _OP) -> true. native(add_read) -> add_return; native(inc_read) -> inc_return; native(dec_read) -> dec_return; native(read_band) -> and_retold; native(read_bor) -> or_retold; native(Op) -> Op. op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set -> [Var, " = ", Val1]; op(read, #op_context{ret = Ret, var = Var}) -> [Ret, " = ", Var]; op(add_read, OpC) -> [op(add, OpC), "; ", op(read, OpC)]; op(add, #op_context{var = Var, val1 = Val1}) -> [Var, " += ", Val1]; op(inc, #op_context{var = Var}) -> ["++(", Var, ")"]; op(dec, #op_context{var = Var}) -> ["--(", Var, ")"]; op(inc_read, #op_context{ret = Ret, var = Var}) -> [Ret, " = ++(", Var, ")"]; op(dec_read, #op_context{ret = Ret, var = Var}) -> [Ret, " = --(", Var, ")"]; op(read_band, #op_context{var = Var, val1 = Val1} = OpC) -> [op(read, OpC), "; ", Var, " &= ", Val1]; op(read_bor, #op_context{var = Var, val1 = Val1} = OpC) -> [op(read, OpC), "; ", Var, " |= ", Val1]; op(xchg, OpC) -> [op(read, OpC), "; ", op(set, OpC)]; op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) -> [Ret, " = (", Var, " == ", Val2, " ? (", Var, " = ", Val1, ", ", Val2, ") : ", Var, ")"]. dw_op(Op, #op_context{var = Var, val1 = Val1}) when Op == init; Op == set -> [Var, "[0] = ", Val1, "[0]; ", Var, "[1] = ", Val1, "[1]"]; dw_op(read, #op_context{var = Var, val1 = Val1}) -> [Val1, "[0] = ", Var, "[0]; ", Val1, "[1] = ", Var, "[1]"]; dw_op(cmpxchg, #op_context{ret = Ret, var = Var, val1 = Val1, val2 = Val2}) -> [" { ", Ret, " = (", Var, "[0] == ", Val2, "[0] && ", Var, "[1] == ", Val2, "[1]); if (", Ret, ") { ", Var, "[0] = ", Val1, "[0]; ", Var, "[1] = ", Val1, "[1]; } else { ", Val2, "[0] = ", Var, "[0]; ", Val2, "[1] = ", Var, "[1]; } }"]. op_head_tail(init) -> {undef, undef}; op_head_tail(set) -> {store, store}; op_head_tail(read) -> {load, load}; op_head_tail(_) -> {load, undef}. op_barrier_ext(none) -> ""; op_barrier_ext(Barrier) -> [$_, a2l(Barrier)]. op_call(addr, _DW, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) -> [Ret, " ", Func, "(", Arg1, ");"]; op_call(Op, false, Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == read; Op == inc_read; Op == inc_return; Op == dec_read; Op == dec_return -> [Ret, " ", Func, "(", Arg1, ");"]; op_call(Op, false, _Ret, Func, Arg1, _Arg2, _Arg3, _TypeCast) when Op == inc; Op == dec -> [Func, "(", Arg1, ");"]; op_call(Op, false, Ret, Func, Arg1, Arg2, _Arg3, TypeCast) when Op == add_return; Op == add_read; Op == read_band; Op == and_retold; Op == read_bor; Op == or_retold; Op == xchg -> [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"]; op_call(cmpxchg, _DW, Ret, Func, Arg1, Arg2, Arg3, TypeCast) -> [Ret, " ", Func, "(", Arg1, ",", TypeCast, " ", Arg2, ",", TypeCast, " ", Arg3, ");"]; op_call(_Op, _DW, _Ret, Func, Arg1, Arg2, _Arg3, TypeCast) -> [Func, "(", Arg1, ",", TypeCast, " ", Arg2, ");"]. % set, init, add (!= dw), read (== dw) native_op_call(#atomic_context{dw = DW, ret_var = RetVar, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, aint_t = AintT, 'NATMC' = NATMC, naint_t = NAintT}, Op, B, TypeCasts) -> op_call(Op, DW, [RetVar, " =", case TypeCasts of true -> [" (", AintT, ")"]; false -> "" end], ["ETHR_", NATMC, "_FUNC__(", opstr(native(Op)), op_barrier_ext(B), ")"], Arg1, Arg2, Arg3, case TypeCasts of true -> [" (", NAintT, ")"]; false -> "" end). simple_fallback(#atomic_context{arg1 = Arg1, arg2 = Arg2, 'ATMC' = ATMC}, init, B) -> %% Also double word [" ETHR_", ATMC, "_FUNC__(set", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"]; simple_fallback(#atomic_context{dw = false, arg1 = Arg1, arg2 = Arg2, 'ATMC' = ATMC}, set, B) -> [" (void) ETHR_", ATMC, "_FUNC__(xchg", op_barrier_ext(B),")(", Arg1, ", ", Arg2, ");\n"]; simple_fallback(#atomic_context{dw = false, arg1 = Arg1, arg2 = Arg2, 'ATMC' = ATMC}, add, B) -> [" (void) ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", ", Arg2, ");\n"]; simple_fallback(#atomic_context{dw = false, ret_var = RetVar, arg1 = Arg1, aint_t = AintT, 'ATMC' = ATMC}, inc_read, B) -> [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") 1);\n"]; simple_fallback(#atomic_context{dw = false, ret_var = RetVar, arg1 = Arg1, aint_t = AintT, 'ATMC' = ATMC}, dec_read, B) -> [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(add_read", op_barrier_ext(B), ")(", Arg1, ", (", AintT,") -1);\n"]; simple_fallback(#atomic_context{dw = false, arg1 = Arg1, 'ATMC' = ATMC}, inc, B) -> [" (void) ETHR_", ATMC, "_FUNC__(inc_read", op_barrier_ext(B), ")(", Arg1, ");\n"]; simple_fallback(#atomic_context{dw = false, arg1 = Arg1, 'ATMC' = ATMC}, dec, B) -> [" (void) ETHR_", ATMC, "_FUNC__(dec_read", op_barrier_ext(B), ")(", Arg1, ");\n"]; simple_fallback(#atomic_context{dw = false, unusual_val = UnusualVal, ret_var = RetVar, arg1 = Arg1, aint_t = AintT, 'ATMC' = ATMC}, read, B) -> [" ", RetVar, " = ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", (", AintT, ") ", UnusualVal, ", (", AintT,") ", UnusualVal, ");\n"]; simple_fallback(#atomic_context{dw = true, unusual_val = UnusualVal, arg1 = Arg1, arg2 = Arg2, aint_t = AintT, 'ATMC' = ATMC}, read, B) -> [" ", AintT, " tmp; tmp.", ?DW_SINT_FIELD, "[0] = ", UnusualVal, "; tmp.", ?DW_SINT_FIELD, "[1] = ", UnusualVal, "; ", Arg2, "->", ?DW_SINT_FIELD, "[0] = ", UnusualVal, "; ", Arg2, "->", ?DW_SINT_FIELD, "[1] = ", UnusualVal, "; (void) ETHR_", ATMC, "_FUNC__(cmpxchg", op_barrier_ext(B), ")(", Arg1, ", &tmp, ", Arg2, "); " ]; simple_fallback(_AC, _Op, _B) -> []. func_header(AC, prototype, MacroName, Op, B) -> [func_header(AC, implementation, MacroName, Op, B), ";"]; func_header(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, _MacroName, Op, B) -> do_func_header(AC, Op, "static ETHR_INLINE ", ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"]); func_header(#atomic_context{atomic = Atomic} = AC, implementation, false, Op, B) -> do_func_header(AC, Op, "", [Atomic, "_", opstr(Op), op_barrier_ext(B)]); func_header(AC, implementation, MacroName, Op, B) -> do_func_header(AC, Op, "", [MacroName, "(", opstr(Op), op_barrier_ext(B), ")"]). do_func_header(#atomic_context{atomic_t = AtomicT, addr_aint_t = AddrAintT, arg1 = Arg1}, addr, Inline, Func) -> [Inline, AddrAintT, " *", Func, "(", AtomicT, " *", Arg1, ")"]; do_func_header(#atomic_context{dw = false, atomic_t = AtomicT, aint_t = AintT, arg1 = Arg1, arg2 = Arg2}, Op, Inline, Func) when Op == init; Op == set; Op == add -> [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"]; do_func_header(#atomic_context{dw = false, atomic_t = AtomicT, arg1 = Arg1}, Op, Inline, Func) when Op == inc; Op == dec -> [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ")"]; do_func_header(#atomic_context{dw = false, atomic_t = AtomicT, aint_t = AintT, arg1 = Arg1}, Op, Inline, Func) when Op == read; Op == inc_read; Op == dec_read -> [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ")"]; do_func_header(#atomic_context{dw = false, atomic_t = AtomicT, aint_t = AintT, arg1 = Arg1, arg2 = Arg2}, Op, Inline, Func) when Op == add_read; Op == read_band; Op == read_bor; Op == xchg -> [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ")"]; do_func_header(#atomic_context{dw = false, atomic_t = AtomicT, aint_t = AintT, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3}, cmpxchg, Inline, Func) -> [Inline, AintT, " ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " ", Arg2, ", ", AintT, " ", Arg3, ")"]; do_func_header(#atomic_context{dw = true, atomic_t = AtomicT, aint_t = AintT, arg1 = Arg1, arg2 = Arg2}, Op, Inline, Func) when Op == init; Op == set; Op == read -> [Inline, "void ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ")"]; do_func_header(#atomic_context{dw = true, atomic_t = AtomicT, aint_t = AintT, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3}, cmpxchg, Inline, Func) -> [Inline, "int ", Func, "(", AtomicT, " *", Arg1, ", ", AintT, " *", Arg2, ", ", AintT, " *", Arg3, ")"]. xbarriers(_Op, none, _NB) -> {"", ""}; xbarriers(_Op, acqb, NB) when NB == acqb; NB == mb -> {"", ""}; xbarriers(Op, acqb, NB) -> case {op_head_tail(Op), NB} of {{_, load}, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore);"}; {{_, load}, _} -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);"}; {{_, store}, _} -> {"", "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);"}; {_, rb} -> {"", "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"}; _ -> {"", "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"} end; xbarriers(_Op, relb, NB) when NB == relb; NB == mb -> {"", ""}; xbarriers(Op, relb, NB) -> case {op_head_tail(Op), NB} of {{store, _}, wb} -> {"ETHR_MEMBAR(ETHR_LoadStore);", ""}; {{store, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);", ""}; {{load, _}, _} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);", ""}; {_, wb} -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);", ""}; _ -> {"ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);", ""} end; xbarriers(_Op, wb, NB) when NB == wb; NB == mb -> {"", ""}; xbarriers(_Op, wb, _NB) -> {"ETHR_MEMBAR(ETHR_StoreStore);", ""}; xbarriers(_Op, rb, NB) when NB == rb; NB == mb -> {"", ""}; xbarriers(_Op, rb, _NB) -> {"", "ETHR_MEMBAR(ETHR_LoadLoad);"}; xbarriers(_Op, mb, mb) -> {"", ""}; xbarriers(Op, mb, NB) -> MB = "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);", {Head, Tail} = op_head_tail(Op), PreOp = case {Head, NB} of {_, relb} -> ""; {store, wb} -> "ETHR_MEMBAR(ETHR_LoadStore);"; {store, _} -> "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);"; {load, _} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);"; {_, wb} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad);"; _ -> MB end, PostOp = case {Tail, NB} of {_, acqb} -> ""; {load, rb} -> "ETHR_MEMBAR(ETHR_LoadStore);"; {load, _} -> "ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);"; {store, _} -> "ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);"; {_, rb} -> "ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);"; _ -> MB end, {PreOp, PostOp}. try_barrier_order_first(none) -> [none, rb, wb, acqb, relb]; try_barrier_order_first(acqb) -> [acqb, rb, none, mb]; try_barrier_order_first(relb) -> [relb, wb, none, mb]; try_barrier_order_first(rb) -> [rb, none, mb]; try_barrier_order_first(wb) -> [wb, none, mb]; try_barrier_order_first(mb) -> [mb, relb, acqb, wb, rb, none]. try_barrier_order(B) -> First = try_barrier_order_first(B), First ++ (?NATIVE_BARRIERS -- First). native_barrier_op(#atomic_context{'NATMC' = NATMC} = AC, If, ExtraDecl, Op, B, NB, TypeCasts) -> NOpStr = opstr(native(Op)), CapNOpStr = to_upper(NOpStr), NBExt = op_barrier_ext(NB), CapNBExt = to_upper(NBExt), {PreB, PostB} = xbarriers(Op, B, NB), [If, " defined(ETHR_HAVE_", NATMC, "_", CapNOpStr, CapNBExt, ")\n", ExtraDecl, case PreB of "" -> ""; _ -> [" ", PreB, "\n"] end, " ", native_op_call(AC, Op, NB, TypeCasts), "\n", case PostB of "" -> ""; _ -> [" ", PostB, "\n"] end]. dw_native_barrier_op(#atomic_context{arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, If, ExtraDecl, Op, B, NB) -> native_barrier_op(AC#atomic_context{arg1 = ["&", Arg1, "->native"], arg2 = [Arg2, "->", ?DW_SINT_FIELD], arg3 = [Arg3, "->", ?DW_SINT_FIELD]}, If, ExtraDecl, Op, B, NB, false). su_dw_native_barrier_op(#atomic_context{dw = true, naint_t = NAintT, ret_var = RetVar, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, 'NATMC' = NATMC} = AC, If, cmpxchg, B, NB) -> SU = ["->", ?SU_DW_SINT_FIELD], TmpVar = "act", SUArg1 = ["&", Arg1, "->native"], SUArg2 = [Arg2, SU], SUArg3 = [Arg3, SU], ExtraDecl = [" ", NAintT, " ", TmpVar, ";\n"], [native_barrier_op(AC#atomic_context{dw = false, ret_var = TmpVar, arg1 = SUArg1, arg2 = SUArg2, arg3 = SUArg3, 'NATMC' = ["SU_", NATMC]}, If, ExtraDecl, cmpxchg, B, NB, false), " ", RetVar, " = (", TmpVar, " == ", SUArg3, "); ", SUArg3, " = ", TmpVar, "; " ]; su_dw_native_barrier_op(#atomic_context{dw = true, arg1 = Arg1, arg2 = Arg2, 'NATMC' = NATMC} = AC, If, Op, B, NB) -> SUArg1 = ["&", Arg1, "->native"], SUArg2 = [Arg2, "->", ?SU_DW_SINT_FIELD], native_barrier_op(AC#atomic_context{dw = false, ret_var = SUArg2, arg1 = SUArg1, arg2 = SUArg2, arg3 = not_used, 'NATMC' = ["SU_", NATMC]}, If, "", Op, B, NB, false). cmpxchg_fallback_define(#atomic_context{dw = false, aint_t = AintT} = AC) -> do_cmpxchg_fallback_define(AC, true, AintT); cmpxchg_fallback_define(#atomic_context{dw = true, 'NATMC' = NATMC, naint_t = NAintT} = AC) -> ["\n\n#if defined(ETHR_HAVE_NATIVE_DW_ATOMIC)\n", do_cmpxchg_fallback_define(AC, false, not_used), "\n\n#elif defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)\n", do_cmpxchg_fallback_define(AC#atomic_context{'NATMC' = ["SU_", NATMC], naint_t = NAintT}, true, NAintT), " #else # error \"?!?\" #endif "]. do_cmpxchg_fallback_define(#atomic_context{'NATMC' = NATMC, aint_t = AintT, naint_t = NAintT}, SU, SUType) -> ReadFunc = fun (IF) -> fun (B) -> BExt = op_barrier_ext(B), CapBExt = to_upper(BExt), [IF, " defined(ETHR_HAVE_", NATMC, "_READ", CapBExt, ")", case SU of true -> [" #define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\ ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR) " ]; false -> [" #define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\ ETHR_", NATMC, "_FUNC__(read", BExt, ")(VAR, VAL) #elif defined(ETHR_HAVE_SU_", NATMC, "_READ", CapBExt, ") #define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\ VAL.", ?SU_DW_SINT_FIELD, " = ETHR_SU_", NATMC, "_FUNC__(read", BExt, ")(VAR) " ] end] end end, NotDefCMPXCHG = fun (B) -> CapBExt = to_upper(op_barrier_ext(B)), ["!defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapBExt, ")"] end, NoneTryBarrierOrder = try_barrier_order(none), %% First a sanity check [" #if (", NotDefCMPXCHG(hd(?NATIVE_BARRIERS)) , lists:map(fun (B) -> [" \\ && ", NotDefCMPXCHG(B)] end, tl(?NATIVE_BARRIERS)), ") # error \"No native cmpxchg() op available\" #endif /* * Read op used together with cmpxchg() fallback when no native op present. */ ", %% Read op to use with cmpxchg fallback (ReadFunc("#if"))(hd(NoneTryBarrierOrder)), lists:map(ReadFunc("#elif"), tl(NoneTryBarrierOrder)), "#else /* * We have no native read() op; guess zero and then use the * the atomics actual value returned from cmpxchg(). */", case SU of true -> [" #define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR) \\ ((", NAintT, ") 0)"]; false -> [" #define ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, VAL) \\ do { \\ VAL.", ?DW_SINT_FIELD, "[0] = (ethr_sint_t) 0; \\ VAL.", ?DW_SINT_FIELD, "[1] = (ethr_sint_t) 0; \\ } while (0)"] end, " #endif ", %% The fallback " /* * Native cmpxchg() fallback used when no native op present. */ #define ETHR_", NATMC, "_CMPXCHG_FALLBACK__(CMPXCHG, VAR, AVAL, OPS) \\ do { \\", case SU of true -> [" ", SUType, " AVAL; \\ ", NAintT, " new__, act__, exp__; \\ act__ = ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR); \\ do { \\ exp__ = act__; \\ AVAL = (", SUType, ") act__; \\ { OPS; } \\ new__ = (", NAintT, ") AVAL; \\ act__ = CMPXCHG(VAR, new__, exp__); \\ } while (__builtin_expect(act__ != exp__, 0)); \\"]; false -> [" int res__; \\ ", AintT, " AVAL, exp_act__; \\ ETHR_", NATMC, "_CMPXCHG_FALLBACK_READ__(VAR, exp_act__); \\ do { \\ AVAL.", ?DW_SINT_FIELD, "[0] = exp_act__.", ?DW_SINT_FIELD, "[0]; \\ AVAL.", ?DW_SINT_FIELD, "[1] = exp_act__.", ?DW_SINT_FIELD, "[1]; \\ { OPS; } \\ res__ = CMPXCHG(VAR, AVAL.", ?DW_SINT_FIELD, ", exp_act__.", ?DW_SINT_FIELD, "); \\ } while (__builtin_expect(res__ == 0, 0)); \\"] end, " } while (0) " ]. cmpxchg_fallbacks(#atomic_context{}, _SUDW, cmpxchg, _B) -> ""; %% No need for a fallback cmpxchg_fallbacks(#atomic_context{dw = DW, ret_var = RetVar, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, 'NATMC' = NATMC}, SUDW, Op, B) -> Operation = case DW of false -> op(Op, #op_context{ret = RetVar, var = "aval", val1 = Arg2, val2 = Arg3}); true -> case SUDW of true -> op(Op, #op_context{ret = [Arg2, "->", ?SU_DW_SINT_FIELD], var = "aval", val1 = [Arg2, "->", ?SU_DW_SINT_FIELD]}); false -> dw_op(Op, #op_context{ret = RetVar, var = ["aval.", ?DW_SINT_FIELD], val1 = [Arg2, "->", ?DW_SINT_FIELD]}) end end, [lists:map(fun (NB) -> NativeVar = case DW of true -> ["&", Arg1, "->native"]; false -> Arg1 end, NBExt = op_barrier_ext(NB), CapNBExt = to_upper(NBExt), {PreB, PostB} = xbarriers(cmpxchg, B, NB), ["#elif defined(ETHR_HAVE_", NATMC, "_CMPXCHG", CapNBExt, ")\n", case PreB of "" -> ""; _ -> [" ", PreB, "\n"] end, " ETHR_", NATMC, "_CMPXCHG_FALLBACK__(ETHR_", NATMC, "_FUNC__(cmpxchg", NBExt, "), ", NativeVar, ", aval, ", Operation, ");\n", case PostB of "" -> ""; _ -> [" ", PostB, "\n"] end] end, try_barrier_order(B))]. translate_have_defs(#atomic_context{dw = DW, 'NATMC' = NATMC}) -> [" #if !defined(ETHR_", NATMC, "_BITS__) # error \"Missing native atomic implementation\"", lists:map(fun (NBits) -> {HaveInPrefix, HaveOutPrefix, HaveInPrefixExtra, HaveOutPrefixExtra, NativeTypeCheck} = case NBits of "dw" -> {"ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC", ["ETHR_HAVE_", NATMC], "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC", ["ETHR_HAVE_SU_", NATMC], "\n#elif defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC)"}; _ -> {[?HAVE_NATIVE_ATOMIC, NBits], case DW of true -> ["ETHR_HAVE_SU_", NATMC]; false -> ["ETHR_HAVE_", NATMC] end, false, ["ETHR_HAVE_", NATMC], ["\n#elif ETHR_", NATMC, "_BITS__ == ", NBits]} end, [NativeTypeCheck, lists:map(fun (Op) -> NOpStr = opstr(native(Op)), CapNOpStr = to_upper(NOpStr), lists:map(fun (B) -> NBExt = op_barrier_ext(B), CapNBExt = to_upper(NBExt), HaveOutDef = [HaveOutPrefix, "_", CapNOpStr, CapNBExt], HaveOutDefExtra = [HaveOutPrefixExtra, "_", CapNOpStr, CapNBExt], [case DW of true -> ["\n# undef ", HaveOutDefExtra]; false -> "" end, " # undef ", HaveOutDef," # ifdef ", HaveInPrefix, "_", CapNOpStr, CapNBExt, " # define ", HaveOutDef, " 1 # endif", case HaveInPrefixExtra of false -> ""; _ -> [" # ifdef ", HaveInPrefixExtra, "_", CapNOpStr, CapNBExt, " # define ", HaveOutDefExtra, " 1 # endif" ] end] end, ?NATIVE_BARRIERS) end, case DW of true -> ?DW_ATOMIC_OPS; false -> ?ATOMIC_OPS end)] end, case DW of true -> ["dw", "64"]; false -> ?POTENTIAL_NBITS end), " #else # error \"Invalid native atomic size\" #endif "]. make_prototypes(#atomic_context{dw = DW, 'ATMC' = ATMC} = AC) -> MkProt = fun (MacroName) -> %% addr() is special [func_header(AC, prototype, MacroName, addr, none), "\n", lists:map(fun (Op) -> lists:map(fun (B) -> [func_header(AC, prototype, MacroName, Op, B), "\n"] end, ?BARRIERS) end, case DW of true -> ?DW_ATOMIC_OPS; false -> ?ATOMIC_OPS end)] end, [" #ifdef ETHR_NEED_", ATMC, "_PROTOTYPES__ ", MkProt(false), case DW of true -> ["#if defined(", ?DW_RTCHK_MACRO, ")\n", MkProt(?DW_FUNC_MACRO), "#endif\n"]; false -> "" end, "#endif /* ETHR_NEED_", ATMC, "_PROTOTYPES__ */\n"]. rtchk_fallback_call(Return, #atomic_context{dw = DW, ret_var = RetVar, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3}, Op, B) -> op_call(Op, DW, case Return of true -> "return"; false -> [RetVar, " ="] end, [?DW_FUNC_MACRO, "(", opstr(Op), op_barrier_ext(B), ")"], Arg1, Arg2, Arg3, ""). non_native_barrier(B) -> lists:member(B, ?NON_NATIVE_BARRIERS). non_native_barrier_impl(AC, inline_implementation = Type, Op, B) -> [" ", func_header(AC, Type, false, Op, B), " {", case B of ddrb -> [" #ifdef ETHR_ORDERED_READ_DEPEND ", func_call(AC, Type, Op, none, true), " #else ", func_call(AC, Type, Op, rb, true), " #endif " ] end, "} " ]; non_native_barrier_impl(#atomic_context{have_native_atomic_ops = HaveNative} = AC, implementation = Type, Op, B) -> [" ", func_header(AC, Type, false, Op, B), " {", case B of ddrb -> [" #if defined(", HaveNative, ") ", func_call(AC, Type, Op, B, true), " #elif defined(ETHR_ORDERED_READ_DEPEND) ", func_call(AC, symbol_implementation, Op, none, true), " #else ", func_call(AC, symbol_implementation, Op, rb, true), " #endif " ] end, "} " ]. func_call(#atomic_context{'ATMC' = ATMC} = AC, inline_implementation, Op, B, RetStatement) -> func_call(AC, Op, ["ETHR_", ATMC, "_FUNC__(", opstr(Op), op_barrier_ext(B), ")"], RetStatement); func_call(#atomic_context{atomic = Atomic} = AC, implementation, Op, B, RetStatement) -> func_call(AC, Op, [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], RetStatement); func_call(#atomic_context{atomic = Atomic} = AC, symbol_implementation, Op, B, RetStatement) -> func_call(AC, Op, [Atomic, "_", opstr(Op), op_barrier_ext(B)], RetStatement). func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, Op, Func, true) -> op_call(Op, DW, case is_return_op(AC, Op) of true -> "return"; false -> "" end, Func, Arg1, Arg2, Arg3, ""); func_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar} = AC, Op, Func, false) -> op_call(Op, DW, case is_return_op(AC, Op) of true -> [RetVar, " = "]; false -> "" end, Func, Arg1, Arg2, Arg3, ""). make_implementations(#atomic_context{dw = DW, ret_type = RetType, ret_var = RetVar, arg1 = Arg1, addr_aint_t = AddrAintT, atomic = Atomic, have_native_atomic_ops = HaveNativeAtomicOps, 'ATMC' = ATMC, 'NATMC' = NATMC} = AC) -> NativeVar = case DW of true -> ["(&", Arg1, "->native)"]; false -> Arg1 end, RtchkBegin = [" #if defined(", ?DW_RTCHK_MACRO, ") if (", ?DW_RTCHK_MACRO, ") { #endif "], RtchkEnd = fun (Return, Operation, Barrier) -> [" #if defined(", ?DW_RTCHK_MACRO, ") } else { ", rtchk_fallback_call(Return, AC, Operation, Barrier), " } #endif\n" ] end, [" #if (defined(", HaveNativeAtomicOps, ") \\ && (defined(ETHR_", ATMC, "_INLINE__) || defined(ETHR_ATOMIC_IMPL__))) ", translate_have_defs(AC), cmpxchg_fallback_define(AC), %% addr() is special " /* --- addr() --- */ ", func_header(AC, inline_implementation, false, addr, none), " {", case DW of true -> RtchkBegin; false -> "" end, " return (", AddrAintT, " *) ETHR_", NATMC, "_ADDR_FUNC__(", NativeVar, "); ",case DW of true -> RtchkEnd(true, addr, none); false -> "" end, " } ", lists:map(fun (Op) -> OpStr = opstr(Op), [" /* --- ", OpStr, "() --- */ ", lists:map(fun (B) -> case non_native_barrier(B) of true -> non_native_barrier_impl(AC, inline_implementation, Op, B); false -> TryBarriers = try_barrier_order(B), [" ", func_header(AC, inline_implementation, false, Op, B), " { ", case is_return_op(AC, Op) of true -> [" ", RetType, " ", RetVar, ";\n"]; _ -> "" end, case DW of true -> [RtchkBegin, "\n", su_dw_native_barrier_op(AC, "#if", Op, B, hd(TryBarriers)), lists:map(fun (NB) -> su_dw_native_barrier_op(AC, "#elif", Op, B, NB) end, tl(TryBarriers)), lists:map(fun (NB) -> dw_native_barrier_op(AC, "#elif", "", Op, B, NB) end, TryBarriers), case simple_fallback(AC, Op, B) of "" -> %% No simple fallback available; %% use cmpxchg() fallbacks... [cmpxchg_fallbacks(AC#atomic_context{'NATMC' = ["SU_", NATMC]}, true, Op, B), cmpxchg_fallbacks(AC, false, Op, B), "#else #error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\" #endif " ]; SimpleFallback -> ["#else\n", SimpleFallback, "#endif\n"] end, RtchkEnd(false, Op, B), "\n"]; false -> [native_barrier_op(AC, "#if", "", Op, B, hd(TryBarriers), true), lists:map(fun (NB) -> native_barrier_op(AC, "#elif", "", Op, B, NB, true) end, tl(TryBarriers)), case simple_fallback(AC, Op, B) of "" -> %% No simple fallback available; %% use cmpxchg() fallbacks... [cmpxchg_fallbacks(AC, false, Op, B), "#else #error \"Missing implementation of ", Atomic, "_", opstr(Op), op_barrier_ext(B), "()!\" #endif " ]; SimpleFallback -> ["#else\n", SimpleFallback, "#endif\n"] end] end, case is_return_op(AC, Op) of true -> [" return ", RetVar, ";\n"]; false -> "" end, "}\n"] end end, ?NATIVE_BARRIERS ++ ?NON_NATIVE_BARRIERS)] %% non-native needs to be after native... end, case DW of true -> ?DW_ATOMIC_OPS; false -> ?ATOMIC_OPS end), " #endif /* ETHR_", ATMC, "_INLINE__ */ " ]. atomic_implementation_comment(AtomicSize) -> CSz = case AtomicSize of "dword" -> "Double word size"; "word" -> "Word size"; _ -> AtomicSize ++ "-bit" end, [" /* ---------- ", CSz, " atomic implementation ---------- */ " ]. write_h_file(FileName) -> {ok, FD} = file:open(FileName, [write, latin1]), ok = file:write(FD, comments()), ok = file:write(FD, " #ifndef ETHR_ATOMICS_H__ #define ETHR_ATOMICS_H__ " ), ok = file:write(FD, h_top()), ok = lists:foreach(fun (AtomicSize) -> AC = atomic_context(AtomicSize), ok = file:write(FD, [atomic_implementation_comment(AtomicSize), make_prototypes(AC), make_implementations(AC)]) end, ?ATOMIC_SIZES), ok = file:write(FD, " #endif /* ETHR_ATOMICS_H__ */ " ), ok = file:close(FD). make_native_impl_op(#atomic_context{dw = DW, atomic = Atomic, have_native_atomic_ops = HaveNativeAtomicOps, ret_var = RetVar, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3}, Op, B) -> ["#if defined(", HaveNativeAtomicOps, ")", case DW of true -> [" && !defined(", ?DW_RTCHK_MACRO, ")"]; false -> "" end, "\n", " ", op_call(Op, DW, [RetVar, " = "], [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""), "\n"]. amc_op_dw_arg(#atomic_context{dw = false}) -> "0"; amc_op_dw_arg(#atomic_context{dw = true}) -> "1". amc_op_arg_prefix(#atomic_context{dw = false}) -> "&"; amc_op_arg_prefix(#atomic_context{dw = true}) -> "". amc_sint_arg(#atomic_context{dw = DW, arg2 = Arg}, arg2) -> amc_sint_arg(DW, Arg); amc_sint_arg(#atomic_context{dw = DW, arg3 = Arg}, arg3) -> amc_sint_arg(DW, Arg); amc_sint_arg(#atomic_context{dw = DW, ret_var = Arg}, ret_var) -> amc_sint_arg(DW, Arg); amc_sint_arg(true, Arg) -> [Arg, "->" ?DW_SINT_FIELD]; amc_sint_arg(false, Arg) -> ["&", Arg]. amc_op_call(#atomic_context{arg1 = Arg1} = AC, init) -> [" amc_init(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; amc_op_call(#atomic_context{arg1 = Arg1} = AC, set) -> [" amc_set(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; amc_op_call(#atomic_context{dw = false, arg1 = Arg1} = AC, read) -> [" amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, ret_var), ");\n"]; amc_op_call(#atomic_context{dw = true, arg1 = Arg1} = AC, read) -> [" amc_read(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ");\n"]; amc_op_call(#atomic_context{dw = false, arg1 = Arg1, arg3 = Arg3, ret_var = RetVar} = AC, cmpxchg) -> [" ", RetVar, " = ", Arg3, "; (void) amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, ret_var), ");\n"]; amc_op_call(#atomic_context{dw = true, arg1 = Arg1, ret_var = RetVar} = AC, cmpxchg) -> [" ", RetVar, " = amc_cmpxchg(&", Arg1, "->amc, ", amc_op_dw_arg(AC), ", ", amc_op_arg_prefix(AC), Arg1, "->sint, ", amc_sint_arg(AC, arg2), ", ", amc_sint_arg(AC, arg3), ");\n"]; amc_op_call(#atomic_context{dw = DW, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3, ret_var = RetVar}, Op) -> OpCtxt = #op_context{ret = RetVar, var = [Arg1,"->sint"], val1 = Arg2, val2 = Arg3}, OpStr = case DW of true -> dw_op(Op, OpCtxt); false -> op(Op, OpCtxt) end, [" ETHR_AMC_MODIFICATION_OPS__(&", Arg1, "->amc, ", OpStr, ");\n"]. make_amc_fallback_op(#atomic_context{amc_fallback = false}, _Op, _B) -> ""; make_amc_fallback_op(#atomic_context{amc_fallback = true} = AC, Op, B) -> NB = case Op of read -> rb; _ -> none end, {PreB, PostB} = xbarriers(Op, B, NB), ["#elif defined(ETHR_AMC_FALLBACK__)\n", case PreB of "" -> ""; _ -> [" ", PreB, "\n"] end, amc_op_call(AC, Op), case PostB of "" -> ""; _ -> [" ", PostB, "\n"] end]. make_locked_fallback_op(#atomic_context{dw = DW, ret_var = RetVar, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3}, Op, B) -> OpStr = case DW of true -> dw_op(Op, #op_context{ret = RetVar, var = [Arg1, "->" ?DW_SINT_FIELD], val1 = [Arg2, "->" ?DW_SINT_FIELD], val2 = [Arg3, "->" ?DW_SINT_FIELD]}); false -> op(Op, #op_context{ret = RetVar, var = ["*", Arg1], val1 = Arg2, val2 = Arg3}) end, {PreB, PostB} = xbarriers(Op, B, none), ["#else\n", case PreB of "" -> ""; _ -> [" ", PreB, "\n"] end, [" ETHR_ATOMIC_OP_FALLBACK_IMPL__(", Arg1, ", ", OpStr, ");\n"], case PostB of "" -> ""; _ -> [" ", PostB, "\n"] end, "#endif\n"]. make_symbol_to_fallback_impl(#atomic_context{dw = true, atomic = Atomic, arg1 = Arg1, arg2 = Arg2, arg3 = Arg3} = AC, Op, B) -> [" #ifdef ", ?DW_RTCHK_MACRO, " ", func_header(AC, implementation, false, Op, B), " {", case Op of init -> ""; _ -> ["\n ETHR_ASSERT(!ethr_not_inited__);"] end, " ETHR_ASSERT(", Arg1, "); ", op_call(Op, true, "return", [Atomic, "_", opstr(Op), op_barrier_ext(B), "__"], Arg1, Arg2, Arg3, ""), " } #endif " ]; make_symbol_to_fallback_impl(_, _, _) -> "". make_symbol_implementations(#atomic_context{dw = DW, amc_fallback = AMC, ret_type = RetType, addr_aint_t = AddrAintT, ret_var = RetVar, arg1 = Arg1} = AC) -> FallbackVar = case DW of true -> ["(&", Arg1, "->fallback)"]; false -> Arg1 end, [" ", case DW of true -> [" /* * Double word atomics need runtime test. */ int ethr_have_native_dw_atomic(void) { return ethr_have_native_dw_atomic__(); } "]; false -> "" end, " /* --- addr() --- */ ", func_header(AC, implementation, case DW of true -> ?DW_FUNC_MACRO; false -> false end, addr, none), " { ", AddrAintT, " *", RetVar, "; ETHR_ASSERT(!ethr_not_inited__); ETHR_ASSERT(", Arg1, "); ", make_native_impl_op(AC, addr, none), case AMC of true -> ["#elif defined(ETHR_AMC_FALLBACK__) ", RetVar ," = (", AddrAintT, " *) (", FallbackVar, ")->sint;"]; false -> "" end, " #else ", RetVar, " = (", AddrAintT, " *) ", FallbackVar, "; #endif return ", RetVar, "; } ", make_symbol_to_fallback_impl(AC, addr, none), lists:map(fun (Op) -> [" /* -- ", opstr(Op), "() -- */ ", lists:map(fun (B) -> Macro = case DW of true -> ?DW_FUNC_MACRO; false -> false end, case non_native_barrier(B) of true -> non_native_barrier_impl(AC, implementation, Op, B); false -> ["\n", func_header(AC, implementation, Macro, Op, B), "\n{\n", case is_return_op(AC, Op) of true -> [" ", RetType, " ", RetVar, ";\n"]; false -> "" end, case Op of init -> ""; _ -> [" ETHR_ASSERT(!ethr_not_inited__);\n"] end, [" ETHR_ASSERT(", Arg1, ");\n"], make_native_impl_op(AC, Op, B), make_amc_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), make_locked_fallback_op(AC#atomic_context{arg1 = FallbackVar}, Op, B), case is_return_op(AC, Op) of true -> [" return ", RetVar, ";" ]; false -> "" end, "\n}\n", make_symbol_to_fallback_impl(AC, Op, B)] end end, ?BARRIERS)] end, case DW of true -> ?DW_ATOMIC_OPS; false -> ?ATOMIC_OPS end)]. make_info_functions() -> [" /* --------- Info functions --------- */ #if defined(", ?DW_RTCHK_MACRO, ") char *zero_ops[] = {NULL}; #endif ", [lists:map(fun (NBits) -> {DW, Bits} = case NBits of "su_dw" -> {"su_dw_", ""}; "dw" -> {"dw_", ""}; _ -> {"", NBits} end, [" static char *native_", DW, "atomic", Bits, "_ops[] = {", lists:map(fun (Op) -> NOpStr = opstr(native(Op)), CapNOpStr = to_upper(NOpStr), lists:map(fun (B) -> HaveNative = case NBits of "dw" -> "ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC"; "su_dw" -> "ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC"; _ -> [?HAVE_NATIVE_ATOMIC, NBits] end, NBExt = op_barrier_ext(B), CapNBExt = to_upper(NBExt), [" #ifdef ", HaveNative, "_", CapNOpStr, CapNBExt, " \"", NOpStr, NBExt, "\", #endif" ] end, ?NATIVE_BARRIERS) end, case NBits of "dw" -> ?DW_ATOMIC_OPS; "su_dw" -> ?DW_ATOMIC_OPS; _ -> ?ATOMIC_OPS end), " NULL }; char ** ethr_native_", DW, "atomic", Bits, "_ops(void) { ", case DW of "" -> ""; _ -> [" #if defined(", ?DW_RTCHK_MACRO, ") if (!", ?DW_RTCHK_MACRO, ") return &zero_ops[0]; #endif" ] end, " return &native_", DW, "atomic", Bits, "_ops[0]; } " ] end, ["su_dw", "dw" | ?POTENTIAL_NBITS])]]. write_c_file(FileName) -> {ok, FD} = file:open(FileName, [write, latin1]), ok = file:write(FD, comments()), ok = file:write(FD, c_top()), lists:foreach(fun (AtomicSize) -> ok = file:write(FD, [atomic_implementation_comment(AtomicSize), make_symbol_implementations(atomic_context(AtomicSize))]) end, ?ATOMIC_SIZES), ok = file:write(FD, make_info_functions()). main([]) -> case os:getenv("ERL_TOP") of false -> io:format("$ERL_TOP not set!~n", []), halt(1); ErlTop -> HFile = filename:join(ErlTop, ?H_FILE), WHFile = fun () -> write_h_file(HFile) end, CFile = filename:join(ErlTop, ?C_FILE), WCFile = fun () -> write_c_file(CFile) end, case erlang:system_info(schedulers_online) of 1 -> WHFile(), WCFile(); _ -> {HPid, HMon} = spawn_monitor(WHFile), {CPid, CMon} = spawn_monitor(WCFile), receive {'DOWN', HMon, process, HPid, HReason} -> normal = HReason end, receive {'DOWN', CMon, process, CPid, CReason} -> normal = CReason end end, io:format("Wrote: ~s~n", [HFile]), io:format("Wrote: ~s~n", [CFile]), init:stop() end. a2l(A) -> atom_to_list(A). opstr(A) -> a2l(A). to_upper([]) -> []; to_upper([C|Cs]) when is_list(C) -> [to_upper(C)|to_upper(Cs)]; to_upper([C|Cs]) when is_integer(C), 97 =< C, C =< 122 -> [C-32|to_upper(Cs)]; to_upper([C|Cs]) -> [C|to_upper(Cs)]. comments() -> Years = case erlang:date() of {2011, _, _} -> "2011"; {Y, _, _} -> "2011-"++integer_to_list(Y) end, ["/* * --------------- DO NOT EDIT THIS FILE! --------------- * This file was automatically generated by the * \$ERL_TOP/erts/lib_src/utils/make_atomics_api script. * If you need to make changes, edit the script and * regenerate this file. * --------------- DO NOT EDIT THIS FILE! --------------- */ /* * %CopyrightBegin% * * Copyright Ericsson AB ", Years, ". All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * %CopyrightEnd% */ /* * Description: The ethread atomics API * Author: Rickard Green */ /* * This file maps native atomic implementations to ethread * API atomics. If no native atomic implementation * is available, a less efficient fallback is used instead. * The API consists of 32-bit size, word size (pointer size), * and double word size atomics. * * The following atomic operations are implemented for * 32-bit size, and word size atomics: ", lists:map(fun (Op) -> [" * - ", opstr(Op), "\n"] end, ?ATOMIC_OPS), " * * The following atomic operations are implemented for * double word size atomics: ", lists:map(fun (Op) -> [" * - ", opstr(Op), "\n"] end, ?DW_ATOMIC_OPS), " * * Appart from a function implementing the atomic operation * with unspecified memory barrier semantics, there are * functions implementing each operation with the following * implied memory barrier semantics:", lists:map(fun (none) -> ""; (mb) -> [" * - mb - Full memory barrier. Orders both loads, and * stores before, and after the atomic operation. * No load or store is allowed to be reordered * over the atomic operation."]; (acqb) -> [" * - acqb - Acquire barrier. Orders both loads, and stores * appearing *after* the atomic operation. These * are not allowed to be reordered over the * atomic operation."]; (relb) -> [" * - relb - Release barrier. Orders both loads, and * stores appearing *before* the atomic * operation. These are not allowed to be * reordered over the atomic operation."]; (rb) -> [" * - rb - Read barrier. Orders *only* loads. These are * not allowed to be reordered over the barrier. * Load in atomic operation is ordered *before* * the barrier. "]; (ddrb) -> [" * - ddrb - Data dependency read barrier. Orders *only* * loads according to data dependency across the * barrier. Load in atomic operation is ordered * before the barrier."]; (wb) -> [" * - wb - Write barrier. Orders *only* stores. These are * not allowed to be reordered over the barrier. * Store in atomic operation is ordered *after* * the barrier."]; (B) -> [" * - ", a2l(B), "\n"] end, lists:reverse(?BARRIERS)), " * * We implement all of these operation/barrier * combinations, regardless of whether they are useful * or not (some of them are useless). * * Double word size atomic functions are on the followning * form: * ethr_dw_atomic_[_] * * Word size atomic functions are on the followning * form: * ethr_atomic_[_] * * 32-bit size atomic functions are on the followning * form: * ethr_atomic32_[_] * * Apart from the operation/barrier functions * described above also 'addr' functions are implemented * which return the actual memory address used of the * atomic variable. The 'addr' functions have no barrier * versions. * * The native atomic implementation does not need to * implement all operation/barrier combinations. * Functions that have no native implementation will be * constructed from existing native functionality. These * functions will perform the wanted operation and will * produce sufficient memory barriers, but may * in some cases be less efficient than pure native * versions. * * When we create ethread API operation/barrier functions by * adding barriers before and after native operations it is * assumed that: * - A native read operation begins, and ends with a load. * - A native set operation begins, and ends with a store. * - An init operation begins with either a load, or a store, * and ends with either a load, or a store. * - All other operations begins with a load, and ends with * either a load, or a store. * * This is the minimum functionality that a native * implementation needs to provide: * * - Functions that need to be implemented: * * - ethr_native_[dw_|su_dw_]atomic[BITS]_addr * - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_] * (at least one cmpxchg of optional barrier) * * - Macros that needs to be defined: * * A macro informing about the presence of the native * implementation: * * - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS] * * A macro naming (a string constant) the implementation: * * - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL * * Each implemented native atomic function has to * be accompanied by a defined macro on the following * form informing about its presence: * * - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_[_] * * A (sparc-v9 style) membar macro: * * - ETHR_MEMBAR(B) * * Which takes a combination of the following macros * or:ed (using |) together: * * - ETHR_LoadLoad * - ETHR_LoadStore * - ETHR_StoreLoad * - ETHR_StoreStore * */ " ]. h_top() -> [" #undef ETHR_AMC_FALLBACK__ #undef ETHR_AMC_NO_ATMCS__ #undef ETHR_AMC_ATMC_T__ #undef ETHR_AMC_ATMC_FUNC__ /* -- 32-bit atomics -- */ #undef ETHR_NAINT32_T__ #undef ETHR_NATMC32_FUNC__ #undef ETHR_NATMC32_ADDR_FUNC__ #undef ETHR_NATMC32_BITS__ #if defined(ETHR_HAVE_NATIVE_ATOMIC32) # define ETHR_NEED_NATMC32_ADDR # define ETHR_NATMC32_ADDR_FUNC__ ethr_native_atomic32_addr typedef ethr_native_atomic32_t ethr_atomic32_t; # define ETHR_NAINT32_T__ ethr_sint32_t # define ETHR_NATMC32_FUNC__(X) ethr_native_atomic32_ ## X # define ETHR_NATMC32_BITS__ 32 #elif defined(ETHR_HAVE_NATIVE_ATOMIC64) # define ETHR_NEED_NATMC64_ADDR #ifdef ETHR_BIGENDIAN # define ETHR_NATMC32_ADDR_FUNC__(VAR) \\ (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) #else # define ETHR_NATMC32_ADDR_FUNC__(VAR) \\ ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) #endif typedef ethr_native_atomic64_t ethr_atomic32_t; # define ETHR_NAINT32_T__ ethr_sint64_t # define ETHR_NATMC32_FUNC__(X) ethr_native_atomic64_ ## X # define ETHR_NATMC32_BITS__ 64 #else /* * No native atomics usable for 32-bits atomics :( * Use fallback... */ typedef ethr_sint32_t ethr_atomic32_t; #endif #undef ETHR_ATMC32_INLINE__ #ifdef ETHR_NATMC32_BITS__ # ifdef ETHR_TRY_INLINE_FUNCS # define ETHR_ATMC32_INLINE__ # endif # define ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS #endif #if !defined(ETHR_ATMC32_INLINE__) || defined(ETHR_ATOMIC_IMPL__) # define ETHR_NEED_ATMC32_PROTOTYPES__ #endif #ifndef ETHR_INLINE_ATMC32_FUNC_NAME_ # define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X #endif #undef ETHR_ATMC32_FUNC__ #define ETHR_ATMC32_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) /* -- Word size atomics -- */ #undef ETHR_NEED_NATMC32_ADDR #undef ETHR_NEED_NATMC64_ADDR #undef ETHR_NAINT_T__ #undef ETHR_NATMC_FUNC__ #undef ETHR_NATMC_ADDR_FUNC__ #undef ETHR_NATMC_BITS__ #if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_NATIVE_ATOMIC64) # ifndef ETHR_NEED_NATMC64_ADDR # define ETHR_NEED_NATMC64_ADDR # endif # define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic64_addr typedef ethr_native_atomic64_t ethr_atomic_t; # define ETHR_NAINT_T__ ethr_sint64_t # define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X # define ETHR_NATMC_BITS__ 64 #elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC32) # ifndef ETHR_NEED_NATMC64_ADDR # define ETHR_NEED_NATMC32_ADDR # endif # define ETHR_NATMC_ADDR_FUNC__ ethr_native_atomic32_addr typedef ethr_native_atomic32_t ethr_atomic_t; # define ETHR_NAINT_T__ ethr_sint32_t # define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X # define ETHR_NATMC_BITS__ 32 #elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) # ifndef ETHR_NEED_NATMC64_ADDR # define ETHR_NEED_NATMC64_ADDR # endif #ifdef ETHR_BIGENDIAN # define ETHR_NATMC_ADDR_FUNC__(VAR) \\ (((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) + 1) #else # define ETHR_NATMC_ADDR_FUNC__(VAR) \\ ((ethr_sint32_t *) ethr_native_atomic64_addr((VAR))) #endif typedef ethr_native_atomic64_t ethr_atomic_t; # define ETHR_NATMC_T__ ethr_native_atomic64_t # define ETHR_NAINT_T__ ethr_sint64_t # define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X # define ETHR_NATMC_BITS__ 64 #else /* * No native atomics usable for pointer size atomics :( * Use fallback... */ # if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) # define ETHR_AMC_FALLBACK__ # define ETHR_AMC_NO_ATMCS__ 2 # define ETHR_AMC_SINT_T__ ethr_sint32_t # define ETHR_AMC_ATMC_T__ ethr_atomic32_t # define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) typedef struct { ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; } ethr_amc_t; typedef struct { ethr_amc_t amc; ethr_sint_t sint; } ethr_atomic_t; # else /* locked fallback */ typedef ethr_sint_t ethr_atomic_t; # endif #endif #undef ETHR_ATMC_INLINE__ #ifdef ETHR_NATMC_BITS__ # ifdef ETHR_TRY_INLINE_FUNCS # define ETHR_ATMC_INLINE__ # endif # define ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS #endif #if !defined(ETHR_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) # define ETHR_NEED_ATMC_PROTOTYPES__ #endif #ifndef ETHR_INLINE_ATMC_FUNC_NAME_ # define ETHR_INLINE_ATMC_FUNC_NAME_(X) X #endif #undef ETHR_ATMC_FUNC__ #define ETHR_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) /* -- Double word atomics -- */ #undef ETHR_SU_DW_NAINT_T__ #undef ETHR_SU_DW_NATMC_FUNC__ #undef ETHR_SU_DW_NATMC_ADDR_FUNC__ #undef ETHR_DW_NATMC_FUNC__ #undef ETHR_DW_NATMC_ADDR_FUNC__ #undef ETHR_DW_NATMC_BITS__ #if defined(ETHR_HAVE_NATIVE_DW_ATOMIC) || defined(ETHR_HAVE_NATIVE_SU_DW_ATOMIC) # define ETHR_NEED_DW_NATMC_ADDR # define ETHR_DW_NATMC_ADDR_FUNC__ ethr_native_dw_atomic_addr # define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_dw_atomic_t # define ETHR_DW_NATMC_FUNC__(X) ethr_native_dw_atomic_ ## X # define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_su_dw_atomic_ ## X # if ETHR_SIZEOF_PTR == 8 # define ETHR_DW_NATMC_BITS__ 128 # elif ETHR_SIZEOF_PTR == 4 # define ETHR_DW_NATMC_BITS__ 64 # else # error \"Word size not supported\" # endif # ifdef ETHR_NATIVE_SU_DW_SINT_T # define ETHR_SU_DW_NAINT_T__ ETHR_NATIVE_SU_DW_SINT_T # endif #elif ETHR_SIZEOF_PTR == 4 && defined(ETHR_HAVE_NATIVE_ATOMIC64) # define ETHR_HAVE_NATIVE_SU_DW_ATOMIC # ifndef ETHR_NEED_NATMC64_ADDR # define ETHR_NEED_NATMC64_ADDR # endif # define ETHR_DW_NATMC_ADDR_FUNC__(VAR) \\ ((ethr_dw_sint_t *) ethr_native_atomic64_addr((VAR))) # define ETHR_NATIVE_DW_ATOMIC_T__ ethr_native_atomic64_t # define ETHR_SU_DW_NAINT_T__ ethr_sint64_t # define ETHR_SU_DW_NATMC_FUNC__(X) ethr_native_atomic64_ ## X # define ETHR_DW_NATMC_BITS__ 64 #endif #if defined(", ?DW_RTCHK_MACRO, ") #define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X ## _fallback__ #else #define ", ?DW_FUNC_MACRO, "(X) ethr_dw_atomic_ ## X #endif #if !defined(ETHR_DW_NATMC_BITS__) || defined(", ?DW_RTCHK_MACRO, ") # define ETHR_NEED_DW_FALLBACK__ #endif #if defined(ETHR_NEED_DW_FALLBACK__) /* * No native atomics usable for double word atomics :( * Use fallback... */ # ifndef ETHR_AMC_FALLBACK__ # if ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) # define ETHR_AMC_FALLBACK__ # define ETHR_AMC_NO_ATMCS__ 1 # define ETHR_AMC_SINT_T__ ethr_sint_t # define ETHR_AMC_ATMC_T__ ethr_atomic_t # define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC_FUNC_NAME_(ethr_atomic_ ## X) # elif defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS) # define ETHR_AMC_FALLBACK__ # define ETHR_AMC_NO_ATMCS__ 2 # define ETHR_AMC_SINT_T__ ethr_sint32_t # define ETHR_AMC_ATMC_T__ ethr_atomic32_t # define ETHR_AMC_ATMC_FUNC__(X) ETHR_INLINE_ATMC32_FUNC_NAME_(ethr_atomic32_ ## X) # endif # ifdef ETHR_AMC_FALLBACK__ typedef struct { ETHR_AMC_ATMC_T__ atomic[ETHR_AMC_NO_ATMCS__]; } ethr_amc_t; # endif # endif typedef struct { #ifdef ETHR_AMC_FALLBACK__ ethr_amc_t amc; #endif ethr_sint_t sint[2]; } ethr_dw_atomic_fallback_t; #endif typedef union { #ifdef ETHR_NATIVE_DW_ATOMIC_T__ ETHR_NATIVE_DW_ATOMIC_T__ native; #endif #ifdef ETHR_NEED_DW_FALLBACK__ ethr_dw_atomic_fallback_t fallback; #endif ethr_sint_t sint[2]; } ethr_dw_atomic_t; typedef union { #ifdef ETHR_SU_DW_NAINT_T__ ETHR_SU_DW_NAINT_T__ ", ?SU_DW_SINT_FIELD, "; #endif ethr_sint_t ", ?DW_SINT_FIELD, "[2]; } ethr_dw_sint_t; #ifdef ETHR_BIGENDIAN # define ETHR_DW_SINT_LOW_WORD 1 # define ETHR_DW_SINT_HIGH_WORD 0 #else # define ETHR_DW_SINT_LOW_WORD 0 # define ETHR_DW_SINT_HIGH_WORD 1 #endif #undef ETHR_DW_ATMC_INLINE__ #ifdef ETHR_DW_NATMC_BITS__ # ifdef ETHR_TRY_INLINE_FUNCS # define ETHR_ATMC32_INLINE__ # endif # define ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS #endif #if !defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) # define ETHR_NEED_DW_ATMC_PROTOTYPES__ #endif #ifndef ETHR_INLINE_DW_ATMC_FUNC_NAME_ # define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X #endif #undef ETHR_DW_ATMC_FUNC__ #define ETHR_DW_ATMC_FUNC__(X) ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_dw_atomic_ ## X) #if defined(ETHR_NEED_DW_ATMC_PROTOTYPES__) int ethr_have_native_dw_atomic(void); #endif #if defined(ETHR_DW_ATMC_INLINE__) || defined(ETHR_ATOMIC_IMPL__) static ETHR_INLINE int ETHR_INLINE_DW_ATMC_FUNC_NAME_(ethr_have_native_dw_atomic)(void) { #if defined(", ?DW_RTCHK_MACRO, ") return ", ?DW_RTCHK_MACRO, "; #elif defined(ETHR_DW_NATMC_BITS__) return 1; #else return 0; #endif } #endif /* -- Misc -- */ #if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) /* * Unusual values are used by read() fallbacks implemented via cmpxchg(). * We want to use an unusual value in hope that it is more efficient * not to match the value in memory. * * - Negative integer values are probably more unusual. * - Very large absolute integer values are probably more unusual. * - Odd pointers are probably more unusual (only char pointers can be odd). */ # define ETHR_UNUSUAL_SINT32_VAL__ ((ethr_sint32_t) 0x81818181) # if ETHR_SIZEOF_PTR == 4 # define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) ETHR_UNUSUAL_SINT32_VAL__) # elif ETHR_SIZEOF_PTR == 8 # define ETHR_UNUSUAL_SINT_VAL__ ((ethr_sint_t) 0x8181818181818181L) # else # error \"Word size not supported\" # endif # if defined(ETHR_NEED_DW_NATMC_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR) # error \"No ethr_native_dw_atomic_addr() available\" # endif # if defined(ETHR_NEED_NATMC32_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADDR) # error \"No ethr_native_atomic32_addr() available\" # endif # if defined(ETHR_NEED_NATMC64_ADDR) && !defined(ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADDR) # error \"No ethr_native_atomic64_addr() available\" # endif #endif #if defined(__GNUC__) # ifndef ETHR_COMPILER_BARRIER # define ETHR_COMPILER_BARRIER __asm__ __volatile__(\"\" : : : \"memory\") # endif #elif defined(ETHR_WIN32_THREADS) # ifndef ETHR_COMPILER_BARRIER # include # pragma intrinsic(_ReadWriteBarrier) # define ETHR_COMPILER_BARRIER _ReadWriteBarrier() # endif #endif void ethr_compiler_barrier_fallback(void); #ifndef ETHR_COMPILER_BARRIER # define ETHR_COMPILER_BARRIER ethr_compiler_barrier_fallback() #endif int ethr_init_atomics(void); /* info */ char **ethr_native_atomic32_ops(void); char **ethr_native_atomic64_ops(void); char **ethr_native_dw_atomic_ops(void); char **ethr_native_su_dw_atomic_ops(void); #if !defined(ETHR_DW_NATMC_BITS__) && !defined(ETHR_NATMC_BITS__) && !defined(ETHR_NATMC32_BITS__) /* * ETHR_*MEMORY_BARRIER orders between locked and atomic accesses only, * i.e. when no native atomic implementation exist and only our lock * based atomic fallback is used, a noop is sufficient. */ # undef ETHR_MEMORY_BARRIER # undef ETHR_WRITE_MEMORY_BARRIER # undef ETHR_READ_MEMORY_BARRIER # undef ETHR_READ_DEPEND_MEMORY_BARRIER # undef ETHR_MEMBAR # define ETHR_MEMBAR(B) do { } while (0) #endif #ifndef ETHR_MEMBAR # error \"No ETHR_MEMBAR defined\" #endif #define ETHR_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore) #define ETHR_WRITE_MEMORY_BARRIER ETHR_MEMBAR(ETHR_StoreStore) #define ETHR_READ_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad) #ifdef ETHR_READ_DEPEND_MEMORY_BARRIER # undef ETHR_ORDERED_READ_DEPEND #else # define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_COMPILER_BARRIER # define ETHR_ORDERED_READ_DEPEND #endif "]. c_top() -> [" #ifdef HAVE_CONFIG_H #include \"config.h\" #endif #define ETHR_TRY_INLINE_FUNCS #define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __ #define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __ #define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __ #define ETHR_ATOMIC_IMPL__ #include \"ethread.h\" #include \"ethr_internal.h\" #if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\ || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) /* * Spinlock based fallback for atomics used in absence of a native * implementation. */ #define ETHR_ATMC_FLLBK_ADDR_BITS ", ?ETHR_ATMC_FLLBK_ADDR_BITS, " #define ETHR_ATMC_FLLBK_ADDR_SHIFT ", ?ETHR_ATMC_FLLBK_ADDR_SHIFT, " typedef struct { union { ethr_spinlock_t lck; char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))]; } u; } ethr_atomic_protection_t; extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; #define ETHR_ATOMIC_PTR2LCK__(PTR) \\ (ðr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \\ & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck) #define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS) \\ do { \\ ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP)); \\ ethr_spin_lock(slp__); \\ { EXPS; } \\ ethr_spin_unlock(slp__); \\ } while (0) ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS]; #endif ", make_amc_fallback(), " int ethr_init_atomics(void) { #if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \\ || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)) int i; for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) { int res = ethr_spinlock_init(ðr_atomic_protection__[i].u.lck); if (res != 0) return res; } #endif return 0; } "]. make_amc_fallback() -> [" #if defined(ETHR_AMC_FALLBACK__) /* * Fallback for large sized (word and/or double word size) atomics using * an \"Atomic Modification Counter\" based on smaller sized native atomics. * * We use a 63-bit modification counter and a one bit exclusive flag. * If 32-bit native atomics are used, we need two 32-bit native atomics. * The exclusive flag is the least significant bit, or if multiple atomics * are used, the least significant bit of the least significant atomic. * * When using the AMC fallback the following is true: * - Reads of the same atomic variable can be done in parallel. * - Uncontended reads doesn't cause any cache line invalidations, * since no modifications are done. * - Assuming that the AMC atomic(s) and the integer(s) containing the * value of the implemented atomic resides in the same cache line, * modifications will only cause invalidations of one cache line. * * When using the spinlock based fallback none of the above is true, * however, the spinlock based fallback consumes less memory. */ # if ETHR_AMC_NO_ATMCS__ != 1 && ETHR_AMC_NO_ATMCS__ != 2 # error \"Not supported\" # endif # define ETHR_AMC_MAX_TRY_READ__ 10 # ifdef ETHR_DEBUG # define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \\ do { \\ ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \\ ETHR_ASSERT(act == (S) + 1); \\ ETHR_ASSERT(act & 1); \\ } while (0) # else # define ETHR_DBG_CHK_EXCL_STATE(ASP, S) # endif static ETHR_INLINE void amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) { avar[0] = val[0]; if (dw) avar[1] = val[1]; #if ETHR_AMC_NO_ATMCS__ == 2 ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0); #endif ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0); } static ETHR_INLINE ETHR_AMC_SINT_T__ amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0) { ETHR_AMC_SINT_T__ state0 = prev_state0; /* Set exclusive flag. */ while (1) { ETHR_AMC_SINT_T__ act_state0, new_state0; while (state0 & 1) { /* Wait until exclusive bit has been cleared */ ETHR_SPIN_BODY; state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); } /* Try to set exclusive bit */ new_state0 = state0 + 1; act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0], new_state0, state0); if (state0 == act_state0) return state0; /* old state0 */ state0 = act_state0; } } static ETHR_INLINE void amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) { ETHR_AMC_SINT_T__ state0 = old_state0; /* Increment modification counter and reset exclusive flag. */ ETHR_DBG_CHK_EXCL_STATE(amc, state0); state0 += 2; ETHR_ASSERT((state0 & 1) == 0); #if ETHR_AMC_NO_ATMCS__ == 2 if (state0 == 0) { /* * state0 wrapped, so we need to increment state1. There is no need * for atomic inc op, since this is always done while having exclusive * flag. */ ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]); state1++; ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1); } #endif ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0); } static ETHR_INLINE void amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0) { ETHR_DBG_CHK_EXCL_STATE(amc, old_state0); /* * Reset exclusive flag, but leave modification counter unchanged, * i.e., restore state to what it was before setting exclusive * flag. */ ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0); } static ETHR_INLINE void amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) { ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); state0 = amc_set_excl(amc, state0); avar[0] = val[0]; if (dw) avar[1] = val[1]; amc_inc_mc_unset_excl(amc, state0); } static ETHR_INLINE int amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p) { /* *state0p should contain last read value if aborting */ ETHR_AMC_SINT_T__ old_state0; #if ETHR_AMC_NO_ATMCS__ == 2 ETHR_AMC_SINT_T__ state1; int abrt; #endif *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); if ((*state0p) & 1) return 0; /* exclusive flag set; abort */ #if ETHR_AMC_NO_ATMCS__ == 2 state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]); #else ETHR_COMPILER_BARRIER; #endif val[0] = avar[0]; if (dw) val[1] = avar[1]; ETHR_READ_MEMORY_BARRIER; /* * Abort if state has changed (i.e, either the exclusive * flag is set, or modification counter changed). */ old_state0 = *state0p; #if ETHR_AMC_NO_ATMCS__ == 2 *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]); abrt = (old_state0 != *state0p); abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1])); return abrt == 0; #else *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); return old_state0 == *state0p; #endif } static ETHR_INLINE void amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val) { ETHR_AMC_SINT_T__ state0; int i; #if ETHR_AMC_MAX_TRY_READ__ == 0 state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]); #else for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) { if (amc_try_read(amc, dw, avar, val, &state0)) return; /* read success */ ETHR_SPIN_BODY; } #endif state0 = amc_set_excl(amc, state0); val[0] = avar[0]; if (dw) val[1] = avar[1]; amc_unset_excl(amc, state0); } static ETHR_INLINE int amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *new, ethr_sint_t *xchg) { ethr_sint_t val[2]; ETHR_AMC_SINT_T__ state0; if (amc_try_read(amc, dw, avar, val, &state0)) { if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) { xchg[0] = val[0]; if (dw) xchg[1] = val[1]; return 0; /* failed */ } /* Operation will succeed if not interrupted */ } state0 = amc_set_excl(amc, state0); if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) { xchg[0] = avar[0]; if (dw) xchg[1] = avar[1]; ETHR_DBG_CHK_EXCL_STATE(amc, state0); amc_unset_excl(amc, state0); return 0; /* failed */ } avar[0] = new[0]; if (dw) avar[1] = new[1]; amc_inc_mc_unset_excl(amc, state0); return 1; } #define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS) \\ do { \\ ETHR_AMC_SINT_T__ state0__; \\ state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]); \\ state0__ = amc_set_excl((AMC), state0__); \\ { OPS; } \\ amc_inc_mc_unset_excl((AMC), state0__); \\ } while (0) #endif /* amc fallback */ "].