aboutsummaryrefslogblamecommitdiffstats
path: root/erts/lib_src/common/ethr_atomics.c
blob: d09387384150ed8e31ebdbb26b81bc6ee5a23e88 (plain) (tree)
1
2
3
4
5
6
7
8
9
10
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
  








                                                         

                   
                                                        















                                                                         
                                       


                        
































                                                             
























                                                          


















































































                                                                      



                    



                                                 




                          









































































































































































































                                                                                         
      












































































                                                                      



                       






                                                                         




             




                                                                  
  
                                         

   
                                    
 
                                          
 


                    
 
                                                               
 

                                    
                     







                                                                                                           

 

                                                       


                                    
                                      
 




                     
 
                                                                                                       
 
            

                                    


















                                                                                                                   

 

                                                                                               


                                    


                                                       
 
                                                                                                    

                               
                                                     
     
                                                        


      
                                                                                                          
 
            

                                    




















                                                                                                                   

 

                                                                                                  


                                    
                                                          
 
      
 
                                                                                                          
 
            

                                    
























                                                                                                                   


                                    
                                                          
 
      
 
                                                                                                            
 
            

                                    




















                                                                                                                   

 

                                                                                                    


                                    
                                                            
 
      
 
                                                                                                            
 
            

                                    




















                                                                                                                   

 

                                                                                                    


                                    


                                                            
 
                                                                                                          
 
            

                                    






















                                                                                                                   

 

                                                                                                  


                                    
                                                          
 



                 
 

                                                                           


                                    







                                                                                                                                         

 

                                                                   


                                    
                                   
 
      
 
                                                                        

                               
                                 
     
                                    


      
                                                                              


                                    









                                                                                                                                         

 

                                                                      


                                    
                                      
 
      
 
                                                                              


                                    









                                                                                                                                         

 

                                                                      


                                    
                                      
 
      
 












                                                                                                                                         
 
 
 

                                                                        
 
                                    
                     
                                        
 
      
 
                                                                                
 
                                    
                     









                                                                                                                                         

 

                                                                        


                                    
                                        
 
      
 
                                                                              


                                    










                                                                                                                                         
 
 
 

                                                                      


                                    






                                      
 
                                                                            


                                    







                                                                                                                                         

 

                                                                    


                                    
                                    
 
      
 
                                                                         

                               
                                  
     
                                     


      
                                                                               


                                    












                                                                                                                                         


                                    
                                       
 
      
 
                                                                               


                                    









                                                                                                                                         

 

                                                                       


                                    
                                       
 
      
 
                                                                                 


                                    









                                                                                                                                         

 

                                                                         


                                    


                                         
 
                                                                                 


                                    









                                                                                                                                         

 

                                                                         


                                    
                                         
 
      
 
                                                                               


                                    











                                                                                                                                         

 

                                                                       


                                    
                                       
 

      
 



                                                                            
 
                     







                                                                                                                                         

 

                                                                    
 
                     
                                    
 
      
 
                                                                         

                               
                                  
     
                                     


      
                                                                               
 
                     









                                                                                                                                         

 













































































                                                                                                                                         
 
                     











                                                                                                                                         

 


















































                                                                                                








                                                                                              














































































































                                                                                                








                                                                      








































































































                                                                             








                                                              


































































































                                                              








                                                               































































































                                                                              








                                                                          









































































































                                                                              








                                                     








































































































                                                                








                                                         









































































































                                                                             








                                                         








































































































                                                                             








                                                              



































































































                                                                             








                                             



































































































                                                                             








                                             




































































































                                                                              








                                                                           









































































































                                                                              








                                                                          





























































































































                                                                                                   








                                                                                                        























































































                                                                                                        








                                                                              






















































































                                                                              








                                                                    
















































































                                                                    








                                                                     













































































                                                                             








                                                                                  























































































                                                                                  








                                                           























































































                                                           








                                                               























































































                                                                             








                                                               






















































































                                                                             








                                                                    

















































































                                                                             








                                                 

















































































                                                                             








                                                 


















































































                                                                              








                                                                                   























































































                                                                                   








                                                                                  









































































































































































































































































































































































































































































































































































































































































































































































                                                                                  
/*
 * --------------- DO NOT EDIT THIS FILE! ---------------
 * This file was automatically generated by the
 * $ERL_TOP/erts/lib_src/utils/make_atomics_api script.
 * If you need to make changes, edit the script and
 * regenerate this file.
 * --------------- DO NOT EDIT THIS FILE! ---------------
 */

/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 2011-2012. All Rights Reserved.
 *
 * The contents of this file are subject to the Erlang Public License,
 * Version 1.1, (the "License"); you may not use this file except in
 * compliance with the License. You should have received a copy of the
 * Erlang Public License along with this software. If not, it can be
 * retrieved online at http://www.erlang.org/.
 *
 * Software distributed under the License is distributed on an "AS IS"
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See
 * the License for the specific language governing rights and limitations
 * under the License.
 *
 * %CopyrightEnd%
 */

/*
 * Description: The ethread atomics API
 * Author: Rickard Green
 */

/*
 * This file maps native atomic implementations to ethread
 * API atomics. If no native atomic implementation
 * is available, a less efficient fallback is used instead.
 * The API consists of 32-bit size, word size (pointer size),
 * and double word size atomics.
 *
 * The following atomic operations are implemented for
 * 32-bit size, and word size atomics:
 * - cmpxchg
 * - xchg
 * - set
 * - init
 * - add_read
 * - read
 * - inc_read
 * - dec_read
 * - add
 * - inc
 * - dec
 * - read_band
 * - read_bor
 *
 * The following atomic operations are implemented for
 * double word size atomics:
 * - cmpxchg
 * - set
 * - read
 * - init
 *
 * Appart from a function implementing the atomic operation
 * with unspecified memory barrier semantics, there are
 * functions implementing each operation with the following
 * implied memory barrier semantics:
 * - mb   - Full memory barrier. Orders both loads, and
 *          stores before, and after the atomic operation.
 *          No load or store is allowed to be reordered
 *          over the atomic operation.
 * - relb - Release barrier. Orders both loads, and
 *          stores appearing *before* the atomic
 *          operation. These are not allowed to be
 *          reordered over the atomic operation.
 * - acqb - Acquire barrier. Orders both loads, and stores
 *          appearing *after* the atomic operation. These
 *          are not allowed to be reordered over the
 *          atomic operation.
 * - wb   - Write barrier. Orders *only* stores. These are
 *          not allowed to be reordered over the barrier.
 *          Store in atomic operation is ordered *after*
 *          the barrier.
 * - rb   - Read barrier. Orders *only* loads. These are
 *          not allowed to be reordered over the barrier.
 *          Load in atomic operation is ordered *before*
 *          the barrier. 
 * - ddrb - Data dependency read barrier. Orders *only*
 *          loads according to data dependency across the
 *          barrier. Load in atomic operation is ordered
 *          before the barrier.
 *
 * We implement all of these operation/barrier
 * combinations, regardless of whether they are useful
 * or not (some of them are useless).
 *
 * Double word size atomic functions are on the followning
 * form:
 *   ethr_dw_atomic_<OP>[_<BARRIER>]
 *
 * Word size atomic functions are on the followning
 * form:
 *   ethr_atomic_<OP>[_<BARRIER>]
 *
 * 32-bit size atomic functions are on the followning
 * form:
 *   ethr_atomic32_<OP>[_<BARRIER>]
 *
 * Apart from the operation/barrier functions
 * described above also 'addr' functions are implemented
 * which return the actual memory address used of the
 * atomic variable. The 'addr' functions have no barrier
 * versions.
 *
 * The native atomic implementation does not need to
 * implement all operation/barrier combinations.
 * Functions that have no native implementation will be
 * constructed from existing native functionality. These
 * functions will perform the wanted operation and will
 * produce sufficient memory barriers, but may
 * in some cases be less efficient than pure native
 * versions.
 *
 * When we create ethread API operation/barrier functions by
 * adding barriers before and after native operations it is
 * assumed that:
 * - A native read operation begins, and ends with a load.
 * - A native set operation begins, and ends with a store.
 * - An init operation begins with either a load, or a store,
 *   and ends with either a load, or a store.
 * - All other operations begins with a load, and ends with
 *   either a load, or a store.
 *
 * This is the minimum functionality that a native
 * implementation needs to provide:
 *
 * - Functions that need to be implemented:
 *
 *   - ethr_native_[dw_|su_dw_]atomic[BITS]_addr
 *   - ethr_native_[dw_|su_dw_]atomic[BITS]_cmpxchg[_<BARRIER>]
 *     (at least one cmpxchg of optional barrier)
 *
 * - Macros that needs to be defined:
 *
 *   A macro informing about the presence of the native
 *   implementation:
 *
 *   - ETHR_HAVE_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]
 *
 *   A macro naming (a string constant) the implementation:
 *
 *   - ETHR_NATIVE_[DW_]ATOMIC[BITS]_IMPL
 *
 *   Each implemented native atomic function has to
 *   be accompanied by a defined macro on the following
 *   form informing about its presence:
 *
 *   - ETHR_HAVE_ETHR_NATIVE_[DW_|SU_DW_]ATOMIC[BITS]_<OP>[_<BARRIER>]
 *
 *   A (sparc-v9 style) membar macro:
 *
 *   - ETHR_MEMBAR(B)
 *
 *     Which takes a combination of the following macros
 *     or:ed (using |) together:
 *
 *     - ETHR_LoadLoad
 *     - ETHR_LoadStore
 *     - ETHR_StoreLoad
 *     - ETHR_StoreStore
 *
 */


#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#define ETHR_TRY_INLINE_FUNCS
#define ETHR_INLINE_DW_ATMC_FUNC_NAME_(X) X ## __
#define ETHR_INLINE_ATMC_FUNC_NAME_(X) X ## __
#define ETHR_INLINE_ATMC32_FUNC_NAME_(X) X ## __
#define ETHR_ATOMIC_IMPL__

#include "ethread.h"
#include "ethr_internal.h"

#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
     || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
/*
 * Spinlock based fallback for atomics used in absence of a native
 * implementation.
 */

#define ETHR_ATMC_FLLBK_ADDR_BITS 10
#define ETHR_ATMC_FLLBK_ADDR_SHIFT 6

typedef struct {
    union {
	ethr_spinlock_t lck;
	char buf[ETHR_CACHE_LINE_ALIGN_SIZE(sizeof(ethr_spinlock_t))];
    } u;
} ethr_atomic_protection_t;

extern ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];

#define ETHR_ATOMIC_PTR2LCK__(PTR) \
(&ethr_atomic_protection__[((((ethr_uint_t) (PTR)) >> ETHR_ATMC_FLLBK_ADDR_SHIFT) \
			   & ((1 << ETHR_ATMC_FLLBK_ADDR_BITS) - 1))].u.lck)


#define ETHR_ATOMIC_OP_FALLBACK_IMPL__(AP, EXPS)			\
do {									\
    ethr_spinlock_t *slp__ = ETHR_ATOMIC_PTR2LCK__((AP));		\
    ethr_spin_lock(slp__);						\
    { EXPS; }								\
    ethr_spin_unlock(slp__);						\
} while (0)

ethr_atomic_protection_t ethr_atomic_protection__[1 << ETHR_ATMC_FLLBK_ADDR_BITS];

#endif


#if defined(ETHR_AMC_FALLBACK__)

/*
 * Fallback for large sized (word and/or double word size) atomics using
 * an "Atomic Modification Counter" based on smaller sized native atomics.
 *
 * We use a 63-bit modification counter and a one bit exclusive flag.
 * If 32-bit native atomics are used, we need two 32-bit native atomics.
 * The exclusive flag is the least significant bit, or if multiple atomics
 * are used, the least significant bit of the least significant atomic.
 *
 * When using the AMC fallback the following is true:
 * - Reads of the same atomic variable can be done in parallel.
 * - Uncontended reads doesn't cause any cache line invalidations,
 *   since no modifications are done.
 * - Assuming that the AMC atomic(s) and the integer(s) containing the
 *   value of the implemented atomic resides in the same cache line,
 *   modifications will only cause invalidations of one cache line.
 *
 * When using the spinlock based fallback none of the above is true,
 * however, the spinlock based fallback consumes less memory.
 */

#  if ETHR_AMC_NO_ATMCS__ != 1  && ETHR_AMC_NO_ATMCS__ != 2
#    error "Not supported"
#  endif
#  define ETHR_AMC_MAX_TRY_READ__ 10
#  ifdef ETHR_DEBUG
#    define ETHR_DBG_CHK_EXCL_STATE(ASP, S) \
do { \
    ETHR_AMC_SINT_T__ act = ETHR_AMC_ATMC_FUNC__(read)(&(ASP)->atomic[0]); \
    ETHR_ASSERT(act == (S) + 1); \
    ETHR_ASSERT(act & 1); \
} while (0)
#  else
#    define ETHR_DBG_CHK_EXCL_STATE(ASP, S)
#  endif

static ETHR_INLINE void
amc_init(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
{
    avar[0] = val[0];
    if (dw)
	avar[1] = val[1];
#if ETHR_AMC_NO_ATMCS__ == 2
    ETHR_AMC_ATMC_FUNC__(init)(&amc->atomic[1], 0);
#endif
    ETHR_AMC_ATMC_FUNC__(init_wb)(&amc->atomic[0], 0);
}

static ETHR_INLINE ETHR_AMC_SINT_T__
amc_set_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ prev_state0)
{
    ETHR_AMC_SINT_T__ state0 = prev_state0;
    /* Set exclusive flag. */
    while (1) {
	ETHR_AMC_SINT_T__ act_state0, new_state0;
	while (state0 & 1) { /* Wait until exclusive bit has been cleared */
	    ETHR_SPIN_BODY;
	    state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
	}
	/* Try to set exclusive bit */
	new_state0 = state0 + 1;
	act_state0 = ETHR_AMC_ATMC_FUNC__(cmpxchg_acqb)(&amc->atomic[0],
							new_state0,
							state0);
	if (state0 == act_state0)
	    return state0; /* old state0 */
	state0 = act_state0;
    }
}

static ETHR_INLINE void
amc_inc_mc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
{
    ETHR_AMC_SINT_T__ state0 = old_state0;

    /* Increment modification counter and reset exclusive flag. */

    ETHR_DBG_CHK_EXCL_STATE(amc, state0);

    state0 += 2;

    ETHR_ASSERT((state0 & 1) == 0);

#if ETHR_AMC_NO_ATMCS__ == 2
    if (state0 == 0) {
	/*
	 * state0 wrapped, so we need to increment state1. There is no need
	 * for atomic inc op, since this is always done while having exclusive
	 * flag.
	 */
	ETHR_AMC_SINT_T__ state1 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]);
	state1++;
	ETHR_AMC_ATMC_FUNC__(set)(&amc->atomic[1], state1);
    }
#endif
    ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], state0);
}

static ETHR_INLINE void
amc_unset_excl(ethr_amc_t *amc, ETHR_AMC_SINT_T__ old_state0)
{
    ETHR_DBG_CHK_EXCL_STATE(amc, old_state0);
    /*
     * Reset exclusive flag, but leave modification counter unchanged,
     * i.e., restore state to what it was before setting exclusive
     * flag.
     */
    ETHR_AMC_ATMC_FUNC__(set_relb)(&amc->atomic[0], old_state0);
}

static ETHR_INLINE void
amc_set(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
{
    ETHR_AMC_SINT_T__ state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);

    state0 = amc_set_excl(amc, state0);

    avar[0] = val[0];
    if (dw)
	avar[1] = val[1];

    amc_inc_mc_unset_excl(amc, state0);
}

static ETHR_INLINE int
amc_try_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
	     ethr_sint_t *val, ETHR_AMC_SINT_T__ *state0p)
{
    /* *state0p should contain last read value if aborting */
    ETHR_AMC_SINT_T__ old_state0;
#if ETHR_AMC_NO_ATMCS__ == 2
    ETHR_AMC_SINT_T__ state1;
    int abrt;
#endif

    *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
    if ((*state0p) & 1) 
	return 0; /* exclusive flag set; abort */
#if ETHR_AMC_NO_ATMCS__ == 2
    state1 = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[1]);
#else
    ETHR_COMPILER_BARRIER;
#endif

    val[0] = avar[0];
    if (dw)
	val[1] = avar[1];

    ETHR_READ_MEMORY_BARRIER;

    /*
     * Abort if state has changed (i.e, either the exclusive
     * flag is set, or modification counter changed).
     */
    old_state0 = *state0p;
#if ETHR_AMC_NO_ATMCS__ == 2
    *state0p = ETHR_AMC_ATMC_FUNC__(read_rb)(&amc->atomic[0]);
    abrt = (old_state0 != *state0p);
    abrt |= (state1 != ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[1]));
    return abrt == 0;
#else
    *state0p = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
    return old_state0 == *state0p;
#endif
}

static ETHR_INLINE void
amc_read(ethr_amc_t *amc, int dw, ethr_sint_t *avar, ethr_sint_t *val)
{
    ETHR_AMC_SINT_T__ state0;
    int i;

#if ETHR_AMC_MAX_TRY_READ__ == 0
    state0 = ETHR_AMC_ATMC_FUNC__(read)(&amc->atomic[0]);
#else
    for (i = 0; i < ETHR_AMC_MAX_TRY_READ__; i++) {
	if (amc_try_read(amc, dw, avar, val, &state0))
	    return; /* read success */
	ETHR_SPIN_BODY;
    }
#endif

    state0 = amc_set_excl(amc, state0);

    val[0] = avar[0];
    if (dw)
	val[1] = avar[1];

    amc_unset_excl(amc, state0);
}

static ETHR_INLINE int
amc_cmpxchg(ethr_amc_t *amc, int dw, ethr_sint_t *avar,
	    ethr_sint_t *new, ethr_sint_t *xchg)
{
    ethr_sint_t val[2];
    ETHR_AMC_SINT_T__ state0;

    if (amc_try_read(amc, dw, avar, val, &state0)) {
	if (val[0] != xchg[0] || (dw && val[1] != xchg[1])) {
	    xchg[0] = val[0];
	    if (dw)
		xchg[1] = val[1];
	    return 0; /* failed */
	}
	/* Operation will succeed if not interrupted */
    }

    state0 = amc_set_excl(amc, state0);

    if (xchg[0] != avar[0] || (dw && xchg[1] != avar[1])) {
	xchg[0] = avar[0];
	if (dw)
	    xchg[1] = avar[1];

	ETHR_DBG_CHK_EXCL_STATE(amc, state0);

	amc_unset_excl(amc, state0);
	return 0; /* failed */
    }

    avar[0] = new[0];
    if (dw)
	avar[1] = new[1];

    amc_inc_mc_unset_excl(amc, state0);
    return 1;
}


#define ETHR_AMC_MODIFICATION_OPS__(AMC, OPS)			\
do {								\
    ETHR_AMC_SINT_T__ state0__;					\
    state0__ = ETHR_AMC_ATMC_FUNC__(read)(&(AMC)->atomic[0]);	\
    state0__ = amc_set_excl((AMC), state0__);			\
    { OPS; }							\
    amc_inc_mc_unset_excl((AMC), state0__);			\
} while (0)

#endif /* amc fallback */


int
ethr_init_atomics(void)
{
#if (!defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS) \
     || !defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS))
    int i;
    for (i = 0; i < (1 << ETHR_ATMC_FLLBK_ADDR_BITS); i++) {
	int res = ethr_spinlock_init(&ethr_atomic_protection__[i].u.lck);
	if (res != 0)
	    return res;
    }
#endif
    return 0;
}


/* ---------- Double word size atomic implementation ---------- */



/*
 * Double word atomics need runtime test.
 */

int ethr_have_native_dw_atomic(void)
{
    return ethr_have_native_dw_atomic__();
}
     

/* --- addr() --- */

ethr_sint_t *ETHR_DW_ATOMIC_FUNC__(addr)(ethr_dw_atomic_t *var)
{
    ethr_sint_t *res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    res =  ethr_dw_atomic_addr__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    res = (ethr_sint_t *) ((&var->fallback))->sint;
#else
    res = (ethr_sint_t *) (&var->fallback);
#endif
    return res;
}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
ethr_sint_t *ethr_dw_atomic_addr(ethr_dw_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    return ethr_dw_atomic_addr__(var);
}
#endif


/* -- cmpxchg() -- */


int ETHR_DW_ATOMIC_FUNC__(cmpxchg)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    res =  ethr_dw_atomic_cmpxchg__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
	{
	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
	    if (res) {
		(&var->fallback)->sint[0] = val->sint[0];
		(&var->fallback)->sint[1] = val->sint[1];
	    }
	    else {
		old_val->sint[0] = (&var->fallback)->sint[0];
		old_val->sint[1] = (&var->fallback)->sint[1];
	    }
	});
#endif
    return res;
}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
int ethr_dw_atomic_cmpxchg(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    return ethr_dw_atomic_cmpxchg__(var, val, old_val);
}
#endif

int ethr_dw_atomic_cmpxchg_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_dw_atomic_cmpxchg(var, val, old_val);
#else
    return ethr_dw_atomic_cmpxchg_rb(var, val, old_val);
#endif
}

int ETHR_DW_ATOMIC_FUNC__(cmpxchg_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    res =  ethr_dw_atomic_cmpxchg_rb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
	{
	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
	    if (res) {
		(&var->fallback)->sint[0] = val->sint[0];
		(&var->fallback)->sint[1] = val->sint[1];
	    }
	    else {
		old_val->sint[0] = (&var->fallback)->sint[0];
		old_val->sint[1] = (&var->fallback)->sint[1];
	    }
	});
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
int ethr_dw_atomic_cmpxchg_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    return ethr_dw_atomic_cmpxchg_rb__(var, val, old_val);
}
#endif

int ETHR_DW_ATOMIC_FUNC__(cmpxchg_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    res =  ethr_dw_atomic_cmpxchg_wb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
	{
	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
	    if (res) {
		(&var->fallback)->sint[0] = val->sint[0];
		(&var->fallback)->sint[1] = val->sint[1];
	    }
	    else {
		old_val->sint[0] = (&var->fallback)->sint[0];
		old_val->sint[1] = (&var->fallback)->sint[1];
	    }
	});
#endif
    return res;
}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
int ethr_dw_atomic_cmpxchg_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    return ethr_dw_atomic_cmpxchg_wb__(var, val, old_val);
}
#endif

int ETHR_DW_ATOMIC_FUNC__(cmpxchg_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    res =  ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
	{
	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
	    if (res) {
		(&var->fallback)->sint[0] = val->sint[0];
		(&var->fallback)->sint[1] = val->sint[1];
	    }
	    else {
		old_val->sint[0] = (&var->fallback)->sint[0];
		old_val->sint[1] = (&var->fallback)->sint[1];
	    }
	});
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
int ethr_dw_atomic_cmpxchg_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    return ethr_dw_atomic_cmpxchg_acqb__(var, val, old_val);
}
#endif

int ETHR_DW_ATOMIC_FUNC__(cmpxchg_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    res =  ethr_dw_atomic_cmpxchg_relb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
	{
	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
	    if (res) {
		(&var->fallback)->sint[0] = val->sint[0];
		(&var->fallback)->sint[1] = val->sint[1];
	    }
	    else {
		old_val->sint[0] = (&var->fallback)->sint[0];
		old_val->sint[1] = (&var->fallback)->sint[1];
	    }
	});
#endif
    return res;
}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
int ethr_dw_atomic_cmpxchg_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    return ethr_dw_atomic_cmpxchg_relb__(var, val, old_val);
}
#endif

int ETHR_DW_ATOMIC_FUNC__(cmpxchg_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    int res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    res =  ethr_dw_atomic_cmpxchg_mb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = amc_cmpxchg(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint, old_val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), 
	{
	    res = ((&var->fallback)->sint[0] == old_val->sint[0] && (&var->fallback)->sint[1] == old_val->sint[1]);
	    if (res) {
		(&var->fallback)->sint[0] = val->sint[0];
		(&var->fallback)->sint[1] = val->sint[1];
	    }
	    else {
		old_val->sint[0] = (&var->fallback)->sint[0];
		old_val->sint[1] = (&var->fallback)->sint[1];
	    }
	});
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
int ethr_dw_atomic_cmpxchg_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val, ethr_dw_sint_t *old_val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    return ethr_dw_atomic_cmpxchg_mb__(var, val, old_val);
}
#endif


/* -- set() -- */


void ETHR_DW_ATOMIC_FUNC__(set)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_set__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_set(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_set__(var, val);
}
#endif

void ethr_dw_atomic_set_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_dw_atomic_set(var, val);
#else
    ethr_dw_atomic_set_rb(var, val);
#endif
}

void ETHR_DW_ATOMIC_FUNC__(set_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_set_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_set_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_set_rb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(set_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_set_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_set_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_set_wb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(set_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_set_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_set_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_set_acqb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(set_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_set_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_set_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_set_relb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(set_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_set_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    amc_set(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_set_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_set_mb__(var, val);
}
#endif


/* -- read() -- */


void ETHR_DW_ATOMIC_FUNC__(read)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_read__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_read(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_read__(var, val);
}
#endif

void ethr_dw_atomic_read_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_dw_atomic_read(var, val);
#else
    ethr_dw_atomic_read_rb(var, val);
#endif
}

void ETHR_DW_ATOMIC_FUNC__(read_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_read_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_read_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_read_rb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(read_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_read_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_read_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_read_wb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(read_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_read_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
    ETHR_MEMBAR(ETHR_LoadStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_read_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_read_acqb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(read_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_read_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_read_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_read_relb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(read_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_read_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    amc_read(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
    ETHR_MEMBAR(ETHR_LoadStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), val->sint[0] = (&var->fallback)->sint[0]; val->sint[1] = (&var->fallback)->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_read_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
    ethr_dw_atomic_read_mb__(var, val);
}
#endif


/* -- init() -- */


void ETHR_DW_ATOMIC_FUNC__(init)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_init__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_init(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
    ethr_dw_atomic_init__(var, val);
}
#endif

void ethr_dw_atomic_init_ddrb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_dw_atomic_init(var, val);
#else
    ethr_dw_atomic_init_rb(var, val);
#endif
}

void ETHR_DW_ATOMIC_FUNC__(init_rb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_init_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_init_rb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
    ethr_dw_atomic_init_rb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(init_wb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_init_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_init_wb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
    ethr_dw_atomic_init_wb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(init_acqb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_init_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_init_acqb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
    ethr_dw_atomic_init_acqb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(init_relb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_init_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_init_relb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
    ethr_dw_atomic_init_relb__(var, val);
}
#endif

void ETHR_DW_ATOMIC_FUNC__(init_mb)(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_DOUBLE_WORD_SZ_NATIVE_ATOMIC_OPS) && !defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    ethr_dw_atomic_init_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    amc_init(&(&var->fallback)->amc, 1, (&var->fallback)->sint, val->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__((&var->fallback), (&var->fallback)->sint[0] = val->sint[0]; (&var->fallback)->sint[1] = val->sint[1]);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

#ifdef ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__
void ethr_dw_atomic_init_mb(ethr_dw_atomic_t *var, ethr_dw_sint_t *val)
{
    ETHR_ASSERT(var);
    ethr_dw_atomic_init_mb__(var, val);
}
#endif


/* ---------- Word size atomic implementation ---------- */




/* --- addr() --- */

ethr_sint_t *ethr_atomic_addr(ethr_atomic_t *var)
{
    ethr_sint_t *res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_addr__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    res = (ethr_sint_t *) (var)->sint;
#else
    res = (ethr_sint_t *) var;
#endif
    return res;
}


/* -- cmpxchg() -- */


ethr_sint_t ethr_atomic_cmpxchg(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_cmpxchg__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    res = old_val;
    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
#endif
    return res;
}

ethr_sint_t ethr_atomic_cmpxchg_ddrb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic_cmpxchg(var, val, old_val);
#else
    return ethr_atomic_cmpxchg_rb(var, val, old_val);
#endif
}

ethr_sint_t ethr_atomic_cmpxchg_rb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_cmpxchg_rb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    res = old_val;
    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint_t ethr_atomic_cmpxchg_wb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_cmpxchg_wb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    res = old_val;
    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
#endif
    return res;
}

ethr_sint_t ethr_atomic_cmpxchg_acqb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_cmpxchg_acqb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    res = old_val;
    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint_t ethr_atomic_cmpxchg_relb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_cmpxchg_relb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = old_val;
    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
#endif
    return res;
}

ethr_sint_t ethr_atomic_cmpxchg_mb(ethr_atomic_t *var, ethr_sint_t val, ethr_sint_t old_val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_cmpxchg_mb__(var, val, old_val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    res = old_val;
    (void) amc_cmpxchg(&var->amc, 0, &var->sint, &val, &res);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- xchg() -- */


ethr_sint_t ethr_atomic_xchg(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_xchg__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
#endif
    return res;
}

ethr_sint_t ethr_atomic_xchg_ddrb(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic_xchg(var, val);
#else
    return ethr_atomic_xchg_rb(var, val);
#endif
}

ethr_sint_t ethr_atomic_xchg_rb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_xchg_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint_t ethr_atomic_xchg_wb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_xchg_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
#endif
    return res;
}

ethr_sint_t ethr_atomic_xchg_acqb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_xchg_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint_t ethr_atomic_xchg_relb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_xchg_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
#endif
    return res;
}

ethr_sint_t ethr_atomic_xchg_mb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_xchg_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- set() -- */


void ethr_atomic_set(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_set__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_set(&var->amc, 0, &var->sint, &val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic_set_ddrb(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic_set(var, val);
#else
    ethr_atomic_set_rb(var, val);
#endif
}

void ethr_atomic_set_rb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_set_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_set(&var->amc, 0, &var->sint, &val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic_set_wb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_set_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    amc_set(&var->amc, 0, &var->sint, &val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic_set_acqb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_set_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_set(&var->amc, 0, &var->sint, &val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic_set_relb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_set_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    amc_set(&var->amc, 0, &var->sint, &val);
#else
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic_set_mb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_set_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    amc_set(&var->amc, 0, &var->sint, &val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- init() -- */


void ethr_atomic_init(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_init__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_init(&var->amc, 0, &var->sint, &val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic_init_ddrb(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic_init(var, val);
#else
    ethr_atomic_init_rb(var, val);
#endif
}

void ethr_atomic_init_rb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_init_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_init(&var->amc, 0, &var->sint, &val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic_init_wb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_init_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    amc_init(&var->amc, 0, &var->sint, &val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic_init_acqb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_init_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_init(&var->amc, 0, &var->sint, &val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic_init_relb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_init_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    amc_init(&var->amc, 0, &var->sint, &val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic_init_mb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_init_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    amc_init(&var->amc, 0, &var->sint, &val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- add_read() -- */


ethr_sint_t ethr_atomic_add_read(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_add_read__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
#endif
    return res;
}

ethr_sint_t ethr_atomic_add_read_ddrb(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic_add_read(var, val);
#else
    return ethr_atomic_add_read_rb(var, val);
#endif
}

ethr_sint_t ethr_atomic_add_read_rb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_add_read_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint_t ethr_atomic_add_read_wb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_add_read_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
#endif
    return res;
}

ethr_sint_t ethr_atomic_add_read_acqb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_add_read_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint_t ethr_atomic_add_read_relb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_add_read_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
#endif
    return res;
}

ethr_sint_t ethr_atomic_add_read_mb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_add_read_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val; res = var->sint);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- read() -- */


ethr_sint_t ethr_atomic_read(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_read(&var->amc, 0, &var->sint, &res);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_ddrb(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic_read(var);
#else
    return ethr_atomic_read_rb(var);
#endif
}

ethr_sint_t ethr_atomic_read_rb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_rb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_read(&var->amc, 0, &var->sint, &res);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_wb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_wb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    amc_read(&var->amc, 0, &var->sint, &res);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_acqb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_acqb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    amc_read(&var->amc, 0, &var->sint, &res);
    ETHR_MEMBAR(ETHR_LoadStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_relb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_relb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    amc_read(&var->amc, 0, &var->sint, &res);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_mb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_mb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    amc_read(&var->amc, 0, &var->sint, &res);
    ETHR_MEMBAR(ETHR_LoadStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#endif
    return res;
}


/* -- inc_read() -- */


ethr_sint_t ethr_atomic_inc_read(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_inc_read__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
#endif
    return res;
}

ethr_sint_t ethr_atomic_inc_read_ddrb(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic_inc_read(var);
#else
    return ethr_atomic_inc_read_rb(var);
#endif
}

ethr_sint_t ethr_atomic_inc_read_rb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_inc_read_rb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint_t ethr_atomic_inc_read_wb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_inc_read_wb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
#endif
    return res;
}

ethr_sint_t ethr_atomic_inc_read_acqb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_inc_read_acqb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint_t ethr_atomic_inc_read_relb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_inc_read_relb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
#endif
    return res;
}

ethr_sint_t ethr_atomic_inc_read_mb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_inc_read_mb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = ++(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- dec_read() -- */


ethr_sint_t ethr_atomic_dec_read(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_dec_read__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
#endif
    return res;
}

ethr_sint_t ethr_atomic_dec_read_ddrb(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic_dec_read(var);
#else
    return ethr_atomic_dec_read_rb(var);
#endif
}

ethr_sint_t ethr_atomic_dec_read_rb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_dec_read_rb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint_t ethr_atomic_dec_read_wb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_dec_read_wb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
#endif
    return res;
}

ethr_sint_t ethr_atomic_dec_read_acqb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_dec_read_acqb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint_t ethr_atomic_dec_read_relb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_dec_read_relb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
#endif
    return res;
}

ethr_sint_t ethr_atomic_dec_read_mb(ethr_atomic_t *var)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_dec_read_mb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = --(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- add() -- */


void ethr_atomic_add(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_add__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
#endif

}

void ethr_atomic_add_ddrb(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic_add(var, val);
#else
    ethr_atomic_add_rb(var, val);
#endif
}

void ethr_atomic_add_rb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_add_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic_add_wb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_add_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
#endif

}

void ethr_atomic_add_acqb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_add_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic_add_relb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_add_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
#endif

}

void ethr_atomic_add_mb(ethr_atomic_t *var, ethr_sint_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_add_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, var->sint += val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- inc() -- */


void ethr_atomic_inc(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_inc__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
#endif

}

void ethr_atomic_inc_ddrb(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic_inc(var);
#else
    ethr_atomic_inc_rb(var);
#endif
}

void ethr_atomic_inc_rb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_inc_rb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic_inc_wb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_inc_wb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
#endif

}

void ethr_atomic_inc_acqb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_inc_acqb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic_inc_relb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_inc_relb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
#endif

}

void ethr_atomic_inc_mb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_inc_mb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, ++(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- dec() -- */


void ethr_atomic_dec(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_dec__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
#endif

}

void ethr_atomic_dec_ddrb(ethr_atomic_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic_dec(var);
#else
    ethr_atomic_dec_rb(var);
#endif
}

void ethr_atomic_dec_rb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_dec_rb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic_dec_wb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_dec_wb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
#endif

}

void ethr_atomic_dec_acqb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_dec_acqb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic_dec_relb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_dec_relb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
#endif

}

void ethr_atomic_dec_mb(ethr_atomic_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    ethr_atomic_dec_mb__(var);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, --(var->sint));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- read_band() -- */


ethr_sint_t ethr_atomic_read_band(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_band__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_band_ddrb(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic_read_band(var, val);
#else
    return ethr_atomic_read_band_rb(var, val);
#endif
}

ethr_sint_t ethr_atomic_read_band_rb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_band_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_band_wb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_band_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_band_acqb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_band_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_band_relb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_band_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_band_mb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_band_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- read_bor() -- */


ethr_sint_t ethr_atomic_read_bor(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_bor__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_bor_ddrb(ethr_atomic_t *var, ethr_sint_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic_read_bor(var, val);
#else
    return ethr_atomic_read_bor_rb(var, val);
#endif
}

ethr_sint_t ethr_atomic_read_bor_rb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_bor_rb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_bor_wb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_bor_wb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_bor_acqb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_bor_acqb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_bor_relb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_bor_relb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
#endif
    return res;
}

ethr_sint_t ethr_atomic_read_bor_mb(ethr_atomic_t *var, ethr_sint_t val)
{
    ethr_sint_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_WORD_SZ_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic_read_bor_mb__(var, val);
#elif defined(ETHR_AMC_FALLBACK__)
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_AMC_MODIFICATION_OPS__(&var->amc, res = var->sint; var->sint |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* ---------- 32-bit atomic implementation ---------- */




/* --- addr() --- */

ethr_sint32_t *ethr_atomic32_addr(ethr_atomic32_t *var)
{
    ethr_sint32_t *res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_addr__(var);

#else
    res = (ethr_sint32_t *) var;
#endif
    return res;
}


/* -- cmpxchg() -- */


ethr_sint32_t ethr_atomic32_cmpxchg(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_cmpxchg__(var, val, old_val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_cmpxchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic32_cmpxchg(var, val, old_val);
#else
    return ethr_atomic32_cmpxchg_rb(var, val, old_val);
#endif
}

ethr_sint32_t ethr_atomic32_cmpxchg_rb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_cmpxchg_rb__(var, val, old_val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_cmpxchg_wb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_cmpxchg_wb__(var, val, old_val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_cmpxchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_cmpxchg_acqb__(var, val, old_val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_cmpxchg_relb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_cmpxchg_relb__(var, val, old_val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_cmpxchg_mb(ethr_atomic32_t *var, ethr_sint32_t val, ethr_sint32_t old_val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_cmpxchg_mb__(var, val, old_val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = (*var == old_val ? (*var = val, old_val) : *var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- xchg() -- */


ethr_sint32_t ethr_atomic32_xchg(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_xchg__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_xchg_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic32_xchg(var, val);
#else
    return ethr_atomic32_xchg_rb(var, val);
#endif
}

ethr_sint32_t ethr_atomic32_xchg_rb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_xchg_rb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_xchg_wb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_xchg_wb__(var, val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_xchg_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_xchg_acqb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_xchg_relb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_xchg_relb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_xchg_mb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_xchg_mb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- set() -- */


void ethr_atomic32_set(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_set__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic32_set_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic32_set(var, val);
#else
    ethr_atomic32_set_rb(var, val);
#endif
}

void ethr_atomic32_set_rb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_set_rb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic32_set_wb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_set_wb__(var, val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic32_set_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_set_acqb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic32_set_relb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_set_relb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic32_set_mb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_set_mb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadStore|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- init() -- */


void ethr_atomic32_init(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_init__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic32_init_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic32_init(var, val);
#else
    ethr_atomic32_init_rb(var, val);
#endif
}

void ethr_atomic32_init_rb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_init_rb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic32_init_wb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_init_wb__(var, val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic32_init_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_init_acqb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic32_init_relb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_init_relb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
#endif

}

void ethr_atomic32_init_mb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_init_mb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var = val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- add_read() -- */


ethr_sint32_t ethr_atomic32_add_read(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_add_read__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_add_read_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic32_add_read(var, val);
#else
    return ethr_atomic32_add_read_rb(var, val);
#endif
}

ethr_sint32_t ethr_atomic32_add_read_rb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_add_read_rb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_add_read_wb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_add_read_wb__(var, val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_add_read_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_add_read_acqb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_add_read_relb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_add_read_relb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_add_read_mb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_add_read_mb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val; res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- read() -- */


ethr_sint32_t ethr_atomic32_read(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_ddrb(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic32_read(var);
#else
    return ethr_atomic32_read_rb(var);
#endif
}

ethr_sint32_t ethr_atomic32_read_rb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_rb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_wb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_wb__(var);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_acqb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_acqb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_relb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_relb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_mb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_mb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore);
#endif
    return res;
}


/* -- inc_read() -- */


ethr_sint32_t ethr_atomic32_inc_read(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_inc_read__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_inc_read_ddrb(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic32_inc_read(var);
#else
    return ethr_atomic32_inc_read_rb(var);
#endif
}

ethr_sint32_t ethr_atomic32_inc_read_rb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_inc_read_rb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_inc_read_wb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_inc_read_wb__(var);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_inc_read_acqb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_inc_read_acqb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_inc_read_relb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_inc_read_relb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_inc_read_mb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_inc_read_mb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- dec_read() -- */


ethr_sint32_t ethr_atomic32_dec_read(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_dec_read__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_dec_read_ddrb(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic32_dec_read(var);
#else
    return ethr_atomic32_dec_read_rb(var);
#endif
}

ethr_sint32_t ethr_atomic32_dec_read_rb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_dec_read_rb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_dec_read_wb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_dec_read_wb__(var);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_dec_read_acqb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_dec_read_acqb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_dec_read_relb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_dec_read_relb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_dec_read_mb(ethr_atomic32_t *var)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_dec_read_mb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- add() -- */


void ethr_atomic32_add(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_add__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
#endif

}

void ethr_atomic32_add_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic32_add(var, val);
#else
    ethr_atomic32_add_rb(var, val);
#endif
}

void ethr_atomic32_add_rb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_add_rb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic32_add_wb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_add_wb__(var, val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
#endif

}

void ethr_atomic32_add_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_add_acqb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic32_add_relb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_add_relb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
#endif

}

void ethr_atomic32_add_mb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_add_mb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, *var += val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- inc() -- */


void ethr_atomic32_inc(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_inc__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
#endif

}

void ethr_atomic32_inc_ddrb(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic32_inc(var);
#else
    ethr_atomic32_inc_rb(var);
#endif
}

void ethr_atomic32_inc_rb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_inc_rb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic32_inc_wb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_inc_wb__(var);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
#endif

}

void ethr_atomic32_inc_acqb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_inc_acqb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic32_inc_relb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_inc_relb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
#endif

}

void ethr_atomic32_inc_mb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_inc_mb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, ++(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- dec() -- */


void ethr_atomic32_dec(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_dec__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
#endif

}

void ethr_atomic32_dec_ddrb(ethr_atomic32_t *var)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    ethr_atomic32_dec(var);
#else
    ethr_atomic32_dec_rb(var);
#endif
}

void ethr_atomic32_dec_rb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_dec_rb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif

}

void ethr_atomic32_dec_wb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_dec_wb__(var);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
#endif

}

void ethr_atomic32_dec_acqb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_dec_acqb__(var);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}

void ethr_atomic32_dec_relb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_dec_relb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
#endif

}

void ethr_atomic32_dec_mb(ethr_atomic32_t *var)
{
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    ethr_atomic32_dec_mb__(var);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, --(*var));
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif

}


/* -- read_band() -- */


ethr_sint32_t ethr_atomic32_read_band(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_band__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_band_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic32_read_band(var, val);
#else
    return ethr_atomic32_read_band_rb(var, val);
#endif
}

ethr_sint32_t ethr_atomic32_read_band_rb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_band_rb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_band_wb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_band_wb__(var, val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_band_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_band_acqb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_band_relb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_band_relb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_band_mb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_band_mb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var &= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}


/* -- read_bor() -- */


ethr_sint32_t ethr_atomic32_read_bor(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_bor__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_bor_ddrb(ethr_atomic32_t *var, ethr_sint32_t val)
{
#ifdef ETHR_ORDERED_READ_DEPEND
    return ethr_atomic32_read_bor(var, val);
#else
    return ethr_atomic32_read_bor_rb(var, val);
#endif
}

ethr_sint32_t ethr_atomic32_read_bor_rb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_bor_rb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
    ETHR_MEMBAR(ETHR_LoadLoad);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_bor_wb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_bor_wb__(var, val);
#else
    ETHR_MEMBAR(ETHR_StoreStore);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_bor_acqb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_bor_acqb__(var, val);
#else
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_bor_relb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_bor_relb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
#endif
    return res;
}

ethr_sint32_t ethr_atomic32_read_bor_mb(ethr_atomic32_t *var, ethr_sint32_t val)
{
    ethr_sint32_t res;
    ETHR_ASSERT(!ethr_not_inited__);
    ETHR_ASSERT(var);
#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
    res =  ethr_atomic32_read_bor_mb__(var, val);
#else
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_StoreLoad);
    ETHR_ATOMIC_OP_FALLBACK_IMPL__(var, res = *var; *var |= val);
    ETHR_MEMBAR(ETHR_LoadLoad|ETHR_LoadStore|ETHR_StoreLoad|ETHR_StoreStore);
#endif
    return res;
}



/* --------- Info functions --------- */

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
char *zero_ops[] = {NULL};
#endif


static char *native_su_dw_atomic_ops[] = {
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG
    "cmpxchg",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB
    "cmpxchg_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB
    "cmpxchg_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB
    "cmpxchg_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB
    "cmpxchg_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB
    "cmpxchg_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET
    "set",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RB
    "set_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB
    "set_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_ACQB
    "set_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB
    "set_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_MB
    "set_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ
    "read",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB
    "read_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_WB
    "read_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB
    "read_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RELB
    "read_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_MB
    "read_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT
    "init",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RB
    "init_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_WB
    "init_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_ACQB
    "init_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_RELB
    "init_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_INIT_MB
    "init_mb",
#endif
    NULL
};

char **
ethr_native_su_dw_atomic_ops(void)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
	return &zero_ops[0];
#endif
    return &native_su_dw_atomic_ops[0];
}


static char *native_dw_atomic_ops[] = {
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG
    "cmpxchg",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB
    "cmpxchg_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB
    "cmpxchg_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB
    "cmpxchg_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB
    "cmpxchg_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB
    "cmpxchg_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET
    "set",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RB
    "set_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB
    "set_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_ACQB
    "set_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB
    "set_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_MB
    "set_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ
    "read",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB
    "read_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_WB
    "read_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB
    "read_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RELB
    "read_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_MB
    "read_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT
    "init",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RB
    "init_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_WB
    "init_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_ACQB
    "init_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_RELB
    "init_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_INIT_MB
    "init_mb",
#endif
    NULL
};

char **
ethr_native_dw_atomic_ops(void)
{

#if defined(ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
    if (!ETHR_RTCHK_USE_NATIVE_DW_ATOMIC_IMPL__)
	return &zero_ops[0];
#endif
    return &native_dw_atomic_ops[0];
}


static char *native_atomic64_ops[] = {
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG
    "cmpxchg",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB
    "cmpxchg_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB
    "cmpxchg_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_ACQB
    "cmpxchg_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RELB
    "cmpxchg_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB
    "cmpxchg_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG
    "xchg",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RB
    "xchg_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_WB
    "xchg_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_ACQB
    "xchg_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_RELB
    "xchg_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_XCHG_MB
    "xchg_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET
    "set",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RB
    "set_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB
    "set_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_ACQB
    "set_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_RELB
    "set_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB
    "set_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT
    "init",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RB
    "init_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_WB
    "init_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_ACQB
    "init_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_RELB
    "init_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INIT_MB
    "init_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN
    "add_return",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RB
    "add_return_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_WB
    "add_return_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB
    "add_return_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB
    "add_return_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB
    "add_return_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ
    "read",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB
    "read_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_WB
    "read_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_ACQB
    "read_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RELB
    "read_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB
    "read_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN
    "inc_return",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RB
    "inc_return_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_WB
    "inc_return_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_ACQB
    "inc_return_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB
    "inc_return_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB
    "inc_return_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN
    "dec_return",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RB
    "dec_return_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_WB
    "dec_return_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB
    "dec_return_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_RELB
    "dec_return_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB
    "dec_return_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD
    "add",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RB
    "add_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_WB
    "add_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_ACQB
    "add_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RELB
    "add_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_MB
    "add_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC
    "inc",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RB
    "inc_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_WB
    "inc_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_ACQB
    "inc_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RELB
    "inc_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_MB
    "inc_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC
    "dec",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RB
    "dec_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_WB
    "dec_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_ACQB
    "dec_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RELB
    "dec_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_MB
    "dec_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD
    "and_retold",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RB
    "and_retold_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_WB
    "and_retold_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_ACQB
    "and_retold_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_RELB
    "and_retold_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_AND_RETOLD_MB
    "and_retold_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD
    "or_retold",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RB
    "or_retold_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_WB
    "or_retold_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_ACQB
    "or_retold_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_RELB
    "or_retold_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC64_OR_RETOLD_MB
    "or_retold_mb",
#endif
    NULL
};

char **
ethr_native_atomic64_ops(void)
{

    return &native_atomic64_ops[0];
}


static char *native_atomic32_ops[] = {
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG
    "cmpxchg",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB
    "cmpxchg_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB
    "cmpxchg_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB
    "cmpxchg_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB
    "cmpxchg_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB
    "cmpxchg_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG
    "xchg",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RB
    "xchg_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_WB
    "xchg_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_ACQB
    "xchg_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB
    "xchg_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_MB
    "xchg_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET
    "set",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RB
    "set_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB
    "set_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_ACQB
    "set_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_RELB
    "set_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB
    "set_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT
    "init",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RB
    "init_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_WB
    "init_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_ACQB
    "init_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_RELB
    "init_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INIT_MB
    "init_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN
    "add_return",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RB
    "add_return_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_WB
    "add_return_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB
    "add_return_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB
    "add_return_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB
    "add_return_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ
    "read",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB
    "read_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_WB
    "read_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_ACQB
    "read_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RELB
    "read_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB
    "read_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN
    "inc_return",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RB
    "inc_return_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_WB
    "inc_return_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_ACQB
    "inc_return_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB
    "inc_return_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB
    "inc_return_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN
    "dec_return",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RB
    "dec_return_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_WB
    "dec_return_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB
    "dec_return_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB
    "dec_return_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB
    "dec_return_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD
    "add",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RB
    "add_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_WB
    "add_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_ACQB
    "add_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RELB
    "add_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_MB
    "add_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC
    "inc",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RB
    "inc_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_WB
    "inc_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_ACQB
    "inc_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RELB
    "inc_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_MB
    "inc_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC
    "dec",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RB
    "dec_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_WB
    "dec_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_ACQB
    "dec_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RELB
    "dec_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_MB
    "dec_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD
    "and_retold",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RB
    "and_retold_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_WB
    "and_retold_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_ACQB
    "and_retold_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB
    "and_retold_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_MB
    "and_retold_mb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD
    "or_retold",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RB
    "or_retold_rb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_WB
    "or_retold_wb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_ACQB
    "or_retold_acqb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB
    "or_retold_relb",
#endif
#ifdef ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_MB
    "or_retold_mb",
#endif
    NULL
};

char **
ethr_native_atomic32_ops(void)
{

    return &native_atomic32_ops[0];
}