diff options
Diffstat (limited to 'erts/include')
| -rw-r--r-- | erts/include/internal/ethread.h | 66 | ||||
| -rw-r--r-- | erts/include/internal/ethread_header_config.h.in | 23 | ||||
| -rw-r--r-- | erts/include/internal/gcc/ethr_atomic.h | 164 | ||||
| -rw-r--r-- | erts/include/internal/gcc/ethread.h | 30 | ||||
| -rw-r--r-- | erts/include/internal/i386/atomic.h | 20 | ||||
| -rw-r--r-- | erts/include/internal/ppc32/atomic.h | 11 | ||||
| -rw-r--r-- | erts/include/internal/sparc32/atomic.h | 14 | ||||
| -rw-r--r-- | erts/include/internal/tile/atomic.h | 2 |
8 files changed, 287 insertions, 43 deletions
diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 934a79c6f9..4e7a38cd5c 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -375,6 +375,12 @@ ETHR_INLINE_FUNC_NAME_(ethr_mutex_unlock)(ethr_mutex *mtx) #define ETHR_HAVE_OPTIMIZED_ATOMIC_OPS 1 #define ETHR_HAVE_OPTIMIZED_LOCKS 1 +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile LONG x___ = 0; \ + (void) _InterlockedCompareExchange(&x___, (LONG) 1, (LONG) 0); \ +} while (0) + typedef struct { volatile LONG value; } ethr_atomic_t; @@ -688,25 +694,33 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) #endif /* For CPU-optimised atomics, spinlocks, and rwlocks. */ -#if !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(__GNUC__) -# if ETHR_SIZEOF_PTR == 4 -# if defined(__i386__) -# include "i386/ethread.h" -# elif (defined(__powerpc__) || defined(__ppc__)) && !defined(__powerpc64__) -# include "ppc32/ethread.h" -# elif defined(__sparc__) -# include "sparc32/ethread.h" -# elif defined(__tile__) -# include "tile/ethread.h" +#if !defined(ETHR_DISABLE_NATIVE_IMPLS) +# if defined(__GNUC__) +# if defined(ETHR_PREFER_GCC_NATIVE_IMPLS) +# include "gcc/ethread.h" # endif -# elif ETHR_SIZEOF_PTR == 8 -# if defined(__x86_64__) -# include "x86_64/ethread.h" -# elif defined(__sparc__) && defined(__arch64__) -# include "sparc64/ethread.h" +# ifndef ETHR_HAVE_NATIVE_ATOMICS +# if ETHR_SIZEOF_PTR == 4 +# if defined(__i386__) +# include "i386/ethread.h" +# elif (defined(__powerpc__)||defined(__ppc__))&&!defined(__powerpc64__) +# include "ppc32/ethread.h" +# elif defined(__sparc__) +# include "sparc32/ethread.h" +# elif defined(__tile__) +# include "tile/ethread.h" +# endif +# elif ETHR_SIZEOF_PTR == 8 +# if defined(__x86_64__) +# include "x86_64/ethread.h" +# elif defined(__sparc__) && defined(__arch64__) +# include "sparc64/ethread.h" +# endif +# endif +# include "gcc/ethread.h" # endif # endif -#endif /* !defined(ETHR_DISABLE_NATIVE_IMPLS) && defined(__GNUC__) */ +#endif /* !defined(ETHR_DISABLE_NATIVE_IMPLS) */ #ifdef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS # undef ETHR_HAVE_NATIVE_ATOMICS @@ -1124,8 +1138,14 @@ ETHR_INLINE_FUNC_NAME_(ethr_write_lock)(ethr_rwlock_t *lock) */ #ifndef ETHR_HAVE_OPTIMIZED_ATOMIC_OPS -#define ETHR_ATOMIC_ADDR_BITS 4 -#define ETHR_ATOMIC_ADDR_SHIFT 3 +/* + * ETHR_MEMORY_BARRIER orders between locked and atomic accesses only, + * i.e. when this atomic fallback is used a noop is sufficient. + */ +#define ETHR_MEMORY_BARRIER + +#define ETHR_ATOMIC_ADDR_BITS 10 +#define ETHR_ATOMIC_ADDR_SHIFT 6 typedef struct { union { diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index e5b4946a53..c9fd87c2f6 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2004-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2004-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -50,6 +50,19 @@ /* Define if you have a pthread_rwlock implementation that can be used */ #undef ETHR_HAVE_PTHREAD_RWLOCK_INIT +/* Define if you have the pthread_rwlockattr_setkind_np() function. */ +#undef ETHR_HAVE_PTHREAD_RWLOCKATTR_SETKIND_NP + +/* Define if you have the PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP rwlock + attribute. */ +#undef ETHR_HAVE_PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP + +/* Define if you have gcc atomic operations */ +#undef ETHR_HAVE_GCC_ATOMIC_OPS + +/* Define if you prefer gcc native ethread implementations */ +#undef ETHR_PREFER_GCC_NATIVE_IMPLS + /* Define if you want to turn on extra sanity checking in the ethread library */ #undef ETHR_XCHK diff --git a/erts/include/internal/gcc/ethr_atomic.h b/erts/include/internal/gcc/ethr_atomic.h new file mode 100644 index 0000000000..775030c8d5 --- /dev/null +++ b/erts/include/internal/gcc/ethr_atomic.h @@ -0,0 +1,164 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomics ethread support using gcc's builtins + * Author: Rickard Green + */ + +#ifndef ETHR_GCC_ATOMIC_H__ +#define ETHR_GCC_ATOMIC_H__ + +#if !defined(ETHR_HAVE_NATIVE_ATOMICS) && defined(ETHR_HAVE_GCC_ATOMIC_OPS) +#define ETHR_HAVE_NATIVE_ATOMICS 1 + +#define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 0 +/* Enable immediate read/write on platforms where we know it is safe */ +#if defined(__i386__) || defined(__x86_64__) || defined(__sparc__) \ + || defined(__powerpc__) || defined(__ppc__) +# undef ETHR_IMMED_ATOMIC_SET_GET_SAFE__ +# define ETHR_IMMED_ATOMIC_SET_GET_SAFE__ 1 +#endif + +typedef struct { + volatile long counter; +} ethr_native_atomic_t; + + +/* + * According to the documentation this is what we want: + * #define ETHR_MEMORY_BARRIER __sync_synchronize() + * However, __sync_synchronize() is known to erroneously be + * a noop on at least some platforms with some gcc versions. + * This has suposedly been fixed in some gcc version, but we + * don't know from which version. Therefore, we use the + * workaround implemented below on all gcc versions. + */ +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile long x___ = 0; \ + (void) __sync_val_compare_and_swap(&x___, (long) 0, (long) 1); \ +} while (0) + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_AUX_IMPL__) + +static ETHR_INLINE void +ethr_native_atomic_set(ethr_native_atomic_t *var, long value) +{ +#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ + var->counter = value; +#else + /* + * Unfortunately no __sync_store() or similar exist in the gcc atomic + * op interface. We therefore have to simulate it this way... + */ + long act = 0, exp; + do { + exp = act; + act = __sync_val_compare_and_swap(&var->counter, exp, value); + } while (act != exp); +#endif +} + +#define ethr_native_atomic_init ethr_native_atomic_set + +static ETHR_INLINE long +ethr_native_atomic_read(ethr_native_atomic_t *var) +{ +#if ETHR_IMMED_ATOMIC_SET_GET_SAFE__ + return var->counter; +#else + /* + * Unfortunately no __sync_fetch() or similar exist in the gcc atomic + * op interface. We therefore have to simulate it this way... + */ + return __sync_add_and_fetch(&var->counter, (long) 0); +#endif +} + +static ETHR_INLINE void +ethr_native_atomic_add(ethr_native_atomic_t *var, long incr) +{ + (void) __sync_add_and_fetch(&var->counter, incr); +} + +static ETHR_INLINE long +ethr_native_atomic_add_return(ethr_native_atomic_t *var, long incr) +{ + return __sync_add_and_fetch(&var->counter, incr); +} + +static ETHR_INLINE void +ethr_native_atomic_inc(ethr_native_atomic_t *var) +{ + (void) __sync_add_and_fetch(&var->counter, (long) 1); +} + +static ETHR_INLINE void +ethr_native_atomic_dec(ethr_native_atomic_t *var) +{ + (void) __sync_sub_and_fetch(&var->counter, (long) 1); +} + +static ETHR_INLINE long +ethr_native_atomic_inc_return(ethr_native_atomic_t *var) +{ + return __sync_add_and_fetch(&var->counter, (long) 1); +} + +static ETHR_INLINE long +ethr_native_atomic_dec_return(ethr_native_atomic_t *var) +{ + return __sync_sub_and_fetch(&var->counter, (long) 1); +} + +static ETHR_INLINE long +ethr_native_atomic_and_retold(ethr_native_atomic_t *var, long mask) +{ + return __sync_fetch_and_and(&var->counter, mask); +} + +static ETHR_INLINE long +ethr_native_atomic_or_retold(ethr_native_atomic_t *var, long mask) +{ + return (long) __sync_fetch_and_or(&var->counter, mask); +} + +static ETHR_INLINE long +ethr_native_atomic_cmpxchg(ethr_native_atomic_t *var, long new, long old) +{ + return __sync_val_compare_and_swap(&var->counter, old, new); +} + +static ETHR_INLINE long +ethr_native_atomic_xchg(ethr_native_atomic_t *var, long new) +{ + long exp, act = 0; + do { + exp = act; + act = __sync_val_compare_and_swap(&var->counter, exp, new); + } while (act != exp); + return act; +} + +#endif + +#endif + +#endif diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h new file mode 100644 index 0000000000..bb378e31e0 --- /dev/null +++ b/erts/include/internal/gcc/ethread.h @@ -0,0 +1,30 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2010. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native atomic ethread support when using gcc + * Author: Rickard Green + */ + +#ifndef ETHREAD_GCC_H__ +#define ETHREAD_GCC_H__ + +#include "ethr_atomic.h" + +#endif diff --git a/erts/include/internal/i386/atomic.h b/erts/include/internal/i386/atomic.h index 3291ad38e5..90b4c5f773 100644 --- a/erts/include/internal/i386/atomic.h +++ b/erts/include/internal/i386/atomic.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -32,6 +32,16 @@ typedef struct { volatile long counter; } ethr_native_atomic_t; +#ifdef __x86_64__ +#define ETHR_MEMORY_BARRIER __asm__ __volatile__("mfence" : : : "memory") +#else +#define ETHR_MEMORY_BARRIER \ +do { \ + volatile long x___ = 0; \ + __asm__ __volatile__("lock; incl %0" : "=m"(x___) : "m"(x___) : "memory"); \ +} while (0) +#endif + #ifdef ETHR_TRY_INLINE_FUNCS #ifdef __x86_64__ diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index fa701c6a92..105d874995 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -32,6 +32,7 @@ typedef struct { volatile int counter; } ethr_native_atomic_t; +#define ETHR_MEMORY_BARRIER __asm__ __volatile__("sync" : : : "memory") #ifdef ETHR_TRY_INLINE_FUNCS diff --git a/erts/include/internal/sparc32/atomic.h b/erts/include/internal/sparc32/atomic.h index d6fdc6b2a4..8fde449a52 100644 --- a/erts/include/internal/sparc32/atomic.h +++ b/erts/include/internal/sparc32/atomic.h @@ -1,19 +1,19 @@ /* * %CopyrightBegin% - * - * Copyright Ericsson AB 2005-2009. All Rights Reserved. - * + * + * Copyright Ericsson AB 2005-2010. All Rights Reserved. + * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in * compliance with the License. You should have received a copy of the * Erlang Public License along with this software. If not, it can be * retrieved online at http://www.erlang.org/. - * + * * Software distributed under the License is distributed on an "AS IS" * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See * the License for the specific language governing rights and limitations * under the License. - * + * * %CopyrightEnd% */ @@ -28,6 +28,10 @@ typedef struct { volatile long counter; } ethr_native_atomic_t; +#define ETHR_MEMORY_BARRIER \ + __asm__ __volatile__("membar #LoadLoad|#LoadStore|#StoreLoad|#StoreStore\n" \ + : : : "memory") + #ifdef ETHR_TRY_INLINE_FUNCS #if defined(__arch64__) diff --git a/erts/include/internal/tile/atomic.h b/erts/include/internal/tile/atomic.h index 59a9250e7c..5e4c7ac9fe 100644 --- a/erts/include/internal/tile/atomic.h +++ b/erts/include/internal/tile/atomic.h @@ -32,6 +32,8 @@ typedef struct { volatile long counter; } ethr_native_atomic_t; +#define ETHR_MEMORY_BARRIER __insn_mf() + #ifdef ETHR_TRY_INLINE_FUNCS static ETHR_INLINE void |