diff options
39 files changed, 2073 insertions, 184 deletions
diff --git a/HOWTO/INSTALL.md b/HOWTO/INSTALL.md index 7a7e63164c..53b1b8cd8a 100644 --- a/HOWTO/INSTALL.md +++ b/HOWTO/INSTALL.md @@ -409,6 +409,30 @@ Some of the available `configure` options are: If you or your system has special requirements please read the `Makefile` for additional configuration information. +#### Atomic Memory Operations and the VM #### + +The VM with SMP support makes quite a heavy use of atomic memory operations. +An implementation providing native atomic memory operations is therefore very +important when building Erlang/OTP. By default the VM will refuse to build +if native atomic memory operations are not available. + +Erlang/OTP itself provides implementations of native atomic memory operations +that can be used when compiling with a `gcc` compatible compiler on 32-bit +and 64-bit x86, 32-bit and 64-bit SPARC V9, and 32-bit PowerPC. When compiling +with a `gcc` compatible compiler on other architectures, the VM may be able to +make use of native atomic operations using the `__sync_*` primitives, but this +should only be used as a last resort since this wont give you optimal +performance. When compiling on Windows using a MicroSoft Visual C++ compiler +native atomic memory operations are provided by Windows APIs. + +You are recommended to use the native atomic implementation provided by +Erlang/OTP, or the API provided by Windows. If these do not provide native +atomics on your platform, you are recommended to build and install +[libatomic_ops][] before building Erlang/OTP. The `libatomic_ops` library +provides native atomic memory operations for a variety of platforms and +compilers. When building Erlang/OTP you need to inform the build system of +where the `libatomic_ops` library is installed using the +`--with-libatomic_ops=PATH` configure switch. ### Building ### @@ -862,3 +886,4 @@ under the License. [Optional Utilities]: #Optional-Utilities [Building on a Mac]: #Advanced-configuration-and-build-of-ErlangOTP_Building_OS-X-Darwin [Building with wxErlang]: #Advanced-configuration-and-build-of-ErlangOTP_Building_Building-with-wxErlang + [libatomic_ops]: https://github.com/ivmai/libatomic_ops/ diff --git a/aclocal.m4 b/aclocal.m4 index ed492d55ff..d78025b0be 100644 --- a/aclocal.m4 +++ b/aclocal.m4 @@ -1421,9 +1421,31 @@ case "$THR_LIB_NAME" in int z; AO_nop_full(); +#if defined(AO_HAVE_store) AO_store(&x, (AO_t) 0); +#elif defined(AO_HAVE_store_release) + AO_store_release(&x, (AO_t) 0); +#else +#error No store +#endif +#if defined(AO_HAVE_load) z = AO_load(&x); +#elif defined(AO_HAVE_load_acquire) + z = AO_load_acquire(&x); +#else +#error No load +#endif +#if defined(AO_HAVE_compare_and_swap_full) z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_release) + z = AO_compare_and_swap_release(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_acquire) + z = AO_compare_and_swap_acquire(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap) + z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); +#else +#error No compare_and_swap +#endif ], [ethr_have_native_atomics=yes ethr_have_libatomic_ops=yes]) diff --git a/configure.in b/configure.in index 008fa38632..0e09af4eff 100644 --- a/configure.in +++ b/configure.in @@ -266,6 +266,13 @@ AC_ARG_WITH(ssl-incl, AS_HELP_STRING([--with-ssl-incl=PATH], [location of OpenSSL include dir, if different than specified by --with-ssl=PATH])) +AC_ARG_WITH(ssl-rpath, +AS_HELP_STRING([--with-ssl-rpath=yes|no|PATHS], + [runtime library path for OpenSSL. Default is 'yes', which equates to a + number of standard locations. If 'no', then no runtime + library paths wil be used. Anything else should be a + comma separated list of paths.])) + AC_ARG_ENABLE(dynamic-ssl-lib, AS_HELP_STRING([--disable-dynamic-ssl-lib], [disable using dynamic openssl libraries])) diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index ed492d55ff..d78025b0be 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -1421,9 +1421,31 @@ case "$THR_LIB_NAME" in int z; AO_nop_full(); +#if defined(AO_HAVE_store) AO_store(&x, (AO_t) 0); +#elif defined(AO_HAVE_store_release) + AO_store_release(&x, (AO_t) 0); +#else +#error No store +#endif +#if defined(AO_HAVE_load) z = AO_load(&x); +#elif defined(AO_HAVE_load_acquire) + z = AO_load_acquire(&x); +#else +#error No load +#endif +#if defined(AO_HAVE_compare_and_swap_full) z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_release) + z = AO_compare_and_swap_release(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_acquire) + z = AO_compare_and_swap_acquire(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap) + z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); +#else +#error No compare_and_swap +#endif ], [ethr_have_native_atomics=yes ethr_have_libatomic_ops=yes]) diff --git a/erts/configure.in b/erts/configure.in index a594d59355..877e0d4c1c 100644 --- a/erts/configure.in +++ b/erts/configure.in @@ -4105,6 +4105,20 @@ esac ], [with_ssl_incl=$with_ssl]) #default +AC_ARG_WITH(ssl-rpath, +AS_HELP_STRING([--with-ssl-rpath=yes|no|PATHS], + [runtime library path for OpenSSL. Default is "yes", which equates to a + number of standard locations. If "no", then no runtime + library paths wil be used. Anything else should be a + comma separated list of paths.]), +[ +case X$with_ssl in + Xno) AC_MSG_ERROR([--with-ssl-rpath set without --with-ssl]);; +esac +], +[with_ssl_rpath=yes]) #default + + AC_ARG_ENABLE(dynamic-ssl-lib, AS_HELP_STRING([--disable-dynamic-ssl-lib], [disable using dynamic openssl libraries]), @@ -4570,7 +4584,12 @@ cc_rflg="$CFLAG_RUNTIME_LIBRARY_PATH" ld_rflg="$LDFLAG_RUNTIME_LIBRARY_PATH" ded_ld_rflg="$DED_LD_FLAG_RUNTIME_LIBRARY_PATH" -if test "$SSL_APP" != "" && test "$SSL_DYNAMIC_ONLY" = "yes" && \ + +case "$with_ssl_rpath" in + +yes) # Use standard lib locations for ssl runtime library path + + if test "$SSL_APP" != "" && test "$SSL_DYNAMIC_ONLY" = "yes" && \ { test "$cc_rflg" != "" || test "$ld_rflg" != "" || test "$ded_ld_rflg" != ""; } ; then AC_MSG_CHECKING(for ssl runtime library path to use) @@ -4654,7 +4673,25 @@ if test "$SSL_APP" != "" && test "$SSL_DYNAMIC_ONLY" = "yes" && \ AC_MSG_RESULT([$rpath]) test "$rpath" != "" || AC_MSG_WARN([Cannot set run path during linking]) -fi + fi + ;; + +no) # Use no ssl runtime library path + SSL_DED_LD_RUNTIME_LIBRARY_PATH= + ;; + +*) # Use ssl runtime library paths set by --with-ssl-rpath (without any check) + ded_ld_rpath= + delimit= + for dir in `echo $with_ssl_rpath | sed "s/,/ /g"`; do + ded_ld_rpath="$ded_ld_rpath$delimit$ded_ld_rflg$dir" + delimit=" " + done + SSL_DED_LD_RUNTIME_LIBRARY_PATH="$ded_ld_rpath" + ;; + +esac + #-------------------------------------------------------------------- # Os mon stuff. diff --git a/erts/emulator/beam/erl_alloc_util.c b/erts/emulator/beam/erl_alloc_util.c index 55052430e1..e3172dc4fb 100644 --- a/erts/emulator/beam/erl_alloc_util.c +++ b/erts/emulator/beam/erl_alloc_util.c @@ -205,7 +205,7 @@ MBC after deallocating first block: ASSERT(((UWord)(F) & (~FLG_MASK|THIS_FREE_BLK_HDR_FLG|PREV_FREE_BLK_HDR_FLG)) == THIS_FREE_BLK_HDR_FLG), \ (B)->bhdr = ((Sz) | (F)), \ (B)->u.carrier = (C)) - + # define IS_MBC_FIRST_ABLK(AP,B) \ ((((UWord)(B) & ~ERTS_SACRR_UNIT_MASK) == MBC_HEADER_SIZE(AP)) \ && ((B)->bhdr & MBC_ABLK_OFFSET_MASK) == 0) @@ -378,9 +378,8 @@ do { \ #ifdef ERTS_SMP #define SBC_HEADER_SIZE \ - (UNIT_CEILING(sizeof(Carrier_t) \ - - sizeof(ErtsAlcCPoolData_t) \ - + ABLK_HDR_SZ) \ + (UNIT_CEILING(offsetof(Carrier_t, cpool) \ + + ABLK_HDR_SZ) \ - ABLK_HDR_SZ) #else #define SBC_HEADER_SIZE \ @@ -929,6 +928,88 @@ unlink_carrier(CarrierList_t *cl, Carrier_t *crr) #ifdef ERTS_SMP +#ifdef DEBUG +static int is_in_list(ErtsDoubleLink_t* sentinel, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* p; + + ASSERT(node != sentinel); + for (p = sentinel->next; p != sentinel; p = p->next) { + if (p == node) + return 1; + } + return 0; +} +#endif /* DEBUG */ + +static ERTS_INLINE void +link_edl_after(ErtsDoubleLink_t* after_me, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* before_me = after_me->next; + ASSERT(node != after_me && node != before_me); + node->next = before_me; + node->prev = after_me; + before_me->prev = node; + after_me->next = node; +} + +static ERTS_INLINE void +link_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) +{ + ErtsDoubleLink_t* after_me = before_me->prev; + ASSERT(node != before_me && node != after_me); + node->next = before_me; + node->prev = after_me; + before_me->prev = node; + after_me->next = node; +} + +static ERTS_INLINE void +unlink_edl(ErtsDoubleLink_t* node) +{ + node->next->prev = node->prev; + node->prev->next = node->next; +} + +static ERTS_INLINE void +relink_edl_before(ErtsDoubleLink_t* before_me, ErtsDoubleLink_t* node) +{ + if (node != before_me && node != before_me->prev) { + unlink_edl(node); + link_edl_before(before_me, node); + } +} + +static ERTS_INLINE int is_abandoned(Carrier_t *crr) +{ + return crr->cpool.abandoned.next != NULL; +} + +static ERTS_INLINE void +link_abandoned_carrier(ErtsDoubleLink_t* list, Carrier_t *crr) +{ + ASSERT(!is_abandoned(crr)); + + link_edl_after(list, &crr->cpool.abandoned); + + ASSERT(crr->cpool.abandoned.next != &crr->cpool.abandoned); + ASSERT(crr->cpool.abandoned.prev != &crr->cpool.abandoned); +} + +static ERTS_INLINE void +unlink_abandoned_carrier(Carrier_t *crr) +{ + ASSERT(is_in_list(&crr->cpool.orig_allctr->cpool.pooled_list, + &crr->cpool.abandoned) || + is_in_list(&crr->cpool.orig_allctr->cpool.traitor_list, + &crr->cpool.abandoned)); + + unlink_edl(&crr->cpool.abandoned); + + crr->cpool.abandoned.next = NULL; + crr->cpool.abandoned.prev = NULL; +} + static ERTS_INLINE void clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) { @@ -955,7 +1036,7 @@ clear_busy_pool_carrier(Allctr_t *allctr, Carrier_t *crr) } } -#endif +#endif /* ERTS_SMP */ #if 0 #define ERTS_DBG_CHK_FIX_LIST(A, FIX, IX, B) \ @@ -2575,10 +2656,9 @@ mbc_realloc(Allctr_t *allctr, void *p, Uint size, Uint32 alcu_flgs, #ifdef ERTS_SMP #define ERTS_ALC_MAX_DEALLOC_CARRIER 10 -#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 10 +#define ERTS_ALC_CPOOL_MAX_FETCH_INSPECT 20 +#define ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT 10 #define ERTS_ALC_CPOOL_CHECK_LIMIT_COUNT 100 -#define ERTS_ALC_CPOOL_MAX_NO_CARRIERS 5 -#define ERTS_ALC_CPOOL_INSERT_ALLOWED_OFFSET 100 #define ERTS_ALC_CPOOL_MAX_FAILED_STAT_READS 3 #define ERTS_ALC_CPOOL_PTR_MOD_MRK (((erts_aint_t) 1) << 0) @@ -2755,9 +2835,6 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) (erts_aint_t) CARRIER_SZ(crr)); erts_atomic_inc_nob(&allctr->cpool.stat.no_carriers); - erts_smp_atomic_set_nob(&crr->allctr, - ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); - /* * We search in 'next' direction and begin by passing * one element before trying to insert. This in order to @@ -2816,6 +2893,9 @@ cpool_insert(Allctr_t *allctr, Carrier_t *crr) cpool_set_mod_marked(&cpd2p->prev, (erts_aint_t) &crr->cpool, (erts_aint_t) cpd1p); + + erts_smp_atomic_set_wb(&crr->allctr, + ((erts_aint_t) allctr)|ERTS_CRR_ALCTR_FLG_IN_POOL); } static void @@ -2916,59 +2996,163 @@ cpool_delete(Allctr_t *allctr, Allctr_t *prev_allctr, Carrier_t *crr) static Carrier_t * cpool_fetch(Allctr_t *allctr, UWord size) { - int i; + int i, i_stop, has_passed_sentinel; Carrier_t *crr; ErtsAlcCPoolData_t *cpdp; - ErtsAlcCPoolData_t *sentinel = &carrier_pool[allctr->alloc_no].sentinel; + ErtsAlcCPoolData_t *cpool_entrance; + ErtsAlcCPoolData_t *sentinel; + ErtsDoubleLink_t* dl; + ErtsDoubleLink_t* first_old_traitor; ERTS_ALC_CPOOL_ASSERT(allctr->alloc_no == ERTS_ALC_A_INVALID /* testcase */ || erts_thr_progress_is_managed_thread()); - i = 0; + i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; + first_old_traitor = allctr->cpool.traitor_list.next; + cpool_entrance = NULL; - /* First; check our own pending dealloc carrier list... */ - crr = allctr->cpool.dc_list.last; - while (crr && i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { - if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { - unlink_carrier(&allctr->cpool.dc_list, crr); -#ifdef ERTS_ALC_CPOOL_DEBUG - ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, - ((erts_aint_t) allctr)) - == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); -#else - erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); -#endif - return crr; + /* + * Search my own pooled_list, + * i.e my abandoned carriers that were in the pool last time I checked. + */ + + dl = allctr->cpool.pooled_list.next; + while(dl != &allctr->cpool.pooled_list) { + erts_aint_t exp, act; + crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); + + ASSERT(!is_in_list(&allctr->cpool.traitor_list, dl)); + ASSERT(crr->cpool.orig_allctr == allctr); + dl = dl->next; + exp = erts_smp_atomic_read_rb(&crr->allctr); + if ((exp & ERTS_CRR_ALCTR_FLG_MASK) == ERTS_CRR_ALCTR_FLG_IN_POOL + && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + unlink_abandoned_carrier(crr); + + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.pooled_list); + return crr; + } + exp = act; + } + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!cpool_entrance) + cpool_entrance = &crr->cpool; + } + else { /* Not in pool, move to traitor_list */ + unlink_abandoned_carrier(crr); + link_abandoned_carrier(&allctr->cpool.traitor_list, crr); + } + if (--i <= 0) { + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.pooled_list); + return NULL; } - crr = crr->prev; - i++; } - /* ... then the pool ... */ + /* Now search traitor_list. + * i.e carriers employed by other allocators last time I checked. + * They might have been abandoned since then. + */ + + i_stop = (i < ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT ? + 0 : i - ERTS_ALC_CPOOL_MAX_TRAITOR_INSPECT); + dl = first_old_traitor; + while(dl != &allctr->cpool.traitor_list) { + erts_aint_t exp, act; + crr = (Carrier_t *) (((char *) dl) - offsetof(Carrier_t, cpool.abandoned)); + ASSERT(dl != &allctr->cpool.pooled_list); + ASSERT(crr->cpool.orig_allctr == allctr); + dl = dl->next; + exp = erts_smp_atomic_read_rb(&crr->allctr); + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!(exp & ERTS_CRR_ALCTR_FLG_BUSY) + && erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + /* Try to fetch it... */ + act = erts_smp_atomic_cmpxchg_mb(&crr->allctr, + (erts_aint_t) allctr, + exp); + if (act == exp) { + cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + unlink_abandoned_carrier(crr); + + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.traitor_list); + return crr; + } + exp = act; + } + if (exp & ERTS_CRR_ALCTR_FLG_IN_POOL) { + if (!cpool_entrance) + cpool_entrance = &crr->cpool; + + /* Move to pooled_list */ + unlink_abandoned_carrier(crr); + link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + } + } + if (--i <= i_stop) { + /* Move sentinel to continue next search from here */ + relink_edl_before(dl, &allctr->cpool.traitor_list); + if (i > 0) + break; + else + return NULL; + } + } /* - * We search in 'prev' direction and begin by passing - * one element before trying to fetch. This in order to - * avoid contention with threads inserting elements. + * Finally search the shared pool and try employ foreign carriers */ - cpdp = cpool_aint2cpd(cpool_read(&sentinel->prev)); - if (cpdp == sentinel) - return NULL; + sentinel = &carrier_pool[allctr->alloc_no].sentinel; + if (cpool_entrance) { + /* We saw a pooled carried above, use it as entrance into the pool + */ + cpdp = cpool_entrance; + } + else { + /* No pooled carried seen above. Start search at cpool sentinel, + * but begin by passing one element before trying to fetch. + * This in order to avoid contention with threads inserting elements. + */ + cpool_entrance = sentinel; + cpdp = cpool_aint2cpd(cpool_read(&cpool_entrance->prev)); + if (cpdp == sentinel) + return NULL; + } - while (i < ERTS_ALC_CPOOL_MAX_FETCH_INSPECT) { + has_passed_sentinel = 0; + while (1) { erts_aint_t exp; cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); - if (cpdp == sentinel) { + if (cpdp == cpool_entrance) { + if (cpool_entrance == sentinel) { + cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); + if (cpdp == sentinel) + return NULL; + } + i = 0; /* Last one to inspect */ + } + else if (cpdp == sentinel) { + if (has_passed_sentinel) { + /* We been here before. cpool_entrance must have been removed */ + return NULL; + } cpdp = cpool_aint2cpd(cpool_read(&cpdp->prev)); if (cpdp == sentinel) return NULL; - i = ERTS_ALC_CPOOL_MAX_FETCH_INSPECT; /* Last one to inspect */ + has_passed_sentinel = 1; } - crr = (Carrier_t *) (((char *) cpdp) - offsetof(Carrier_t, cpool)); + crr = (Carrier_t *)(((char *)cpdp) - offsetof(Carrier_t, cpool)); exp = erts_smp_atomic_read_rb(&crr->allctr); - if (((exp & (ERTS_CRR_ALCTR_FLG_IN_POOL|ERTS_CRR_ALCTR_FLG_BUSY)) - == ERTS_CRR_ALCTR_FLG_IN_POOL) + if (((exp & (ERTS_CRR_ALCTR_FLG_MASK)) == ERTS_CRR_ALCTR_FLG_IN_POOL) && (erts_atomic_read_nob(&cpdp->max_size) >= size)) { erts_aint_t act; /* Try to fetch it... */ @@ -2977,11 +3161,35 @@ cpool_fetch(Allctr_t *allctr, UWord size) exp); if (act == exp) { cpool_delete(allctr, ((Allctr_t *) (act & ~ERTS_CRR_ALCTR_FLG_MASK)), crr); + if (crr->cpool.orig_allctr == allctr) { + unlink_abandoned_carrier(crr); + } return crr; } } - i++; + if (--i <= 0) + return NULL; } + + /* Last; check our own pending dealloc carrier list... */ + crr = allctr->cpool.dc_list.last; + while (crr) { + if (erts_atomic_read_nob(&crr->cpool.max_size) >= size) { + unlink_carrier(&allctr->cpool.dc_list, crr); +#ifdef ERTS_ALC_CPOOL_DEBUG + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_xchg_nob(&crr->allctr, + ((erts_aint_t) allctr)) + == (((erts_aint_t) allctr) & ~ERTS_CRR_ALCTR_FLG_MASK)); +#else + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); +#endif + return crr; + } + crr = crr->prev; + if (--i <= 0) + return NULL; + } + return NULL; } @@ -3078,6 +3286,9 @@ schedule_dealloc_carrier(Allctr_t *allctr, Carrier_t *crr) return; } + if (is_abandoned(crr)) + unlink_abandoned_carrier(crr); + if (crr->cpool.thr_prgr == ERTS_THR_PRGR_INVALID || erts_thr_progress_has_reached(crr->cpool.thr_prgr)) { dealloc_carrier(allctr, crr, 1); @@ -3124,6 +3335,8 @@ cpool_init_carrier_data(Allctr_t *allctr, Carrier_t *crr) limit = (csz/100)*allctr->cpool.util_limit; crr->cpool.abandon_limit = limit; } + crr->cpool.abandoned.next = NULL; + crr->cpool.abandoned.prev = NULL; } static void @@ -3154,6 +3367,9 @@ abandon_carrier(Allctr_t *allctr, Carrier_t *crr) STAT_MBC_CPOOL_INSERT(allctr, crr); unlink_carrier(&allctr->mbc_list, crr); + if (crr->cpool.orig_allctr == allctr) { + link_abandoned_carrier(&allctr->cpool.pooled_list, crr); + } allctr->remove_mbc(allctr, crr); @@ -3661,6 +3877,11 @@ destroy_carrier(Allctr_t *allctr, Block_t *blk, Carrier_t **busy_pcrr_pp) if (busy_pcrr_pp && *busy_pcrr_pp) { ERTS_ALC_CPOOL_ASSERT(*busy_pcrr_pp == crr); *busy_pcrr_pp = NULL; + ERTS_ALC_CPOOL_ASSERT(erts_smp_atomic_read_nob(&crr->allctr) + == (((erts_aint_t) allctr) + | ERTS_CRR_ALCTR_FLG_IN_POOL + | ERTS_CRR_ALCTR_FLG_BUSY)); + erts_smp_atomic_set_nob(&crr->allctr, ((erts_aint_t) allctr)); cpool_delete(allctr, allctr, crr); } else @@ -5540,6 +5761,10 @@ erts_alcu_start(Allctr_t *allctr, AllctrInit_t *init) allctr->min_block_size = sz; } + allctr->cpool.pooled_list.next = &allctr->cpool.pooled_list; + allctr->cpool.pooled_list.prev = &allctr->cpool.pooled_list; + allctr->cpool.traitor_list.next = &allctr->cpool.traitor_list; + allctr->cpool.traitor_list.prev = &allctr->cpool.traitor_list; allctr->cpool.dc_list.first = NULL; allctr->cpool.dc_list.last = NULL; allctr->cpool.abandon_limit = 0; diff --git a/erts/emulator/beam/erl_alloc_util.h b/erts/emulator/beam/erl_alloc_util.h index 7be6b1ed9d..eee920e66c 100644 --- a/erts/emulator/beam/erl_alloc_util.h +++ b/erts/emulator/beam/erl_alloc_util.h @@ -268,6 +268,11 @@ typedef union {char c[ERTS_ALLOC_ALIGN_BYTES]; long l; double d;} Unit_t; #ifdef ERTS_SMP +typedef struct ErtsDoubleLink_t_ { + struct ErtsDoubleLink_t_ *next; + struct ErtsDoubleLink_t_ *prev; +}ErtsDoubleLink_t; + typedef struct { erts_atomic_t next; erts_atomic_t prev; @@ -277,6 +282,7 @@ typedef struct { UWord abandon_limit; UWord blocks; UWord blocks_size; + ErtsDoubleLink_t abandoned; /* node in pooled_list or traitor_list */ } ErtsAlcCPoolData_t; #endif @@ -500,7 +506,12 @@ struct Allctr_t_ { CarrierList_t sbc_list; #ifdef ERTS_SMP struct { - CarrierList_t dc_list; + /* pooled_list, traitor list and dc_list contain only + carriers _created_ by this allocator */ + ErtsDoubleLink_t pooled_list; + ErtsDoubleLink_t traitor_list; + CarrierList_t dc_list; + UWord abandon_limit; int disable_abandon; int check_limit_count; diff --git a/erts/emulator/beam/erl_bif_port.c b/erts/emulator/beam/erl_bif_port.c index 64bd598ba6..7ce950e090 100644 --- a/erts/emulator/beam/erl_bif_port.c +++ b/erts/emulator/beam/erl_bif_port.c @@ -472,7 +472,7 @@ cleanup_old_port_data(erts_aint_t data) ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; size_t size; ERTS_SMP_DATA_DEPENDENCY_READ_MEMORY_BARRIER; - size = sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm) - 1); + size = sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm); erts_schedule_thr_prgr_later_cleanup_op(free_port_data_heap, (void *) pdhp, &pdhp->later_op, @@ -508,7 +508,7 @@ erts_port_data_size(Port *prt) } else { ErtsPortDataHeap *pdhp = (ErtsPortDataHeap *) data; - return (Uint) sizeof(ErtsPortDataHeap) + pdhp->hsize*(sizeof(Eterm)-1); + return (Uint) sizeof(ErtsPortDataHeap) + (pdhp->hsize-1)*sizeof(Eterm); } } @@ -550,7 +550,7 @@ BIF_RETTYPE port_set_data_2(BIF_ALIST_2) hsize = size_object(BIF_ARG_2); pdhp = erts_alloc(ERTS_ALC_T_PORT_DATA_HEAP, - sizeof(ErtsPortDataHeap) + hsize*(sizeof(Eterm)-1)); + sizeof(ErtsPortDataHeap) + (hsize-1)*sizeof(Eterm)); hp = &pdhp->heap[0]; pdhp->off_heap.first = NULL; pdhp->off_heap.overhead = 0; diff --git a/erts/emulator/beam/erl_process.c b/erts/emulator/beam/erl_process.c index 7b272885a7..b0e0cf13f8 100644 --- a/erts/emulator/beam/erl_process.c +++ b/erts/emulator/beam/erl_process.c @@ -3244,11 +3244,11 @@ chk_wake_sched(ErtsRunQueue *crq, int ix, int activate) return 0; wrq = ERTS_RUNQ_IX(ix); flags = ERTS_RUNQ_FLGS_GET(wrq); + if (activate && !(flags & ERTS_RUNQ_FLG_SUSPENDED)) { + if (try_inc_no_active_runqs(ix+1)) + (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); + } if (!(flags & (ERTS_RUNQ_FLG_SUSPENDED|ERTS_RUNQ_FLG_NONEMPTY))) { - if (activate) { - if (try_inc_no_active_runqs(ix+1)) - (void) ERTS_RUNQ_FLGS_UNSET(wrq, ERTS_RUNQ_FLG_INACTIVE); - } wake_scheduler(wrq); return 1; } diff --git a/erts/emulator/internal_doc/CarrierMigration.md b/erts/emulator/internal_doc/CarrierMigration.md index b93c11c6ec..7afdb70aef 100644 --- a/erts/emulator/internal_doc/CarrierMigration.md +++ b/erts/emulator/internal_doc/CarrierMigration.md @@ -146,28 +146,53 @@ Since the carrier has been unlinked from the data structure of available free blocks, no more allocations will be made in the carrier. The allocator instance putting the carrier into the pool, however, still has the responsibility of performing deallocations in -it while it remains in the pool. +it while it remains in the pool. The allocator instance with this +deallocation responsibility is here called the **employer**. -Each carrier has a flag field containing information about allocator -instance owning the carrier, a flag indicating if the carrier is in +Each carrier has a flag field containing information about the +employing allocator instance, a flag indicating if the carrier is in the pool or not, and a flag indicating if it is busy or not. When the -carrier is in the pool, the owning allocator instance needs to mark it +carrier is in the pool, the employing allocator instance needs to mark it as busy while operating on it. If another thread inspects it in order -to try to fetch it from the pool, it will abort the fetch if it is -busy. When fetching the carrier from the pool, ownership will changed -and further deallocations in the carrier will be redirected to the new -owner using the delayed dealloc functionality. +to try to fetch it from the pool, it will skip it if it is busy. When +fetching the carrier from the pool, employment will change and further +deallocations in the carrier will be redirected to the new +employer using the delayed dealloc functionality. If a carrier in the pool becomes empty, it will be withdrawn from the pool. All carriers that become empty are also always passed to its -originating allocator instance for deallocation using the delayed +**owning** allocator instance for deallocation using the delayed dealloc functionality. Since carriers this way always will be -deallocated by the allocator instance that allocated the carrier the +deallocated by the owner, that allocated the carrier, the underlying functionality of allocating and deallocating carriers can remain simple and doesn't have to bother about multiple threads. In a NUMA system we will also not mix carriers originating from multiple NUMA nodes. +In short: + +* The allocator instance that created a carrier **owns** it. +* An empty carrier is always deallocated by its **owner**. +* **Ownership** never changes. +* The allocator instance that uses a carrier **employs** it. +* An **employer** can abandon a carrier into the pool. +* Pooled carriers are not allocated from. +* Deallocation in a pooled carrier is still performed by its **employer**. +* **Employment** can only change when a carrier is fetched from the pool. + +### Searching the pool ### + +To harbor real time characteristics, searching the pool is +limited. We only inspect a limited number of carriers. If none of +those carriers had a free block large enough to satisfy the allocation +request, the search will fail. A carrier in the pool can also be busy, +if another thread is currently doing block deallocation work on the +carrier. A busy carrier will also be skipped by the search as it can +not satisfy the request. The pool is lock free and we do not want to +block, waiting for the other thread to finish. + +#### Before OTP 17.4 #### + When an allocator instance needs more carrier space, it always begins by inspecting its own carriers that are waiting for thread progress before they can be deallocated. If no such carrier could be found, it @@ -176,6 +201,65 @@ it will allocate a new carrier. Regardless of where the allocator instance gets the carrier from it the just links in the carrier into its data structure of free blocks. +#### After OTP 17.4 #### + +The old search algorithm had a problem as the search always started at +the same position in the pool, the sentinel. This could lead to +contention from concurrent searching processes. But even worse, it +could lead to a "bad" state when searches fail with a high rate +leading to new carriers instead being allocated. These new carriers +may later be inserted into the pool due to bad utilization. If the +frequency of insertions into the pool is higher than successful +fetching from the pool, memory will eventually get exhausted. + +This "bad" state, consist of a cluster of small and/or highly +fragmented carriers located at the sentinel in the pool. The largest free +block in such a "bad" carrier is rather small, making it not able to satisfy +most allocation requests. As the search always started at the +sentinel, any such "bad" carriers that had been left in the pool would +eventually cluster together at the sentinel. All searches first +have to skip past this cluster of "bad" carriers to reach a "good" +carrier. When the cluster gets to the same size as the search limit, +all searches will essentially fail. + +To counter the "bad cluster" problem and also ease the contention, the +search will now always start by first looking at the allocators **own** +carriers. That is, carriers that were initially created by the +allocator itself and later had been abandoned to the pool. If none of +our own abandoned carrier would do, then the search continues into the +pool, as before, to look for carriers created by other +allocators. However, if we have at least one abandoned carrier of our +own, that could not satisfy the request, we can use that as entry point +into the pool. + +The result is that we prefer carriers created by the thread itself, +which is good for NUMA performance. And we get more entry points when +searching the pool, which will ease contention and clustering. + +To do the first search among own carriers, every allocator instance +has two new lists; `pooled_list` and `traitor_list`. These lists are only +accessed by the allocator itself and they only contain the allocators +own carriers. When an owned carrier is abandoned and put in the +pool, it is also linked into `pooled_list`. When we search our +`pooled_list` and find a carrier that is no longer in the pool, we +move that carrier from `pooled_list` to `traitor_list` as it is now +employed by another allocator. If searching `pooled_list` fails, we +also do a limited search of `traitor_list`. When finding an abandoned +carrier in `traitor_list` it is either employed, or moved back to +`pooled_list` if it could not satisfy the allocation request. + +When searching `pooled_list` and `traitor_list` we always start at the +point where the last search ended. This to avoid clustering +problems and increase the probability to find a "good" carrier. As +`pooled_list` and `traitor_list` are only accessed by the owning +allocator instance, they need no thread synchronization at all. + +Furthermore, the search for own carriers that are scheduled +for deallocation is now done as the last search option. The idea is +that it is better to reuse a poorly utilized carrier, than to +resurrect an empty carrier that was just about to be released back to +the OS. + ### Result ### The use of this strategy of abandoning carriers with poor utilization diff --git a/erts/emulator/internal_doc/SuperCarrier.md b/erts/emulator/internal_doc/SuperCarrier.md new file mode 100644 index 0000000000..0ad6af41de --- /dev/null +++ b/erts/emulator/internal_doc/SuperCarrier.md @@ -0,0 +1,191 @@ +Super Carrier +============= + +A super carrier is large memory area, allocated at VM start, which can +be used during runtime to allocate normal carriers from. + +The super carrier feature was introduced in OTP R16B03. It is +enabled with command line option +MMscs <size in Mb> +and can be configured with other options. + +Problem +------- + +The initial motivation for this feature was customers asking for a way +to pre-allocate physcial memory at VM start for it to use. + +Other problems were different experienced limitations of the OS +implementation of mmap: + +* Increasingly bad performance of mmap/munmap as the number of mmap'ed areas grow. +* Fragmentation problem between mmap'ed areas. + +A third problem was management of low memory in the halfword +emulator. The implementation used a naive linear search structure to +hold free segments which would lead to poor performance when +fragmentation increased. + + +Solution +-------- + +Allocate one large continious area of address space at VM start and +then use that area to satisfy our dynamic memory need during +runtime. In other words: implement our own mmap. + +### Use cases ### + +If command line option +MMscrpm (Reserve Physical Memory) is set to +false, only virtual space is allocated for the super carrier from +start. The super carrier then acts as an "alternative mmap" implementation +without changing the consumption of physical memory pages. Physical +pages will be reserved on demand when an allocation is done from the super +carrier and be unreserved when the memory is released back to the +super carrier. + +If +MMscrpm is set to true, which is default, the initial allocation +will reserve physical memory for the entire super carrier. This can be +used by users that want to ensure a certain *minimum* amount of +physical memory for the VM. + +However, what reservation of physical memory actually means highly +depends on the operating system, and how it is configured. For +example, different memory overcommit settings on Linux drastically +change the behaviour. + +A third feature is to have the super carrier limit the *maximum* +amount of memory used by the VM. If +MMsco (Super Carrier Only) is set +to true, which is default, allocations will only be done from the +super carrier. When the super carrier gets full, the VM will fail due +to out of memory. +If +MMsco is false, allocations will use mmap directly if the super +carrier is full. + + + +### Implementation ### + +The entire super carrier implementation is kept in erl_mmap.c. The +name suggest that it can be viewed as our own mmap implementation. + +A super carrier needs to satisfy two slightly different kinds of +allocation requests; multi block carriers (MBC) and single block +carriers (SBC). They are both rather large blocks of continious +memory, but MBCs and SBCs have different demands on alignment and +size. + +SBCs can have arbitrary size and do only need minimum 8-byte +alignment. + +MBCs are more restricted. They can only have a number of fixed +sizes that are powers of 2. The start address need to have a very +large aligment (currently 256 kb, called "super alignment"). This is a +design choice that allows very low overhead per allocated block in the +MBC. + +To reduce fragmentation within the super carrier, it is good to keep SBCs +and MBCs apart. MBCs with their uniform alignment and sizes can be +packed very efficiently together. SBCs without demand for aligment can +also be allocated quite efficiently together. But mixing them can lead +to a lot of memory wasted when we need to create large holes of +padding to the next alignment limit. + +The super carrier thus contains two areas. One area for MBCs growing from +the bottom and up. And one area for SBCs growing from the top and +down. Like a process with a heap and a stack growing towards each +other. + + +### Data structures ### + +The MBC area is called **sa** as in super aligned and the SBC area is +called **sua** as in super un-aligned. + +Note that the "super" in super alignment and the "super" in super +carrier has nothing to do with each other. We could have choosen +another naming to avoid confusion, such as "meta" carrier or "giant" +aligment. + + +-------+ <---- sua.top + | sua | + | | + |-------| <---- sua.bot + | | + | | + | | + |-------| <---- sa.top + | | + | sa | + | | + +-------+ <---- sa.bot + + +When a carrier is deallocated a free memory segment will be created +inside the corresponding area, unless the carrier was at the very top +(in `sa`) or bottom (in `sua`) in which case the area will just shrink +down or up. + +We need to keep track of all the free segments in order to reuse them +for new carrier allocations. One initial idea was to use the same +mechanism that is used to keep track of free blocks within MBCs +(alloc_util and the different strategies). However, that would not be +as straight forward as one can think and can also waste quite a lot of +memory as it uses prepended block headers. The granularity of the +super carrier is one memory page (usually 4kb). We want to allocate +and free entire pages and we don't want to waste an entire page just +to hold the block header of the following pages. + +Instead we store the meta information about all the free segments in a +dedicated area apart from the `sa` and `sua` areas. Every free segment is +represented by a descriptor struct (`ErtsFreeSegDesc`). + + typedef struct { + RBTNode snode; /* node in 'stree' */ + RBTNode anode; /* node in 'atree' */ + char* start; + char* end; + }ErtsFreeSegDesc; + +To find the smallest free segment that will satisfy a carrier allocation +(best fit), the free segments are organized in a tree sorted by +size (`stree`). We search in this tree at allocation. If no free segment of +sufficient size was found, the area (`sa` or `sua`) is instead expanded. +If two or more free segments with equal size exist, the one at lowest +address is choosen for `sa` and highest address for `sua`. + +At carrier deallocation, we want to coalesce with any adjacent free +segments, to form one large free segment. To do that, all free +segments are also organized in a tree sorted in address order (`atree`). + +So, in total we keep four trees of free descriptors for the super +carrier; two for `sa` and two for `sua`. They all use the same +red-black-tree implementation that support the different sorting +orders used. + +When allocating a new MBC we first search after a free segment in `sa`, +then try to raise `sa.top`, and then as a fallback try to search after a +free segment in `sua`. When an MBC is allocated in `sua`, a larger segment +is allocated which is then trimmed to obtain the right +alignment. Allocation search for an SBC is done in reverse order. When +an SBC is allocated in `sa`, the size is aligned up to super aligned +size. + +### The free descriptor area ### + +As mentioned above, the descriptors for the free segments are +allocated in a separate area. This area has a constant configurable +size (+MMscrfsd) that defaults to 65536 descriptors. This should be +more than enough in most cases. If the descriptors area should fill up, +new descriptor areas will be allocated first directly from the OS, and +then from `sua` and `sa` in the super carrier, and lastly from the memory +segment itself which is being deallocated. Allocating free descriptor +areas from the super carrier is only a last resort, and should be +avoided, as it creates fragmentation. + +### Halfword emulator ### + +The halfword emulator uses the super carrier implementation to manage +its low memory mappings thar are needed for all term storage. The +super carrier can here not be configured by command line options. One +could imagine a second configurable instance of the super carrier used +by high memory allocation, but that has not been implemented. diff --git a/erts/emulator/test/port_SUITE.erl b/erts/emulator/test/port_SUITE.erl index 1bb4cb3637..6bbf93b7d7 100644 --- a/erts/emulator/test/port_SUITE.erl +++ b/erts/emulator/test/port_SUITE.erl @@ -2349,8 +2349,10 @@ port_setget_data(Config) when is_list(Config) -> Port = erlang:open_port({spawn_driver, "echo_drv"}, []), NSched = erlang:system_info(schedulers_online), + HeapData = {1,2,3,<<"A heap binary">>,fun()->"This is fun"end, + list_to_binary(lists:seq(1,100))}, PRs = lists:map(fun(I) -> - spawn_opt(fun() -> port_setget_data_hammer(Port,1) end, + spawn_opt(fun() -> port_setget_data_hammer(Port,HeapData,false,1) end, [monitor, {scheduler, I rem NSched}]) end, lists:seq(1,10)), @@ -2368,13 +2370,17 @@ port_setget_data(Config) when is_list(Config) -> PRs), ok. -port_setget_data_hammer(Port, N) -> +port_setget_data_hammer(Port, HeapData, IsSet0, N) -> Rand = random:uniform(3), - try case Rand of - 1 -> true = erlang:port_set_data(Port, atom); - 2 -> true = erlang:port_set_data(Port, {1,2,3}); - 3 -> erlang:port_get_data(Port) - end + IsSet1 = try case Rand of + 1 -> true = erlang:port_set_data(Port, atom), true; + 2 -> true = erlang:port_set_data(Port, HeapData), true; + 3 -> case erlang:port_get_data(Port) of + atom -> true; + HeapData -> true; + undefined -> false=IsSet0 + end + end catch error:badarg -> true = get(prepare_for_close), @@ -2387,7 +2393,7 @@ port_setget_data_hammer(Port, N) -> after 0 -> ok end, - port_setget_data_hammer(Port, N+1). + port_setget_data_hammer(Port, HeapData, IsSet1, N+1). wait_until(Fun) -> diff --git a/erts/include/internal/ethread.h b/erts/include/internal/ethread.h index 72c054b588..ad5d05704c 100644 --- a/erts/include/internal/ethread.h +++ b/erts/include/internal/ethread.h @@ -364,8 +364,8 @@ extern ethr_runtime_t ethr_runtime__; # include "sparc64/ethread.h" # endif # endif -# include "gcc/ethread.h" # include "libatomic_ops/ethread.h" +# include "gcc/ethread.h" # endif # elif defined(ETHR_HAVE_LIBATOMIC_OPS) # include "libatomic_ops/ethread.h" diff --git a/erts/include/internal/libatomic_ops/ethr_atomic.h b/erts/include/internal/libatomic_ops/ethr_atomic.h index fb1288c330..734cdf0890 100644 --- a/erts/include/internal/libatomic_ops/ethr_atomic.h +++ b/erts/include/internal/libatomic_ops/ethr_atomic.h @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2010-2011. All Rights Reserved. + * Copyright Ericsson AB 2010-2014. All Rights Reserved. * * The contents of this file are subject to the Erlang Public License, * Version 1.1, (the "License"); you may not use this file except in @@ -32,22 +32,23 @@ * These operations need to be defined by libatomic_ops; * otherwise, we won't compile: * - AO_nop_full() - * - AO_load() - * - AO_store() - * - AO_compare_and_swap() + * - AO_load() || AO_load_aquire() + * - AO_store() || AO_store_release() + * - AO_compare_and_swap() || AO_compare_and_swap_acquire() + * || AO_compare_and_swap_release() || AO_compare_and_swap_full() * */ #if ETHR_SIZEOF_AO_T == 4 #define ETHR_HAVE_NATIVE_ATOMIC32 1 -#define ETHR_NATIVE_ATOMIC32_IMPL "libatomic_ops" +#define ETHR_NATIVE_ATOMIC32_IMPL ETHR_NATIVE_IMPL__ #define ETHR_NATMC_FUNC__(X) ethr_native_atomic32_ ## X #define ETHR_ATMC_T__ ethr_native_atomic32_t #define ETHR_AINT_T__ ethr_sint32_t #define ETHR_AINT_SUFFIX__ "l" #elif ETHR_SIZEOF_AO_T == 8 #define ETHR_HAVE_NATIVE_ATOMIC64 1 -#define ETHR_NATIVE_ATOMIC64_IMPL "libatomic_ops" +#define ETHR_NATIVE_ATOMIC64_IMPL ETHR_NATIVE_IMPL__ #define ETHR_NATMC_FUNC__(X) ethr_native_atomic64_ ## X #define ETHR_ATMC_T__ ethr_native_atomic64_t #define ETHR_AINT_T__ ethr_sint64_t @@ -74,6 +75,8 @@ ETHR_NATMC_FUNC__(addr)(ETHR_ATMC_T__ *var) return (ETHR_AINT_T__ *) &var->counter; } +#ifdef AO_HAVE_store + #if ETHR_SIZEOF_AO_T == 4 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET 1 #else @@ -86,6 +89,24 @@ ETHR_NATMC_FUNC__(set)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) AO_store(&var->counter, (AO_t) value); } +#endif + +#ifdef AO_HAVE_store_write + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_WB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_WB 1 +#endif + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_wb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +{ + AO_store_write(&var->counter, (AO_t) value); +} + +#endif + #ifdef AO_HAVE_store_release #if ETHR_SIZEOF_AO_T == 4 @@ -102,6 +123,24 @@ ETHR_NATMC_FUNC__(set_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) #endif +#ifdef AO_HAVE_store_full + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_SET_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_SET_MB 1 +#endif + +static ETHR_INLINE void +ETHR_NATMC_FUNC__(set_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ value) +{ + AO_store_full(&var->counter, (AO_t) value); +} + +#endif + +#ifdef AO_HAVE_load + #if ETHR_SIZEOF_AO_T == 4 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ 1 #else @@ -114,6 +153,24 @@ ETHR_NATMC_FUNC__(read)(ETHR_ATMC_T__ *var) return (ETHR_AINT_T__) AO_load(&var->counter); } +#endif + +#ifdef AO_HAVE_load_read + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_RB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_RB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_rb)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__) AO_load_read(&var->counter); +} + +#endif + #ifdef AO_HAVE_load_acquire #if ETHR_SIZEOF_AO_T == 4 @@ -130,6 +187,22 @@ ETHR_NATMC_FUNC__(read_acqb)(ETHR_ATMC_T__ *var) #endif +#ifdef AO_HAVE_load_full + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_READ_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_READ_MB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(read_mb)(ETHR_ATMC_T__ *var) +{ + return (ETHR_AINT_T__) AO_load_full(&var->counter); +} + +#endif + #ifdef AO_HAVE_fetch_and_add #if ETHR_SIZEOF_AO_T == 4 @@ -146,6 +219,54 @@ ETHR_NATMC_FUNC__(add_return)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) #endif +#ifdef AO_HAVE_fetch_and_add_acquire + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_ACQB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_ACQB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +{ + return ((ETHR_AINT_T__) AO_fetch_and_add_acquire(&var->counter, (AO_t) incr)) + incr; +} + +#endif + +#ifdef AO_HAVE_fetch_and_add_release + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_RELB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +{ + return ((ETHR_AINT_T__) AO_fetch_and_add_release(&var->counter, (AO_t) incr)) + incr; +} + +#endif + +#ifdef AO_HAVE_fetch_and_add_full + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_ADD_RETURN_MB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(add_return_mb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ incr) +{ + return ((ETHR_AINT_T__) AO_fetch_and_add_full(&var->counter, (AO_t) incr)) + incr; +} + +#endif + #ifdef AO_HAVE_fetch_and_add1 #if ETHR_SIZEOF_AO_T == 4 @@ -178,6 +299,38 @@ ETHR_NATMC_FUNC__(inc_return_acqb)(ETHR_ATMC_T__ *var) #endif +#ifdef AO_HAVE_fetch_and_add1_release + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_RELB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_relb)(ETHR_ATMC_T__ *var) +{ + return ((ETHR_AINT_T__) AO_fetch_and_add1_release(&var->counter)) + 1; +} + +#endif + +#ifdef AO_HAVE_fetch_and_add1_full + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_INC_RETURN_MB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(inc_return_mb)(ETHR_ATMC_T__ *var) +{ + return ((ETHR_AINT_T__) AO_fetch_and_add1_full(&var->counter)) + 1; +} + +#endif + #ifdef AO_HAVE_fetch_and_sub1 #if ETHR_SIZEOF_AO_T == 4 @@ -194,6 +347,22 @@ ETHR_NATMC_FUNC__(dec_return)(ETHR_ATMC_T__ *var) #endif +#ifdef AO_HAVE_fetch_and_sub1_acquire + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_ACQB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_ACQB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_acqb)(ETHR_ATMC_T__ *var) +{ + return ((ETHR_AINT_T__) AO_fetch_and_sub1_acquire(&var->counter)) - 1; +} + +#endif + #ifdef AO_HAVE_fetch_and_sub1_release #if ETHR_SIZEOF_AO_T == 4 @@ -210,7 +379,60 @@ ETHR_NATMC_FUNC__(dec_return_relb)(ETHR_ATMC_T__ *var) #endif -#ifdef AO_HAVE_compare_and_swap +#ifdef AO_HAVE_fetch_and_sub1_full + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_DEC_RETURN_MB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(dec_return_mb)(ETHR_ATMC_T__ *var) +{ + return ((ETHR_AINT_T__) AO_fetch_and_sub1_full(&var->counter)) - 1; +} + +#endif + +#if defined(AO_HAVE_compare_and_swap_full) || defined(AO_HAVE_fetch_compare_and_swap_full) + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_MB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_MB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_mb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) +{ +#if defined(AO_HAVE_fetch_compare_and_swap_full) + return (ETHR_AINT_T__) AO_fetch_compare_and_swap_full(&var->counter, + (AO_t) exp, + (AO_t) new); +#else + ETHR_AINT_T__ act; + do { + if (AO_compare_and_swap_full(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; +#ifdef AO_HAVE_load_acquire + act = (ETHR_AINT_T__) AO_load_acquire(&var->counter); +#else + act = (ETHR_AINT_T__) AO_load(&var->counter); +#endif + } while (act == exp); +#ifndef AO_HAVE_load_acquire + AO_nop_full(); +#endif + return act; +#endif +} + +#endif + +#if defined(AO_HAVE_compare_and_swap) || defined(AO_HAVE_fetch_compare_and_swap) #if ETHR_SIZEOF_AO_T == 4 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1 @@ -223,18 +445,28 @@ ETHR_NATMC_FUNC__(cmpxchg)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ exp) { +#if defined(AO_HAVE_fetch_compare_and_swap) + return (ETHR_AINT_T__) AO_fetch_compare_and_swap(&var->counter, + (AO_t) exp, + (AO_t) new); +#else ETHR_AINT_T__ act; do { if (AO_compare_and_swap(&var->counter, (AO_t) exp, (AO_t) new)) return exp; +#ifdef AO_HAVE_load act = (ETHR_AINT_T__) AO_load(&var->counter); +#else + act = (ETHR_AINT_T__) AO_load_aquire(&var->counter); +#endif } while (act == exp); return act; +#endif } #endif -#ifdef AO_HAVE_compare_and_swap_acquire +#if defined(AO_HAVE_compare_and_swap_acquire) || defined(AO_HAVE_fetch_compare_and_swap_acquire) #if ETHR_SIZEOF_AO_T == 4 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_ACQB 1 @@ -247,6 +479,11 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ exp) { +#if defined(AO_HAVE_fetch_compare_and_swap_acquire) + return (ETHR_AINT_T__) AO_fetch_compare_and_swap_acquire(&var->counter, + (AO_t) exp, + (AO_t) new); +#else ETHR_AINT_T__ act; do { if (AO_compare_and_swap_acquire(&var->counter, (AO_t) exp, (AO_t) new)) @@ -261,11 +498,55 @@ ETHR_NATMC_FUNC__(cmpxchg_acqb)(ETHR_ATMC_T__ *var, AO_nop_full(); #endif return act; +#endif +} + +#endif + +#if defined(AO_HAVE_compare_and_swap_read) || defined(AO_HAVE_fetch_compare_and_swap_read) + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_RB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_rb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) +{ +#if defined(AO_HAVE_fetch_compare_and_swap_read) + return (ETHR_AINT_T__) AO_fetch_compare_and_swap_read(&var->counter, + (AO_t) exp, + (AO_t) new); +#else + ETHR_AINT_T__ act; + do { + if (AO_compare_and_swap_read(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; +#if defined(AO_HAVE_load_read) + act = (ETHR_AINT_T__) AO_load_read(&var->counter); +#elif defined(AO_HAVE_load) + act = (ETHR_AINT_T__) AO_load(&var->counter); +#else + act = (ETHR_AINT_T__) AO_load_acquire(&var->counter); +#endif + } while (act == exp); +#ifndef AO_HAVE_load_read +#ifdef AO_HAVE_nop_read + AO_nop_read(); +#else + AO_nop_full(); +#endif +#endif + return act; +#endif } #endif -#ifdef AO_HAVE_compare_and_swap_release +#if defined(AO_HAVE_compare_and_swap_release) || defined(AO_HAVE_fetch_compare_and_swap_release) #if ETHR_SIZEOF_AO_T == 4 # define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1 @@ -278,13 +559,57 @@ ETHR_NATMC_FUNC__(cmpxchg_relb)(ETHR_ATMC_T__ *var, ETHR_AINT_T__ new, ETHR_AINT_T__ exp) { +#if defined(AO_HAVE_fetch_compare_and_swap_release) + return (ETHR_AINT_T__) AO_fetch_compare_and_swap_release(&var->counter, + (AO_t) exp, + (AO_t) new); +#else ETHR_AINT_T__ act; do { if (AO_compare_and_swap_release(&var->counter, (AO_t) exp, (AO_t) new)) return exp; +#ifdef AO_HAVE_load act = (ETHR_AINT_T__) AO_load(&var->counter); +#else + act = (ETHR_AINT_T__) AO_load_acquire(&var->counter); +#endif + } while (act == exp); + return act; +#endif +} + +#endif + +#if defined(AO_HAVE_compare_and_swap_write) || defined(AO_HAVE_fetch_compare_and_swap_write) + +#if ETHR_SIZEOF_AO_T == 4 +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_WB 1 +#else +# define ETHR_HAVE_ETHR_NATIVE_ATOMIC64_CMPXCHG_WB 1 +#endif + +static ETHR_INLINE ETHR_AINT_T__ +ETHR_NATMC_FUNC__(cmpxchg_wb)(ETHR_ATMC_T__ *var, + ETHR_AINT_T__ new, + ETHR_AINT_T__ exp) +{ +#if defined(AO_HAVE_fetch_compare_and_swap_write) + return (ETHR_AINT_T__) AO_fetch_compare_and_swap_write(&var->counter, + (AO_t) exp, + (AO_t) new); +#else + ETHR_AINT_T__ act; + do { + if (AO_compare_and_swap_write(&var->counter, (AO_t) exp, (AO_t) new)) + return exp; +#ifdef AO_HAVE_load + act = (ETHR_AINT_T__) AO_load(&var->counter); +#else + act = (ETHR_AINT_T__) AO_load_acquire(&var->counter); +#endif } while (act == exp); return act; +#endif } #endif diff --git a/erts/include/internal/libatomic_ops/ethr_dw_atomic.h b/erts/include/internal/libatomic_ops/ethr_dw_atomic.h new file mode 100644 index 0000000000..4dd9f41e96 --- /dev/null +++ b/erts/include/internal/libatomic_ops/ethr_dw_atomic.h @@ -0,0 +1,567 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2014. All Rights Reserved. + * + * The contents of this file are subject to the Erlang Public License, + * Version 1.1, (the "License"); you may not use this file except in + * compliance with the License. You should have received a copy of the + * Erlang Public License along with this software. If not, it can be + * retrieved online at http://www.erlang.org/. + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See + * the License for the specific language governing rights and limitations + * under the License. + * + * %CopyrightEnd% + */ + +/* + * Description: Native double word atomics using libatomic_ops + * Author: Rickard Green + */ + +#ifndef ETHR_LIBATOMIC_OPS_DW_ATOMIC_H__ +#define ETHR_LIBATOMIC_OPS_DW_ATOMIC_H__ + +#if defined(AO_HAVE_double_t) \ + && (defined(AO_HAVE_double_load_acquire) \ + || defined(AO_HAVE_double_load)) \ + && (defined(AO_HAVE_compare_double_and_swap_double) \ + || defined(AO_HAVE_compare_double_and_swap_double_full) \ + || defined(AO_HAVE_compare_double_and_swap_double_acquire) \ + || defined(AO_HAVE_compare_double_and_swap_double_release) \ + || defined(AO_HAVE_double_compare_and_swap) \ + || defined(AO_HAVE_double_compare_and_swap_full) \ + || defined(AO_HAVE_double_compare_and_swap_acquire) \ + || defined(AO_HAVE_double_compare_and_swap_release)) + +#if ETHR_SIZEOF_PTR == 4 +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint64_t +#elif ETHR_SIZEOF_PTR == 8 && defined(ETHR_HAVE_INT128_T) +# define ETHR_NATIVE_SU_DW_SINT_T ethr_sint128_t +#endif + +typedef union { + volatile AO_double_t dw_mem; +#if defined(ETHR_NATIVE_SU_DW_SINT_T) + ETHR_NATIVE_SU_DW_SINT_T su_dw_sint; +#endif +} ethr_native_dw_atomic_t; + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_NATIVE_SU_DW_ATOMIC +#else +# define ETHR_HAVE_NATIVE_DW_ATOMIC +#endif + +#define ETHR_NATIVE_DW_ATOMIC_IMPL ETHR_NATIVE_IMPL__ + +#if defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) + + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_NDWA_FUNC__(Func) ethr_native_su_dw_atomic_ ## Func +# define ETHR_NDWA_RET_3_TYPE__ ETHR_NATIVE_SU_DW_SINT_T +# define ETHR_NDWA_RET_2_TYPE__ ETHR_NATIVE_SU_DW_SINT_T +# define ETHR_NDWA_VAL_ARG_TYPE__ ETHR_NATIVE_SU_DW_SINT_T +# define ETHR_NDWA_DECL_ARG__(Arg) +# if defined(AO_HAVE_DOUBLE_PTR_STORAGE) +# define ETHR_NDWA_VAL2AOVAL__(AOV, V) \ + ((AOV).AO_whole = (double_ptr_storage) (V)) +# define ETHR_NDWA_AOVAL2VAL__(AOV, V) \ + ((V) = (ETHR_NATIVE_SU_DW_SINT_T) (AOV).AO_whole) +# define ETHR_NDWA_RETURN_VAL_3__(SUCCESS, AOVAL, VAL) \ + do { \ + return (ETHR_NATIVE_SU_DW_SINT_T) (AOVAL).AO_whole; \ + } while (0) +# define ETHR_NDWA_RETURN_VAL_2__(AOVAL, VAL) \ + do { \ + return (ETHR_NATIVE_SU_DW_SINT_T) (AOVAL).AO_whole; \ + } while (0) +# define ETHR_NDWA_AOVAL_EQ__(AOV1, AOV2) \ + ((AOV1).AO_whole == (AOV2).AO_whole) +# else +typedef union { + ethr_sint_t sint[2]; + ETHR_NATIVE_SU_DW_SINT_T dw_sint; +} ethr_dw_splitter_t; +# define ETHR_NDWA_VAL2AOVAL__(AOV, V) \ + do { \ + ethr_dw_splitter_t tmp__; \ + tmp__.dw_sint = (V); \ + (AOV).AO_val1 = (AO_t) tmp__.sint[0]; \ + (AOV).AO_val2 = (AO_t) tmp__.sint[1]; \ + } while (0) +# define ETHR_NDWA_AOVAL2VAL__(AOV, V) \ + do { \ + ethr_dw_splitter_t tmp__; \ + tmp__.sint[0] = (ethr_sint_t) (AOV).AO_val1; \ + tmp__.sint[1] = (ethr_sint_t) (AOV).AO_val2; \ + (V) = tmp__.dw_sint; \ + } while (0) +# define ETHR_NDWA_RETURN_VAL_3__(SUCCESS, AOVAL, VAL) \ + do { \ + ethr_dw_splitter_t tmp__; \ + tmp__.sint[0] = (ethr_sint_t) (AOVAL).AO_val1; \ + tmp__.sint[1] = (ethr_sint_t) (AOVAL).AO_val2; \ + return tmp__.dw_sint; \ + } while (0) +# define ETHR_NDWA_AOVAL_EQ__(AOV1, AOV2) \ + ((AOV1).AO_val1 == (AOV2).AO_val1 \ + && (AOV1).AO_val2 == (AOV2).AO_val2) +# endif +#else +# define ETHR_NDWA_FUNC__(Func) ethr_native_dw_atomic_ ## Func +# define ETHR_NDWA_RET_3_TYPE__ int +# define ETHR_NDWA_RET_2_TYPE__ void +# define ETHR_NDWA_VAL_ARG_TYPE__ ethr_sint_t * +# define ETHR_NDWA_DECL_ARG__(Arg) , ETHR_NDWA_VAL_ARG_TYPE__ Arg +# define ETHR_NDWA_VAL2AOVAL__(AOV, V) \ + do { \ + (AOV).AO_val1 = (AO_t) (V)[0]; \ + (AOV).AO_val2 = (AO_t) (V)[1]; \ + } while (0) +# define ETHR_NDWA_AOVAL2VAL__(AOV, V) \ + do { \ + ethr_dw_splitter_t tmp__; \ + (V)[0] = (ethr_sint_t) (AOV).AO_val1; \ + (V)[1] = (ethr_sint_t) (AOV).AO_val2; \ + } while (0) +# define ETHR_NDWA_RETURN_VAL_3__(SUCCESS, AOVAL, VAL) \ + do { \ + (VAL)[0] = (ethr_sint_t) (AOVAL).AO_val1; \ + (VAL)[1] = (ethr_sint_t) (AOVAL).AO_val2; \ + return (SUCCESS); \ + } while (0) +# define ETHR_NDWA_RETURN_VAL_2__(AOVAL, VAL) \ + do { \ + (VAL)[0] = (ethr_sint_t) (AOVAL).AO_val1; \ + (VAL)[1] = (ethr_sint_t) (AOVAL).AO_val2; \ + return; \ + } while (0) +# if defined(AO_HAVE_DOUBLE_PTR_STORAGE) +# define ETHR_NDWA_AOVAL_EQ__(AOV1, AOV2) \ + ((AOV1).AO_whole == (AOV2).AO_whole) +# else +# define ETHR_NDWA_AOVAL_EQ__(AOV1, AOV2) \ + ((AOV1).AO_val1 == (AOV2).AO_val1 \ + && (AOV1).AO_val2 == (AOV2).AO_val2) +# endif +#endif + +#define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_ADDR +static ETHR_INLINE ethr_sint_t * +ethr_native_dw_atomic_addr(ethr_native_dw_atomic_t *var) +{ + return (ethr_sint_t *) &var->dw_mem; +} + +#ifdef AO_HAVE_double_load + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ +#endif + +static ETHR_INLINE ETHR_NDWA_RET_2_TYPE__ +ETHR_NDWA_FUNC__(read)(ethr_native_dw_atomic_t *var + ETHR_NDWA_DECL_ARG__(val)) +{ + AO_double_t act = AO_double_load(&var->dw_mem); + ETHR_NDWA_RETURN_VAL_2__(act, val); +} + +#endif + +#ifdef AO_HAVE_double_load_read + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_RB +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_RB +#endif + +static ETHR_INLINE ETHR_NDWA_RET_2_TYPE__ +ETHR_NDWA_FUNC__(read_rb)(ethr_native_dw_atomic_t *var + ETHR_NDWA_DECL_ARG__(val)) +{ + AO_double_t act = AO_double_load_read(&var->dw_mem); + ETHR_NDWA_RETURN_VAL_2__(act, val); +} + +#endif + +#ifdef AO_HAVE_double_load_acquire + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_READ_ACQB +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_READ_ACQB +#endif + +static ETHR_INLINE ETHR_NDWA_RET_2_TYPE__ +ETHR_NDWA_FUNC__(read_acqb)(ethr_native_dw_atomic_t *var + ETHR_NDWA_DECL_ARG__(val)) +{ + AO_double_t act = AO_double_load_acquire(&var->dw_mem); + ETHR_NDWA_RETURN_VAL_2__(act, val); +} + +#endif + +#ifdef AO_HAVE_double_store + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET +#endif + +static ETHR_INLINE void +ETHR_NDWA_FUNC__(set)(ethr_native_dw_atomic_t *var, + ETHR_NDWA_VAL_ARG_TYPE__ val) +{ + AO_double_t new; + ETHR_NDWA_VAL2AOVAL__(new, val); + AO_double_store(&var->dw_mem, new); +} + +#endif + +#ifdef AO_HAVE_double_store_write + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_WB +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_WB +#endif + +static ETHR_INLINE void +ETHR_NDWA_FUNC__(set_wb)(ethr_native_dw_atomic_t *var, + ETHR_NDWA_VAL_ARG_TYPE__ val) +{ + AO_double_t new; + ETHR_NDWA_VAL2AOVAL__(new, val); + AO_double_store_write(&var->dw_mem, new); +} + +#endif + +#ifdef AO_HAVE_double_store_release + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_SET_RELB +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_SET_RELB +#endif + +static ETHR_INLINE void +ETHR_NDWA_FUNC__(set_relb)(ethr_native_dw_atomic_t *var, + ETHR_NDWA_VAL_ARG_TYPE__ val) +{ + AO_double_t new; + ETHR_NDWA_VAL2AOVAL__(new, val); + AO_double_store_release(&var->dw_mem, new); +} + +#endif + +#if defined(AO_HAVE_double_compare_and_swap_full) || defined(AO_HAVE_compare_double_and_swap_double_full) + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_MB +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_MB +#endif + +static ETHR_INLINE ETHR_NDWA_RET_3_TYPE__ +ETHR_NDWA_FUNC__(cmpxchg_mb)(ethr_native_dw_atomic_t *var, + ETHR_NDWA_VAL_ARG_TYPE__ new, + ETHR_NDWA_VAL_ARG_TYPE__ exp) +{ + AO_double_t ao_act, ao_new, ao_exp; + + ETHR_NDWA_VAL2AOVAL__(ao_exp, exp); + ETHR_NDWA_VAL2AOVAL__(ao_new, new); + + do { + int xchgd; +#if defined(AO_HAVE_double_compare_and_swap_full) + xchgd = AO_double_compare_and_swap_full(&var->dw_mem, ao_exp, ao_new); +#elif defined(AO_HAVE_compare_double_and_swap_double_full) + xchgd = AO_compare_double_and_swap_double_full(&var->dw_mem, + ao_exp.AO_val1, + ao_exp.AO_val2, + ao_new.AO_val1, + ao_new.AO_val2); +#endif + + if (xchgd) + ETHR_NDWA_RETURN_VAL_3__(1, ao_exp, exp); + +#ifdef AO_HAVE_double_load_acquire + ao_act = AO_double_load_acquire(&var->dw_mem); +#else + ao_act = AO_double_load(&var->dw_mem); +#endif + + } while (ETHR_NDWA_AOVAL_EQ__(ao_exp, ao_act)); + +#ifndef AO_HAVE_double_load_acquire + AO_nop_full(); +#endif + + ETHR_NDWA_RETURN_VAL_3__(1, ao_act, exp); +} + +#endif + +#if defined(AO_HAVE_double_compare_and_swap) || defined(AO_HAVE_compare_double_and_swap_double) + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG +#endif + +static ETHR_INLINE ETHR_NDWA_RET_3_TYPE__ +ETHR_NDWA_FUNC__(cmpxchg)(ethr_native_dw_atomic_t *var, + ETHR_NDWA_VAL_ARG_TYPE__ new, + ETHR_NDWA_VAL_ARG_TYPE__ exp) +{ + AO_double_t ao_act, ao_new, ao_exp; + + ETHR_NDWA_VAL2AOVAL__(ao_exp, exp); + ETHR_NDWA_VAL2AOVAL__(ao_new, new); + + do { + int xchgd; +#if defined(AO_HAVE_double_compare_and_swap) + xchgd = AO_double_compare_and_swap(&var->dw_mem, ao_exp, ao_new); +#elif defined(AO_HAVE_compare_double_and_swap_double) + xchgd = AO_compare_double_and_swap_double(&var->dw_mem, + ao_exp.AO_val1, + ao_exp.AO_val2, + ao_new.AO_val1, + ao_new.AO_val2); +#endif + + if (xchgd) + ETHR_NDWA_RETURN_VAL_3__(1, ao_exp, exp); + +#ifdef AO_HAVE_double_load + ao_act = AO_double_load(&var->dw_mem); +#else + ao_act = AO_double_load_acquire(&var->dw_mem); +#endif + + } while (ETHR_NDWA_AOVAL_EQ__(ao_exp, ao_act)); + + ETHR_NDWA_RETURN_VAL_3__(1, ao_act, exp); +} + +#endif + +#if defined(AO_HAVE_double_compare_and_swap_read) || defined(AO_HAVE_compare_double_and_swap_double_read) + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RB +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RB +#endif + +static ETHR_INLINE ETHR_NDWA_RET_3_TYPE__ +ETHR_NDWA_FUNC__(cmpxchg_rb)(ethr_native_dw_atomic_t *var, + ETHR_NDWA_VAL_ARG_TYPE__ new, + ETHR_NDWA_VAL_ARG_TYPE__ exp) +{ + AO_double_t ao_act, ao_new, ao_exp; + + ETHR_NDWA_VAL2AOVAL__(ao_exp, exp); + ETHR_NDWA_VAL2AOVAL__(ao_new, new); + + do { + int xchgd; +#if defined(AO_HAVE_double_compare_and_swap_read) + xchgd = AO_double_compare_and_swap_read(&var->dw_mem, ao_exp, ao_new); +#elif defined(AO_HAVE_compare_double_and_swap_double_read) + xchgd = AO_compare_double_and_swap_double_read(&var->dw_mem, + ao_exp.AO_val1, + ao_exp.AO_val2, + ao_new.AO_val1, + ao_new.AO_val2); +#endif + + if (xchgd) + ETHR_NDWA_RETURN_VAL_3__(1, ao_exp, exp); + +#if defined(AO_HAVE_double_load_read) + ao_act = AO_double_load_read(&var->dw_mem); +#elif defined(AO_HAVE_double_load) + ao_act = AO_double_load(&var->dw_mem); +#else + ao_act = AO_double_load_acquire(&var->dw_mem); +#endif + + } while (ETHR_NDWA_AOVAL_EQ__(ao_exp, ao_act)); + +#ifndef AO_HAVE_double_load_read +#ifdef AO_HAVE_nop_read + AO_nop_read(); +#else + AO_nop_full(); +#endif +#endif + + ETHR_NDWA_RETURN_VAL_3__(1, ao_act, exp); +} + +#endif + +#if defined(AO_HAVE_double_compare_and_swap_acquire) || defined(AO_HAVE_compare_double_and_swap_double_acquire) + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_ACQB +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_ACQB +#endif + +static ETHR_INLINE ETHR_NDWA_RET_3_TYPE__ +ETHR_NDWA_FUNC__(cmpxchg_acqb)(ethr_native_dw_atomic_t *var, + ETHR_NDWA_VAL_ARG_TYPE__ new, + ETHR_NDWA_VAL_ARG_TYPE__ exp) +{ + AO_double_t ao_act, ao_new, ao_exp; + + ETHR_NDWA_VAL2AOVAL__(ao_exp, exp); + ETHR_NDWA_VAL2AOVAL__(ao_new, new); + + do { + int xchgd; +#if defined(AO_HAVE_double_compare_and_swap_acquire) + xchgd = AO_double_compare_and_swap_acquire(&var->dw_mem, ao_exp, ao_new); +#elif defined(AO_HAVE_compare_double_and_swap_double_acquire) + xchgd = AO_compare_double_and_swap_double_acquire(&var->dw_mem, + ao_exp.AO_val1, + ao_exp.AO_val2, + ao_new.AO_val1, + ao_new.AO_val2); +#endif + + if (xchgd) + ETHR_NDWA_RETURN_VAL_3__(1, ao_exp, exp); + +#ifdef AO_HAVE_double_load_acquire + ao_act = AO_double_load_acquire(&var->dw_mem); +#else + ao_act = AO_double_load(&var->dw_mem); +#endif + + } while (ETHR_NDWA_AOVAL_EQ__(ao_exp, ao_act)); + +#ifndef AO_HAVE_double_load_acquire + AO_nop_full(); +#endif + + ETHR_NDWA_RETURN_VAL_3__(1, ao_act, exp); +} + +#endif + +#if defined(AO_HAVE_double_compare_and_swap_write) || defined(AO_HAVE_compare_double_and_swap_double_write) + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_WB +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_WB +#endif + +static ETHR_INLINE ETHR_NDWA_RET_3_TYPE__ +ETHR_NDWA_FUNC__(cmpxchg_wb)(ethr_native_dw_atomic_t *var, + ETHR_NDWA_VAL_ARG_TYPE__ new, + ETHR_NDWA_VAL_ARG_TYPE__ exp) +{ + AO_double_t ao_act, ao_new, ao_exp; + + ETHR_NDWA_VAL2AOVAL__(ao_exp, exp); + ETHR_NDWA_VAL2AOVAL__(ao_new, new); + + do { + int xchgd; +#if defined(AO_HAVE_double_compare_and_swap_write) + xchgd = AO_double_compare_and_swap_write(&var->dw_mem, ao_exp, ao_new); +#elif defined(AO_HAVE_compare_double_and_swap_double_write) + xchgd = AO_compare_double_and_swap_double_write(&var->dw_mem, + ao_exp.AO_val1, + ao_exp.AO_val2, + ao_new.AO_val1, + ao_new.AO_val2); +#endif + + if (xchgd) + ETHR_NDWA_RETURN_VAL_3__(1, ao_exp, exp); + +#ifdef AO_HAVE_double_load + ao_act = AO_double_load(&var->dw_mem); +#else + ao_act = AO_double_load_acquire(&var->dw_mem); +#endif + + } while (ETHR_NDWA_AOVAL_EQ__(ao_exp, ao_act)); + + ETHR_NDWA_RETURN_VAL_3__(1, ao_act, exp); +} + +#endif + +#if defined(AO_HAVE_double_compare_and_swap_release) || defined(AO_HAVE_compare_double_and_swap_double_release) + +#if defined(ETHR_NATIVE_SU_DW_SINT_T) +# define ETHR_HAVE_ETHR_NATIVE_SU_DW_ATOMIC_CMPXCHG_RELB +#else +# define ETHR_HAVE_ETHR_NATIVE_DW_ATOMIC_CMPXCHG_RELB +#endif + +static ETHR_INLINE ETHR_NDWA_RET_3_TYPE__ +ETHR_NDWA_FUNC__(cmpxchg_relb)(ethr_native_dw_atomic_t *var, + ETHR_NDWA_VAL_ARG_TYPE__ new, + ETHR_NDWA_VAL_ARG_TYPE__ exp) +{ + AO_double_t ao_act, ao_new, ao_exp; + + ETHR_NDWA_VAL2AOVAL__(ao_exp, exp); + ETHR_NDWA_VAL2AOVAL__(ao_new, new); + + do { + int xchgd; +#if defined(AO_HAVE_double_compare_and_swap_release) + xchgd = AO_double_compare_and_swap_release(&var->dw_mem, ao_exp, ao_new); +#elif defined(AO_HAVE_compare_double_and_swap_double_release) + xchgd = AO_compare_double_and_swap_double_release(&var->dw_mem, + ao_exp.AO_val1, + ao_exp.AO_val2, + ao_new.AO_val1, + ao_new.AO_val2); +#endif + + if (xchgd) + ETHR_NDWA_RETURN_VAL_3__(1, ao_exp, exp); + + ao_act = AO_double_load(&var->dw_mem); + + } while (ETHR_NDWA_AOVAL_EQ__(ao_exp, ao_act)); + + ETHR_NDWA_RETURN_VAL_3__(1, ao_act, exp); +} + +#endif + +#endif /* defined(ETHR_TRY_INLINE_FUNCS) || defined(ETHR_ATOMIC_IMPL__) */ + +#endif /* Have AO double functionality ... */ + +#endif /* ETHR_LIBATOMIC_OPS_DW_ATOMIC_H__ */ + diff --git a/erts/include/internal/libatomic_ops/ethread.h b/erts/include/internal/libatomic_ops/ethread.h index e1fdd588bb..d65ee19b04 100644 --- a/erts/include/internal/libatomic_ops/ethread.h +++ b/erts/include/internal/libatomic_ops/ethread.h @@ -33,9 +33,12 @@ #define AO_USE_PENTIUM4_INSTRS #endif +#define ETHR_NATIVE_IMPL__ "libatomic_ops" + #include "atomic_ops.h" #include "ethr_membar.h" #include "ethr_atomic.h" +#include "ethr_dw_atomic.h" #endif diff --git a/erts/include/internal/ppc32/atomic.h b/erts/include/internal/ppc32/atomic.h index 6001620677..b558626b09 100644 --- a/erts/include/internal/ppc32/atomic.h +++ b/erts/include/internal/ppc32/atomic.h @@ -91,6 +91,20 @@ ethr_native_atomic32_add_return_acqb(ethr_native_atomic32_t *var, ethr_sint32_t return res; } + +#ifndef ETHR_PPC_HAVE_NO_LWSYNC + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_ADD_RETURN_RELB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_add_return_relb(ethr_native_atomic32_t *var, ethr_sint32_t incr) +{ + ethr_lwsync__(); + return ethr_native_atomic32_add_return(var, incr); +} + +#endif + #define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN 1 static ETHR_INLINE ethr_sint32_t @@ -120,7 +134,19 @@ ethr_native_atomic32_inc_return_acqb(ethr_native_atomic32_t *var) __asm__ __volatile("isync\n\t" : : : "memory"); return res; } - + +#ifndef ETHR_PPC_HAVE_NO_LWSYNC + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_INC_RETURN_RELB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_inc_return_relb(ethr_native_atomic32_t *var) +{ + ethr_lwsync__(); + return ethr_native_atomic32_inc_return(var); +} + +#endif #define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN 1 @@ -152,6 +178,19 @@ ethr_native_atomic32_dec_return_acqb(ethr_native_atomic32_t *var) return res; } +#ifndef ETHR_PPC_HAVE_NO_LWSYNC + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_DEC_RETURN_RELB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_dec_return_relb(ethr_native_atomic32_t *var) +{ + ethr_lwsync__(); + return ethr_native_atomic32_dec_return(var); +} + +#endif + #define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD 1 static ETHR_INLINE ethr_sint32_t @@ -182,6 +221,19 @@ ethr_native_atomic32_and_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t return res; } +#ifndef ETHR_PPC_HAVE_NO_LWSYNC + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_AND_RETOLD_RELB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_and_retold_relb(ethr_native_atomic32_t *var, ethr_sint32_t mask) +{ + ethr_lwsync__(); + return ethr_native_atomic32_and_retold(var, mask); +} + +#endif + #define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD 1 static ETHR_INLINE ethr_sint32_t @@ -212,6 +264,18 @@ ethr_native_atomic32_or_retold_acqb(ethr_native_atomic32_t *var, ethr_sint32_t m return res; } +#ifndef ETHR_PPC_HAVE_NO_LWSYNC + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_OR_RETOLD_RELB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_or_retold_relb(ethr_native_atomic32_t *var, ethr_sint32_t mask) +{ + ethr_lwsync__(); + return ethr_native_atomic32_or_retold(var, mask); +} + +#endif #define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG 1 @@ -242,6 +306,19 @@ ethr_native_atomic32_xchg_acqb(ethr_native_atomic32_t *var, ethr_sint32_t val) return res; } +#ifndef ETHR_PPC_HAVE_NO_LWSYNC + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_XCHG_RELB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_xchg_relb(ethr_native_atomic32_t *var, ethr_sint32_t val) +{ + ethr_lwsync__(); + return ethr_native_atomic32_xchg(var, val); +} + +#endif + #define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG 1 static ETHR_INLINE ethr_sint32_t @@ -291,6 +368,73 @@ ethr_native_atomic32_cmpxchg_acqb(ethr_native_atomic32_t *var, return old; } +#if !defined(ETHR_DISABLE_LWSYNC_FOR_CMPXCHG_RELB) && !defined(ETHR_PPC_HAVE_NO_LWSYNC) + +#define ETHR_HAVE_ETHR_NATIVE_ATOMIC32_CMPXCHG_RELB 1 + +static ETHR_INLINE ethr_sint32_t +ethr_native_atomic32_cmpxchg_relb(ethr_native_atomic32_t *var, + ethr_sint32_t new, + ethr_sint32_t expected) +{ + ethr_sint32_t actual; + + /* + * We want to implement the release barrier using the + * 'lwsync' instruction instead of using the more + * expensive 'sync' instruction. + * + * cmpxchg looks something like this: + * + * lwarx # Load + * ... + * if (fail) + * goto done; + * stwcx # Store + * if (fail) + * goto done; + * ... + * + * In the case we succeeded, 'lwsync' will have + * ordered all previously issued loads and stores + * against the successful store to this variable. + * That is everything is fine! + * + * In the case we did not succeed, we need to order + * all previously issued loads and stores against + * the load of this variable. 'lwsync' does not + * guarantee this. In order to solve this we issue + * a 'sync' and redo the load. If the value has + * changed to what the user passed as expected value + * we need to try the cmpxchg operation again, since + * this value indicates success. + */ + + ethr_lwsync__(); + + actual = ethr_native_atomic32_cmpxchg(var, new, expected); + +#ifndef ETHR_PPC_HAVE_LWSYNC + /* We checked for lwsync support in runtime... */ + if (ETHR_PPC_RUNTIME_CONF_HAVE_NO_LWSYNC__) + return actual; /* No need to; ethr_lwsync__() issued a sync... */ +#endif + + /* ethr_lwsync__() issued an lwsync... */ + if (actual == expected) + return actual; /* Successful operation */ + + /* Failure... need to issue a sync... */ + ethr_sync__(); + actual = ethr_native_atomic32_read(var); + if (actual != expected) + return actual; /* Fail... */ + /* Try again... */ + return ethr_native_atomic32_cmpxchg(var, new, expected); +} + +#endif + #endif /* ETHR_TRY_INLINE_FUNCS */ #endif /* ETHREAD_PPC_ATOMIC_H */ diff --git a/lib/asn1/src/asn1ct_check.erl b/lib/asn1/src/asn1ct_check.erl index 5d8740b92e..240f1cbb16 100644 --- a/lib/asn1/src/asn1ct_check.erl +++ b/lib/asn1/src/asn1ct_check.erl @@ -2148,11 +2148,9 @@ check_valuedef(#state{recordtopname=TopName}=S0, V0) -> {'INTEGER',NamedNumberList} -> ok = validate_integer(SVal, Value, NamedNumberList, Constr), V#valuedef{value=normalize_value(SVal, Vtype, Value, [])}; - #'SEQUENCE'{components=Components} -> - {ok,SeqVal} = validate_sequence(SVal, Value, - Components, Constr), - V#valuedef{value=normalize_value(SVal, Vtype, - SeqVal, TopName)}; + #'SEQUENCE'{} -> + {ok,SeqVal} = convert_external(SVal, Value), + V#valuedef{value=normalize_value(SVal, Vtype, SeqVal, TopName)}; {'SelectionType',SelName,SelT} -> CheckedT = check_selectiontype(SVal, SelName, SelT), NewV = V#valuedef{type=CheckedT}, @@ -2412,13 +2410,13 @@ valid_objectid(o_id,_I,[1]) -> false; valid_objectid(o_id,_I,[2]) -> true; valid_objectid(_,_,_) -> true. -validate_sequence(S=#state{type=Vtype},Value,_Components,_Constr) -> +convert_external(S=#state{type=Vtype}, Value) -> case Vtype of #type{tag=[{tag,'UNIVERSAL',8,'IMPLICIT',32}]} -> %% this is an 'EXTERNAL' (or INSTANCE OF) case Value of - [{identification,_}|_RestVal] -> - {ok,to_EXTERNAL1990(S,Value)}; + [{#seqtag{val=identification},_}|_] -> + {ok,to_EXTERNAL1990(S, Value)}; _ -> {ok,Value} end; @@ -2426,21 +2424,25 @@ validate_sequence(S=#state{type=Vtype},Value,_Components,_Constr) -> {ok,Value} end. -to_EXTERNAL1990(S,[{identification,{'CHOICE',{syntax,Stx}}}|Rest]) -> - to_EXTERNAL1990(S,Rest,[{'direct-reference',Stx}]); -to_EXTERNAL1990(S,[{identification,{'CHOICE',{'presentation-context-id',I}}}|Rest]) -> - to_EXTERNAL1990(S,Rest,[{'indirect-reference',I}]); -to_EXTERNAL1990(S,[{identification,{'CHOICE',{'context-negotiation',[{_,PCid},{_,TrStx}]}}}|Rest]) -> - to_EXTERNAL1990(S,Rest,[{'indirect-reference',PCid},{'direct-reference',TrStx}]); -to_EXTERNAL1990(S,_) -> +to_EXTERNAL1990(S, [{#seqtag{val=identification}=T, + {'CHOICE',{syntax,Stx}}}|Rest]) -> + to_EXTERNAL1990(S, Rest, [{T#seqtag{val='direct-reference'},Stx}]); +to_EXTERNAL1990(S, [{#seqtag{val=identification}=T, + {'CHOICE',{'presentation-context-id',I}}}|Rest]) -> + to_EXTERNAL1990(S, Rest, [{T#seqtag{val='indirect-reference'},I}]); +to_EXTERNAL1990(S, [{#seqtag{val=identification}=T, + {'CHOICE',{'context-negotiation',[{_,PCid},{_,TrStx}]}}}|Rest]) -> + to_EXTERNAL1990(S, Rest, [{T#seqtag{val='indirect-reference'},PCid}, + {T#seqtag{val='direct-reference'},TrStx}]); +to_EXTERNAL1990(S, _) -> error({value,"illegal value in EXTERNAL type",S}). -to_EXTERNAL1990(S,[V={'data-value-descriptor',_}|Rest],Acc) -> - to_EXTERNAL1990(S,Rest,[V|Acc]); -to_EXTERNAL1990(_S,[{'data-value',Val}],Acc) -> - Encoding = {encoding,{'CHOICE',{'octet-aligned',Val}}}, +to_EXTERNAL1990(S, [V={#seqtag{val='data-value-descriptor'},_}|Rest], Acc) -> + to_EXTERNAL1990(S, Rest, [V|Acc]); +to_EXTERNAL1990(_S, [{#seqtag{val='data-value'}=T,Val}], Acc) -> + Encoding = {T#seqtag{val=encoding},{'CHOICE',{'octet-aligned',Val}}}, lists:reverse([Encoding|Acc]); -to_EXTERNAL1990(S,_,_) -> +to_EXTERNAL1990(S, _, _) -> error({value,"illegal value in EXTERNAL type",S}). %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -6581,6 +6583,8 @@ merge_tags2([T1= #tag{type='IMPLICIT'}, T2 |Rest], Acc) -> merge_tags2([T1#tag{type=T2#tag.type, form=T2#tag.form}|Rest],Acc); merge_tags2([T1= #tag{type={default,'IMPLICIT'}}, T2 |Rest], Acc) -> merge_tags2([T1#tag{type=T2#tag.type, form=T2#tag.form}|Rest],Acc); +merge_tags2([T1= #tag{type={default,'AUTOMATIC'}}, T2 |Rest], Acc) -> + merge_tags2([T1#tag{type=T2#tag.type, form=T2#tag.form}|Rest],Acc); merge_tags2([H|T],Acc) -> merge_tags2(T, [H|Acc]); merge_tags2([], Acc) -> diff --git a/lib/asn1/test/asn1_SUITE.erl b/lib/asn1/test/asn1_SUITE.erl index 888339a4d2..432197eec0 100644 --- a/lib/asn1/test/asn1_SUITE.erl +++ b/lib/asn1/test/asn1_SUITE.erl @@ -60,7 +60,8 @@ groups() -> {ber, Parallel, [ber_choiceinseq, % Uses 'SOpttest' - ber_optional]}, + ber_optional, + tagdefault_automatic]}, {app_test, [], [{asn1_app_test, all}]}, @@ -659,6 +660,19 @@ ber_optional(Config, Rule, Opts) -> {'C', asn1_NOVALUE, 111, asn1_NOVALUE}}, asn1_test_lib:roundtrip('SOpttest', 'S', V). +tagdefault_automatic(Config) -> + test(Config, fun tagdefault_automatic/3, [ber]). +tagdefault_automatic(Config, Rule, Opts) -> + asn1_test_lib:compile("TAGDEFAULT-AUTOMATIC", Config, [Rule|Opts]), + << 48,8,128,2,100,101,129,2,110,111 >> = + asn1_test_lib:roundtrip_enc('TAGDEFAULT-AUTOMATIC', 'Tagged', {'Tagged', << 100,101 >>, << 110,111 >>}), + << 48,8,128,2,100,101,129,2,110,111 >> = + asn1_test_lib:roundtrip_enc('TAGDEFAULT-AUTOMATIC', 'Untagged', {'Untagged', << 100,101 >>, << 110,111 >>}), + << 48,8,4,2,100,101,130,2,110,111 >> = + asn1_test_lib:roundtrip_enc('TAGDEFAULT-AUTOMATIC', 'Mixed', {'Mixed', << 100,101 >>, << 110,111 >>}), + + ok. + %% records used by test-case default -record('Def1', {bool0, bool1 = asn1_DEFAULT, diff --git a/lib/asn1/test/asn1_SUITE_data/ContextSwitchingTypes.asn1 b/lib/asn1/test/asn1_SUITE_data/ContextSwitchingTypes.asn1 index c8145bad63..e279a180c1 100644 --- a/lib/asn1/test/asn1_SUITE_data/ContextSwitchingTypes.asn1 +++ b/lib/asn1/test/asn1_SUITE_data/ContextSwitchingTypes.asn1 @@ -21,6 +21,10 @@ val3-T T ::= {identification context-negotiation:{presentation-context-id 12, transfer-syntax {1 2 3}}, data-value '123'H} +val4-T T ::= {identification presentation-context-id:42, + data-value-descriptor "FooBar", + data-value '123'H} + -- EMBEDDED PDV type EP ::= EMBEDDED PDV diff --git a/lib/asn1/test/asn1_SUITE_data/TAGDEFAULT-AUTOMATIC.asn b/lib/asn1/test/asn1_SUITE_data/TAGDEFAULT-AUTOMATIC.asn new file mode 100644 index 0000000000..2fcba1f71e --- /dev/null +++ b/lib/asn1/test/asn1_SUITE_data/TAGDEFAULT-AUTOMATIC.asn @@ -0,0 +1,25 @@ +TAGDEFAULT-AUTOMATIC DEFINITIONS AUTOMATIC TAGS ::= +BEGIN + +EXPORTS + Tagged, + Untagged, + Mixed +; + +Tagged ::= SEQUENCE { + o0 [0] OCTET STRING, + o1 [1] OCTET STRING +} + +Untagged ::= SEQUENCE { + o0 OCTET STRING, + o1 OCTET STRING +} + +Mixed ::= SEQUENCE { + o0 OCTET STRING, + o2 [2] OCTET STRING +} + +END diff --git a/lib/asn1/test/testContextSwitchingTypes.erl b/lib/asn1/test/testContextSwitchingTypes.erl index bdd6883dac..61d1fbdd69 100644 --- a/lib/asn1/test/testContextSwitchingTypes.erl +++ b/lib/asn1/test/testContextSwitchingTypes.erl @@ -24,11 +24,21 @@ -include_lib("test_server/include/test_server.hrl"). test(Config) -> - ValT = 'ContextSwitchingTypes':'val1-T'(), - check_EXTERNAL(enc_dec('T', ValT)), + ValT_1 = 'ContextSwitchingTypes':'val1-T'(), + check_EXTERNAL(enc_dec('T', ValT_1)), + + ValT_2 = 'ContextSwitchingTypes':'val2-T'(), + check_EXTERNAL(enc_dec('T', ValT_2)), + + ValT_3 = 'ContextSwitchingTypes':'val3-T'(), + check_EXTERNAL(enc_dec('T', ValT_3)), + + ValT_4 = 'ContextSwitchingTypes':'val4-T'(), + check_EXTERNAL(enc_dec('T', ValT_4)), {ok,ValT2} = asn1ct:value('ContextSwitchingTypes', 'T', [{i,?config(case_dir, Config)}]), + io:format("ValT2 ~p~n",[ValT2]), check_EXTERNAL(enc_dec('T', ValT2)), ValEP = 'ContextSwitchingTypes':'val1-EP'(), diff --git a/lib/erl_interface/aclocal.m4 b/lib/erl_interface/aclocal.m4 index ed492d55ff..d78025b0be 100644 --- a/lib/erl_interface/aclocal.m4 +++ b/lib/erl_interface/aclocal.m4 @@ -1421,9 +1421,31 @@ case "$THR_LIB_NAME" in int z; AO_nop_full(); +#if defined(AO_HAVE_store) AO_store(&x, (AO_t) 0); +#elif defined(AO_HAVE_store_release) + AO_store_release(&x, (AO_t) 0); +#else +#error No store +#endif +#if defined(AO_HAVE_load) z = AO_load(&x); +#elif defined(AO_HAVE_load_acquire) + z = AO_load_acquire(&x); +#else +#error No load +#endif +#if defined(AO_HAVE_compare_and_swap_full) z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_release) + z = AO_compare_and_swap_release(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_acquire) + z = AO_compare_and_swap_acquire(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap) + z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); +#else +#error No compare_and_swap +#endif ], [ethr_have_native_atomics=yes ethr_have_libatomic_ops=yes]) diff --git a/lib/hipe/llvm/elf_format.hrl b/lib/hipe/llvm/elf_format.hrl index 78592e6e2a..7a3cdfead6 100644 --- a/lib/hipe/llvm/elf_format.hrl +++ b/lib/hipe/llvm/elf_format.hrl @@ -68,7 +68,7 @@ -define(E_IDENT, {?E_IDENT_OFFSET, ?E_IDENT_SIZE}). -define(E_TYPE, {?E_TYPE_OFFSET, ?E_TYPE_SIZE}). -define(E_MACHINE, {?E_MACHINE_OFFSET, ?E_MACHINE_SIZE}). --define(E_VERSION, {?E_VERSION_OFFSET, ?E_VERSION_SIZE}) +-define(E_VERSION, {?E_VERSION_OFFSET, ?E_VERSION_SIZE}). -define(E_ENTRY, {?E_ENTRY_OFFSET, ?E_ENTRY_SIZE}). -define(E_PHOFF, {?E_PHOFF_OFFSET, ?E_PHOFF_SIZE}). -define(E_SHOFF, {?E_SHOFF_OFFSET, ?E_SHOFF_SIZE}). diff --git a/lib/kernel/doc/src/file.xml b/lib/kernel/doc/src/file.xml index 6976457897..338d62e82b 100644 --- a/lib/kernel/doc/src/file.xml +++ b/lib/kernel/doc/src/file.xml @@ -1283,6 +1283,8 @@ or before unix time epoch which is 1970-01-01 00:00 UTC. Default is <c>{time, local}</c>. </p> + <p>If the <c>raw</c> option is set, the file server will not be called + and only informations about local files will be returned.</p> <note> <p> Since file times is stored in posix time on most OS it is @@ -1509,6 +1511,8 @@ the link will be returned in the <c>file_info</c> record and the <c>type</c> field of the record will be set to <c>symlink</c>.</p> + <p>If the <c>raw</c> option is set, the file server will not be called + and only informations about local files will be returned.</p> <p>If <c><anno>Name</anno></c> is not a symbolic link, this function returns exactly the same result as <c>read_file_info/1</c>. On platforms that do not support symbolic links, this function @@ -1847,6 +1851,8 @@ interpret it as universal time and <c>posix</c> must be seconds since or before unix time epoch which is 1970-01-01 00:00 UTC. Default is <c>{time, local}</c>. + <p>If the <c>raw</c> option is set, the file server will not be called + and only informations about local files will be returned.</p> </p> <p>The following fields are used from the record, if they are given.</p> diff --git a/lib/kernel/src/file.erl b/lib/kernel/src/file.erl index 7b2750846e..b6b153ae56 100644 --- a/lib/kernel/src/file.erl +++ b/lib/kernel/src/file.erl @@ -114,7 +114,7 @@ -type sendfile_option() :: {chunk_size, non_neg_integer()} | {use_threads, boolean()}. -type file_info_option() :: {'time', 'local'} | {'time', 'universal'} - | {'time', 'posix'}. + | {'time', 'posix'} | raw. %%% BIFs -export([native_name_encoding/0]). @@ -242,7 +242,19 @@ read_file_info(Name) -> Reason :: posix() | badarg. read_file_info(Name, Opts) when is_list(Opts) -> - check_and_call(read_file_info, [file_name(Name), Opts]). + Args = [file_name(Name), Opts], + case check_args(Args) of + ok -> + case lists:member(raw, Opts) of + true -> + [FileName|_] = Args, + ?PRIM_FILE:read_file_info(FileName, Opts); + false -> + call(read_file_info, Args) + end; + Error -> + Error + end. -spec altname(Name :: name_all()) -> any(). @@ -264,7 +276,19 @@ read_link_info(Name) -> Reason :: posix() | badarg. read_link_info(Name, Opts) when is_list(Opts) -> - check_and_call(read_link_info, [file_name(Name),Opts]). + Args = [file_name(Name), Opts], + case check_args(Args) of + ok -> + case lists:member(raw, Opts) of + true -> + [FileName|_] = Args, + ?PRIM_FILE:read_link_info(FileName, Opts); + false -> + call(read_link_info, Args) + end; + Error -> + Error + end. -spec read_link(Name) -> {ok, Filename} | {error, Reason} when @@ -298,7 +322,19 @@ write_file_info(Name, Info = #file_info{}) -> Reason :: posix() | badarg. write_file_info(Name, Info = #file_info{}, Opts) when is_list(Opts) -> - check_and_call(write_file_info, [file_name(Name), Info, Opts]). + Args = [file_name(Name), Info, Opts], + case check_args(Args) of + ok -> + case lists:member(raw, Opts) of + true -> + [FileName|_] = Args, + ?PRIM_FILE:write_file_info(FileName, Info, Opts); + false -> + call(write_file_info, Args) + end; + Error -> + Error + end. -spec list_dir(Dir) -> {ok, Filenames} | {error, Reason} when Dir :: name_all(), @@ -384,26 +420,12 @@ write_file(Name, Bin, ModeList) when is_list(ModeList) -> %% Obsolete, undocumented, local node only, don't use!. %% XXX to be removed. raw_read_file_info(Name) -> - Args = [file_name(Name)], - case check_args(Args) of - ok -> - [FileName] = Args, - ?PRIM_FILE:read_file_info(FileName); - Error -> - Error - end. + read_file_info(Name, [raw]). %% Obsolete, undocumented, local node only, don't use!. %% XXX to be removed. raw_write_file_info(Name, #file_info{} = Info) -> - Args = [file_name(Name)], - case check_args(Args) of - ok -> - [FileName] = Args, - ?PRIM_FILE:write_file_info(FileName, Info); - Error -> - Error - end. + write_file_info(Name, Info, [raw]). %%%----------------------------------------------------------------- %%% File io server functions. diff --git a/lib/kernel/test/file_SUITE.erl b/lib/kernel/test/file_SUITE.erl index f6d6cd94ab..56c35678b6 100644 --- a/lib/kernel/test/file_SUITE.erl +++ b/lib/kernel/test/file_SUITE.erl @@ -1236,9 +1236,10 @@ file_info_basic_file(Config) when is_list(Config) -> %% Test that the file has the expected attributes. %% The times are tricky, so we will save them to a separate test case. - ?line {ok,#file_info{size=Size,type=Type,access=Access, - atime=AccessTime,mtime=ModifyTime}} = - ?FILE_MODULE:read_file_info(Name), + {ok,FileInfo} = ?FILE_MODULE:read_file_info(Name), + {ok,FileInfo} = ?FILE_MODULE:read_file_info(Name, [raw]), + #file_info{size=Size,type=Type,access=Access, + atime=AccessTime,mtime=ModifyTime} = FileInfo, ?line io:format("Access ~p, Modify ~p", [AccessTime, ModifyTime]), ?line Size = 7, ?line Type = regular, @@ -1280,9 +1281,10 @@ file_info_basic_directory(Config) when is_list(Config) -> test_server:timetrap_cancel(Dog). test_directory(Name, ExpectedAccess) -> - ?line {ok,#file_info{size=Size,type=Type,access=Access, - atime=AccessTime,mtime=ModifyTime}} = - ?FILE_MODULE:read_file_info(Name), + {ok,FileInfo} = ?FILE_MODULE:read_file_info(Name), + {ok,FileInfo} = ?FILE_MODULE:read_file_info(Name, [raw]), + #file_info{size=Size,type=Type,access=Access, + atime=AccessTime,mtime=ModifyTime} = FileInfo, ?line io:format("Testing directory ~s", [Name]), ?line io:format("Directory size is ~p", [Size]), ?line io:format("Access ~p", [Access]), @@ -1307,11 +1309,11 @@ file_info_bad(doc) -> []; file_info_bad(Config) when is_list(Config) -> ?line Dog = test_server:timetrap(test_server:seconds(5)), ?line RootDir = filename:join([?config(priv_dir, Config)]), - ?line {error, enoent} = - ?FILE_MODULE:read_file_info( - filename:join(RootDir, - atom_to_list(?MODULE)++ "_nonexistent")), + FileName = filename:join(RootDir, atom_to_list(?MODULE) ++ "_nonexistent"), + {error,enoent} = ?FILE_MODULE:read_file_info(FileName), + {error,enoent} = ?FILE_MODULE:read_file_info(FileName, [raw]), ?line {error, enoent} = ?FILE_MODULE:read_file_info(""), + {error, enoent} = ?FILE_MODULE:read_file_info("", [raw]), ?line [] = flush(), ?line test_server:timetrap_cancel(Dog), ok. @@ -1346,8 +1348,16 @@ file_info_int(Config) -> ?line io:put_chars(Fd1,"foo"), %% check that the file got a modify date max a few seconds away from now - ?line {ok,#file_info{type=regular,atime=AccTime1,mtime=ModTime1}} = - ?FILE_MODULE:read_file_info(Name), + {ok,FileInfo1} = ?FILE_MODULE:read_file_info(Name), + {ok,FileInfo1Raw} = ?FILE_MODULE:read_file_info(Name, [raw]), + + %% We assert that everything but the size is the same, on some OSs the + %% size may not have been flushed to disc and we do not want to do a + %% sync to force it. + FileInfo1Raw = FileInfo1#file_info{ size = FileInfo1Raw#file_info.size }, + + #file_info{type=regular,atime=AccTime1,mtime=ModTime1} = FileInfo1, + ?line Now = erlang:localtime(), %??? ?line io:format("Now ~p",[Now]), ?line io:format("Open file Acc ~p Mod ~p",[AccTime1,ModTime1]), @@ -1363,9 +1373,10 @@ file_info_int(Config) -> %% close the file, and watch the modify date change ?line ok = ?FILE_MODULE:close(Fd1), - ?line {ok,#file_info{size=Size,type=regular,access=Access, - atime=AccTime2,mtime=ModTime2}} = - ?FILE_MODULE:read_file_info(Name), + {ok,FileInfo2} = ?FILE_MODULE:read_file_info(Name), + {ok,FileInfo2} = ?FILE_MODULE:read_file_info(Name, [raw]), + #file_info{size=Size,type=regular,access=Access, + atime=AccTime2,mtime=ModTime2} = FileInfo2, ?line io:format("Closed file Acc ~p Mod ~p",[AccTime2,ModTime2]), ?line true = time_dist(ModTime1,ModTime2) >= 0, @@ -1374,9 +1385,10 @@ file_info_int(Config) -> ?line Access = read_write, %% Do some directory checking - ?line {ok,#file_info{size=DSize,type=directory,access=DAccess, - atime=AccTime3,mtime=ModTime3}} = - ?FILE_MODULE:read_file_info(RootDir), + {ok,FileInfo3} = ?FILE_MODULE:read_file_info(RootDir), + {ok,FileInfo3} = ?FILE_MODULE:read_file_info(RootDir, [raw]), + #file_info{size=DSize,type=directory,access=DAccess, + atime=AccTime3,mtime=ModTime3} = FileInfo3, %% this dir was modified only a few secs ago ?line io:format("Dir Acc ~p; Mod ~p; Now ~p", [AccTime3, ModTime3, Now]), ?line true = abs(time_dist(Now,ModTime3)) < 5, @@ -1449,6 +1461,12 @@ file_write_file_info(Config) when is_list(Config) -> ?line ?FILE_MODULE:write_file_info(Name1, #file_info{mode=8#400}), ?line {error, eacces} = ?FILE_MODULE:write_file(Name1, "hello again"), + %% Same with raw. + ?FILE_MODULE:write_file_info(Name1, #file_info{mode=8#600}, [raw]), + ok = ?FILE_MODULE:write_file(Name1, "hello again"), + ?FILE_MODULE:write_file_info(Name1, #file_info{mode=8#400}, [raw]), + {error,eacces} = ?FILE_MODULE:write_file(Name1, "hello again"), + %% Write the times again. %% Note: Seconds must be even; see note in file_info_times/1. @@ -2650,7 +2668,9 @@ make_link(Config) when is_list(Config) -> %% since they are not used on symbolic links. ?line {ok, Info} = ?FILE_MODULE:read_link_info(Name), + {ok,Info} = ?FILE_MODULE:read_link_info(Name, [raw]), ?line {ok, Info} = ?FILE_MODULE:read_link_info(Alias), + {ok,Info} = ?FILE_MODULE:read_link_info(Alias, [raw]), ?line #file_info{links = 2, type = regular} = Info, ?line {error, eexist} = ?FILE_MODULE:make_link(Name, Alias), @@ -2670,6 +2690,7 @@ read_link_info_for_non_link(Config) when is_list(Config) -> ?line {ok, #file_info{type=directory}} = ?FILE_MODULE:read_link_info("."), + {ok, #file_info{type=directory}} = ?FILE_MODULE:read_link_info(".", [raw]), ?line [] = flush(), ?line test_server:timetrap_cancel(Dog), @@ -2700,11 +2721,15 @@ symlinks(Config) when is_list(Config) -> {skipped, "Windows user not privileged to create symlinks"}; ok -> ?line {ok, Info1} = ?FILE_MODULE:read_file_info(Name), + {ok,Info1} = ?FILE_MODULE:read_file_info(Name, [raw]), ?line {ok, Info1} = ?FILE_MODULE:read_file_info(Alias), + {ok,Info1} = ?FILE_MODULE:read_file_info(Alias, [raw]), ?line {ok, Info1} = ?FILE_MODULE:read_link_info(Name), + {ok,Info1} = ?FILE_MODULE:read_link_info(Name, [raw]), ?line #file_info{links = 1, type = regular} = Info1, ?line {ok, Info2} = ?FILE_MODULE:read_link_info(Alias), + {ok,Info2} = ?FILE_MODULE:read_link_info(Alias, [raw]), ?line #file_info{links=1, type=symlink} = Info2, ?line {ok, Name} = ?FILE_MODULE:read_link(Alias), {ok, Name} = ?FILE_MODULE:read_link_all(Alias), diff --git a/lib/megaco/aclocal.m4 b/lib/megaco/aclocal.m4 index ed492d55ff..d78025b0be 100644 --- a/lib/megaco/aclocal.m4 +++ b/lib/megaco/aclocal.m4 @@ -1421,9 +1421,31 @@ case "$THR_LIB_NAME" in int z; AO_nop_full(); +#if defined(AO_HAVE_store) AO_store(&x, (AO_t) 0); +#elif defined(AO_HAVE_store_release) + AO_store_release(&x, (AO_t) 0); +#else +#error No store +#endif +#if defined(AO_HAVE_load) z = AO_load(&x); +#elif defined(AO_HAVE_load_acquire) + z = AO_load_acquire(&x); +#else +#error No load +#endif +#if defined(AO_HAVE_compare_and_swap_full) z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_release) + z = AO_compare_and_swap_release(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_acquire) + z = AO_compare_and_swap_acquire(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap) + z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); +#else +#error No compare_and_swap +#endif ], [ethr_have_native_atomics=yes ethr_have_libatomic_ops=yes]) diff --git a/lib/mnesia/src/mnesia.erl b/lib/mnesia/src/mnesia.erl index b7d80c1370..8f14831ad3 100644 --- a/lib/mnesia/src/mnesia.erl +++ b/lib/mnesia/src/mnesia.erl @@ -1140,10 +1140,12 @@ match_object(_Tid, _Ts, Tab, Pat, _LockKind) -> add_written_match(S, Pat, Tab, Objs) -> Ops = find_ops(S, Tab, Pat), - add_match(Ops, Objs, val({Tab, setorbag})). + FixedRes = add_match(Ops, Objs, val({Tab, setorbag})), + MS = ets:match_spec_compile([{Pat, [], ['$_']}]), + ets:match_spec_run(FixedRes, MS). find_ops(S, Tab, Pat) -> - GetWritten = [{{{Tab, '_'}, Pat, write}, [], ['$_']}, + GetWritten = [{{{Tab, '_'}, '_', write}, [], ['$_']}, {{{Tab, '_'}, '_', delete}, [], ['$_']}, {{{Tab, '_'}, Pat, delete_object}, [], ['$_']}], ets:select(S, GetWritten). diff --git a/lib/mnesia/src/mnesia_subscr.erl b/lib/mnesia/src/mnesia_subscr.erl index 9272211ad2..866a57e370 100644 --- a/lib/mnesia/src/mnesia_subscr.erl +++ b/lib/mnesia/src/mnesia_subscr.erl @@ -225,7 +225,7 @@ call(Msg) -> Res = gen_server:call(Pid, Msg, infinity), %% We get an exit signal if server dies receive - {'EXIT', _Pid, _Reason} -> + {'EXIT', Pid, _Reason} -> {error, {node_not_running, node()}} after 0 -> Res diff --git a/lib/mnesia/test/mnesia_isolation_test.erl b/lib/mnesia/test/mnesia_isolation_test.erl index d57f976d1f..8468472cf2 100644 --- a/lib/mnesia/test/mnesia_isolation_test.erl +++ b/lib/mnesia/test/mnesia_isolation_test.erl @@ -1584,7 +1584,8 @@ write_shadows(Config) when is_list(Config) -> ?match([RecA2], mnesia:read({Tab, a})), ?match([RecA2], mnesia:wread({Tab, a})), - ?match([RecA2], mnesia:match_object(PatA2)), %% delete shadow old but not new write - is the new value visable + ?match([], mnesia:match_object(PatA1)), %% delete shadow old but not new write + ?match([RecA2], mnesia:match_object(PatA2)), %% is the new value visable ?match([a], mnesia:all_keys(Tab)), ?match([RecA2], mnesia:index_match_object(PatA2, ValPos)), @@ -1643,6 +1644,7 @@ delete_shadows(Config) when is_list(Config) -> ?match([RecA2], mnesia:read({Tab, a})), ?match([RecA2], mnesia:wread({Tab, a})), + ?match([], mnesia:match_object(PatA1)), ?match([RecA2], mnesia:match_object(PatA2)), ?match([a], mnesia:all_keys(Tab)), ?match([RecA2], mnesia:index_match_object(PatA2, ValPos)), diff --git a/lib/odbc/aclocal.m4 b/lib/odbc/aclocal.m4 index ed492d55ff..d78025b0be 100644 --- a/lib/odbc/aclocal.m4 +++ b/lib/odbc/aclocal.m4 @@ -1421,9 +1421,31 @@ case "$THR_LIB_NAME" in int z; AO_nop_full(); +#if defined(AO_HAVE_store) AO_store(&x, (AO_t) 0); +#elif defined(AO_HAVE_store_release) + AO_store_release(&x, (AO_t) 0); +#else +#error No store +#endif +#if defined(AO_HAVE_load) z = AO_load(&x); +#elif defined(AO_HAVE_load_acquire) + z = AO_load_acquire(&x); +#else +#error No load +#endif +#if defined(AO_HAVE_compare_and_swap_full) z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_release) + z = AO_compare_and_swap_release(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_acquire) + z = AO_compare_and_swap_acquire(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap) + z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); +#else +#error No compare_and_swap +#endif ], [ethr_have_native_atomics=yes ethr_have_libatomic_ops=yes]) diff --git a/lib/odbc/c_src/odbcserver.c b/lib/odbc/c_src/odbcserver.c index 84c201a656..f4b0a5d8d0 100644 --- a/lib/odbc/c_src/odbcserver.c +++ b/lib/odbc/c_src/odbcserver.c @@ -564,7 +564,6 @@ static db_result_msg db_connect(byte *args, db_state *state) /* Close the connection to the database. Returns an ok or error message. */ static db_result_msg db_close_connection(db_state *state) { - int index; SQLRETURN result; diagnos diagnos; @@ -613,11 +612,7 @@ static db_result_msg db_end_tran(byte compleationtype, db_state *state) erlang term into the message buffer of the returned message-struct. */ static db_result_msg db_query(byte *sql, db_state *state) { - char *atom; - int num_of_rows, elements, update; - SQLSMALLINT num_of_columns; SQLRETURN result; - SQLINTEGER RowCountPtr; db_result_msg msg; diagnos diagnos; byte is_error[6]; @@ -701,12 +696,9 @@ static db_result_msg db_query(byte *sql, db_state *state) set. */ static db_result_msg db_select_count(byte *sql, db_state *state) { - SQLSMALLINT num_of_columns, intresult; + SQLSMALLINT num_of_columns; SQLLEN num_of_rows; - SQLRETURN result; diagnos diagnos; - db_result_msg msg; - int index; if (associated_result_set(state)) { clean_state(state); @@ -792,6 +784,9 @@ static db_result_msg db_select(byte *args, db_state *state) orientation = SQL_FETCH_NEXT; offset = atoi(strtok((char *)(args + sizeof(byte)), ";")); n = atoi(strtok(NULL, ";")); + break; + default: + DO_EXIT(EXIT_PARAM_ARRAY); } msg = encode_empty_message(); @@ -942,7 +937,7 @@ static db_result_msg db_describe_table(byte *sql, db_state *state) SQLSMALLINT num_of_columns; SQLCHAR name[MAX_NAME]; SQLSMALLINT name_len, sql_type, dec_digits, nullable; - SQLLEN size; + SQLULEN size; diagnos diagnos; int i; @@ -1294,8 +1289,7 @@ static db_result_msg encode_column_name_list(SQLSMALLINT num_of_columns, db_result_msg msg; SQLCHAR name[MAX_NAME]; SQLSMALLINT name_len, sql_type, dec_digits, nullable; - SQLLEN size; - SQLRETURN result; + SQLULEN size; msg = encode_empty_message(); @@ -1357,9 +1351,8 @@ static db_result_msg encode_column_name_list(SQLSMALLINT num_of_columns, static db_result_msg encode_value_list(SQLSMALLINT num_of_columns, db_state *state) { - int i, msg_len; + int i; SQLRETURN result; - db_result_msg list_result; db_result_msg msg; msg = encode_empty_message(); @@ -1402,9 +1395,8 @@ static db_result_msg encode_value_list_scroll(SQLSMALLINT num_of_columns, SQLINTEGER OffSet, int N, db_state *state) { - int i, j, msg_len; + int i, j; SQLRETURN result; - db_result_msg list_result; db_result_msg msg; msg = encode_empty_message(); @@ -1810,10 +1802,23 @@ static int read_exact(byte *buffer, int len) { #endif +static size_t length_buffer_to_size(byte length_buffer[LENGTH_INDICATOR_SIZE]) +{ + size_t size = 0, i; + + for (i = 0; i < LENGTH_INDICATOR_SIZE; ++i) { + size <<= 8; + size |= (unsigned char)length_buffer[i]; + } + + return size; +} + + /* Recieive (read) data from erlang on stdin */ static byte * receive_erlang_port_msg(void) { - int i, len = 0; + size_t len; byte *buffer; byte lengthstr[LENGTH_INDICATOR_SIZE]; @@ -1822,10 +1827,8 @@ static byte * receive_erlang_port_msg(void) { DO_EXIT(EXIT_STDIN_HEADER); } - for(i=0; i < LENGTH_INDICATOR_SIZE; i++) { - len <<= 8; - len |= lengthstr[i]; - } + + len = length_buffer_to_size(lengthstr); if (len <= 0 || len > 1024) { DO_EXIT(EXIT_STDIN_HEADER); @@ -1926,8 +1929,7 @@ static byte * receive_msg(int socket) #endif { byte lengthstr[LENGTH_INDICATOR_SIZE]; - size_t msg_len = 0; - int i; + size_t msg_len; byte *buffer = NULL; if(!receive_msg_part(socket, lengthstr, LENGTH_INDICATOR_SIZE)) { @@ -1935,10 +1937,7 @@ static byte * receive_msg(int socket) DO_EXIT(EXIT_SOCKET_RECV_HEADER); } - for(i = 0; i < LENGTH_INDICATOR_SIZE; i++) { - msg_len <<= 8; - msg_len |= lengthstr[i]; - } + msg_len = length_buffer_to_size(lengthstr); buffer = (byte *)safe_malloc(msg_len); @@ -2200,8 +2199,7 @@ static void init_driver(int erl_auto_commit_mode, int erl_trace_driver, static void init_param_column(param_array *params, byte *buffer, int *index, int num_param_values, db_state* state) { - int size, erl_type; - long user_type, precision, scale, length, dummy; + long user_type, precision, scale, length; long in_or_out; ei_decode_long(buffer, index, &user_type); @@ -2514,8 +2512,7 @@ static param_array * bind_parameter_arrays(byte *buffer, int *index, int cols, int num_param_values, db_state *state) { - int i, j, k, size, erl_type; - db_result_msg msg; + int i, j, size, erl_type; long dummy; void *Values; param_array *params; @@ -2601,7 +2598,6 @@ static db_column retrive_binary_data(db_column column, int column_nr, db_state *state) { char *outputptr; - char *sqlState; int blocklen, outputlen, result; diagnos diagnos; diff --git a/lib/odbc/test/odbc_start_SUITE.erl b/lib/odbc/test/odbc_start_SUITE.erl index a7bb1d0ffe..d75e615ed4 100644 --- a/lib/odbc/test/odbc_start_SUITE.erl +++ b/lib/odbc/test/odbc_start_SUITE.erl @@ -109,8 +109,8 @@ suite() -> [{ct_hooks,[ts_install_cth]}]. all() -> case odbc_test_lib:odbc_check() of - ok -> [app, appup, start]; - Other -> [app, appup] + ok -> [app, appup, start, long_connection_line]; + _Other -> [app, appup] end. groups() -> @@ -168,3 +168,14 @@ start_odbc(Type) -> {error, odbc_not_started} -> test_server:fail(start_failed) end. + + +long_connection_line(doc)-> + ["Test a connection line longer than 127 characters"]; +long_connection_line(suite) -> []; +long_connection_line(_Config) -> + odbc:start(), + String133 = "unknown_odbc_parameter=01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", + {error, Reason} = odbc:connect(String133, []), + odbc:stop(), + ct:pal("Driver error reason: ~p",[Reason]). diff --git a/lib/stdlib/test/gen_fsm_SUITE.erl b/lib/stdlib/test/gen_fsm_SUITE.erl index 75796ab1b6..dabc10aec4 100644 --- a/lib/stdlib/test/gen_fsm_SUITE.erl +++ b/lib/stdlib/test/gen_fsm_SUITE.erl @@ -538,7 +538,7 @@ terminate_crash_format(Config) when is_list(Config) -> io:format("Timeout: expected error logger msg", []), ?t:fail() end, - [] = ?t:messages_get(), + _ = ?t:messages_get(), process_flag(trap_exit, OldFl), ok. diff --git a/lib/tools/src/cover.erl b/lib/tools/src/cover.erl index 113fa24bd5..31754015f7 100644 --- a/lib/tools/src/cover.erl +++ b/lib/tools/src/cover.erl @@ -1696,6 +1696,8 @@ fix_expr(T, Line, Bump) when is_tuple(T) -> fix_expr(E, _Line, _Bump) -> E. +fix_clauses([], _Line, _Bump) -> + []; fix_clauses(Cs, Line, Bump) -> case bumps_line(lists:last(Cs), Line) of true -> diff --git a/lib/tools/test/cover_SUITE_data/b.erl b/lib/tools/test/cover_SUITE_data/b.erl index 13f39b8cb9..0a418a58d8 100644 --- a/lib/tools/test/cover_SUITE_data/b.erl +++ b/lib/tools/test/cover_SUITE_data/b.erl @@ -1,5 +1,5 @@ -module(b). --export([start/0, loop/0]). +-export([start/0, loop/0, wait/0]). start() -> spawn(?MODULE, loop, []). @@ -12,3 +12,9 @@ loop() -> stop -> done end. + +%% This checks for a bug in expressions which have no +%% "main" clauses (only after and friends) followed by +%% a return value in the same line. +wait() -> + receive after 1000 -> done end, ok. diff --git a/lib/wx/aclocal.m4 b/lib/wx/aclocal.m4 index ed492d55ff..d78025b0be 100644 --- a/lib/wx/aclocal.m4 +++ b/lib/wx/aclocal.m4 @@ -1421,9 +1421,31 @@ case "$THR_LIB_NAME" in int z; AO_nop_full(); +#if defined(AO_HAVE_store) AO_store(&x, (AO_t) 0); +#elif defined(AO_HAVE_store_release) + AO_store_release(&x, (AO_t) 0); +#else +#error No store +#endif +#if defined(AO_HAVE_load) z = AO_load(&x); +#elif defined(AO_HAVE_load_acquire) + z = AO_load_acquire(&x); +#else +#error No load +#endif +#if defined(AO_HAVE_compare_and_swap_full) z = AO_compare_and_swap_full(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_release) + z = AO_compare_and_swap_release(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap_acquire) + z = AO_compare_and_swap_acquire(&x, (AO_t) 0, (AO_t) 1); +#elif defined(AO_HAVE_compare_and_swap) + z = AO_compare_and_swap(&x, (AO_t) 0, (AO_t) 1); +#else +#error No compare_and_swap +#endif ], [ethr_have_native_atomics=yes ethr_have_libatomic_ops=yes]) diff --git a/system/doc/reference_manual/typespec.xml b/system/doc/reference_manual/typespec.xml index 321f0a24d5..e4aa2ceda6 100644 --- a/system/doc/reference_manual/typespec.xml +++ b/system/doc/reference_manual/typespec.xml @@ -217,6 +217,9 @@ <cell><c>iolist()</c></cell><cell><c>maybe_improper_list(byte() | binary() | iolist(), binary() | [])</c></cell> </row> <row> + <cell><c>function()</c></cell><cell><c>fun()</c></cell> + </row> + <row> <cell><c>module()</c></cell><cell><c>atom()</c></cell> </row> <row> |