From 1315c6457e49595fdd3f91693c0506964416c9f0 Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Thu, 15 Nov 2018 15:24:32 +0100 Subject: erts: Add new module 'atomics' --- erts/emulator/Makefile.in | 2 + erts/emulator/beam/atom.names | 1 + erts/emulator/beam/bif.tab | 9 ++ erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/erl_bif_atomics.c | 256 +++++++++++++++++++++++++++++++++++ erts/emulator/beam/sys.h | 3 + erts/emulator/test/Makefile | 1 + erts/emulator/test/atomics_SUITE.erl | 147 ++++++++++++++++++++ 8 files changed, 420 insertions(+) create mode 100644 erts/emulator/beam/erl_bif_atomics.c create mode 100644 erts/emulator/test/atomics_SUITE.erl (limited to 'erts/emulator') diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index d5ceb4b00b..7fa29a7f2e 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -648,6 +648,7 @@ PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erl_tracer.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam \ + $(ERL_TOP)/erts/preloaded/ebin/atomics.beam \ $(ERL_TOP)/erts/preloaded/ebin/persistent_term.beam ifeq ($(TARGET),win32) @@ -841,6 +842,7 @@ RUN_OBJS += \ $(OBJDIR)/erl_bif_info.o $(OBJDIR)/erl_bif_op.o \ $(OBJDIR)/erl_bif_os.o $(OBJDIR)/erl_bif_lists.o \ $(OBJDIR)/erl_bif_persistent.o \ + $(OBJDIR)/erl_bif_atomics.o \ $(OBJDIR)/erl_bif_trace.o $(OBJDIR)/erl_bif_unique.o \ $(OBJDIR)/erl_bif_wrap.o $(OBJDIR)/erl_nfunc_sched.o \ $(OBJDIR)/erl_guard_bifs.o $(OBJDIR)/erl_dirty_bif_wrap.o \ diff --git a/erts/emulator/beam/atom.names b/erts/emulator/beam/atom.names index 82bb620d62..a14f22b19e 100644 --- a/erts/emulator/beam/atom.names +++ b/erts/emulator/beam/atom.names @@ -396,6 +396,7 @@ atom microsecond atom microstate_accounting atom milli_seconds atom millisecond +atom min atom min_heap_size atom min_bin_vheap_size atom minor diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index f1f34ae323..0577c5722f 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -710,3 +710,12 @@ bif persistent_term:get/0 bif persistent_term:erase/1 bif persistent_term:info/0 bif erts_internal:erase_persistent_terms/0 + +bif erts_internal:atomics_new/2 +bif atomics:get/2 +bif atomics:put/3 +bif atomics:add/3 +bif atomics:add_get/3 +bif atomics:exchange/3 +bif atomics:compare_exchange/4 +bif atomics:info/1 diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index 3fc036a13c..aefa04a031 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -338,6 +338,7 @@ type GC_INFO_REQ SHORT_LIVED SYSTEM gc_info_request type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap type MSACC DRIVER SYSTEM microstate_accounting type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request +type ATOMICS STANDARD SYSTEM erl_bif_atomics # # Types used by system specific code diff --git a/erts/emulator/beam/erl_bif_atomics.c b/erts/emulator/beam/erl_bif_atomics.c new file mode 100644 index 0000000000..092dbb3bd3 --- /dev/null +++ b/erts/emulator/beam/erl_bif_atomics.c @@ -0,0 +1,256 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Purpose: High performance atomics. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include /* offsetof */ + +#include "sys.h" +#include "export.h" +#include "bif.h" +#include "erl_threads.h" +#include "big.h" +#include "erl_binary.h" +#include "erl_bif_unique.h" +#include "erl_map.h" + +typedef struct +{ + int is_signed; + UWord vlen; + erts_atomic64_t v[1]; +}AtomicsRef; + +static int atomics_destructor(Binary *unused) +{ + return 1; +} + +#define OPT_SIGNED (1 << 0) + +BIF_RETTYPE erts_internal_atomics_new_2(BIF_ALIST_2) +{ + AtomicsRef* p; + Binary* mbin; + UWord i, cnt, opts; + Uint bytes; + Eterm* hp; + + if (!term_to_UWord(BIF_ARG_1, &cnt) + || cnt == 0 + || !term_to_UWord(BIF_ARG_2, &opts)) { + + BIF_ERROR(BIF_P, BADARG); + } + + if (cnt > (ERTS_UWORD_MAX / sizeof(p->v[0]))) + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + + bytes = offsetof(AtomicsRef, v) + cnt*sizeof(p->v[0]); + mbin = erts_create_magic_binary_x(bytes, + atomics_destructor, + ERTS_ALC_T_ATOMICS, + 0); + p = ERTS_MAGIC_BIN_DATA(mbin); + p->is_signed = opts & OPT_SIGNED; + p->vlen = cnt; + for (i=0; i < cnt; i++) + erts_atomic64_init_nob(&p->v[i], 0); + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin); +} + +static ERTS_INLINE int get_ref(Eterm ref, AtomicsRef** pp) +{ + Binary* mbin; + if (!is_internal_magic_ref(ref)) + return 0; + + mbin = erts_magic_ref2bin(ref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != atomics_destructor) + return 0; + *pp = ERTS_MAGIC_BIN_DATA(mbin); + return 1; +} + +static ERTS_INLINE int get_ref_ix(Eterm ref, Eterm ix, + AtomicsRef** pp, UWord* ixp) +{ + return (get_ref(ref, pp) + && term_to_UWord(ix, ixp) + && --(*ixp) < (*pp)->vlen); +} + +static ERTS_INLINE int get_value(AtomicsRef* p, Eterm term, erts_aint64_t *valp) +{ + return (p->is_signed ? + term_to_Sint64(term, (Sint64*)valp) : + term_to_Uint64(term, (Uint64*)valp)); +} + +static ERTS_INLINE int get_incr(AtomicsRef* p, Eterm term, erts_aint64_t *valp) +{ + return (term_to_Sint64(term, (Sint64*)valp) + || term_to_Uint64(term, (Uint64*)valp)); +} + +static ERTS_INLINE Eterm bld_atomic(Process* proc, AtomicsRef* p, + erts_aint64_t val) +{ + if (p->is_signed) { + if (IS_SSMALL(val)) + return make_small((Sint) val); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(val); + Eterm* hp = HAlloc(proc, hsz); + return erts_sint64_to_big(val, &hp); + } + } + else { + if ((Uint64)val <= MAX_SMALL) + return make_small((Sint) val); + else { + Uint hsz = ERTS_UINT64_HEAP_SIZE(val); + Eterm* hp = HAlloc(proc, hsz); + return erts_uint64_to_big(val, &hp); + } + } +} + +BIF_RETTYPE atomics_put_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t val; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_value(p, BIF_ARG_3, &val)) { + BIF_ERROR(BIF_P, BADARG); + } + erts_atomic64_set_mb(&p->v[ix], val); + return am_ok; +} + +BIF_RETTYPE atomics_get_2(BIF_ALIST_2) +{ + AtomicsRef* p; + UWord ix; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix)) { + BIF_ERROR(BIF_P, BADARG); + } + return bld_atomic(BIF_P, p, erts_atomic64_read_mb(&p->v[ix])); +} + +BIF_RETTYPE atomics_add_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t incr; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_incr(p, BIF_ARG_3, &incr)) { + BIF_ERROR(BIF_P, BADARG); + } + erts_atomic64_add_mb(&p->v[ix], incr); + return am_ok; +} + +BIF_RETTYPE atomics_add_get_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t incr; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_incr(p, BIF_ARG_3, &incr)) { + BIF_ERROR(BIF_P, BADARG); + } + return bld_atomic(BIF_P, p, erts_atomic64_add_read_mb(&p->v[ix], incr)); +} + +BIF_RETTYPE atomics_exchange_3(BIF_ALIST_3) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t desired, was; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_value(p, BIF_ARG_3, &desired)) { + BIF_ERROR(BIF_P, BADARG); + } + was = erts_atomic64_xchg_mb(&p->v[ix], desired); + return bld_atomic(BIF_P, p, was); +} + +BIF_RETTYPE atomics_compare_exchange_4(BIF_ALIST_4) +{ + AtomicsRef* p; + UWord ix; + erts_aint64_t expected, desired, was; + + if (!get_ref_ix(BIF_ARG_1, BIF_ARG_2, &p, &ix) + || !get_value(p, BIF_ARG_3, &expected) + || !get_value(p, BIF_ARG_4, &desired)) { + BIF_ERROR(BIF_P, BADARG); + } + was = erts_atomic64_cmpxchg_mb(&p->v[ix], desired, expected); + return was == expected ? am_ok : bld_atomic(BIF_P, p, was); +} + +BIF_RETTYPE atomics_info_1(BIF_ALIST_1) +{ + AtomicsRef* p; + Uint hsz = MAP4_SZ; + Eterm *hp; + Uint64 max; + Sint64 min; + UWord memory; + Eterm max_val, min_val, sz_val, mem_val; + + if (!get_ref(BIF_ARG_1, &p)) + BIF_ERROR(BIF_P, BADARG); + + max = p->is_signed ? ERTS_SINT64_MAX : ERTS_UINT64_MAX; + min = p->is_signed ? ERTS_SINT64_MIN : 0; + memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size; + + erts_bld_uint64(NULL, &hsz, max); + erts_bld_sint64(NULL, &hsz, min); + erts_bld_uword(NULL, &hsz, p->vlen); + erts_bld_uword(NULL, &hsz, memory); + + hp = HAlloc(BIF_P, hsz); + max_val = erts_bld_uint64(&hp, NULL, max); + min_val = erts_bld_sint64(&hp, NULL, min); + sz_val = erts_bld_uword(&hp, NULL, p->vlen); + mem_val = erts_bld_uword(&hp, NULL, memory); + + return MAP4(hp, am_max, max_val, + am_memory, mem_val, + am_min, min_val, + am_size, sz_val); +} diff --git a/erts/emulator/beam/sys.h b/erts/emulator/beam/sys.h index bb22548587..869a575cb4 100644 --- a/erts/emulator/beam/sys.h +++ b/erts/emulator/beam/sys.h @@ -325,6 +325,7 @@ typedef long Sint erts_align_attribute(sizeof(long)); #define UWORD_CONSTANT(Const) Const##UL #define ERTS_UWORD_MAX ULONG_MAX #define ERTS_SWORD_MAX LONG_MAX +#define ERTS_SWORD_MIN LONG_MIN #define ERTS_SIZEOF_ETERM SIZEOF_LONG #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_INT @@ -335,6 +336,7 @@ typedef int Sint erts_align_attribute(sizeof(int)); #define UWORD_CONSTANT(Const) Const##U #define ERTS_UWORD_MAX UINT_MAX #define ERTS_SWORD_MAX INT_MAX +#define ERTS_SWORD_MIN INT_MIN #define ERTS_SIZEOF_ETERM SIZEOF_INT #define ErtsStrToSint strtol #elif SIZEOF_VOID_P == SIZEOF_LONG_LONG @@ -345,6 +347,7 @@ typedef long long Sint erts_align_attribute(sizeof(long long)); #define UWORD_CONSTANT(Const) Const##ULL #define ERTS_UWORD_MAX ULLONG_MAX #define ERTS_SWORD_MAX LLONG_MAX +#define ERTS_SWORD_MIN LLONG_MIN #define ERTS_SIZEOF_ETERM SIZEOF_LONG_LONG #if defined(__WIN32__) #define ErtsStrToSint _strtoi64 diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index d201ea6ee1..fc395899b8 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -33,6 +33,7 @@ MODULES= \ after_SUITE \ alloc_SUITE \ async_ports_SUITE \ + atomics_SUITE \ beam_SUITE \ beam_literals_SUITE \ bif_SUITE \ diff --git a/erts/emulator/test/atomics_SUITE.erl b/erts/emulator/test/atomics_SUITE.erl new file mode 100644 index 0000000000..8c42354770 --- /dev/null +++ b/erts/emulator/test/atomics_SUITE.erl @@ -0,0 +1,147 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(atomics_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [signed, unsigned, bad, signed_limits, unsigned_limits]. + +signed(Config) when is_list(Config) -> + Size = 10, + Ref = atomics:new(Size,[]), + #{size:=Size, memory:=Memory} = atomics:info(Ref), + {_,true} = {Memory, Memory > Size*8}, + {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100}, + [signed_do(Ref, Ix) || Ix <- lists:seq(1, Size)], + ok. + +signed_do(Ref, Ix) -> + 0 = atomics:get(Ref, Ix), + ok = atomics:put(Ref, Ix, 3), + ok = atomics:add(Ref, Ix, 14), + 17 = atomics:get(Ref, Ix), + 20 = atomics:add_get(Ref, Ix, 3), + -3 = atomics:add_get(Ref, Ix, -23), + 17 = atomics:add_get(Ref, Ix, 20), + ok = atomics:sub(Ref, Ix, 4), + 13 = atomics:get(Ref, Ix), + -7 = atomics:sub_get(Ref, Ix, 20), + 3 = atomics:sub_get(Ref, Ix, -10), + 3 = atomics:exchange(Ref, Ix, 666), + ok = atomics:compare_exchange(Ref, Ix, 666, 777), + 777 = atomics:compare_exchange(Ref, Ix, 666, -666), + ok. + +unsigned(Config) when is_list(Config) -> + Size = 10, + Ref = atomics:new(Size,[{signed, false}]), + #{size:=Size, memory:=Memory} = atomics:info(Ref), + true = Memory > Size*8, + true = Memory < Size*max_atomic_sz() + 100, + [unsigned_do(Ref, Ix) || Ix <- lists:seq(1, Size)], + ok. + +unsigned_do(Ref, Ix) -> + 0 = atomics:get(Ref, Ix), + ok = atomics:put(Ref, Ix, 3), + ok = atomics:add(Ref, Ix, 14), + 17 = atomics:get(Ref, Ix), + 20 = atomics:add_get(Ref, Ix, 3), + ok = atomics:sub(Ref, Ix, 7), + 13 = atomics:get(Ref, Ix), + 3 = atomics:sub_get(Ref, Ix, 10), + 3 = atomics:exchange(Ref, Ix, 666), + ok = atomics:compare_exchange(Ref, Ix, 666, 777), + 777 = atomics:compare_exchange(Ref, Ix, 666, 888), + ok. + +bad(Config) when is_list(Config) -> + {'EXIT',{badarg,_}} = (catch atomics:new(0,[])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[bad])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,bad}])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,true}, bad])), + {'EXIT',{badarg,_}} = (catch atomics:new(10,[{signed,false} | bad])), + Ref = atomics:new(10,[]), + {'EXIT',{badarg,_}} = (catch atomics:get(1742, 7)), + {'EXIT',{badarg,_}} = (catch atomics:get(make_ref(), 7)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, -1)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 0)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 11)), + {'EXIT',{badarg,_}} = (catch atomics:get(Ref, 7.0)), + ok. + + +signed_limits(Config) when is_list(Config) -> + Bits = 64, + Max = (1 bsl (Bits-1)) - 1, + Min = -(1 bsl (Bits-1)), + + Ref = atomics:new(1,[{signed, true}]), + #{max:=Max, min:=Min} = atomics:info(Ref), + 0 = atomics:get(Ref, 1), + ok = atomics:add(Ref, 1, Max), + Min = atomics:add_get(Ref, 1, 1), + Max = atomics:sub_get(Ref, 1, 1), + + IncrMax = (Max bsl 1) bor 1, + ok = atomics:put(Ref, 1, 0), + ok = atomics:add(Ref, 1, IncrMax), + -1 = atomics:get(Ref, 1), + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, IncrMax+1)), + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, Min-1)), + + ok. + +unsigned_limits(Config) when is_list(Config) -> + Bits = 64, + Max = (1 bsl Bits) - 1, + Min = 0, + + Ref = atomics:new(1,[{signed,false}]), + #{max:=Max, min:=Min} = atomics:info(Ref), + 0 = atomics:get(Ref, 1), + ok = atomics:add(Ref, 1, Max), + Min = atomics:add_get(Ref, 1, 1), + Max = atomics:sub_get(Ref, 1, 1), + + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, Max+1)), + IncrMin = -(1 bsl (Bits-1)), + ok = atomics:put(Ref, 1, -IncrMin), + ok = atomics:add(Ref, 1, IncrMin), + 0 = atomics:get(Ref, 1), + {'EXIT',{badarg,_}} = (catch atomics:add(Ref, 1, IncrMin-1)), + + ok. + +max_atomic_sz() -> + case erlang:system_info({wordsize, external}) of + 4 -> 16; + 8 -> + EI = erlang:system_info(ethread_info), + case lists:keyfind("64-bit native atomics", 1, EI) of + {_, "no", _} -> 16; + _ -> 8 + end + end. -- cgit v1.2.3 From fefb5d039e87ff7137e78b3d5f2eaf01e498ec4d Mon Sep 17 00:00:00 2001 From: Sverker Eriksson Date: Tue, 25 Sep 2018 13:34:52 +0200 Subject: erts: Add new module 'counters' --- erts/emulator/Makefile.in | 3 +- erts/emulator/beam/bif.tab | 5 + erts/emulator/beam/erl_alloc.types | 1 + erts/emulator/beam/erl_bif_counters.c | 219 ++++++++++++++++++++++++++++++++++ erts/emulator/test/Makefile | 1 + erts/emulator/test/counters_SUITE.erl | 112 +++++++++++++++++ 6 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 erts/emulator/beam/erl_bif_counters.c create mode 100644 erts/emulator/test/counters_SUITE.erl (limited to 'erts/emulator') diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in index 7fa29a7f2e..57a9d45887 100644 --- a/erts/emulator/Makefile.in +++ b/erts/emulator/Makefile.in @@ -649,6 +649,7 @@ PRELOAD_BEAM = $(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \ $(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam \ $(ERL_TOP)/erts/preloaded/ebin/atomics.beam \ + $(ERL_TOP)/erts/preloaded/ebin/counters.beam \ $(ERL_TOP)/erts/preloaded/ebin/persistent_term.beam ifeq ($(TARGET),win32) @@ -842,7 +843,7 @@ RUN_OBJS += \ $(OBJDIR)/erl_bif_info.o $(OBJDIR)/erl_bif_op.o \ $(OBJDIR)/erl_bif_os.o $(OBJDIR)/erl_bif_lists.o \ $(OBJDIR)/erl_bif_persistent.o \ - $(OBJDIR)/erl_bif_atomics.o \ + $(OBJDIR)/erl_bif_atomics.o $(OBJDIR)/erl_bif_counters.o \ $(OBJDIR)/erl_bif_trace.o $(OBJDIR)/erl_bif_unique.o \ $(OBJDIR)/erl_bif_wrap.o $(OBJDIR)/erl_nfunc_sched.o \ $(OBJDIR)/erl_guard_bifs.o $(OBJDIR)/erl_dirty_bif_wrap.o \ diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab index 0577c5722f..aa3c3acd9f 100644 --- a/erts/emulator/beam/bif.tab +++ b/erts/emulator/beam/bif.tab @@ -719,3 +719,8 @@ bif atomics:add_get/3 bif atomics:exchange/3 bif atomics:compare_exchange/4 bif atomics:info/1 + +bif erts_internal:counters_new/1 +bif erts_internal:counters_get/2 +bif erts_internal:counters_add/3 +bif erts_internal:counters_info/1 diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types index aefa04a031..4f03a34390 100644 --- a/erts/emulator/beam/erl_alloc.types +++ b/erts/emulator/beam/erl_alloc.types @@ -339,6 +339,7 @@ type PORT_DATA_HEAP STANDARD SYSTEM port_data_heap type MSACC DRIVER SYSTEM microstate_accounting type SYS_CHECK_REQ SHORT_LIVED SYSTEM system_check_request type ATOMICS STANDARD SYSTEM erl_bif_atomics +type COUNTERS STANDARD SYSTEM erl_bif_counters # # Types used by system specific code diff --git a/erts/emulator/beam/erl_bif_counters.c b/erts/emulator/beam/erl_bif_counters.c new file mode 100644 index 0000000000..a46b462225 --- /dev/null +++ b/erts/emulator/beam/erl_bif_counters.c @@ -0,0 +1,219 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2018. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +/* + * Purpose: High performance atomics. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include /* offsetof */ + +#include "sys.h" +#include "export.h" +#include "bif.h" +#include "erl_threads.h" +#include "big.h" +#include "erl_binary.h" +#include "erl_bif_unique.h" +#include "erl_map.h" + + +#define COUNTERS_PER_CACHE_LINE (ERTS_CACHE_LINE_SIZE / sizeof(erts_atomic64_t)) + +typedef struct +{ + UWord arity; +#ifdef DEBUG + UWord ulen; +#endif + union { + erts_atomic64_t v[COUNTERS_PER_CACHE_LINE]; + byte cache_line__[ERTS_CACHE_LINE_SIZE]; + } u[1]; +}CountersRef; + +static int counters_destructor(Binary *unused) +{ + return 1; +} + + +static UWord ERTS_INLINE div_ceil(UWord dividend, UWord divisor) +{ + return (dividend + divisor - 1) / divisor; +} + +BIF_RETTYPE erts_internal_counters_new_1(BIF_ALIST_1) +{ + CountersRef* p; + Binary* mbin; + UWord ui, vi, cnt; + Uint bytes, cache_lines; + Eterm* hp; + + if (!term_to_UWord(BIF_ARG_1, &cnt) + || cnt == 0) { + BIF_ERROR(BIF_P, BADARG); + } + + if (cnt > (ERTS_UWORD_MAX / (sizeof(erts_atomic64_t)*2*erts_no_schedulers))) + BIF_ERROR(BIF_P, SYSTEM_LIMIT); + + cache_lines = erts_no_schedulers * div_ceil(cnt, COUNTERS_PER_CACHE_LINE); + bytes = offsetof(CountersRef, u) + cache_lines * ERTS_CACHE_LINE_SIZE; + mbin = erts_create_magic_binary_x(bytes, + counters_destructor, + ERTS_ALC_T_ATOMICS, + 0); + p = ERTS_MAGIC_BIN_DATA(mbin); + p->arity = cnt; +#ifdef DEBUG + p->ulen = cache_lines; +#endif + ASSERT((byte*)&p->u[cache_lines] <= ((byte*)p + bytes)); + for (ui=0; ui < cache_lines; ui++) + for (vi=0; vi < COUNTERS_PER_CACHE_LINE; vi++) + erts_atomic64_init_nob(&p->u[ui].v[vi], 0); + hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE); + return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin); +} + +static ERTS_INLINE int get_ref(Eterm ref, CountersRef** pp) +{ + Binary* mbin; + if (!is_internal_magic_ref(ref)) + return 0; + + mbin = erts_magic_ref2bin(ref); + if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != counters_destructor) + return 0; + *pp = ERTS_MAGIC_BIN_DATA(mbin); + return 1; +} + +static ERTS_INLINE int get_ref_cnt(Eterm ref, Eterm index, + CountersRef** pp, + erts_atomic64_t** app, + UWord sched_ix) +{ + CountersRef* p; + UWord ix, ui, vi; + if (!get_ref(ref, &p) || !term_to_UWord(index, &ix) || --ix >= p->arity) + return 0; + ui = (ix / COUNTERS_PER_CACHE_LINE) * erts_no_schedulers + sched_ix; + vi = ix % COUNTERS_PER_CACHE_LINE; + ASSERT(ui < p->ulen); + *pp = p; + *app = &p->u[ui].v[vi]; + return 1; +} + +static ERTS_INLINE int get_ref_my_cnt(Eterm ref, Eterm index, + CountersRef** pp, + erts_atomic64_t** app) +{ + ErtsSchedulerData *esdp = erts_get_scheduler_data(); + ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp)); + return get_ref_cnt(ref, index, pp, app, esdp->no - 1); +} + +static ERTS_INLINE int get_ref_first_cnt(Eterm ref, Eterm index, + CountersRef** pp, + erts_atomic64_t** app) +{ + return get_ref_cnt(ref, index, pp, app, 0); +} + +static ERTS_INLINE int get_incr(CountersRef* p, Eterm term, erts_aint64_t *valp) +{ + return (term_to_Sint64(term, (Sint64*)valp) + || term_to_Uint64(term, (Uint64*)valp)); +} + +static ERTS_INLINE Eterm bld_counter(Process* proc, CountersRef* p, + erts_aint64_t val) +{ + if (IS_SSMALL(val)) + return make_small((Sint) val); + else { + Uint hsz = ERTS_SINT64_HEAP_SIZE(val); + Eterm* hp = HAlloc(proc, hsz); + return erts_sint64_to_big(val, &hp); + } +} + +BIF_RETTYPE erts_internal_counters_get_2(BIF_ALIST_2) +{ + CountersRef* p; + erts_atomic64_t* ap; + erts_aint64_t acc = 0; + int j; + + if (!get_ref_first_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap)) { + BIF_ERROR(BIF_P, BADARG); + } + for (j = erts_no_schedulers; j ; --j) { + acc += erts_atomic64_read_nob(ap); + ap = (erts_atomic64_t*) ((byte*)ap + ERTS_CACHE_LINE_SIZE); + } + return bld_counter(BIF_P, p, acc); +} + +BIF_RETTYPE erts_internal_counters_add_3(BIF_ALIST_3) +{ + CountersRef* p; + erts_atomic64_t* ap; + erts_aint64_t incr, sum; + + if (!get_ref_my_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap) + || !get_incr(p, BIF_ARG_3, &incr)) { + BIF_ERROR(BIF_P, BADARG); + } + sum = incr + erts_atomic64_read_nob(ap); + erts_atomic64_set_nob(ap, sum); + return am_ok; +} + + +BIF_RETTYPE erts_internal_counters_info_1(BIF_ALIST_1) +{ + CountersRef* p; + Uint hsz = MAP2_SZ; + Eterm *hp; + UWord memory; + Eterm sz_val, mem_val; + + if (!get_ref(BIF_ARG_1, &p)) + BIF_ERROR(BIF_P, BADARG); + + memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size; + erts_bld_uword(NULL, &hsz, p->arity); + erts_bld_uword(NULL, &hsz, memory); + + hp = HAlloc(BIF_P, hsz); + sz_val = erts_bld_uword(&hp, NULL, p->arity); + mem_val = erts_bld_uword(&hp, NULL, memory); + + return MAP2(hp, am_memory, mem_val, + am_size, sz_val); +} diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile index fc395899b8..6a064ec8d4 100644 --- a/erts/emulator/test/Makefile +++ b/erts/emulator/test/Makefile @@ -51,6 +51,7 @@ MODULES= \ call_trace_SUITE \ code_SUITE \ code_parallel_load_SUITE \ + counters_SUITE \ crypto_SUITE \ ddll_SUITE \ decode_packet_SUITE \ diff --git a/erts/emulator/test/counters_SUITE.erl b/erts/emulator/test/counters_SUITE.erl new file mode 100644 index 0000000000..7de164096b --- /dev/null +++ b/erts/emulator/test/counters_SUITE.erl @@ -0,0 +1,112 @@ +%% +%% %CopyrightBegin% +%% +%% Copyright Ericsson AB 2018. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% %CopyrightEnd% +%% +-module(counters_SUITE). + +-include_lib("common_test/include/ct.hrl"). + +-compile(export_all). + +suite() -> [{ct_hooks,[ts_install_cth]}]. + +all() -> + [basic, bad, limits]. + +basic(Config) when is_list(Config) -> + Size = 10, + [begin + Ref = counters:new(Size,[Type]), + #{size:=Size, memory:=Memory} = counters:info(Ref), + check_memory(Type, Memory, Size), + [basic_do(Ref, Ix) || Ix <- lists:seq(1, Size)] + end + || Type <- [atomics, write_concurrency]], + ok. + +basic_do(Ref, Ix) -> + 0 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, 3), + 3 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, 14), + 17 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, -20), + -3 = counters:get(Ref, Ix), + ok = counters:add(Ref, Ix, 100), + 97 = counters:get(Ref, Ix), + ok = counters:sub(Ref, Ix, 20), + 77 = counters:get(Ref, Ix), + ok = counters:sub(Ref, Ix, -10), + 87 = counters:get(Ref, Ix), + ok. + +check_memory(atomics, Memory, Size) -> + {_,true} = {Memory, Memory > Size*8}, + {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100}; +check_memory(write_concurrency, Memory, Size) -> + NScheds = erlang:system_info(schedulers), + {_,true} = {Memory, Memory > NScheds*Size*8}, + {_,true} = {Memory, Memory < NScheds*(Size+7)*max_atomic_sz() + 100}. + +max_atomic_sz() -> + case erlang:system_info({wordsize, external}) of + 4 -> 16; + 8 -> + EI = erlang:system_info(ethread_info), + case lists:keyfind("64-bit native atomics", 1, EI) of + {_, "no", _} -> 16; + _ -> 8 + end + end. + +bad(Config) when is_list(Config) -> + {'EXIT',{badarg,_}} = (catch counters:new(0,[])), + {'EXIT',{badarg,_}} = (catch counters:new(10,[bad])), + {'EXIT',{badarg,_}} = (catch counters:new(10,[atomic, bad])), + {'EXIT',{badarg,_}} = (catch counters:new(10,[write_concurrency | bad])), + Ref = counters:new(10,[]), + {'EXIT',{badarg,_}} = (catch counters:get(1742, 7)), + {'EXIT',{badarg,_}} = (catch counters:get(make_ref(), 7)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, -1)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, 0)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, 11)), + {'EXIT',{badarg,_}} = (catch counters:get(Ref, 7.0)), + ok. + + +limits(Config) when is_list(Config) -> + Bits = 64, + Max = (1 bsl (Bits-1)) - 1, + Min = -(1 bsl (Bits-1)), + + Ref = counters:new(1,[]), + 0 = counters:get(Ref, 1), + ok = counters:add(Ref, 1, Max), + ok = counters:add(Ref, 1, 1), + Min = counters:get(Ref, 1), + ok = counters:sub(Ref, 1, 1), + Max = counters:get(Ref, 1), + + IncrMax = (Max bsl 1) bor 1, + ok = counters:sub(Ref, 1, counters:get(Ref, 1)), + ok = counters:add(Ref, 1, IncrMax), + -1 = counters:get(Ref, 1), + {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, IncrMax+1)), + {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, Min-1)), + + ok. -- cgit v1.2.3