erts: Add new module 'counters'

6 files changed, 340 insertions, 1 deletions

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index 7fa29a7f2e..57a9d45887 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -649,6 +649,7 @@ PRELOAD_BEAM =	$(ERL_TOP)/erts/preloaded/ebin/otp_ring0.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/erts_literal_area_collector.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/erts_dirty_process_signal_handler.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/atomics.beam \
+		$(ERL_TOP)/erts/preloaded/ebin/counters.beam \
 		$(ERL_TOP)/erts/preloaded/ebin/persistent_term.beam
 
 ifeq ($(TARGET),win32)
@@ -842,7 +843,7 @@ RUN_OBJS += \
 	$(OBJDIR)/erl_bif_info.o	$(OBJDIR)/erl_bif_op.o \
 	$(OBJDIR)/erl_bif_os.o		$(OBJDIR)/erl_bif_lists.o \
 	$(OBJDIR)/erl_bif_persistent.o \
-	$(OBJDIR)/erl_bif_atomics.o \
+	$(OBJDIR)/erl_bif_atomics.o    	$(OBJDIR)/erl_bif_counters.o \
 	$(OBJDIR)/erl_bif_trace.o	$(OBJDIR)/erl_bif_unique.o \
 	$(OBJDIR)/erl_bif_wrap.o	$(OBJDIR)/erl_nfunc_sched.o \
 	$(OBJDIR)/erl_guard_bifs.o 	$(OBJDIR)/erl_dirty_bif_wrap.o \
diff --git a/erts/emulator/beam/bif.tab b/erts/emulator/beam/bif.tab
index 0577c5722f..aa3c3acd9f 100644
--- a/erts/emulator/beam/bif.tab
+++ b/erts/emulator/beam/bif.tab
@@ -719,3 +719,8 @@ bif atomics:add_get/3
 bif atomics:exchange/3
 bif atomics:compare_exchange/4
 bif atomics:info/1
+
+bif erts_internal:counters_new/1
+bif erts_internal:counters_get/2
+bif erts_internal:counters_add/3
+bif erts_internal:counters_info/1
diff --git a/erts/emulator/beam/erl_alloc.types b/erts/emulator/beam/erl_alloc.types
index aefa04a031..4f03a34390 100644
--- a/erts/emulator/beam/erl_alloc.types
+++ b/erts/emulator/beam/erl_alloc.types
@@ -339,6 +339,7 @@ type	PORT_DATA_HEAP	STANDARD	SYSTEM		port_data_heap
 type    MSACC           DRIVER          SYSTEM          microstate_accounting
 type	SYS_CHECK_REQ	SHORT_LIVED	SYSTEM		system_check_request
 type	ATOMICS		STANDARD	SYSTEM		erl_bif_atomics
+type	COUNTERS	STANDARD	SYSTEM		erl_bif_counters
 
 #
 # Types used by system specific code
diff --git a/erts/emulator/beam/erl_bif_counters.c b/erts/emulator/beam/erl_bif_counters.c
new file mode 100644
index 0000000000..a46b462225
--- /dev/null
+++ b/erts/emulator/beam/erl_bif_counters.c
@@ -0,0 +1,219 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2018. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/*
+ * Purpose:  High performance atomics.
+ */
+
+#ifdef HAVE_CONFIG_H
+#  include "config.h"
+#endif
+
+#include <stddef.h> /* offsetof */
+
+#include "sys.h"
+#include "export.h"
+#include "bif.h"
+#include "erl_threads.h"
+#include "big.h"
+#include "erl_binary.h"
+#include "erl_bif_unique.h"
+#include "erl_map.h"
+
+
+#define COUNTERS_PER_CACHE_LINE (ERTS_CACHE_LINE_SIZE / sizeof(erts_atomic64_t))
+
+typedef struct
+{
+    UWord arity;
+#ifdef DEBUG
+    UWord ulen;
+#endif
+    union {
+        erts_atomic64_t v[COUNTERS_PER_CACHE_LINE];
+        byte cache_line__[ERTS_CACHE_LINE_SIZE];
+    } u[1];
+}CountersRef;
+
+static int counters_destructor(Binary *unused)
+{
+    return 1;
+}
+
+
+static UWord ERTS_INLINE div_ceil(UWord dividend, UWord divisor)
+{
+    return (dividend + divisor - 1) / divisor;
+}
+
+BIF_RETTYPE erts_internal_counters_new_1(BIF_ALIST_1)
+{
+    CountersRef* p;
+    Binary* mbin;
+    UWord ui, vi, cnt;
+    Uint bytes, cache_lines;
+    Eterm* hp;
+
+    if (!term_to_UWord(BIF_ARG_1, &cnt)
+        || cnt == 0) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+
+    if (cnt > (ERTS_UWORD_MAX / (sizeof(erts_atomic64_t)*2*erts_no_schedulers)))
+        BIF_ERROR(BIF_P, SYSTEM_LIMIT);
+
+    cache_lines = erts_no_schedulers * div_ceil(cnt, COUNTERS_PER_CACHE_LINE);
+    bytes = offsetof(CountersRef, u) + cache_lines * ERTS_CACHE_LINE_SIZE;
+    mbin = erts_create_magic_binary_x(bytes,
+                                      counters_destructor,
+                                      ERTS_ALC_T_ATOMICS,
+                                      0);
+    p = ERTS_MAGIC_BIN_DATA(mbin);
+    p->arity = cnt;
+#ifdef DEBUG
+    p->ulen = cache_lines;
+#endif
+    ASSERT((byte*)&p->u[cache_lines] <= ((byte*)p + bytes));
+    for (ui=0; ui < cache_lines; ui++)
+        for (vi=0; vi < COUNTERS_PER_CACHE_LINE; vi++)
+            erts_atomic64_init_nob(&p->u[ui].v[vi], 0);
+    hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE);
+    return erts_mk_magic_ref(&hp, &MSO(BIF_P), mbin);
+}
+
+static ERTS_INLINE int get_ref(Eterm ref, CountersRef** pp)
+{
+    Binary* mbin;
+    if (!is_internal_magic_ref(ref))
+        return 0;
+
+    mbin = erts_magic_ref2bin(ref);
+    if (ERTS_MAGIC_BIN_DESTRUCTOR(mbin) != counters_destructor)
+        return 0;
+    *pp = ERTS_MAGIC_BIN_DATA(mbin);
+    return 1;
+}
+
+static ERTS_INLINE int get_ref_cnt(Eterm ref, Eterm index,
+                                   CountersRef** pp,
+                                   erts_atomic64_t** app,
+                                   UWord sched_ix)
+{
+    CountersRef* p;
+    UWord ix, ui, vi;
+    if (!get_ref(ref, &p) || !term_to_UWord(index, &ix) || --ix >= p->arity)
+        return 0;
+    ui = (ix / COUNTERS_PER_CACHE_LINE) * erts_no_schedulers + sched_ix;
+    vi = ix % COUNTERS_PER_CACHE_LINE;
+    ASSERT(ui < p->ulen);
+    *pp = p;
+    *app = &p->u[ui].v[vi];
+    return 1;
+}
+
+static ERTS_INLINE int get_ref_my_cnt(Eterm ref, Eterm index,
+                                      CountersRef** pp,
+                                      erts_atomic64_t** app)
+{
+    ErtsSchedulerData *esdp = erts_get_scheduler_data();
+    ASSERT(esdp && !ERTS_SCHEDULER_IS_DIRTY(esdp));
+    return get_ref_cnt(ref, index, pp, app, esdp->no - 1);
+}
+
+static ERTS_INLINE int get_ref_first_cnt(Eterm ref, Eterm index,
+                                         CountersRef** pp,
+                                         erts_atomic64_t** app)
+{
+    return get_ref_cnt(ref, index, pp, app, 0);
+}
+
+static ERTS_INLINE int get_incr(CountersRef* p, Eterm term, erts_aint64_t *valp)
+{
+    return (term_to_Sint64(term, (Sint64*)valp)
+            || term_to_Uint64(term, (Uint64*)valp));
+}
+
+static ERTS_INLINE Eterm bld_counter(Process* proc, CountersRef* p,
+                                     erts_aint64_t val)
+{
+    if (IS_SSMALL(val))
+        return make_small((Sint) val);
+    else {
+        Uint hsz = ERTS_SINT64_HEAP_SIZE(val);
+        Eterm* hp = HAlloc(proc, hsz);
+        return erts_sint64_to_big(val, &hp);
+    }
+}
+
+BIF_RETTYPE erts_internal_counters_get_2(BIF_ALIST_2)
+{
+    CountersRef* p;
+    erts_atomic64_t* ap;
+    erts_aint64_t acc = 0;
+    int j;
+
+    if (!get_ref_first_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    for (j = erts_no_schedulers; j ; --j) {
+        acc += erts_atomic64_read_nob(ap);
+        ap = (erts_atomic64_t*) ((byte*)ap + ERTS_CACHE_LINE_SIZE);
+    }
+    return bld_counter(BIF_P, p, acc);
+}
+
+BIF_RETTYPE erts_internal_counters_add_3(BIF_ALIST_3)
+{
+    CountersRef* p;
+    erts_atomic64_t* ap;
+    erts_aint64_t incr, sum;
+
+    if (!get_ref_my_cnt(BIF_ARG_1, BIF_ARG_2, &p, &ap)
+        || !get_incr(p, BIF_ARG_3, &incr)) {
+        BIF_ERROR(BIF_P, BADARG);
+    }
+    sum = incr + erts_atomic64_read_nob(ap);
+    erts_atomic64_set_nob(ap, sum);
+    return am_ok;
+}
+
+
+BIF_RETTYPE erts_internal_counters_info_1(BIF_ALIST_1)
+{
+    CountersRef* p;
+    Uint hsz = MAP2_SZ;
+    Eterm *hp;
+    UWord memory;
+    Eterm sz_val, mem_val;
+
+    if (!get_ref(BIF_ARG_1, &p))
+        BIF_ERROR(BIF_P, BADARG);
+
+    memory = erts_magic_ref2bin(BIF_ARG_1)->orig_size;
+    erts_bld_uword(NULL, &hsz, p->arity);
+    erts_bld_uword(NULL, &hsz, memory);
+
+    hp = HAlloc(BIF_P, hsz);
+    sz_val  = erts_bld_uword(&hp, NULL, p->arity);
+    mem_val = erts_bld_uword(&hp, NULL, memory);
+
+    return MAP2(hp, am_memory, mem_val,
+                am_size, sz_val);
+}
diff --git a/erts/emulator/test/Makefile b/erts/emulator/test/Makefile
index fc395899b8..6a064ec8d4 100644
--- a/erts/emulator/test/Makefile
+++ b/erts/emulator/test/Makefile
@@ -51,6 +51,7 @@ MODULES= \
 	call_trace_SUITE \
 	code_SUITE \
 	code_parallel_load_SUITE \
+	counters_SUITE \
 	crypto_SUITE \
 	ddll_SUITE \
 	decode_packet_SUITE \
diff --git a/erts/emulator/test/counters_SUITE.erl b/erts/emulator/test/counters_SUITE.erl
new file mode 100644
index 0000000000..7de164096b
--- /dev/null
+++ b/erts/emulator/test/counters_SUITE.erl
@@ -0,0 +1,112 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2018. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+-module(counters_SUITE).
+
+-include_lib("common_test/include/ct.hrl").
+
+-compile(export_all).
+
+suite() -> [{ct_hooks,[ts_install_cth]}].
+
+all() ->
+    [basic, bad, limits].
+
+basic(Config) when is_list(Config) ->
+    Size = 10,
+    [begin
+         Ref = counters:new(Size,[Type]),
+         #{size:=Size, memory:=Memory} = counters:info(Ref),
+         check_memory(Type, Memory, Size),
+         [basic_do(Ref, Ix) || Ix <- lists:seq(1, Size)]
+     end
+     || Type <- [atomics, write_concurrency]],
+    ok.
+
+basic_do(Ref, Ix) ->
+    0 = counters:get(Ref, Ix),
+    ok = counters:add(Ref, Ix, 3),
+    3  = counters:get(Ref, Ix),
+    ok = counters:add(Ref, Ix, 14),
+    17  = counters:get(Ref, Ix),
+    ok = counters:add(Ref, Ix, -20),
+    -3  = counters:get(Ref, Ix),
+    ok = counters:add(Ref, Ix, 100),
+    97 = counters:get(Ref, Ix),
+    ok = counters:sub(Ref, Ix, 20),
+    77 = counters:get(Ref, Ix),
+    ok = counters:sub(Ref, Ix, -10),
+    87 = counters:get(Ref, Ix),
+    ok.
+
+check_memory(atomics, Memory, Size) ->
+    {_,true} = {Memory, Memory > Size*8},
+    {_,true} = {Memory, Memory < Size*max_atomic_sz() + 100};
+check_memory(write_concurrency, Memory, Size) ->
+    NScheds = erlang:system_info(schedulers),
+    {_,true} = {Memory, Memory > NScheds*Size*8},
+    {_,true} = {Memory, Memory < NScheds*(Size+7)*max_atomic_sz() + 100}.
+
+max_atomic_sz() ->
+    case erlang:system_info({wordsize, external}) of
+        4 -> 16;
+        8 ->
+            EI = erlang:system_info(ethread_info),
+            case lists:keyfind("64-bit native atomics", 1, EI) of
+                {_, "no", _} -> 16;
+                _ -> 8
+            end
+    end.
+
+bad(Config) when is_list(Config) ->
+    {'EXIT',{badarg,_}} = (catch counters:new(0,[])),
+    {'EXIT',{badarg,_}} = (catch counters:new(10,[bad])),
+    {'EXIT',{badarg,_}} = (catch counters:new(10,[atomic, bad])),
+    {'EXIT',{badarg,_}} = (catch counters:new(10,[write_concurrency | bad])),
+    Ref = counters:new(10,[]),
+    {'EXIT',{badarg,_}} = (catch counters:get(1742, 7)),
+    {'EXIT',{badarg,_}} = (catch counters:get(make_ref(), 7)),
+    {'EXIT',{badarg,_}} = (catch counters:get(Ref, -1)),
+    {'EXIT',{badarg,_}} = (catch counters:get(Ref, 0)),
+    {'EXIT',{badarg,_}} = (catch counters:get(Ref, 11)),
+    {'EXIT',{badarg,_}} = (catch counters:get(Ref, 7.0)),
+    ok.
+
+
+limits(Config) when is_list(Config) ->
+    Bits = 64,
+    Max = (1 bsl (Bits-1)) - 1,
+    Min = -(1 bsl (Bits-1)),
+
+    Ref = counters:new(1,[]),
+    0 = counters:get(Ref, 1),
+    ok = counters:add(Ref, 1, Max),
+    ok = counters:add(Ref, 1, 1),
+    Min = counters:get(Ref, 1),
+    ok  = counters:sub(Ref, 1, 1),
+    Max = counters:get(Ref, 1),
+
+    IncrMax = (Max bsl 1) bor 1,
+    ok = counters:sub(Ref, 1, counters:get(Ref, 1)),
+    ok = counters:add(Ref, 1, IncrMax),
+    -1 = counters:get(Ref, 1),
+    {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, IncrMax+1)),
+    {'EXIT',{badarg,_}} = (catch counters:add(Ref, 1, Min-1)),
+
+    ok.